diff --git a/.github/workflows/approveAndMergeReleasePR.js b/.github/workflows/approveAndMergeReleasePR.js index 6f4435aaa..32a25a3ea 100644 --- a/.github/workflows/approveAndMergeReleasePR.js +++ b/.github/workflows/approveAndMergeReleasePR.js @@ -3,6 +3,7 @@ const { Command } = require("commander"); const { GraphQLClient } = require("graphql-request"); const program = new Command(); +const client = new GraphQLClient("https://dashboard.cypress.io/graphql"); const repo = "saleor-cloud-deployments"; const owner = "saleor"; @@ -32,12 +33,66 @@ program const commitId = pullRequest.data.merge_commit_sha; - const testsStatus = await getTestsStatus(options.dashboard_url); + const data = await getTestsStatusAndId(options.dashboard_url); + + let testsStatus = data.status; + + let requestBody = `Cypress tests passed. See results at ${options.dashboard_url}`; + + if (testsStatus === "FAILED") { + const failedNewTests = []; + const listOfTestIssues = await getListOfTestsIssues(octokit); + const testCases = await getFailedTestCases(data.runId); + testCases.forEach(testCase => { + if (testCase.titleParts) { + const issue = issueOnGithub(listOfTestIssues, testCase.titleParts[1]); + if (issue) { + const knownBug = isIssueAKnownBugForReleaseVersion( + issue, + options.version, + ); + if (!knownBug) { + failedNewTests.push({ + title: testCase.titleParts[1], + url: issue.url, + spec: testCase.titleParts[0], + }); + } + } else { + failedNewTests.push({ + title: testCase.titleParts[1], + spec: testCase.titleParts[0], + }); + } + } + }); + + if (failedNewTests.length === 0) { + requestBody = `All failed tests are known bugs, can be merged. See results at ${options.dashboard_url}`; + testsStatus = "PASSED"; + } else if (failedNewTests.length > 10) { + //If there are more than 10 new bugs it's probably caused by something else. Server responses with 500, or test user was deleted, etc. + + requestBody = + "There is more than 10 new bugs, check results manually and create issues for them if necessary"; + } else { + requestBody = `New bugs found, results at: ${options.dashboard_url}. List of issues to check: `; + for (const newBug of failedNewTests) { + if (!newBug.url) { + const issueUrl = await createIssue( + newBug, + options.version, + octokit, + ); + requestBody += `\n${newBug.title} - ${issueUrl}`; + } else { + requestBody += `\n${newBug.title} - ${newBug.url}`; + } + } + requestBody += `\nIf this bugs won't be fixed in next patch release for this version mark them as known issues`; + } + } - const requestBody = - testsStatus === "PASSED" - ? `Cypress tests passed. See results at ${options.dashboard_url}` - : `Some tests failed, need manual approve. See results at ${options.dashboard_url}`; const event = "COMMENT"; await octokit.request( @@ -75,7 +130,7 @@ function isPatchRelease(version) { return version.match(regex) ? true : false; } -async function getTestsStatus(dashboardUrl) { +async function getTestsStatusAndId(dashboardUrl) { const getProjectRegex = /\/projects\/([^\/]*)/; const getRunRegex = /\/runs\/([^\/]*)/; @@ -84,25 +139,24 @@ async function getTestsStatus(dashboardUrl) { buildNumber: dashboardUrl.match(getRunRegex)[1], }; - const client = new GraphQLClient("https://dashboard.cypress.io/graphql"); - const throwErrorAfterTimeout = setTimeout(function() { throw new Error("Run have still running status, after all tests executed"); }, 1200000); - const status = await waitForTestsToFinish(client, requestVariables); + const data = await waitForTestsToFinish(requestVariables); clearTimeout(throwErrorAfterTimeout); - return status; + return { status: data.status, runId: data.id }; } -async function waitForTestsToFinish(client, requestVariables) { +async function waitForTestsToFinish(requestVariables) { return new Promise((resolve, reject) => { client .request( `query ($projectId: String!, $buildNumber: ID!) { runByBuildNumber(buildNumber: $buildNumber, projectId: $projectId) { - status + status, + id } }`, requestVariables, @@ -113,8 +167,90 @@ async function waitForTestsToFinish(client, requestVariables) { resolve(await waitForTestsToFinish(client, requestVariables)); }, 10000); } else { - resolve(response.runByBuildNumber.status); + resolve(response.runByBuildNumber); } }); }); } + +async function getFailedTestCases(runId) { + const requestVariables = { + input: { + runId, + testResultState: ["FAILED"], + }, + }; + + return new Promise((resolve, reject) => { + client + .request( + `query RunTestResults($input: TestResultsTableInput!) { + testResults(input: $input) { + ... on TestResult { + ...RunTestResult + } + } + } + fragment RunTestResult on TestResult { id titleParts state}`, + requestVariables, + ) + .then(response => { + resolve(response.testResults); + }); + }); +} + +async function getListOfTestsIssues(octokit) { + const result = await octokit.request( + "GET /repos/{owner}/saleor-dashboard/issues?labels=tests", + { + owner, + }, + ); + return result.data; +} + +function issueOnGithub(listOfTestIssues, testCaseTitle) { + if (listOfTestIssues.length > 0) { + return listOfTestIssues.find(issue => { + return issue.title.includes(testCaseTitle); + }); + } +} + +function isIssueAKnownBugForReleaseVersion(issue, releaseVersion) { + const issueBody = issue.body; + const regex = /Known bug for versions:([\s\S]*)Additional/; + const lines = issueBody.match(regex)[1].split("\n"); + const lineContainReleaseVersionRegex = /v(\d{2,3}).*(true|false)/; + const releaseVersionLine = lines.find(line => { + if (line.match(lineContainReleaseVersionRegex)) { + const version = line.match(lineContainReleaseVersionRegex)[1]; + if (version === getFormattedVersion(releaseVersion)) { + return line; + } + } + }); + const knownBugOnReleaseVersion = releaseVersionLine + ? releaseVersionLine.match(lineContainReleaseVersionRegex)[2] + : false; + return knownBugOnReleaseVersion === "true" ? true : false; +} + +function getFormattedVersion(version) { + const regex = /^\d+\.\d+\./; + return version.match(regex)[0].replace(/\./g, ""); +} + +async function createIssue(newBug, version, octokit) { + const issue = await octokit.request("POST /repos/{owner}/{repo}/issues", { + owner, + repo: "saleor-dashboard", + title: `Cypress test fail: ${newBug.title}`, + body: `**Known bug for versions:**\nv${getFormattedVersion( + version, + )}: false\n**Additional Info:**\nSpec: ${newBug.spec}`, + labels: ["tests"], + }); + return issue.data.url; +}