Speed up releases (#2527)

* create issues on github, a add comment that can be merged if known bugs

* remove old lines for comment

.github/workflows/approveAndMergeReleasePR.js

@@ -3,6 +3,7 @@ const { Command } = require("commander");
 const { GraphQLClient } = require("graphql-request");
 
 const program = new Command();
+const client = new GraphQLClient("https://dashboard.cypress.io/graphql");
 
 const repo = "saleor-cloud-deployments";
 const owner = "saleor";
@@ -32,12 +33,66 @@ program
 
     const commitId = pullRequest.data.merge_commit_sha;
 
-    const testsStatus = await getTestsStatus(options.dashboard_url);
+    const data = await getTestsStatusAndId(options.dashboard_url);
+
+    let testsStatus = data.status;
+
+    let requestBody = `Cypress tests passed. See results at ${options.dashboard_url}`;
+
+    if (testsStatus === "FAILED") {
+      const failedNewTests = [];
+      const listOfTestIssues = await getListOfTestsIssues(octokit);
+      const testCases = await getFailedTestCases(data.runId);
+      testCases.forEach(testCase => {
+        if (testCase.titleParts) {
+          const issue = issueOnGithub(listOfTestIssues, testCase.titleParts[1]);
+          if (issue) {
+            const knownBug = isIssueAKnownBugForReleaseVersion(
+              issue,
+              options.version,
+            );
+            if (!knownBug) {
+              failedNewTests.push({
+                title: testCase.titleParts[1],
+                url: issue.url,
+                spec: testCase.titleParts[0],
+              });
+            }
+          } else {
+            failedNewTests.push({
+              title: testCase.titleParts[1],
+              spec: testCase.titleParts[0],
+            });
+          }
+        }
+      });
+
+      if (failedNewTests.length === 0) {
+        requestBody = `All failed tests are known bugs, can be merged. See results at ${options.dashboard_url}`;
+        testsStatus = "PASSED";
+      } else if (failedNewTests.length > 10) {
+        //If there are more than 10 new bugs it's probably caused by something else. Server responses with 500, or test user was deleted, etc.
+
+        requestBody =
+          "There is more than 10 new bugs, check results manually and create issues for them if necessary";
+      } else {
+        requestBody = `New bugs found, results at: ${options.dashboard_url}. List of issues to check: `;
+        for (const newBug of failedNewTests) {
+          if (!newBug.url) {
+            const issueUrl = await createIssue(
+              newBug,
+              options.version,
+              octokit,
+            );
+            requestBody += `\n${newBug.title} - ${issueUrl}`;
+          } else {
+            requestBody += `\n${newBug.title} - ${newBug.url}`;
+          }
+        }
+        requestBody += `\nIf this bugs won't be fixed in next patch release for this version mark them as known issues`;
+      }
+    }
 
-    const requestBody =
-      testsStatus === "PASSED"
-        ? `Cypress tests passed. See results at ${options.dashboard_url}`
-        : `Some tests failed, need manual approve. See results at ${options.dashboard_url}`;
     const event = "COMMENT";
 
     await octokit.request(
@@ -75,7 +130,7 @@ function isPatchRelease(version) {
   return version.match(regex) ? true : false;
 }
 
-async function getTestsStatus(dashboardUrl) {
+async function getTestsStatusAndId(dashboardUrl) {
   const getProjectRegex = /\/projects\/([^\/]*)/;
   const getRunRegex = /\/runs\/([^\/]*)/;
 
@@ -84,25 +139,24 @@ async function getTestsStatus(dashboardUrl) {
     buildNumber: dashboardUrl.match(getRunRegex)[1],
   };
 
-  const client = new GraphQLClient("https://dashboard.cypress.io/graphql");
-
   const throwErrorAfterTimeout = setTimeout(function() {
     throw new Error("Run have still running status, after all tests executed");
   }, 1200000);
 
-  const status = await waitForTestsToFinish(client, requestVariables);
+  const data = await waitForTestsToFinish(requestVariables);
 
   clearTimeout(throwErrorAfterTimeout);
-  return status;
+  return { status: data.status, runId: data.id };
 }
 
-async function waitForTestsToFinish(client, requestVariables) {
+async function waitForTestsToFinish(requestVariables) {
   return new Promise((resolve, reject) => {
     client
       .request(
         `query ($projectId: String!, $buildNumber: ID!) {
       runByBuildNumber(buildNumber: $buildNumber, projectId: $projectId) {
-        status
+        status,
+        id
       }
     }`,
         requestVariables,
@@ -113,8 +167,90 @@ async function waitForTestsToFinish(client, requestVariables) {
             resolve(await waitForTestsToFinish(client, requestVariables));
           }, 10000);
         } else {
-          resolve(response.runByBuildNumber.status);
+          resolve(response.runByBuildNumber);
         }
       });
   });
 }
+
+async function getFailedTestCases(runId) {
+  const requestVariables = {
+    input: {
+      runId,
+      testResultState: ["FAILED"],
+    },
+  };
+
+  return new Promise((resolve, reject) => {
+    client
+      .request(
+        `query RunTestResults($input: TestResultsTableInput!) {
+          testResults(input: $input) {
+            ... on TestResult {
+            ...RunTestResult
+            }
+          }
+        }
+        fragment RunTestResult on TestResult {  id  titleParts  state}`,
+        requestVariables,
+      )
+      .then(response => {
+        resolve(response.testResults);
+      });
+  });
+}
+
+async function getListOfTestsIssues(octokit) {
+  const result = await octokit.request(
+    "GET /repos/{owner}/saleor-dashboard/issues?labels=tests",
+    {
+      owner,
+    },
+  );
+  return result.data;
+}
+
+function issueOnGithub(listOfTestIssues, testCaseTitle) {
+  if (listOfTestIssues.length > 0) {
+    return listOfTestIssues.find(issue => {
+      return issue.title.includes(testCaseTitle);
+    });
+  }
+}
+
+function isIssueAKnownBugForReleaseVersion(issue, releaseVersion) {
+  const issueBody = issue.body;
+  const regex = /Known bug for versions:([\s\S]*)Additional/;
+  const lines = issueBody.match(regex)[1].split("\n");
+  const lineContainReleaseVersionRegex = /v(\d{2,3}).*(true|false)/;
+  const releaseVersionLine = lines.find(line => {
+    if (line.match(lineContainReleaseVersionRegex)) {
+      const version = line.match(lineContainReleaseVersionRegex)[1];
+      if (version === getFormattedVersion(releaseVersion)) {
+        return line;
+      }
+    }
+  });
+  const knownBugOnReleaseVersion = releaseVersionLine
+    ? releaseVersionLine.match(lineContainReleaseVersionRegex)[2]
+    : false;
+  return knownBugOnReleaseVersion === "true" ? true : false;
+}
+
+function getFormattedVersion(version) {
+  const regex = /^\d+\.\d+\./;
+  return version.match(regex)[0].replace(/\./g, "");
+}
+
+async function createIssue(newBug, version, octokit) {
+  const issue = await octokit.request("POST /repos/{owner}/{repo}/issues", {
+    owner,
+    repo: "saleor-dashboard",
+    title: `Cypress test fail: ${newBug.title}`,
+    body: `**Known bug for versions:**\nv${getFormattedVersion(
+      version,
+    )}: false\n**Additional Info:**\nSpec: ${newBug.spec}`,
+    labels: ["tests"],
+  });
+  return issue.data.url;
+}