3481 if there is more than 20 failed tests add another notification to saleor-qa channel (#3622)

* Add slack notification

* Add correct webhook url

* Create new file with helpers

* add default

* add missing checkout

.github/workflows/cypressTestsHelpers/approveAndMergeReleasePR.js

@@ -1,6 +1,8 @@
 const { Octokit } = require("@octokit/core");
 const { Command } = require("commander");
 const { GraphQLClient } = require("graphql-request");
+const { statusAndID } = require("./getTestsResults");
+const { failedTestCases } = require("./getTestsResults");
 
 const program = new Command();
 const client = new GraphQLClient("https://dashboard.cypress.io/graphql");
@@ -33,7 +35,7 @@ program
 
     const commitId = pullRequest.data.merge_commit_sha;
 
-    const data = await getTestsStatusAndId(options.dashboard_url);
+    const data = await statusAndID(options.dashboard_url);
 
     let testsStatus = data.status;
 
@@ -42,7 +44,7 @@ program
     if (testsStatus === "FAILED") {
       const failedNewTests = [];
       const listOfTestIssues = await getListOfTestsIssues(octokit);
-      const testCases = await getFailedTestCases(data.runId);
+      const testCases = await failedTestCases(data.runId);
       testCases.forEach(testCase => {
         if (testCase.titleParts) {
           const issue = issueOnGithub(listOfTestIssues, testCase.titleParts[1]);
@@ -133,76 +135,6 @@ function isPatchRelease(version) {
   return version.match(regex) ? true : false;
 }
 
-async function getTestsStatusAndId(dashboardUrl) {
-  const getProjectRegex = /\/projects\/([^\/]*)/;
-  const getRunRegex = /\/runs\/([^\/]*)/;
-
-  const requestVariables = {
-    projectId: dashboardUrl.match(getProjectRegex)[1],
-    buildNumber: dashboardUrl.match(getRunRegex)[1],
-  };
-
-  const throwErrorAfterTimeout = setTimeout(function () {
-    throw new Error("Run have still running status, after all tests executed");
-  }, 1200000);
-
-  const data = await waitForTestsToFinish(requestVariables);
-
-  clearTimeout(throwErrorAfterTimeout);
-  return { status: data.status, runId: data.id };
-}
-
-async function waitForTestsToFinish(requestVariables) {
-  return new Promise((resolve, reject) => {
-    client
-      .request(
-        `query ($projectId: String!, $buildNumber: ID!) {
-      runByBuildNumber(buildNumber: $buildNumber, projectId: $projectId) {
-        status,
-        id
-      }
-    }`,
-        requestVariables,
-      )
-      .then(response => {
-        if (response.runByBuildNumber.status === "RUNNING") {
-          setTimeout(async function () {
-            resolve(await waitForTestsToFinish(requestVariables));
-          }, 10000);
-        } else {
-          resolve(response.runByBuildNumber);
-        }
-      });
-  });
-}
-
-async function getFailedTestCases(runId) {
-  const requestVariables = {
-    input: {
-      runId,
-      testResultState: ["FAILED"],
-    },
-  };
-
-  return new Promise((resolve, reject) => {
-    client
-      .request(
-        `query RunTestResults($input: TestResultsTableInput!) {
-          testResults(input: $input) {
-            ... on TestResult {
-            ...RunTestResult
-            }
-          }
-        }
-        fragment RunTestResult on TestResult {  id  titleParts  state}`,
-        requestVariables,
-      )
-      .then(response => {
-        resolve(response.testResults);
-      });
-  });
-}
-
 async function getListOfTestsIssues(octokit) {
   const result = await octokit.request(
     "GET /repos/{owner}/saleor-dashboard/issues?labels=tests",

.github/workflows/cypressTestsHelpers/getFailedTests.js

@@ -0,0 +1,26 @@
+const core = require("@actions/core");
+const { Command } = require("commander");
+const { statusAndID } = require("./getTestsResults");
+const { failedTestCases } = require("./getTestsResults");
+
+const program = new Command();
+
+program
+  .name("Get failed test")
+  .description("Get info if notify on slack if tests failed")
+  .option("--dashboard_url <dashboard_url>", "Cypress dashboard url")
+  .action(async options => {
+    const data = await statusAndID(options.dashboard_url);
+
+    let testsStatus = data.status;
+
+    if (testsStatus === "FAILED") {
+      const testCases = await failedTestCases(data.runId);
+      if (testCases.length >= 20) {
+        core.setOutput("notifySlack", "true");
+      }
+    } else if (testsStatus === "FAILED") {
+      core.setOutput("notifySlack", "true");
+    }
+  })
+  .parse();

.github/workflows/cypressTestsHelpers/getTestsResults.js

@@ -0,0 +1,79 @@
+const { Command } = require("commander");
+const { GraphQLClient } = require("graphql-request");
+
+const program = new Command();
+const client = new GraphQLClient("https://dashboard.cypress.io/graphql");
+
+const statusAndID = dashboardUrl => getTestsStatusAndId(dashboardUrl);
+const failedTestCases = runId => getFailedTestCases(runId);
+
+async function getTestsStatusAndId(dashboardUrl) {
+  const getProjectRegex = /\/projects\/([^\/]*)/;
+  const getRunRegex = /\/runs\/([^\/]*)/;
+  const requestVariables = {
+    projectId: dashboardUrl.match(getProjectRegex)[1],
+    buildNumber: dashboardUrl.match(getRunRegex)[1],
+  };
+
+  const throwErrorAfterTimeout = setTimeout(function () {
+    throw new Error("Run have still running status, after all tests executed");
+  }, 1200000);
+
+  const data = await waitForTestsToFinish(requestVariables);
+
+  clearTimeout(throwErrorAfterTimeout);
+  return { status: data.status, runId: data.id };
+}
+
+async function waitForTestsToFinish(requestVariables) {
+  return new Promise((resolve, reject) => {
+    client
+      .request(
+        `query ($projectId: String!, $buildNumber: ID!) {
+      runByBuildNumber(buildNumber: $buildNumber, projectId: $projectId) {
+        status,
+        id
+      }
+    }`,
+        requestVariables,
+      )
+      .then(response => {
+        if (response.runByBuildNumber.status === "RUNNING") {
+          setTimeout(async function () {
+            resolve(await waitForTestsToFinish(requestVariables));
+          }, 10000);
+        } else {
+          resolve(response.runByBuildNumber);
+        }
+      });
+  });
+}
+
+async function getFailedTestCases(runId) {
+  const requestVariables = {
+    input: {
+      runId,
+      testResultState: ["FAILED"],
+    },
+  };
+
+  return new Promise((resolve, reject) => {
+    client
+      .request(
+        `query RunTestResults($input: TestResultsTableInput!) {
+          testResults(input: $input) {
+            ... on TestResult {
+            ...RunTestResult
+            }
+          }
+        }
+        fragment RunTestResult on TestResult {  id  titleParts  state}`,
+        requestVariables,
+      )
+      .then(response => {
+        resolve(response.testResults);
+      });
+  });
+}
+
+module.exports = { statusAndID, failedTestCases };

.github/workflows/tests-nightly.yml

@@ -98,6 +98,8 @@ jobs:
       GREP_TAGS: ${{ github.event.inputs.tags || '@allEnv'}}
     outputs:
       status: ${{ steps.cypress.outcome }}
+      dashboard_url: ${{ steps.cypress.outputs.dashboardUrl }}
+      environment: ${{  steps.get-env-uri.outputs.ENV_URI }}
     container: cypress/browsers:node18.12.0-chrome106-ff106
     strategy:
       fail-fast: false
@@ -190,7 +192,7 @@ jobs:
       - name: get environment variables
         id: get-environment-variables
         run: |
-              node .github/workflows/getEnvironmentVariables.js \
+              node .github/workflows/cypressTestsHelpers/getEnvironmentVariables.js \
               --version $VERSION \
               --token "$TOKEN" \
               --repo_token "$REPO_TOKEN" \
@@ -297,7 +299,7 @@ jobs:
             export GITHUB_TOKEN=$( \
             curl --request GET --url ${{ secrets.VAULT_URL}} --header "Authorization: JWT ${{ secrets.VAULT_JWT }}" | jq -r .token \
             )
-            node .github/workflows/approveAndMergeReleasePR.js \
+            node .github/workflows/cypressTestsHelpers/approveAndMergeReleasePR.js \
             --version $version \
             --pull_request_number $pull_request_number \
             --dashboard_url $dashboard_url \
@@ -308,6 +310,9 @@ jobs:
     runs-on: ubuntu-latest
     needs: [get-environment-variables, run-tests-on-release, add-review-and-merge-patch]
     steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        
       - name: Notify Slack
         env:
           JOB_DEPLOYMENT_KIND: staging
@@ -315,5 +320,46 @@ jobs:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
           JOB_TITLE: "Test release workflow - ${{github.event.client_payload.project}} ${{github.event.client_payload.version}}"
           JOB_KIND: "release tests"
+        run: |
+          python3 .github/workflows/notify/notify-slack.py
+
+  send-slack-notification-scheduled-and-manually:
+    if: ${{always() && github.event_name != 'repository_dispatch'}}
+    defaults:
+      run:
+       shell: bash
+       working-directory: .github/workflows
+    runs-on: ubuntu-latest
+    needs: [run-tests-in-parallel]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+
+      - name: Install dependencies
+        run: |
+              npm ci
+
+      - name: Get tests results
+        id: get-tests-results
+        env: 
+         tests_status: ${{ needs.run-tests-in-parallel.outputs.status }}
+         dashboard_url: ${{ needs.run-tests-in-parallel.outputs.dashboard_url }}
+        run: |
+            node cypressTestsHelpers/getFailedTests.js \
+            --dashboard_url $dashboard_url 
+
+      - name: Notify Slack
+        if: steps.get-tests-results.outputs.testStatus == 'true'
+        env:
+          JOB_DEPLOYMENT_KIND: ${{ needs.run-tests-in-parallel.outputs.environment }}
+          JOB_STATUS: ${{ job.status }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_SALEOR_QA_WEBHOOK_URL }}
+          JOB_TITLE: "A lot of tests failed, something is probably broken - ${{ needs.run-tests-in-parallel.outputs.dashboard_url }}"
+          JOB_KIND: "Tests"
         run: |
           python3 .github/workflows/notify/notify-slack.py