From 36c14bd9a0fa95bbe376bc3267b815ea8201b7e5 Mon Sep 17 00:00:00 2001 From: Karolina Rakoczy Date: Wed, 17 May 2023 10:24:43 +0200 Subject: [PATCH] 3481 if there is more than 20 failed tests add another notification to saleor-qa channel (#3622) * Add slack notification * Add correct webhook url * Create new file with helpers * add default * add missing checkout --- .../approveAndMergeReleasePR.js | 76 +----------------- .../getEnvironmentVariables.js | 0 .../cypressTestsHelpers/getFailedTests.js | 26 ++++++ .../cypressTestsHelpers/getTestsResults.js | 79 +++++++++++++++++++ .github/workflows/tests-nightly.yml | 50 +++++++++++- 5 files changed, 157 insertions(+), 74 deletions(-) rename .github/workflows/{ => cypressTestsHelpers}/approveAndMergeReleasePR.js (74%) rename .github/workflows/{ => cypressTestsHelpers}/getEnvironmentVariables.js (100%) create mode 100644 .github/workflows/cypressTestsHelpers/getFailedTests.js create mode 100644 .github/workflows/cypressTestsHelpers/getTestsResults.js diff --git a/.github/workflows/approveAndMergeReleasePR.js b/.github/workflows/cypressTestsHelpers/approveAndMergeReleasePR.js similarity index 74% rename from .github/workflows/approveAndMergeReleasePR.js rename to .github/workflows/cypressTestsHelpers/approveAndMergeReleasePR.js index 80cd804d0..2cf103d43 100644 --- a/.github/workflows/approveAndMergeReleasePR.js +++ b/.github/workflows/cypressTestsHelpers/approveAndMergeReleasePR.js @@ -1,6 +1,8 @@ const { Octokit } = require("@octokit/core"); const { Command } = require("commander"); const { GraphQLClient } = require("graphql-request"); +const { statusAndID } = require("./getTestsResults"); +const { failedTestCases } = require("./getTestsResults"); const program = new Command(); const client = new GraphQLClient("https://dashboard.cypress.io/graphql"); @@ -33,7 +35,7 @@ program const commitId = pullRequest.data.merge_commit_sha; - const data = await getTestsStatusAndId(options.dashboard_url); + const data = await statusAndID(options.dashboard_url); let testsStatus = data.status; @@ -42,7 +44,7 @@ program if (testsStatus === "FAILED") { const failedNewTests = []; const listOfTestIssues = await getListOfTestsIssues(octokit); - const testCases = await getFailedTestCases(data.runId); + const testCases = await failedTestCases(data.runId); testCases.forEach(testCase => { if (testCase.titleParts) { const issue = issueOnGithub(listOfTestIssues, testCase.titleParts[1]); @@ -133,76 +135,6 @@ function isPatchRelease(version) { return version.match(regex) ? true : false; } -async function getTestsStatusAndId(dashboardUrl) { - const getProjectRegex = /\/projects\/([^\/]*)/; - const getRunRegex = /\/runs\/([^\/]*)/; - - const requestVariables = { - projectId: dashboardUrl.match(getProjectRegex)[1], - buildNumber: dashboardUrl.match(getRunRegex)[1], - }; - - const throwErrorAfterTimeout = setTimeout(function () { - throw new Error("Run have still running status, after all tests executed"); - }, 1200000); - - const data = await waitForTestsToFinish(requestVariables); - - clearTimeout(throwErrorAfterTimeout); - return { status: data.status, runId: data.id }; -} - -async function waitForTestsToFinish(requestVariables) { - return new Promise((resolve, reject) => { - client - .request( - `query ($projectId: String!, $buildNumber: ID!) { - runByBuildNumber(buildNumber: $buildNumber, projectId: $projectId) { - status, - id - } - }`, - requestVariables, - ) - .then(response => { - if (response.runByBuildNumber.status === "RUNNING") { - setTimeout(async function () { - resolve(await waitForTestsToFinish(requestVariables)); - }, 10000); - } else { - resolve(response.runByBuildNumber); - } - }); - }); -} - -async function getFailedTestCases(runId) { - const requestVariables = { - input: { - runId, - testResultState: ["FAILED"], - }, - }; - - return new Promise((resolve, reject) => { - client - .request( - `query RunTestResults($input: TestResultsTableInput!) { - testResults(input: $input) { - ... on TestResult { - ...RunTestResult - } - } - } - fragment RunTestResult on TestResult { id titleParts state}`, - requestVariables, - ) - .then(response => { - resolve(response.testResults); - }); - }); -} - async function getListOfTestsIssues(octokit) { const result = await octokit.request( "GET /repos/{owner}/saleor-dashboard/issues?labels=tests", diff --git a/.github/workflows/getEnvironmentVariables.js b/.github/workflows/cypressTestsHelpers/getEnvironmentVariables.js similarity index 100% rename from .github/workflows/getEnvironmentVariables.js rename to .github/workflows/cypressTestsHelpers/getEnvironmentVariables.js diff --git a/.github/workflows/cypressTestsHelpers/getFailedTests.js b/.github/workflows/cypressTestsHelpers/getFailedTests.js new file mode 100644 index 000000000..4ee999564 --- /dev/null +++ b/.github/workflows/cypressTestsHelpers/getFailedTests.js @@ -0,0 +1,26 @@ +const core = require("@actions/core"); +const { Command } = require("commander"); +const { statusAndID } = require("./getTestsResults"); +const { failedTestCases } = require("./getTestsResults"); + +const program = new Command(); + +program + .name("Get failed test") + .description("Get info if notify on slack if tests failed") + .option("--dashboard_url ", "Cypress dashboard url") + .action(async options => { + const data = await statusAndID(options.dashboard_url); + + let testsStatus = data.status; + + if (testsStatus === "FAILED") { + const testCases = await failedTestCases(data.runId); + if (testCases.length >= 20) { + core.setOutput("notifySlack", "true"); + } + } else if (testsStatus === "FAILED") { + core.setOutput("notifySlack", "true"); + } + }) + .parse(); diff --git a/.github/workflows/cypressTestsHelpers/getTestsResults.js b/.github/workflows/cypressTestsHelpers/getTestsResults.js new file mode 100644 index 000000000..194870c63 --- /dev/null +++ b/.github/workflows/cypressTestsHelpers/getTestsResults.js @@ -0,0 +1,79 @@ +const { Command } = require("commander"); +const { GraphQLClient } = require("graphql-request"); + +const program = new Command(); +const client = new GraphQLClient("https://dashboard.cypress.io/graphql"); + +const statusAndID = dashboardUrl => getTestsStatusAndId(dashboardUrl); +const failedTestCases = runId => getFailedTestCases(runId); + +async function getTestsStatusAndId(dashboardUrl) { + const getProjectRegex = /\/projects\/([^\/]*)/; + const getRunRegex = /\/runs\/([^\/]*)/; + const requestVariables = { + projectId: dashboardUrl.match(getProjectRegex)[1], + buildNumber: dashboardUrl.match(getRunRegex)[1], + }; + + const throwErrorAfterTimeout = setTimeout(function () { + throw new Error("Run have still running status, after all tests executed"); + }, 1200000); + + const data = await waitForTestsToFinish(requestVariables); + + clearTimeout(throwErrorAfterTimeout); + return { status: data.status, runId: data.id }; +} + +async function waitForTestsToFinish(requestVariables) { + return new Promise((resolve, reject) => { + client + .request( + `query ($projectId: String!, $buildNumber: ID!) { + runByBuildNumber(buildNumber: $buildNumber, projectId: $projectId) { + status, + id + } + }`, + requestVariables, + ) + .then(response => { + if (response.runByBuildNumber.status === "RUNNING") { + setTimeout(async function () { + resolve(await waitForTestsToFinish(requestVariables)); + }, 10000); + } else { + resolve(response.runByBuildNumber); + } + }); + }); +} + +async function getFailedTestCases(runId) { + const requestVariables = { + input: { + runId, + testResultState: ["FAILED"], + }, + }; + + return new Promise((resolve, reject) => { + client + .request( + `query RunTestResults($input: TestResultsTableInput!) { + testResults(input: $input) { + ... on TestResult { + ...RunTestResult + } + } + } + fragment RunTestResult on TestResult { id titleParts state}`, + requestVariables, + ) + .then(response => { + resolve(response.testResults); + }); + }); +} + +module.exports = { statusAndID, failedTestCases }; diff --git a/.github/workflows/tests-nightly.yml b/.github/workflows/tests-nightly.yml index acee9c3bf..b2f48a13b 100644 --- a/.github/workflows/tests-nightly.yml +++ b/.github/workflows/tests-nightly.yml @@ -98,6 +98,8 @@ jobs: GREP_TAGS: ${{ github.event.inputs.tags || '@allEnv'}} outputs: status: ${{ steps.cypress.outcome }} + dashboard_url: ${{ steps.cypress.outputs.dashboardUrl }} + environment: ${{ steps.get-env-uri.outputs.ENV_URI }} container: cypress/browsers:node18.12.0-chrome106-ff106 strategy: fail-fast: false @@ -190,7 +192,7 @@ jobs: - name: get environment variables id: get-environment-variables run: | - node .github/workflows/getEnvironmentVariables.js \ + node .github/workflows/cypressTestsHelpers/getEnvironmentVariables.js \ --version $VERSION \ --token "$TOKEN" \ --repo_token "$REPO_TOKEN" \ @@ -297,7 +299,7 @@ jobs: export GITHUB_TOKEN=$( \ curl --request GET --url ${{ secrets.VAULT_URL}} --header "Authorization: JWT ${{ secrets.VAULT_JWT }}" | jq -r .token \ ) - node .github/workflows/approveAndMergeReleasePR.js \ + node .github/workflows/cypressTestsHelpers/approveAndMergeReleasePR.js \ --version $version \ --pull_request_number $pull_request_number \ --dashboard_url $dashboard_url \ @@ -308,6 +310,9 @@ jobs: runs-on: ubuntu-latest needs: [get-environment-variables, run-tests-on-release, add-review-and-merge-patch] steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Notify Slack env: JOB_DEPLOYMENT_KIND: staging @@ -315,5 +320,46 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }} JOB_TITLE: "Test release workflow - ${{github.event.client_payload.project}} ${{github.event.client_payload.version}}" JOB_KIND: "release tests" + run: | + python3 .github/workflows/notify/notify-slack.py + + send-slack-notification-scheduled-and-manually: + if: ${{always() && github.event_name != 'repository_dispatch'}} + defaults: + run: + shell: bash + working-directory: .github/workflows + runs-on: ubuntu-latest + needs: [run-tests-in-parallel] + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version-file: '.nvmrc' + + - name: Install dependencies + run: | + npm ci + + - name: Get tests results + id: get-tests-results + env: + tests_status: ${{ needs.run-tests-in-parallel.outputs.status }} + dashboard_url: ${{ needs.run-tests-in-parallel.outputs.dashboard_url }} + run: | + node cypressTestsHelpers/getFailedTests.js \ + --dashboard_url $dashboard_url + + - name: Notify Slack + if: steps.get-tests-results.outputs.testStatus == 'true' + env: + JOB_DEPLOYMENT_KIND: ${{ needs.run-tests-in-parallel.outputs.environment }} + JOB_STATUS: ${{ job.status }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_SALEOR_QA_WEBHOOK_URL }} + JOB_TITLE: "A lot of tests failed, something is probably broken - ${{ needs.run-tests-in-parallel.outputs.dashboard_url }}" + JOB_KIND: "Tests" run: | python3 .github/workflows/notify/notify-slack.py \ No newline at end of file