Clean environment db (#4112)

* clean environment db

* fix revert to snapshot

* Working workflow for cleaning environments

* clean also automation-dashboard

* Update .github/workflows/clean-envs.yml

Co-authored-by: Mikail <6186720+NyanKiyoshi@users.noreply.github.com>

* Update .github/workflows/cleanEnvironments.js

Co-authored-by: Mikail <6186720+NyanKiyoshi@users.noreply.github.com>

* fix workflow

---------

Co-authored-by: Mikail <6186720+NyanKiyoshi@users.noreply.github.com>

.github/workflows/clean-envs.yml

@@ -10,7 +10,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       TOKEN: ${{ secrets.CLOUD_ACCESS_TOKEN }}
-      SNAPSHOT: PvsIXENJ
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -25,9 +24,38 @@ jobs:
           cd .github/workflows
           npm ci
 
-      - name: clean environments
+      - name: clean release environments
         id: clean-environments
         run: |
           node .github/workflows/cleanEnvironments.js \
           --token "$TOKEN" \
-          --snapshot "$SNAPSHOT"
+          --environments_to_clean_regex "^v\d+.staging"
+
+      - name: clean master environment
+        id: clean-master-environment
+        run: |
+          node .github/workflows/cleanEnvironments.js \
+          --token "$TOKEN" \
+          --environments_to_clean_regex "master.staging.saleor.cloud"
+
+      - name: Notify Slack
+        if: steps.clean-environments.outputs.sendWarningOnSlack == 'true'
+        env:
+          JOB_DEPLOYMENT_KIND: "release and master envs"
+          JOB_STATUS: 'failure'
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_SALEOR_QA_WEBHOOK_URL }}
+          JOB_TITLE: ${{ steps.clean-environments.outputs.warningMessage }}
+          JOB_KIND: "Clean Environments"
+        run: |
+          python3 .github/workflows/notify/notify-slack.py
+
+      - name: Notify Slack on qa-private
+        if: steps.clean-environments.outputs.sendWarningOnSlack == 'true'
+        env:
+          JOB_DEPLOYMENT_KIND: "release and master envs"
+          JOB_STATUS: 'failure'
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_SALEOR_QA_PRIVATE_WEBHOOK_URL }}
+          JOB_TITLE: ${{ steps.clean-environments.outputs.warningMessage }}
+          JOB_KIND: "Clean Environments"
+        run: |
+          python3 .github/workflows/notify/notify-slack.py

.github/workflows/cleanEnvironments.js

@@ -1,32 +1,56 @@
 const { Command } = require("commander");
 const fetch = require("node-fetch");
+const core = require("@actions/core");
 
 const program = new Command();
 
 const pathToCloudAPI = "https://staging-cloud.saleor.io/platform/api/";
 
+const snapshotName = "snapshot-automation-tests";
+
+let sendWarningOnSlack = "false";
+let warningMessage = "";
+
 program
   .name("cleanEnvironments")
   .description("Clean environments")
   .option("--token <token>", "token fo login to cloud")
-  .option("--snapshot <snapshot>", "snapshot to revert to")
+  .option(
+    "--environments_to_clean_regex <environments_to_clean_regex>",
+    "Regex for environment which need cleaning",
+  )
   .action(async options => {
     const token = options.token;
-    const snapshot = options.snapshot;
-    const environmentsToClean = await getEnvironmentsForReleaseTesting(token);
+    const environmentsToCleanRegex = new RegExp(
+      options.environments_to_clean_regex,
+    );
+    const environmentsToClean = await getEnvironmentsToClean(
+      token,
+      environmentsToCleanRegex,
+    );
+    const snapshotsForRestore = await getSnapshotsForRestore(token);
+    const sortedSnapshotList = sortSnapshots(snapshotsForRestore);
     environmentsToClean.forEach(environment => {
-      cleanEnvironment(environment, snapshot, token);
+      const latestSnapshot = getLatestSnapshotForEnvironment(
+        environment.service.version,
+        sortedSnapshotList,
+      );
+      if (latestSnapshot) {
+        cleanEnvironment(environment, latestSnapshot, token);
+      } else {
+        sendWarningOnSlack = "true";
+        warningMessage += `Snapshot compatible with environment ${environment.domain} does not exist, please create snapshot on cloud staging.\n`;
+      }
     });
+    core.setOutput("sendWarningOnSlack", sendWarningOnSlack);
+    core.setOutput("warningMessage", warningMessage);
   })
   .parse();
 
-async function getEnvironmentsForReleaseTesting(token) {
+async function getEnvironmentsToClean(token, environmentsToCleanRegex) {
   const environments = await getEnvironments(token);
   const environmentsForReleaseTesting = environments.filter(environment => {
-    return (
-      environment.domain.match(/^v\d*.staging/) ||
-      environment.domain == "master.staging.saleor.cloud"
-    );
+    return environment.domain.match(environmentsToCleanRegex)
   });
   return environmentsForReleaseTesting;
 }
@@ -51,7 +75,7 @@ async function cleanEnvironment(environment, snapshot, token) {
     `${pathToCloudAPI}organizations/saleor/environments/${environment.key}/restore/`,
     {
       method: "PUT",
-      body: JSON.stringify({ restore_from: snapshot }),
+      body: JSON.stringify({ restore_from: snapshot.key }),
       headers: {
         Authorization: `Token ${token}`,
         Accept: "application/json",
@@ -65,18 +89,82 @@ async function cleanEnvironment(environment, snapshot, token) {
       ? responseInJson.non_field_errors
       : responseInJson.__all__
   ) {
-    console.warn(
-      `${environment.name}: ${
-        responseInJson.non_field_errors
-          ? responseInJson.non_field_errors
-          : responseInJson.__all__
-      }`,
-    );
+    const warning = responseInJson.non_field_errors
+      ? responseInJson.non_field_errors
+      : responseInJson.__all__;
+    console.warn(`${environment.name}: ${warning}`);
+    sendWarningOnSlack = "true";
+    warningMessage += `Could not revert snapshot on ${environment.domain}: ${warning}.\n`;
   } else {
     await waitUntilTaskInProgress(responseInJson.task_id, environment.name);
   }
 }
 
+async function getSnapshotsForRestore(token) {
+  const snapshotsResponse = await fetch(
+    `${pathToCloudAPI}organizations/saleor/backups/`,
+    {
+      method: "GET",
+      headers: {
+        Authorization: `Token ${token}`,
+        Accept: "application/json",
+        "Content-Type": "application/json;charset=UTF-8",
+      },
+    },
+  );
+  const allSnapshots = await snapshotsResponse.json();
+  return allSnapshots.filter(snapshot => {
+    return snapshot.name.includes(snapshotName);
+  });
+}
+
+function sortSnapshots(snapshotList) {
+  // This function is used to sort snapshots by their version
+  // It returns sorted list of snapshots in descending order
+
+  return snapshotList.sort(function (a, b) {
+    return compareVersions(a.saleor_version, b.saleor_version);
+  });
+}
+
+function compareVersions(versionA, versionB) {
+  // Convert version from string to array eg. from "3.5.7" to [3, 5, 7]
+  // Where 3 is main version, 5 is major version and 7 is patch version
+
+  const versionASplittedToArray = versionA.split(/\D/);
+  const versionBSplittedToArray = versionB.split(/\D/);
+  const mainVersionNumberA = versionASplittedToArray[0];
+  const mainVersionNumberB = versionBSplittedToArray[0];
+  const majorVersionNumberA = versionASplittedToArray[1];
+  const majorVersionNumberB = versionBSplittedToArray[1];
+  const patchVersionNumberA = versionASplittedToArray[2];
+  const patchVersionNumberB = versionBSplittedToArray[2];
+
+  //Compare two versions
+  if (mainVersionNumberA !== mainVersionNumberB) {
+    return mainVersionNumberB - mainVersionNumberA;
+  } else if (majorVersionNumberA !== majorVersionNumberB) {
+    return majorVersionNumberB - majorVersionNumberA;
+  } else if (patchVersionNumberA !== patchVersionNumberB) {
+    return patchVersionNumberB - patchVersionNumberA;
+  } else return 0;
+}
+
+function getLatestSnapshotForEnvironment(environmentVersion, snapshotList) {
+  const compatibleSnapshots = snapshotList.filter(snapshot => {
+    return compareVersions(environmentVersion, snapshot.saleor_version) <= 0;
+  });
+  if (compatibleSnapshots.length > 0) {
+    const latestSnapshot = compatibleSnapshots[0];
+    return latestSnapshot;
+  } else {
+    console.warn(
+      `Could not find snapshot for environment on version: ${environmentVersion}. Environment won't be cleaned`,
+    );
+    return null;
+  }
+}
+
 async function waitUntilTaskInProgress(taskId, environment) {
   const throwErrorAfterTimeout = setTimeout(function () {
     throw new Error("Environment didn't upgrade after 30 minutes");

.github/workflows/tests-nightly.yml

@@ -51,41 +51,28 @@ jobs:
   revert-automation-env-to-snap:
     if: ${{ github.event.inputs.environment == null && github.event_name != 'repository_dispatch' }}
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    env:
+      TOKEN: ${{ secrets.CLOUD_ACCESS_TOKEN }}
     steps:
+      - name: Checkout
+        uses: actions/checkout@v3
 
       - name: Setup Node
         uses: actions/setup-node@v3
         with:
-          # use explicit version as this job does not checkout code
-          node-version: '18'
+          node-version-file: ".nvmrc"
 
-      - name: Install saleor cli
-        id: install-saleor-cli
-        run: npm i -g @saleor/cli
-
-      - name: Cache node modules
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-node-modules-cli
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-qa-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-qa-${{ env.cache-name }}-
-            ${{ runner.os }}-qa-
-            ${{ runner.os }}-
-      - name: Write config file
-        id: write-config-file
-        env:
-          ACCESS_TOKEN: ${{ secrets.CLOUD_ACCESS_TOKEN }}
-        run: echo '{"token":"Token ${{ secrets.CLOUD_ACCESS_TOKEN }}","telemetry":"false","organization_slug":"qa","organization_name":"QA","environment_id":"lHECN87U"}' > ~/.config/saleor.json
-
-      - name: revert snapshot
-        env:
-          CI: true
-          SALEOR_CLI_ENV: staging
-        run: npx saleor backup restore 3R5IPRr6 --skip-webhooks-update
+      - name: Install dependencies
+        run: |
+          cd .github/workflows
+          npm ci
+      
+      - name: clean automation environment
+        id: clean-automation-environment
+        run: |
+          node .github/workflows/cleanEnvironments.js \
+          --token "$TOKEN" \
+          --environments_to_clean_regex "automation-dashboard.staging.saleor.cloud"
 
       - name: Notify Slack
         if: ${{ failure() }}