E2E tests (#668)

* playwright install

* Configrations

* assertions

* basic tests for apps installations

* tests for product feed

* Adjust PF assertions to use test-id

* Taxes smoke test

* moved files around

* Add smoke for Klaviyo

* More taxes tests

* remove workflow

* add example

* extract separate test for pf

* Improve PF test

* cr fixes
This commit is contained in:
Lukasz Ostrowski 2023-06-28 17:08:00 +02:00 committed by GitHub
parent d34c2a1b84
commit 7c98e384fd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 566 additions and 6 deletions

View file

@ -0,0 +1,3 @@
INSTANCE_URL=
DASHBOARD_USER_EMAIL=
DASHBOARD_USER_PASSWORD=

4
packages/e2e/.eslintrc Normal file
View file

@ -0,0 +1,4 @@
{
"root": true,
"extends": ["saleor"]
}

3
packages/e2e/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
/test-results/
/playwright-report/
/playwright/.cache/

20
packages/e2e/package.json Normal file
View file

@ -0,0 +1,20 @@
{
"name": "@saleor/e2e",
"description": "",
"version": "1.0.0",
"author": "",
"scripts": {
"e2e": "playwright test",
"e2e:ui": "playwright test --ui"
},
"devDependencies": {
"@playwright/test": "^1.35.1",
"@saleor/app-sdk": "0.40.1",
"dotenv": "^16.3.1",
"eslint-config-saleor": "workspace:*",
"zod": "3.20.2"
},
"keywords": [],
"license": "ISC",
"main": "index.js"
}

View file

@ -0,0 +1,62 @@
import { defineConfig, devices } from "@playwright/test";
/**
* Read environment variables from file.
* Check setup/configuration
*/
require("dotenv").config();
/**
* See https://playwright.dev/docs/test-configuration.
*/
// eslint-disable-next-line import/no-default-export
export default defineConfig({
testDir: "./tests",
/* Run tests in files in parallel */
fullyParallel: false,
/* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI,
/* Retry on CI only */
retries: process.env.CI ? 2 : 0,
/* Opt out of parallel tests on CI. */
workers: process.env.CI ? 1 : undefined,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
// reporter: "html",
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
/* Base URL to use in actions like `await page.goto('/')`. */
// baseURL: 'http://127.0.0.1:3000',
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
trace: "on-first-retry",
},
/* Configure projects for major browsers */
projects: [
{
name: "chromium",
use: { ...devices["Desktop Chrome"] },
},
/*
*
* {
* name: 'firefox',
* use: { ...devices['Desktop Firefox'] },
* },
*
* {
* name: 'webkit',
* use: { ...devices['Desktop Safari'] },
* },
*/
],
/* Run your local dev server before starting the tests */
/*
* webServer: {
* command: 'npm run start',
* url: 'http://127.0.0.1:3000',
* reuseExistingServer: !process.env.CI,
* },
*/
});

View file

@ -0,0 +1,17 @@
import { z } from "zod";
const instanceUrl = process.env.INSTANCE_URL;
const dashboardUserEmail = process.env.DASHBOARD_USER_EMAIL;
const dashboardUserPassword = process.env.DASHBOARD_USER_PASSWORD;
export const configuration = z
.object({
instanceUrl: z.string().nonempty().url(),
dashboardUserEmail: z.string().nonempty(),
dashboardUserPassword: z.string().nonempty(),
})
.parse({
instanceUrl,
dashboardUserEmail,
dashboardUserPassword,
});

View file

@ -0,0 +1,20 @@
import { configuration } from "./configuration";
export const appUrls = (appUrl: string) => ({
manifest: new URL("/api/manifest", appUrl).href,
register: new URL("/api/register", appUrl).href,
});
const saleorUrls = (dashboardUrl: string) => ({
dashboard: {
homepage: new URL("/dashboard", dashboardUrl).href,
apps: new URL("/dashboard/apps", dashboardUrl).href,
appInstallPage: (appManifest: string) =>
new URL(`/dashboard/apps/install?manifestUrl=${appManifest}`, dashboardUrl).href,
},
api: new URL("/graphql/", dashboardUrl).href,
});
export const routing = {
saleor: saleorUrls(configuration.instanceUrl),
};

View file

@ -0,0 +1,66 @@
import { test, expect, Page } from "@playwright/test";
import { logInIntoDashboard } from "./operations/log-in-to-dashboard";
import { installTheApp } from "./operations/install-app";
import { appUrls, routing } from "../setup/routing";
import { AppManifest } from "@saleor/app-sdk/types";
import { assertAppAvailable } from "./assertions/assert-app-available";
/**
* Hardcoded list of every app deployed on staging and production.
* TODO: Eventually this should be the entry point and the list should be provided via env
*/
const apps: string[] = [
"taxes",
"crm",
"cms",
"emails-and-messages",
"product-feed",
"search",
"klaviyo",
"slack",
"invoices",
"data-importer",
].reduce((urls, appSegment) => {
urls.push(`https://${appSegment}.saleor.app`);
urls.push(`https://${appSegment}.staging.saleor.app`);
return urls;
}, []);
/*
*
* test.describe.configure({
* mode: "parallel",
* });
*/
/**
* TODO Enable parallel mode. It cant work with beforeAll.
*/
test.describe("Apps Installation", () => {
let page: Page;
test.beforeAll(async ({ browser }) => {
if (page) {
return;
}
console.log("beforeAll run");
page = await browser.newPage();
await logInIntoDashboard({ page });
});
for (const appUrl of apps) {
test(`App: "${appUrl}" can be installed in the dashboard`, async () => {
const appManifestUrl = appUrl + "/api/manifest";
await installTheApp({ page, appManifest: appManifestUrl }); // todo extract to helper
const appManifest = (await fetch(appManifestUrl).then((r) => r.json())) as AppManifest;
const appName = appManifest.name;
await assertAppAvailable({ page, appName });
});
}
});

View file

@ -0,0 +1,33 @@
import { Page, test, expect } from "@playwright/test";
import { logInIntoDashboard } from "../../operations/log-in-to-dashboard";
import { openTheApp } from "../../operations/open-app";
test.describe("Klaviyo Configuration", () => {
let page: Page;
test.beforeAll(async ({ browser }) => {
if (page) {
return;
}
console.log("beforeAll run");
page = await browser.newPage();
await logInIntoDashboard({ page });
});
// Test assumes app is installed
test("App can be configured @stable @critical", async () => {
await openTheApp({ page, appName: "Klaviyo" });
// todo make more strict selector
const iframeLocator = page.frameLocator("iframe");
await expect(iframeLocator.getByLabel("CUSTOMER_CREATED_METRIC")).toBeVisible();
await expect(iframeLocator.getByLabel("FULFILLMENT_CREATED_METRIC")).toBeVisible();
await expect(iframeLocator.getByLabel("ORDER_CREATED_METRIC")).toBeVisible();
await expect(iframeLocator.getByLabel("ORDER_FULLY_PAID_METRIC")).toBeVisible();
await expect(iframeLocator.getByLabel("PUBLIC_TOKEN")).toBeVisible();
});
});

View file

@ -0,0 +1,8 @@
import { expect, FrameLocator } from "@playwright/test";
export const assertAppRender = async (iframeLocator: FrameLocator) => {
await expect(iframeLocator.getByTestId("root-heading")).toBeVisible();
await expect(iframeLocator.getByTestId("s3-configuration-section")).toBeVisible();
await expect(iframeLocator.getByTestId("channels-configuration-section")).toBeVisible();
await expect(iframeLocator.getByTestId("categories-mapping-section")).toBeVisible();
};

View file

@ -0,0 +1,10 @@
import { FrameLocator, Locator } from "@playwright/test";
export const fillAwsS3Form = async (iframeLocator: FrameLocator) => {
await iframeLocator.getByLabel("Amazon access key ID").fill("test-id");
await iframeLocator.getByLabel("Amazon secret access key").fill("test-secret");
await iframeLocator.getByLabel("Bucket name").fill("test-bucket");
await iframeLocator.getByLabel("Bucket region").fill("eu-west-1");
await iframeLocator.getByText("Save bucket configuration").click();
};

View file

@ -0,0 +1,16 @@
import { FrameLocator } from "@playwright/test";
export const fillChannelConfig = async (iframeLocator: FrameLocator) => {
const sectionsSelector = await iframeLocator.getByTestId("channels-configuration-section");
const channelRow = sectionsSelector.getByText("Default channel"); // todo add test-id
channelRow.click();
await sectionsSelector.getByLabel("Storefront URL").fill("https://www.example.com");
await sectionsSelector
.getByLabel("Storefront product URL")
.fill("https://www.example.com/{productId}");
await sectionsSelector.getByText("Save channel settings").click();
};

View file

@ -0,0 +1,12 @@
import { FrameLocator, expect } from "@playwright/test";
export const navigateToCategoryMapping = async (iframeLocator: FrameLocator) => {
await iframeLocator.getByText("Map categories").click({ force: true });
await expect(iframeLocator.getByTestId("categories-mapping-container")).toBeVisible();
// todo doesnt load, probably app must be optimized
await expect(iframeLocator.getByText("Accessories")).toBeVisible({
timeout: 120000,
});
};

View file

@ -0,0 +1,7 @@
import { FrameLocator } from "@playwright/test";
export const setCategoryMapping = async (iframeLocator: FrameLocator) => {
await iframeLocator.locator("select").first().selectOption({ index: 0 });
await iframeLocator.getByText("Save").first().click();
};

View file

@ -0,0 +1,63 @@
import { Page, test, expect } from "@playwright/test";
import { logInIntoDashboard } from "../../operations/log-in-to-dashboard";
import { openTheApp } from "../../operations/open-app";
import { fillAwsS3Form } from "./operations/fill-aws-s3-form";
import { assertAppRender } from "./assertions/assert-app-render";
import { fillChannelConfig } from "./operations/fill-channel-config";
import { setCategoryMapping } from "./operations/set-category-mapping";
import { navigateToCategoryMapping } from "./operations/navigate-to-category-mapping";
test.describe("Product Feed Configuration", () => {
let page: Page;
test.beforeAll(async ({ browser }) => {
if (page) {
return;
}
console.log("beforeAll run");
page = await browser.newPage();
await logInIntoDashboard({ page });
});
// Test assumes app is installed
test("App can be configured @stable @critical", async () => {
await openTheApp({ page, appName: "Product Feed" });
// todo make more strict selector
const iframeLocator = page.frameLocator("iframe");
await assertAppRender(iframeLocator);
await fillAwsS3Form(iframeLocator);
await expect(page.getByText("Updated S3 configuration")).toBeVisible({ timeout: 10000 });
await fillChannelConfig(iframeLocator);
await expect(page.getByText("Success")).toBeVisible({
timeout: 10000,
}); // todo add more meaningul message, only "success" is set
});
/**
* Test fails. Probably because of a very big list of Google categories that are fetched.
* TODO: Fix this in the app
*/
test.skip("App can be configured with categories mapping", async () => {
await openTheApp({ page, appName: "Product Feed" });
// todo make more strict selector
const iframeLocator = page.frameLocator("iframe");
await navigateToCategoryMapping(iframeLocator);
await setCategoryMapping(iframeLocator);
await expect(page.getByText("Success")).toBeVisible({ timeout: 10000 }); // todo add more meaningul message, only "success" is set
await iframeLocator.getByText("Configuration").click();
});
});

View file

@ -0,0 +1,21 @@
import { expect, FrameLocator } from "@playwright/test";
export const assertAppRender = async (iframeLocator: FrameLocator) => {
/*
* TODO Add test-ids assertions after added to app
* todo assert empty state, but these tests must ensure app has fresh install
*/
await expect(
iframeLocator.getByRole("heading", {
name: "Tax providers",
})
).toBeVisible();
await expect(iframeLocator.getByRole("heading", { name: "Available channels" })).toBeVisible();
// await expect(iframeLocator.getByRole("heading", { name: "Tax code matcher" })).toBeVisible(); // todo enable when app enables
const addProviderButton = await iframeLocator.getByRole("button", {
name: new RegExp(/Add new|Add first provider/),
});
await expect(addProviderButton).toBeVisible();
};

View file

@ -0,0 +1,57 @@
import { Page, test, expect } from "@playwright/test";
import { logInIntoDashboard } from "../../operations/log-in-to-dashboard";
import { openTheApp } from "../../operations/open-app";
import { assertAppRender } from "./assertions/assert-app-render";
// Test assumes app is installed
test.describe("Taxes Configuration", () => {
let page: Page;
test.beforeAll(async ({ browser }) => {
if (page) {
return;
}
page = await browser.newPage();
await logInIntoDashboard({ page });
});
test("App renders config screen @stable @critical", async () => {
await openTheApp({ page, appName: "Taxes" });
// todo make more strict selector
const iframeLocator = page.frameLocator("iframe");
await assertAppRender(iframeLocator);
});
test("App can configure new Taxjar provider @taxjar", async () => {
await openTheApp({ page, appName: "Taxes" });
// todo make more strict selector
const iframeLocator = page.frameLocator("iframe");
await iframeLocator
.getByRole("button", {
name: new RegExp(/Add new|Add first provider/),
})
.click();
await iframeLocator.getByRole("button", { name: "Choose" }).first().click(); // todo - test id
await iframeLocator.getByLabel("Configuration name").fill("Test Taxjar provider");
await iframeLocator.getByLabel("API key").fill("TEST");
await iframeLocator.getByLabel("Street").fill("Street");
await iframeLocator.getByLabel("City").fill("City");
await iframeLocator.getByLabel("State").fill("State");
await iframeLocator.getByRole("combobox", { name: "Country *" }).click();
await iframeLocator.getByText("Albania").click();
await iframeLocator.getByLabel("Zip").fill("Zip");
await iframeLocator.getByRole("button", { name: "Save" }).first().click(); // todo - test id
});
// todo
test.skip("App can configure new Avalara provider @avalara", async () => {});
});

View file

@ -0,0 +1,19 @@
import { expect, Page } from "@playwright/test";
import { routing } from "../../setup/routing";
import { configuration } from "../../setup/configuration";
interface checkIfAppIsAvailableArgs {
page: Page;
appName: string;
}
export const assertAppAvailable = async ({ page, appName }: checkIfAppIsAvailableArgs) => {
await page.goto(routing.saleor.dashboard.apps, { timeout: 20000, waitUntil: "load" });
// todo need data-testid. this element is not unique
const appEntry = await page.getByText(appName).first();
await expect(appEntry).toBeVisible();
await expect(await page.getByText("Problem occured during installation.")).toBeHidden();
};

View file

@ -0,0 +1,22 @@
import { Page } from "@playwright/test";
import { routing } from "../../setup/routing";
interface InstallTheAppArgs {
page: Page;
appManifest: string;
}
export const installTheApp = async ({ page, appManifest }: InstallTheAppArgs) => {
// got to Apps page, assuming user is logged in
await page.goto(routing.saleor.dashboard.appInstallPage(appManifest), {
timeout: 20000,
waitUntil: "load",
});
console.log("Navigated to", page.url());
await page.getByRole("button", { name: "Install App" }).click();
// wait for the toast
await page.getByText("App installed").isVisible();
};

View file

@ -0,0 +1,25 @@
import { Page, expect } from "@playwright/test";
import { configuration } from "../../setup/configuration";
import { routing } from "../../setup/routing";
interface LogInIntoDashboardArgs {
page: Page;
}
export const logInIntoDashboard = async ({ page }: LogInIntoDashboardArgs) => {
console.log("Will redirect to", routing.saleor.dashboard.homepage);
await page.goto(routing.saleor.dashboard.homepage, { timeout: 20000, waitUntil: "load" });
const url = page.url();
await page.locator('[data-test-id="email"]').click();
await page.locator('[data-test-id="email"]').fill(configuration.dashboardUserEmail);
await page.locator('[data-test-id="email"]').press("Tab");
await page.locator('[data-test-id="password"]').fill(configuration.dashboardUserPassword);
await page.locator('[data-test-id="submit"]').click();
await expect(page.locator('[data-test-id="welcome-header"]')).toBeVisible();
console.log("Logged in");
};

View file

@ -0,0 +1,21 @@
import { Page, expect } from "@playwright/test";
import { routing } from "../../setup/routing";
interface InstallTheAppArgs {
page: Page;
appName: string;
}
export const openTheApp = async ({ page, appName }: InstallTheAppArgs) => {
// got to Apps page, assuming user is logged in
await page.goto(routing.saleor.dashboard.apps, {
waitUntil: "load",
});
console.log("Navigated to", page.url());
// todo it must be test-id because we also have same name in appstore list
await page.getByText(appName).first().click();
await expect(page.url()).toMatch(new RegExp("https:\\/\\/.*\\/dashboard\\/apps\\/.*\\/app"));
};

8
packages/e2e/turbo.json Normal file
View file

@ -0,0 +1,8 @@
{
"$schema": "https://turbo.build/schema.json",
"extends": ["//"],
"pipeline": {
"e2e": {},
"e2e:ui": {}
}
}

View file

@ -1546,6 +1546,24 @@ importers:
specifier: 5.1.3
version: 5.1.3
packages/e2e:
devDependencies:
'@playwright/test':
specifier: ^1.35.1
version: 1.35.1
'@saleor/app-sdk':
specifier: 0.40.1
version: 0.40.1(next@13.3.0)(react-dom@18.2.0)(react@18.2.0)
dotenv:
specifier: ^16.3.1
version: 16.3.1
eslint-config-saleor:
specifier: workspace:*
version: link:../eslint-config-saleor
zod:
specifier: 3.20.2
version: 3.20.2
packages/eslint-config-saleor:
devDependencies:
eslint:
@ -6507,7 +6525,7 @@ packages:
'@whatwg-node/fetch': 0.8.8
chalk: 4.1.2
debug: 4.3.4
dotenv: 16.2.0
dotenv: 16.3.1
graphql: 16.6.0
graphql-request: 6.1.0(graphql@16.6.0)
http-proxy-agent: 6.1.1
@ -7180,6 +7198,17 @@ packages:
tslib: 2.5.3
dev: true
/@playwright/test@1.35.1:
resolution: {integrity: sha512-b5YoFe6J9exsMYg0pQAobNDR85T1nLumUYgUTtKm4d21iX2L7WqKq9dW8NGJ+2vX0etZd+Y7UeuqsxDXm9+5ZA==}
engines: {node: '>=16'}
hasBin: true
dependencies:
'@types/node': 18.15.3
playwright-core: 1.35.1
optionalDependencies:
fsevents: 2.3.2
dev: true
/@popperjs/core@2.11.8:
resolution: {integrity: sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==}
dev: false
@ -12091,7 +12120,7 @@ packages:
supports-color:
optional: true
dependencies:
ms: 2.1.2
ms: 2.1.3
/debug@4.3.4:
resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
@ -12425,6 +12454,11 @@ packages:
resolution: {integrity: sha512-jcq2vR1DY1+QA+vH58RIrWLDZOifTGmyQJWzP9arDUbgZcySdzuBb1WvhWZzZtiXgfm+GW2pjBqStqlfpzq7wQ==}
engines: {node: '>=12'}
/dotenv@16.3.1:
resolution: {integrity: sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==}
engines: {node: '>=12'}
dev: true
/dotenv@8.6.0:
resolution: {integrity: sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==}
engines: {node: '>=10'}
@ -15363,7 +15397,7 @@ packages:
engines: {node: '>=14.0.0'}
dependencies:
app-root-dir: 1.0.2
dotenv: 16.2.0
dotenv: 16.3.1
dotenv-expand: 10.0.0
dev: true
@ -16639,7 +16673,6 @@ packages:
/ms@2.1.3:
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
dev: true
/multimatch@4.0.0:
resolution: {integrity: sha512-lDmx79y1z6i7RNx0ZGCPq1bzJ6ZoDDKbvh7jxr9SJcWLkShMzXrHbYVpTdnhNM5MXpDUxCQ4DgqVttVXlBgiBQ==}
@ -17486,6 +17519,12 @@ packages:
mlly: 1.3.0
pathe: 1.1.1
/playwright-core@1.35.1:
resolution: {integrity: sha512-pNXb6CQ7OqmGDRspEjlxE49w+4YtR6a3X6mT1hZXeJHWmsEz7SunmvZeiG/+y1yyMZdHnnn73WKYdtV1er0Xyg==}
engines: {node: '>=16'}
hasBin: true
dev: true
/png-js@1.0.0:
resolution: {integrity: sha512-k+YsbhpA9e+EFfKjTCH3VW6aoKlyNYI6NYdTfDL4CIvFnvsuO84ttonmZE7rc+v23SLTH8XX+5w/Ak9v0xGY4g==}
dev: false
@ -18614,7 +18653,7 @@ packages:
resolution: {integrity: sha512-I6V1G2JkJ2JFIFSVuultNXepf7BW8SCaSUOq5IETM2fDjFim5Dg5F1zU/QbplNW0mqkk8QCw+I722v3nPkpRlA==}
dependencies:
busboy: 1.6.0
zod: 3.21.4
zod: 3.20.2
/reusify@1.0.4:
resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==}
@ -21089,5 +21128,9 @@ packages:
readable-stream: 3.6.2
dev: false
/zod@3.20.2:
resolution: {integrity: sha512-1MzNQdAvO+54H+EaK5YpyEy0T+Ejo/7YLHS93G3RnYWh5gaotGHwGeN/ZO687qEDU2y4CdStQYXVHIgrUl5UVQ==}
/zod@3.21.4:
resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==}
dev: false

View file

@ -1,7 +1,7 @@
{
"$schema": "https://turbo.build/schema.json",
"globalDependencies": ["**/.env.*local"],
"globalEnv": ["VERCEL_ENV", "APP_LOG_LEVEL", "NODE_ENV"],
"globalEnv": ["VERCEL_ENV", "APP_LOG_LEVEL", "NODE_ENV", "CI"],
"pipeline": {
"build": {
"env": ["NEXT_PUBLIC_VERCEL_ENV"],