chore: switch playwright tests to use playwright test runner instead of vitest (podman-desktop#8650)

* chore(test): switch e2e tests to use playwright test runner
Signed-off-by: Vladimir Lazar <[email protected]>

Author: cbr7

package.json

@@ -38,11 +38,13 @@
     "test:unit": "npm run test:main && npm run test:preload && npm run test:preload-docker-extension && npm run test:preload-webview && npm run test:ui && npm run test:renderer && npm run test:scripts:stylesheet && npm run test:tools && npm run test:extensions",
     "test:e2e": "npm run test:e2e:build && npm run test:e2e:run",
     "test:e2e:build": "cross-env NODE_ENV=development MODE=development DEBUG=pw:browser npm run build",
-    "test:e2e:run": "xvfb-maybe --auto-servernum --server-args='-screen 0 1280x960x24' -- vitest run tests/playwright/src/specs/ --pool=threads --poolOptions.threads.singleThread --poolOptions.threads.isolate --no-file-parallelism",
+    "test:e2e:run": "xvfb-maybe --auto-servernum --server-args='-screen 0 1280x960x24' -- npx playwright test tests/playwright/src/specs/",
     "test:e2e:smoke": "npm run test:e2e:build && npm run test:e2e:smoke:run",
-    "test:e2e:smoke:run": "xvfb-maybe --auto-servernum --server-args='-screen 0 1280x960x24' -- vitest run tests/playwright/src/specs/*smoke.spec.ts --pool=threads --poolOptions.threads.singleThread --poolOptions.threads.isolate --no-file-parallelism",
+    "test:e2e:smoke:run": "xvfb-maybe --auto-servernum --server-args='-screen 0 1280x960x24' -- npx playwright test tests/playwright/src/specs/ -g @smoke",
     "test:e2e:extension": "npm run test:e2e:build && npm run test:e2e:extension:run",
-    "test:e2e:extension:run": "xvfb-maybe --auto-servernum --server-args='-screen 0 1280x960x24' -- vitest run tests/playwright/src/specs/extension-installation.spec.ts --pool=threads --poolOptions.threads.singleThread --poolOptions.threads.isolate --no-file-parallelism",
+    "test:e2e:extension:run": "xvfb-maybe --auto-servernum --server-args='-screen 0 1280x960x24' -- npx playwright test tests/playwright/src/specs/extension-installation.spec.ts",
+    "test:e2e:pw": "npm run test:e2e:build && npm run test:e2e:pw:run",
+    "test:e2e:pw:run": "xvfb-maybe --auto-servernum --server-args='-screen 0 1280x960x24' -- npx playwright test tests/playwright/src/specs/container-smoke.spec.ts",
     "test:e2e:copy": "cp ../podman-desktop-extension-bootc/tests/src/bootc-extension.spec.ts tests/src/",
     "test:main": "vitest run -r packages/main --passWithNoTests --coverage",
     "test:preload": "vitest run -r packages/preload --passWithNoTests --coverage",

tests/playwright/src/setupFiles/extended-hooks.ts playwright.config.ts

@@ -1,5 +1,5 @@
 /**********************************************************************
- * Copyright (C) 2023-2024-2024 Red Hat, Inc.
+ * Copyright (C) 2024 Red Hat, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,15 +16,25 @@
  * SPDX-License-Identifier: Apache-2.0
  ***********************************************************************/
 
-import { afterEach, beforeEach, onTestFailed } from 'vitest';
+import { defineConfig, devices } from '@playwright/test';
 
-import type { RunnerTestContext } from '../testContext/runner-test-context';
-import { takeScreenshotHook } from './extended-hooks-utils';
+export default defineConfig({
+  outputDir: 'tests/playwright/output/',
+  workers: 1,
 
-afterEach(async (context: RunnerTestContext) => {
-  onTestFailed(async () => await takeScreenshotHook(context.pdRunner, context.task.name));
-});
+  reporter: [
+    ['list'],
+    ['junit', { outputFile: 'tests/playwright/output/junit-results.xml' }],
+    ['json', { outputFile: 'tests/playwright/output/json-results.json' }],
+    ['html', { open: 'never', outputFolder: 'tests/playwright/output/html-results/' }],
+  ],
 
-beforeEach(async (context: RunnerTestContext) => {
-  context?.pdRunner?.setTestPassed(true);
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+  ],
 });

tests/playwright/src/index.ts

@@ -19,7 +19,6 @@
 // export core modules
 export * from './globalSetup/global-setup';
 export * from './runner/podman-desktop-runner';
-export * from './setupFiles/extended-hooks-utils';
 export * from './setupFiles/setup-registry';
 export type { RunnerTestContext } from './testContext/runner-test-context';
 export * from './utility/cleanup';

tests/playwright/src/model/pages/extensions-page.ts

@@ -21,9 +21,9 @@ import { expect as playExpect } from '@playwright/test';
 
 import { ExtensionCardPage } from './extension-card-page';
 import type { ExtensionDetailsPage } from './extension-details-page';
-import { MainPage } from './main-page';
 
-export class ExtensionsPage extends MainPage {
+export class ExtensionsPage {
+  readonly page: Page;
   readonly heading: Locator;
   readonly header: Locator;
   readonly content: Locator;
@@ -33,7 +33,7 @@ export class ExtensionsPage extends MainPage {
   readonly installExtensionFromOCIImageButton: Locator;
 
   constructor(page: Page) {
-    super(page, 'extensions');
+    this.page = page;
     this.header = page.getByRole('region', { name: 'header' });
     this.content = page.getByRole('region', { name: 'content' });
     this.heading = this.header.getByLabel('Title').getByText('extensions');

tests/playwright/src/runner/podman-desktop-runner.ts

@@ -20,7 +20,7 @@ import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
 
 import type { ElectronApplication, JSHandle, Page } from '@playwright/test';
-import { _electron as electron } from '@playwright/test';
+import { _electron as electron, test } from '@playwright/test';
 import type { BrowserWindow } from 'electron';
 
 import { waitWhile } from '../utility/wait';
@@ -42,8 +42,6 @@ export class PodmanDesktopRunner {
   private _videoAndTraceName: string | undefined;
   private _autoUpdate: boolean;
   private _autoCheckUpdate: boolean;
-  private _testPassed: boolean;
-  private _suitePassed: boolean;
 
   constructor({
     profile = '',
@@ -53,13 +51,11 @@ export class PodmanDesktopRunner {
   }: { profile?: string; customFolder?: string; autoUpdate?: boolean; autoCheckUpdate?: boolean } = {}) {
     this._running = false;
     this._profile = profile;
-    this._testOutput = join('tests', 'output', this._profile);
+    this._testOutput = join('tests', 'playwright', 'output', this._profile);
     this._customFolder = join(this._testOutput, customFolder);
     this._videoAndTraceName = undefined;
     this._autoUpdate = autoUpdate;
     this._autoCheckUpdate = autoCheckUpdate;
-    this._testPassed = false;
-    this._suitePassed = true;
 
     // Options setting always needs to be last action in constructor in order to apply settings correctly
     this._options = this.defaultOptions();
@@ -138,7 +134,7 @@ export class PodmanDesktopRunner {
   }
 
   public async startTracing(): Promise<void> {
-    await this.getPage().context().tracing.start({ screenshots: true, snapshots: true });
+    await this.getPage().context().tracing.start({ screenshots: true, snapshots: true, sources: true });
   }
 
   public async stopTracing(): Promise<void> {
@@ -237,7 +233,7 @@ export class PodmanDesktopRunner {
       rmSync(rawTracesPath, { recursive: true, force: true, maxRetries: 5 });
     }
 
-    if (!this._testPassed || !this._suitePassed || !this._videoAndTraceName) return;
+    if (test.info().status === 'failed') return;
 
     if (!process.env.KEEP_TRACES_ON_PASS) {
       const tracesPath = join(this._testOutput, 'traces', `${this._videoAndTraceName}_trace.zip`);
@@ -342,18 +338,6 @@ export class PodmanDesktopRunner {
     return this._testOutput;
   }
 
-  public getTestPassed(): boolean {
-    return this._testPassed;
-  }
-
-  public setTestPassed(value: boolean): void {
-    this._testPassed = value;
-  }
-
-  public setSuitePassed(value: boolean): void {
-    this._suitePassed = value;
-  }
-
   public get options(): object {
     return this._options;
   }

tests/playwright/src/setupFiles/extended-hooks-utils.ts

@@ -16,11 +16,8 @@
  * SPDX-License-Identifier: Apache-2.0
  ***********************************************************************/
 
-import { afterEach } from 'node:test';
-
 import type { Page } from '@playwright/test';
-import { expect as playExpect } from '@playwright/test';
-import { afterAll, beforeAll, beforeEach, describe, test } from 'vitest';
+import { expect as playExpect, test } from '@playwright/test';
 
 import { CLIToolsPage } from '../model/pages/cli-tools-page';
 import { ComposeLocalInstallPage } from '../model/pages/compose-onboarding/compose-local-install-page';
@@ -33,7 +30,6 @@ import { SettingsBar } from '../model/pages/settings-bar';
 import { WelcomePage } from '../model/pages/welcome-page';
 import { NavigationBar } from '../model/workbench/navigation';
 import { PodmanDesktopRunner } from '../runner/podman-desktop-runner';
-import type { RunnerTestContext } from '../testContext/runner-test-context';
 import { isCI, isLinux } from '../utility/platform';
 
 const RESOURCE_NAME: string = 'Compose';
@@ -45,7 +41,9 @@ let composeVersion: string;
 // property that will make sure that on linux we can run only partial tests, by default this is turned off
 const composePartialInstallation = process.env.COMPOSE_PARTIAL_INSTALL ? process.env.COMPOSE_PARTIAL_INSTALL : false;
 
-beforeAll(async () => {
+test.skip(!!isCI && isLinux, 'Tests suite should not run on Linux platform');
+
+test.beforeAll(async () => {
   pdRunner = new PodmanDesktopRunner();
   page = await pdRunner.start();
   pdRunner.setVideoAndTraceName('compose-onboarding-e2e');
@@ -55,16 +53,12 @@ beforeAll(async () => {
   navBar = new NavigationBar(page);
 });
 
-afterAll(async () => {
+test.afterAll(async () => {
   await pdRunner.close();
 });
 
-beforeEach<RunnerTestContext>(async ctx => {
-  ctx.pdRunner = pdRunner;
-});
-
-describe.skipIf(isCI && isLinux)('Compose onboarding workflow verification', async () => {
-  afterEach(async () => {
+test.describe.serial('Compose onboarding workflow verification @smoke', () => {
+  test.afterEach(async () => {
     await navBar.openDashboard();
   });
 
@@ -125,7 +119,9 @@ describe.skipIf(isCI && isLinux)('Compose onboarding workflow verification', asy
     await skipOkButton.click();
   });
 
-  test.skipIf(composePartialInstallation)('Can install Compose system-wide', async () => {
+  test('Can install Compose system-wide', async () => {
+    test.skip(!!composePartialInstallation, 'Partial installation of Compose is enabled');
+
     const onboardingPage = await openComposeOnboarding(page);
     await onboardingPage.nextStepButton.click();
 
@@ -140,7 +136,9 @@ describe.skipIf(isCI && isLinux)('Compose onboarding workflow verification', asy
     await playExpect(resourcesPage.heading).toBeVisible();
   });
 
-  test.skipIf(composePartialInstallation)('Verify Compose was installed', async () => {
+  test('Verify Compose was installed', async () => {
+    test.skip(!!composePartialInstallation, 'Partial installation of Compose is enabled');
+
     await navBar.openSettings();
     const settingsBar = new SettingsBar(page);
     const resourcesPage = await settingsBar.openTabPage(ResourcesPage);

tests/playwright/src/specs/compose-onboarding-smoke.spec.ts

@@ -17,8 +17,7 @@
  ***********************************************************************/
 
 import type { Page } from '@playwright/test';
-import { expect as playExpect } from '@playwright/test';
-import { afterAll, beforeAll, beforeEach, describe, test } from 'vitest';
+import { expect as playExpect, test } from '@playwright/test';
 
 import { ContainerState } from '../model/core/states';
 import type { ContainerInteractiveParams } from '../model/core/types';
@@ -28,7 +27,6 @@ import type { ImagesPage } from '../model/pages/images-page';
 import { WelcomePage } from '../model/pages/welcome-page';
 import { NavigationBar } from '../model/workbench/navigation';
 import { PodmanDesktopRunner } from '../runner/podman-desktop-runner';
-import type { RunnerTestContext } from '../testContext/runner-test-context';
 import { deleteContainer, deleteImage } from '../utility/operations';
 import { waitForPodmanMachineStartup, waitWhile } from '../utility/wait';
 
@@ -40,7 +38,7 @@ const containerToRun = 'alpine-container';
 const containerList = ['first', 'second', 'third'];
 const containerStartParams: ContainerInteractiveParams = { attachTerminal: false };
 
-beforeAll(async () => {
+test.beforeAll(async () => {
   pdRunner = new PodmanDesktopRunner();
   page = await pdRunner.start();
   pdRunner.setVideoAndTraceName('containers-e2e');
@@ -67,11 +65,9 @@ beforeAll(async () => {
   }
 });
 
-beforeEach<RunnerTestContext>(async ctx => {
-  ctx.pdRunner = pdRunner;
-});
+test.afterAll(async () => {
+  test.setTimeout(90000);
 
-afterAll(async () => {
   try {
     await deleteContainer(page, containerToRun);
     for (const container of containerList) {
@@ -82,10 +78,14 @@ afterAll(async () => {
   } finally {
     await pdRunner.close();
   }
-}, 90000);
+});
+
+test.describe.serial('Verification of container creation workflow @smoke', () => {
+  test.describe.configure({ retries: 2 });
+
+  test(`Pulling of '${imageToPull}:${imageTag}' image`, async () => {
+    test.setTimeout(90000);
 
-describe('Verification of container creation workflow', async () => {
-  test(`Pulling of '${imageToPull}:${imageTag}' image`, { retry: 2, timeout: 90000 }, async () => {
     const navigationBar = new NavigationBar(page);
     let images = await navigationBar.openImages();
     const pullImagePage = await images.openPullImage();
@@ -114,6 +114,7 @@ describe('Verification of container creation workflow', async () => {
     images = await navigationBar.openImages();
     playExpect(await images.getCurrentStatusOfImage(imageToPull)).toBe('USED');
   });
+
   test('Test navigation between pages', async () => {
     const navigationBar = new NavigationBar(page);
     const containers = await navigationBar.openContainers();
@@ -253,6 +254,8 @@ describe('Verification of container creation workflow', async () => {
   });
 
   test('Prune containers', async () => {
+    test.setTimeout(120000);
+
     const navigationBar = new NavigationBar(page);
     //Start 3 containers
     for (const container of containerList) {
@@ -300,5 +303,5 @@ describe('Verification of container creation workflow', async () => {
         .poll(async () => await containersPage.containerExists(container), { timeout: 30000 })
         .toBeFalsy();
     }
-  }, 120000);
+  });
 });