test(a11y): e2e baseline tests for a11y screen reader descriptions

.buildkite/pipelines/pull_request/pipeline.ts

@@ -18,7 +18,8 @@ import {
   e2eServerStep,
   eslintStep,
   jestStep,
-  playwrightStep,
+  playwrightVrtStep,
+  playwrightA11yStep,
   prettierStep,
   docsStep,
   storybookStep,
@@ -60,7 +61,8 @@ void (async () => {
       e2eServerStep(),
       firebasePreDeployStep(),
       ghpDeployStep(),
-      playwrightStep(),
+      playwrightVrtStep(),
+      playwrightA11yStep(),
       firebaseDeployStep(),
     ].map((step) => step(changeCtx));
 

.buildkite/scripts/pre_exit.ts

@@ -9,7 +9,7 @@
 import { yarnInstall } from './../utils/exec';
 import { bkEnv, buildkiteGQLQuery, codeCheckIsCompleted, getJobMetadata, updateCheckStatus } from '../utils';
 
-const skipChecks = new Set(['playwright']);
+const skipChecks = new Set(['playwright_vrt', 'playwright_a11y']);
 
 void (async function () {
   const { checkId, jobId, jobUrl } = bkEnv;

.buildkite/scripts/steps/e2e_reports.ts

@@ -38,7 +38,9 @@ async function setGroupStatus() {
     return;
   }
 
-  const e2eJobs = await getBuildJobs('playwright__parallel-step');
+  const parallelKey =
+    checkId === 'playwright_a11y' ? 'playwright_a11y__parallel-step' : 'playwright_vrt__parallel-step';
+  const e2eJobs = await getBuildJobs(parallelKey);
   const jobStateMap = new Map<string, number>();
   jobStateMap.set('Success', 0);
   jobStateMap.set('Failed', 0);
@@ -174,9 +176,17 @@ void (async () => {
 
   await setGroupStatus();
 
-  await downloadArtifacts('.buildkite/artifacts/e2e_reports/*');
+  const { checkId } = bkEnv;
+  const isA11y = checkId === 'playwright_a11y';
+  const artifactPath = isA11y ? '.buildkite/artifacts/a11y_reports/*' : '.buildkite/artifacts/vrt_reports/*';
+  const reportDir = isA11y ? '.buildkite/artifacts/a11y_reports' : '.buildkite/artifacts/vrt_reports';
+  const outputDir = isA11y ? 'merged_a11y_html_report' : 'merged_vrt_html_report';
+  const outputArtifact = isA11y
+    ? '.buildkite/artifacts/merged_a11y_html_report.gz'
+    : '.buildkite/artifacts/merged_vrt_html_report.gz';
+
+  await downloadArtifacts(artifactPath);
 
-  const reportDir = '.buildkite/artifacts/e2e_reports';
   const files = fs.readdirSync(reportDir);
   await Promise.all<void>(
     files
@@ -189,18 +199,18 @@ void (async () => {
       ),
   );
 
-  startGroup('Merging e2e reports');
+  startGroup(`Merging ${isA11y ? 'a11y' : 'e2e'} reports`);
 
   await exec('yarn merge:reports', {
     cwd: 'e2e',
     env: {
-      HTML_REPORT_DIR: 'merged_html_report',
+      HTML_REPORT_DIR: outputDir,
     },
   });
 
   await compress({
-    src: 'e2e/merged_html_report',
-    dest: '.buildkite/artifacts/merged_html_report.gz',
+    src: path.join('e2e', outputDir),
+    dest: outputArtifact,
   });
 
   if (bkEnv.steps.playwright.updateScreenshots) {

.buildkite/scripts/steps/firebase_deploy.ts

@@ -40,24 +40,33 @@ void (async () => {
     dest: path.join(outDir, 'e2e'),
   });
 
-  const e2eReportSrc = '.buildkite/artifacts/merged_html_report.gz';
-  await downloadArtifacts(e2eReportSrc, 'playwright_merge_and_status');
+  const vrtReportSrc = '.buildkite/artifacts/merged_vrt_html_report.gz';
+  await downloadArtifacts(vrtReportSrc, 'playwright_vrt_merge_and_status');
   await decompress({
-    src: e2eReportSrc,
-    dest: path.join(outDir, 'e2e-report'),
+    src: vrtReportSrc,
+    dest: path.join(outDir, 'vrt-report'),
+  });
+
+  const a11yReportSrc = '.buildkite/artifacts/merged_a11y_html_report.gz';
+  await downloadArtifacts(a11yReportSrc, 'playwright_a11y_merge_and_status');
+  await decompress({
+    src: a11yReportSrc,
+    dest: path.join(outDir, 'a11y-report'),
   });
 
   startGroup('Check deployment files');
 
   const hasDocsIndex = fs.existsSync(path.join(outDir, 'index.html'));
   const hasStorybookIndex = fs.existsSync(path.join(outDir, 'storybook/index.html'));
   const hasE2EIndex = fs.existsSync(path.join(outDir, 'e2e/index.html'));
-  const hasE2EReportIndex = fs.existsSync(path.join(outDir, 'e2e-report/index.html'));
+  const hasVrtReportIndex = fs.existsSync(path.join(outDir, 'vrt-report/index.html'));
+  const hasA11yReportIndex = fs.existsSync(path.join(outDir, 'a11y-report/index.html'));
   const missingFiles = [
     ['docs', hasDocsIndex],
     ['storybook', hasStorybookIndex],
     ['e2e server', hasE2EIndex],
-    ['e2e report', hasE2EReportIndex],
+    ['vrt report', hasVrtReportIndex],
+    ['a11y report', hasA11yReportIndex],
   ]
     .filter(([, exists]) => !exists)
     .map<string>(([f]) => f as string);

.buildkite/scripts/steps/playwright_a11y.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import path from 'path';
+
+import { getMetadata, setMetadata } from 'buildkite-agent-node';
+
+import { updateCheckStatus } from './../../utils/github';
+import { exec, downloadArtifacts, startGroup, yarnInstall, getNumber, decompress, compress, bkEnv } from '../../utils';
+import { ENV_URL } from '../../utils/constants';
+
+const jobIndex = getNumber(process.env.BUILDKITE_PARALLEL_JOB);
+const shardIndex = jobIndex ? jobIndex + 1 : 1;
+const jobTotal = getNumber(process.env.BUILDKITE_PARALLEL_JOB_COUNT);
+
+const pwFlags = ['--project=Chrome', '--config=playwright.a11y.config.ts'];
+
+if (jobIndex !== null && jobTotal !== null) {
+  pwFlags.push(`--shard=${shardIndex}/${jobTotal}`);
+}
+
+void (async () => {
+  await yarnInstall('e2e');
+
+  const key = `${bkEnv.checkId}--activeJobs`;
+  const value = shardIndex === jobTotal ? jobTotal - 1 : Number(await getMetadata(key));
+  // TODO improve this status logic, not easy to communicate state of parallel steps
+  const activeJobs = Math.min((Number.isNaN(value) ? 0 : value) + 1, jobTotal ?? 1);
+  await setMetadata(key, String(activeJobs));
+
+  await updateCheckStatus(
+    {
+      status: 'in_progress',
+    },
+    'playwright_a11y',
+    `${activeJobs} of ${jobTotal ?? 1} a11y jobs started`,
+  );
+
+  const src = '.buildkite/artifacts/e2e_server.gz';
+  await downloadArtifacts(src, 'build_e2e');
+  await decompress({
+    src,
+    dest: 'e2e/server',
+  });
+
+  startGroup('Check Architecture');
+  await exec('arch');
+
+  startGroup('Running e2e a11y playwright job');
+  const reportDir = `reports/a11y_report_${shardIndex}`;
+  async function postCommandTasks() {
+    await compress({
+      src: path.join('e2e', reportDir),
+      dest: `.buildkite/artifacts/a11y_reports/report_${shardIndex}.gz`,
+    });
+  }
+
+  const command = `yarn playwright test ${pwFlags.join(' ')}`;
+
+  try {
+    await exec(command, {
+      cwd: 'e2e',
+      env: {
+        [ENV_URL]: 'http://127.0.0.1:9002',
+        PLAYWRIGHT_HTML_REPORT: reportDir,
+        PLAYWRIGHT_JSON_OUTPUT_NAME: `reports/a11y-json/report_${shardIndex}.json`,
+      },
+    });
+    await postCommandTasks();
+  } catch (error) {
+    await postCommandTasks();
+    throw error;
+  }
+})();

.buildkite/scripts/steps/playwright.ts → .buildkite/scripts/steps/playwright_vrt.ts

@@ -80,7 +80,7 @@ void (async () => {
     {
       status: 'in_progress',
     },
-    'playwright',
+    'playwright_vrt',
     `${activeJobs} of ${jobTotal ?? 1} jobs started`,
   );
 
@@ -98,12 +98,12 @@ void (async () => {
   // TODO Fix this duplicate script that allows us to skip root node install on all e2e test runners
   await exec('node ./e2e/scripts/extract_examples.js');
 
-  startGroup('Running e2e playwright job');
-  const reportDir = `reports/report_${shardIndex}`;
+  startGroup('Running e2e vrt playwright job');
+  const reportDir = `reports/vrt_report_${shardIndex}`;
   async function postCommandTasks() {
     await compress({
       src: path.join('e2e', reportDir),
-      dest: `.buildkite/artifacts/e2e_reports/report_${shardIndex}.gz`,
+      dest: `.buildkite/artifacts/vrt_reports/report_${shardIndex}.gz`,
     });
 
     if (bkEnv.steps.playwright.updateScreenshots) {
@@ -119,7 +119,7 @@ void (async () => {
       env: {
         [ENV_URL]: 'http://127.0.0.1:9002',
         PLAYWRIGHT_HTML_REPORT: reportDir,
-        PLAYWRIGHT_JSON_OUTPUT_NAME: `reports/json/report_${shardIndex}.json`,
+        PLAYWRIGHT_JSON_OUTPUT_NAME: `reports/vrt-json/report_${shardIndex}.json`,
       },
     });
     await postCommandTasks();

.buildkite/steps/firebase_deploy.ts

@@ -15,7 +15,13 @@ export const firebaseDeployStep = createStep<CustomCommandStep>(() => {
     label: ':firebase: Deploy - firebase',
     key: 'deploy_fb',
     allow_dependency_failure: true,
-    depends_on: ['build_docs', 'build_storybook', 'build_e2e', 'playwright_merge_and_status'],
+    depends_on: [
+      'build_docs',
+      'build_storybook',
+      'build_e2e',
+      'playwright_vrt_merge_and_status',
+      'playwright_a11y_merge_and_status',
+    ],
     commands: ['npx ts-node .buildkite/scripts/steps/firebase_deploy.ts'],
     env: {
       ECH_CHECK_ID: 'deploy_fb',

.buildkite/steps/index.ts

@@ -11,7 +11,8 @@ export * from './eslint';
 export * from './api_check';
 export * from './type_check';
 export * from './prettier';
-export * from './playwright';
+export * from './playwright_vrt';
+export * from './playwright_a11y';
 export * from './docs';
 export * from './storybook';
 export * from './e2e_server';

.buildkite/steps/playwright_a11y.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { CustomGroupStep } from '../utils';
+import { createStep, commandStepDefaults, Plugins } from '../utils';
+
+export const playwrightA11yStep = createStep<CustomGroupStep>(() => {
+  const skip = false;
+  const parallelKey = 'playwright_a11y__parallel-step';
+  return {
+    group: ':playwright: Playwright e2e A11Y',
+    key: 'playwright_a11y',
+    skip,
+    steps: [
+      {
+        ...commandStepDefaults,
+        label: ':playwright: Playwright e2e A11Y',
+        skip,
+        parallelism: 1,
+        retry: {
+          automatic: [
+            {
+              // Playwright tests likely failed correctly
+              exit_status: 1,
+              limit: 0,
+            },
+            {
+              // Something went wrong with step command setup, retry once
+              exit_status: '*',
+              limit: 1,
+            },
+          ],
+        },
+        timeout_in_minutes: 10, // Shorter timeout for a11y tests
+        key: parallelKey,
+        depends_on: ['build_e2e'],
+        plugins: [Plugins.docker.playwright()],
+        artifact_paths: ['.buildkite/artifacts/a11y_reports/*', 'e2e/reports/a11y-json/*'],
+        commands: ['npx ts-node .buildkite/scripts/steps/playwright_a11y.ts'],
+      },
+      {
+        ...commandStepDefaults,
+        key: 'playwright_a11y_merge_and_status',
+        label: ':playwright: Set a11y group status and merge reports',
+        skip,
+        allow_dependency_failure: true,
+        depends_on: [{ step: parallelKey, allow_failure: true }],
+        commands: ['npx ts-node .buildkite/scripts/steps/e2e_reports.ts'],
+        env: {
+          ECH_CHECK_ID: 'playwright_a11y',
+        },
+      },
+    ],
+  };
+});

.buildkite/steps/playwright.ts → .buildkite/steps/playwright_vrt.ts

@@ -9,17 +9,17 @@
 import type { CustomGroupStep } from '../utils';
 import { createStep, commandStepDefaults, Plugins } from '../utils';
 
-export const playwrightStep = createStep<CustomGroupStep>(() => {
+export const playwrightVrtStep = createStep<CustomGroupStep>(() => {
   const skip = false;
-  const parallelKey = 'playwright__parallel-step';
+  const parallelKey = 'playwright_vrt__parallel-step';
   return {
-    group: ':playwright: Playwright e2e',
-    key: 'playwright',
+    group: ':playwright: Playwright e2e VRT',
+    key: 'playwright_vrt',
     skip,
     steps: [
       {
         ...commandStepDefaults,
-        label: ':playwright: Playwright e2e',
+        label: ':playwright: Playwright e2e VRT',
         skip,
         parallelism: 10,
         retry: {
@@ -41,23 +41,23 @@ export const playwrightStep = createStep<CustomGroupStep>(() => {
         depends_on: ['build_e2e'],
         plugins: [Plugins.docker.playwright()],
         artifact_paths: [
-          '.buildkite/artifacts/e2e_reports/*',
+          '.buildkite/artifacts/vrt_reports/*',
           '.buildkite/artifacts/screenshots/*',
           '.buildkite/artifacts/screenshot_meta/*',
           'e2e/reports/json/*',
         ],
-        commands: ['npx ts-node .buildkite/scripts/steps/playwright.ts'],
+        commands: ['npx ts-node .buildkite/scripts/steps/playwright_vrt.ts'],
       },
       {
         ...commandStepDefaults,
-        key: 'playwright_merge_and_status',
-        label: ':playwright: Set group status and merge reports',
+        key: 'playwright_vrt_merge_and_status',
+        label: ':playwright: Set vrt group status and merge reports',
         skip,
         allow_dependency_failure: true,
         depends_on: [{ step: parallelKey, allow_failure: true }],
         commands: ['npx ts-node .buildkite/scripts/steps/e2e_reports.ts'],
         env: {
-          ECH_CHECK_ID: 'playwright',
+          ECH_CHECK_ID: 'playwright_vrt',
         },
       },
     ],

.buildkite/utils/build.ts

@@ -38,7 +38,8 @@ export const getBuildConfig = (): BuildConfig => {
       { name: 'Deploy - firebase', id: 'deploy_fb' },
       ...(bkEnv.isMainBranch ? [{ name: 'Deploy - GitHub Pages', id: 'deploy_ghp' }] : []),
       { name: 'Jest', id: 'jest' },
-      { name: 'Playwright e2e', id: 'playwright' },
+      { name: 'Playwright e2e VRT', id: 'playwright_vrt' },
+      { name: 'Playwright e2e A11Y', id: 'playwright_a11y' },
     ],
   };
 };