diff --git a/.buildkite/pipelines/pull_request/pipeline.ts b/.buildkite/pipelines/pull_request/pipeline.ts index 61d665b790..6b2b6a07b1 100644 --- a/.buildkite/pipelines/pull_request/pipeline.ts +++ b/.buildkite/pipelines/pull_request/pipeline.ts @@ -18,7 +18,8 @@ import { e2eServerStep, eslintStep, jestStep, - playwrightStep, + playwrightVrtStep, + playwrightA11yStep, prettierStep, docsStep, storybookStep, @@ -60,7 +61,8 @@ void (async () => { e2eServerStep(), firebasePreDeployStep(), ghpDeployStep(), - playwrightStep(), + playwrightVrtStep(), + playwrightA11yStep(), firebaseDeployStep(), ].map((step) => step(changeCtx)); diff --git a/.buildkite/scripts/pre_exit.ts b/.buildkite/scripts/pre_exit.ts index e69846fb8a..8d6f44c3ee 100644 --- a/.buildkite/scripts/pre_exit.ts +++ b/.buildkite/scripts/pre_exit.ts @@ -9,7 +9,7 @@ import { yarnInstall } from './../utils/exec'; import { bkEnv, buildkiteGQLQuery, codeCheckIsCompleted, getJobMetadata, updateCheckStatus } from '../utils'; -const skipChecks = new Set(['playwright']); +const skipChecks = new Set(['playwright_vrt', 'playwright_a11y']); void (async function () { const { checkId, jobId, jobUrl } = bkEnv; diff --git a/.buildkite/scripts/steps/e2e_reports.ts b/.buildkite/scripts/steps/e2e_reports.ts index 7d59f6098f..3783cbcb0b 100644 --- a/.buildkite/scripts/steps/e2e_reports.ts +++ b/.buildkite/scripts/steps/e2e_reports.ts @@ -38,7 +38,9 @@ async function setGroupStatus() { return; } - const e2eJobs = await getBuildJobs('playwright__parallel-step'); + const parallelKey = + checkId === 'playwright_a11y' ? 'playwright_a11y__parallel-step' : 'playwright_vrt__parallel-step'; + const e2eJobs = await getBuildJobs(parallelKey); const jobStateMap = new Map(); jobStateMap.set('Success', 0); jobStateMap.set('Failed', 0); @@ -174,9 +176,17 @@ void (async () => { await setGroupStatus(); - await downloadArtifacts('.buildkite/artifacts/e2e_reports/*'); + const { checkId } = bkEnv; + const isA11y = checkId === 'playwright_a11y'; + const artifactPath = isA11y ? '.buildkite/artifacts/a11y_reports/*' : '.buildkite/artifacts/vrt_reports/*'; + const reportDir = isA11y ? '.buildkite/artifacts/a11y_reports' : '.buildkite/artifacts/vrt_reports'; + const outputDir = isA11y ? 'merged_a11y_html_report' : 'merged_vrt_html_report'; + const outputArtifact = isA11y + ? '.buildkite/artifacts/merged_a11y_html_report.gz' + : '.buildkite/artifacts/merged_vrt_html_report.gz'; + + await downloadArtifacts(artifactPath); - const reportDir = '.buildkite/artifacts/e2e_reports'; const files = fs.readdirSync(reportDir); await Promise.all( files @@ -189,18 +199,18 @@ void (async () => { ), ); - startGroup('Merging e2e reports'); + startGroup(`Merging ${isA11y ? 'a11y' : 'e2e'} reports`); await exec('yarn merge:reports', { cwd: 'e2e', env: { - HTML_REPORT_DIR: 'merged_html_report', + HTML_REPORT_DIR: outputDir, }, }); await compress({ - src: 'e2e/merged_html_report', - dest: '.buildkite/artifacts/merged_html_report.gz', + src: path.join('e2e', outputDir), + dest: outputArtifact, }); if (bkEnv.steps.playwright.updateScreenshots) { diff --git a/.buildkite/scripts/steps/firebase_deploy.ts b/.buildkite/scripts/steps/firebase_deploy.ts index ecbf53b441..48164afb21 100644 --- a/.buildkite/scripts/steps/firebase_deploy.ts +++ b/.buildkite/scripts/steps/firebase_deploy.ts @@ -40,11 +40,18 @@ void (async () => { dest: path.join(outDir, 'e2e'), }); - const e2eReportSrc = '.buildkite/artifacts/merged_html_report.gz'; - await downloadArtifacts(e2eReportSrc, 'playwright_merge_and_status'); + const vrtReportSrc = '.buildkite/artifacts/merged_vrt_html_report.gz'; + await downloadArtifacts(vrtReportSrc, 'playwright_vrt_merge_and_status'); await decompress({ - src: e2eReportSrc, - dest: path.join(outDir, 'e2e-report'), + src: vrtReportSrc, + dest: path.join(outDir, 'vrt-report'), + }); + + const a11yReportSrc = '.buildkite/artifacts/merged_a11y_html_report.gz'; + await downloadArtifacts(a11yReportSrc, 'playwright_a11y_merge_and_status'); + await decompress({ + src: a11yReportSrc, + dest: path.join(outDir, 'a11y-report'), }); startGroup('Check deployment files'); @@ -52,12 +59,14 @@ void (async () => { const hasDocsIndex = fs.existsSync(path.join(outDir, 'index.html')); const hasStorybookIndex = fs.existsSync(path.join(outDir, 'storybook/index.html')); const hasE2EIndex = fs.existsSync(path.join(outDir, 'e2e/index.html')); - const hasE2EReportIndex = fs.existsSync(path.join(outDir, 'e2e-report/index.html')); + const hasVrtReportIndex = fs.existsSync(path.join(outDir, 'vrt-report/index.html')); + const hasA11yReportIndex = fs.existsSync(path.join(outDir, 'a11y-report/index.html')); const missingFiles = [ ['docs', hasDocsIndex], ['storybook', hasStorybookIndex], ['e2e server', hasE2EIndex], - ['e2e report', hasE2EReportIndex], + ['vrt report', hasVrtReportIndex], + ['a11y report', hasA11yReportIndex], ] .filter(([, exists]) => !exists) .map(([f]) => f as string); diff --git a/.buildkite/scripts/steps/playwright_a11y.ts b/.buildkite/scripts/steps/playwright_a11y.ts new file mode 100644 index 0000000000..cdcb2f0e8e --- /dev/null +++ b/.buildkite/scripts/steps/playwright_a11y.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import path from 'path'; + +import { getMetadata, setMetadata } from 'buildkite-agent-node'; + +import { updateCheckStatus } from './../../utils/github'; +import { exec, downloadArtifacts, startGroup, yarnInstall, getNumber, decompress, compress, bkEnv } from '../../utils'; +import { ENV_URL } from '../../utils/constants'; + +const jobIndex = getNumber(process.env.BUILDKITE_PARALLEL_JOB); +const shardIndex = jobIndex ? jobIndex + 1 : 1; +const jobTotal = getNumber(process.env.BUILDKITE_PARALLEL_JOB_COUNT); + +const pwFlags = ['--project=Chrome', '--config=playwright.a11y.config.ts']; + +if (jobIndex !== null && jobTotal !== null) { + pwFlags.push(`--shard=${shardIndex}/${jobTotal}`); +} + +void (async () => { + await yarnInstall('e2e'); + + const key = `${bkEnv.checkId}--activeJobs`; + const value = shardIndex === jobTotal ? jobTotal - 1 : Number(await getMetadata(key)); + // TODO improve this status logic, not easy to communicate state of parallel steps + const activeJobs = Math.min((Number.isNaN(value) ? 0 : value) + 1, jobTotal ?? 1); + await setMetadata(key, String(activeJobs)); + + await updateCheckStatus( + { + status: 'in_progress', + }, + 'playwright_a11y', + `${activeJobs} of ${jobTotal ?? 1} a11y jobs started`, + ); + + const src = '.buildkite/artifacts/e2e_server.gz'; + await downloadArtifacts(src, 'build_e2e'); + await decompress({ + src, + dest: 'e2e/server', + }); + + startGroup('Check Architecture'); + await exec('arch'); + + startGroup('Running e2e a11y playwright job'); + const reportDir = `reports/a11y_report_${shardIndex}`; + async function postCommandTasks() { + await compress({ + src: path.join('e2e', reportDir), + dest: `.buildkite/artifacts/a11y_reports/report_${shardIndex}.gz`, + }); + } + + const command = `yarn playwright test ${pwFlags.join(' ')}`; + + try { + await exec(command, { + cwd: 'e2e', + env: { + [ENV_URL]: 'http://127.0.0.1:9002', + PLAYWRIGHT_HTML_REPORT: reportDir, + PLAYWRIGHT_JSON_OUTPUT_NAME: `reports/a11y-json/report_${shardIndex}.json`, + }, + }); + await postCommandTasks(); + } catch (error) { + await postCommandTasks(); + throw error; + } +})(); diff --git a/.buildkite/scripts/steps/playwright.ts b/.buildkite/scripts/steps/playwright_vrt.ts similarity index 93% rename from .buildkite/scripts/steps/playwright.ts rename to .buildkite/scripts/steps/playwright_vrt.ts index d01074699c..cfac7d8c95 100644 --- a/.buildkite/scripts/steps/playwright.ts +++ b/.buildkite/scripts/steps/playwright_vrt.ts @@ -80,7 +80,7 @@ void (async () => { { status: 'in_progress', }, - 'playwright', + 'playwright_vrt', `${activeJobs} of ${jobTotal ?? 1} jobs started`, ); @@ -98,12 +98,12 @@ void (async () => { // TODO Fix this duplicate script that allows us to skip root node install on all e2e test runners await exec('node ./e2e/scripts/extract_examples.js'); - startGroup('Running e2e playwright job'); - const reportDir = `reports/report_${shardIndex}`; + startGroup('Running e2e vrt playwright job'); + const reportDir = `reports/vrt_report_${shardIndex}`; async function postCommandTasks() { await compress({ src: path.join('e2e', reportDir), - dest: `.buildkite/artifacts/e2e_reports/report_${shardIndex}.gz`, + dest: `.buildkite/artifacts/vrt_reports/report_${shardIndex}.gz`, }); if (bkEnv.steps.playwright.updateScreenshots) { @@ -119,7 +119,7 @@ void (async () => { env: { [ENV_URL]: 'http://127.0.0.1:9002', PLAYWRIGHT_HTML_REPORT: reportDir, - PLAYWRIGHT_JSON_OUTPUT_NAME: `reports/json/report_${shardIndex}.json`, + PLAYWRIGHT_JSON_OUTPUT_NAME: `reports/vrt-json/report_${shardIndex}.json`, }, }); await postCommandTasks(); diff --git a/.buildkite/steps/firebase_deploy.ts b/.buildkite/steps/firebase_deploy.ts index 31d879d9b6..73f8ad5662 100644 --- a/.buildkite/steps/firebase_deploy.ts +++ b/.buildkite/steps/firebase_deploy.ts @@ -15,7 +15,13 @@ export const firebaseDeployStep = createStep(() => { label: ':firebase: Deploy - firebase', key: 'deploy_fb', allow_dependency_failure: true, - depends_on: ['build_docs', 'build_storybook', 'build_e2e', 'playwright_merge_and_status'], + depends_on: [ + 'build_docs', + 'build_storybook', + 'build_e2e', + 'playwright_vrt_merge_and_status', + 'playwright_a11y_merge_and_status', + ], commands: ['npx ts-node .buildkite/scripts/steps/firebase_deploy.ts'], env: { ECH_CHECK_ID: 'deploy_fb', diff --git a/.buildkite/steps/index.ts b/.buildkite/steps/index.ts index 978f6c4b3d..54881d2c3f 100644 --- a/.buildkite/steps/index.ts +++ b/.buildkite/steps/index.ts @@ -11,7 +11,8 @@ export * from './eslint'; export * from './api_check'; export * from './type_check'; export * from './prettier'; -export * from './playwright'; +export * from './playwright_vrt'; +export * from './playwright_a11y'; export * from './docs'; export * from './storybook'; export * from './e2e_server'; diff --git a/.buildkite/steps/playwright_a11y.ts b/.buildkite/steps/playwright_a11y.ts new file mode 100644 index 0000000000..7d723998c2 --- /dev/null +++ b/.buildkite/steps/playwright_a11y.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { CustomGroupStep } from '../utils'; +import { createStep, commandStepDefaults, Plugins } from '../utils'; + +export const playwrightA11yStep = createStep(() => { + const skip = false; + const parallelKey = 'playwright_a11y__parallel-step'; + return { + group: ':playwright: Playwright e2e A11Y', + key: 'playwright_a11y', + skip, + steps: [ + { + ...commandStepDefaults, + label: ':playwright: Playwright e2e A11Y', + skip, + parallelism: 1, + retry: { + automatic: [ + { + // Playwright tests likely failed correctly + exit_status: 1, + limit: 0, + }, + { + // Something went wrong with step command setup, retry once + exit_status: '*', + limit: 1, + }, + ], + }, + timeout_in_minutes: 10, // Shorter timeout for a11y tests + key: parallelKey, + depends_on: ['build_e2e'], + plugins: [Plugins.docker.playwright()], + artifact_paths: ['.buildkite/artifacts/a11y_reports/*', 'e2e/reports/a11y-json/*'], + commands: ['npx ts-node .buildkite/scripts/steps/playwright_a11y.ts'], + }, + { + ...commandStepDefaults, + key: 'playwright_a11y_merge_and_status', + label: ':playwright: Set a11y group status and merge reports', + skip, + allow_dependency_failure: true, + depends_on: [{ step: parallelKey, allow_failure: true }], + commands: ['npx ts-node .buildkite/scripts/steps/e2e_reports.ts'], + env: { + ECH_CHECK_ID: 'playwright_a11y', + }, + }, + ], + }; +}); diff --git a/.buildkite/steps/playwright.ts b/.buildkite/steps/playwright_vrt.ts similarity index 78% rename from .buildkite/steps/playwright.ts rename to .buildkite/steps/playwright_vrt.ts index 38e5b7a143..3ff1bc8c1b 100644 --- a/.buildkite/steps/playwright.ts +++ b/.buildkite/steps/playwright_vrt.ts @@ -9,17 +9,17 @@ import type { CustomGroupStep } from '../utils'; import { createStep, commandStepDefaults, Plugins } from '../utils'; -export const playwrightStep = createStep(() => { +export const playwrightVrtStep = createStep(() => { const skip = false; - const parallelKey = 'playwright__parallel-step'; + const parallelKey = 'playwright_vrt__parallel-step'; return { - group: ':playwright: Playwright e2e', - key: 'playwright', + group: ':playwright: Playwright e2e VRT', + key: 'playwright_vrt', skip, steps: [ { ...commandStepDefaults, - label: ':playwright: Playwright e2e', + label: ':playwright: Playwright e2e VRT', skip, parallelism: 10, retry: { @@ -41,23 +41,23 @@ export const playwrightStep = createStep(() => { depends_on: ['build_e2e'], plugins: [Plugins.docker.playwright()], artifact_paths: [ - '.buildkite/artifacts/e2e_reports/*', + '.buildkite/artifacts/vrt_reports/*', '.buildkite/artifacts/screenshots/*', '.buildkite/artifacts/screenshot_meta/*', 'e2e/reports/json/*', ], - commands: ['npx ts-node .buildkite/scripts/steps/playwright.ts'], + commands: ['npx ts-node .buildkite/scripts/steps/playwright_vrt.ts'], }, { ...commandStepDefaults, - key: 'playwright_merge_and_status', - label: ':playwright: Set group status and merge reports', + key: 'playwright_vrt_merge_and_status', + label: ':playwright: Set vrt group status and merge reports', skip, allow_dependency_failure: true, depends_on: [{ step: parallelKey, allow_failure: true }], commands: ['npx ts-node .buildkite/scripts/steps/e2e_reports.ts'], env: { - ECH_CHECK_ID: 'playwright', + ECH_CHECK_ID: 'playwright_vrt', }, }, ], diff --git a/.buildkite/utils/build.ts b/.buildkite/utils/build.ts index 3bd2dbc83d..6b78612b96 100644 --- a/.buildkite/utils/build.ts +++ b/.buildkite/utils/build.ts @@ -38,7 +38,8 @@ export const getBuildConfig = (): BuildConfig => { { name: 'Deploy - firebase', id: 'deploy_fb' }, ...(bkEnv.isMainBranch ? [{ name: 'Deploy - GitHub Pages', id: 'deploy_ghp' }] : []), { name: 'Jest', id: 'jest' }, - { name: 'Playwright e2e', id: 'playwright' }, + { name: 'Playwright e2e VRT', id: 'playwright_vrt' }, + { name: 'Playwright e2e A11Y', id: 'playwright_a11y' }, ], }; }; diff --git a/.buildkite/utils/github.ts b/.buildkite/utils/github.ts index a4662cd914..9eca0a22f3 100644 --- a/.buildkite/utils/github.ts +++ b/.buildkite/utils/github.ts @@ -461,10 +461,12 @@ Failure${jobLink ? ` - [failed job](${jobLink})` : ''}${err} - [Docs](${deploymentUrl}) - [Storybook](${deploymentUrl}/storybook) - [e2e server](${deploymentUrl}/e2e) -- ([Playwright report](${deploymentUrl}/e2e-report)` +- ([Playwright VRT report](${deploymentUrl}/vrt-report) +- ([Playwright A11Y report](${deploymentUrl}/a11y-report)` : `- ⏳ Storybook - ⏳ e2e server -- ⏳ Playwright report`; +- ⏳ Playwright VRT report +- ⏳ Playwright A11Y report`; return `## ⏳ Pending Deployment${buildText} - ${sha}${updateComment} ${deploymentMsg}`; @@ -475,7 +477,8 @@ ${deploymentMsg}`; - [Docs](${deploymentUrl}) - [Storybook](${deploymentUrl}/storybook) - [e2e server](${deploymentUrl}/e2e) -${preDeploy ? '- ⏳ Playwright report - Running e2e tests' : `- [Playwright report](${deploymentUrl}/e2e-report)`}`; +${preDeploy ? '- ⏳ Playwright VRT report - Running e2e tests' : `- [Playwright VRT report](${deploymentUrl}/vrt-report)`} +${preDeploy ? '- ⏳ Playwright A11Y report - Running a11y tests' : `- [Playwright A11Y report](${deploymentUrl}/a11y-report)`}`; }, }; diff --git a/e2e/.gitignore b/e2e/.gitignore index 92c55b79c0..ff31475d38 100644 --- a/e2e/.gitignore +++ b/e2e/.gitignore @@ -1,5 +1,6 @@ reports/ -merged_html_report/ +merged_vrt_html_report/ +merged_a11y_html_report/ test_failures/ server/ diff --git a/e2e/package.json b/e2e/package.json index 5124507325..8b222de6dc 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -6,8 +6,10 @@ "scripts": { "clean": "./scripts/clean.sh", "test": "./scripts/start.sh", + "test:a11y": "./scripts/start.sh --a11y", "start": "npx http-server ./server --port 9002", "test:playwright": "./scripts/test.sh", + "test:playwright:a11y": "./scripts/test.sh --a11y", "merge:reports": "ts-node ./merge_html_reports.ts", "playwright": "playwright" }, diff --git a/e2e/page_objects/common.ts b/e2e/page_objects/common.ts index 29508dc331..790e66550f 100644 --- a/e2e/page_objects/common.ts +++ b/e2e/page_objects/common.ts @@ -600,6 +600,44 @@ export class CommonPage { if (width !== undefined) element.style.width = typeof width === 'number' ? `${width}px` : width; }, dimensions); }; + + /** + * Wait for accessibility content to be rendered + * @param timeout timeout for waiting on element to appear in DOM + */ + waitForA11yContent = + (page: Page) => + async (timeout = 5000) => { + await page.locator('.echScreenReaderOnly').first().waitFor({ state: 'attached', timeout }); + }; + + /** + * Get accessibility summary text from screen reader elements + */ + getA11ySummaryText = (page: Page) => async (): Promise => { + const elements = page.locator('.echScreenReaderOnly'); + const count = await elements.count(); + + const texts = await Promise.all(Array.from({ length: count }, (_, i) => elements.nth(i).textContent())); + + return texts.filter((text): text is string => text !== null).join(' '); + }; + + /** + * Test accessibility summary for a chart at a given URL + * @param url Storybook URL for the chart + * @param expectedSummary Expected accessibility summary text + */ + testA11ySummary = (page: Page) => async (url: string, expectedSummary: string) => { + await this.loadElementFromURL(page)(url, '.echChart'); + + // Wait for the chart to load + await page.waitForSelector('.echChart', { timeout: 5000 }); + await this.waitForA11yContent(page)(); + + const summaryText = await this.getA11ySummaryText(page)(); + expect(summaryText).toBe(expectedSummary); + }; } function getSnapshotOptions(options?: ScreenshotDOMElementOptions) { diff --git a/e2e/playwright.a11y.config.ts b/e2e/playwright.a11y.config.ts new file mode 100644 index 0000000000..51ee4d6887 --- /dev/null +++ b/e2e/playwright.a11y.config.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { PlaywrightTestConfig } from '@playwright/test'; + +import baseConfig from './playwright.config'; + +const config: PlaywrightTestConfig = { + ...baseConfig, + testDir: 'tests_a11y', + testMatch: ['**/tests_a11y/**/*.test.ts'], + reporter: [ + ['list'], + ['html', { open: 'never', outputFolder: 'reports/a11y-html' }], + ['json', { outputFile: 'reports/a11y-json/report.json' }], + ], + use: { + ...baseConfig.use, + // Disable visual regression for a11y tests since we're testing text content + screenshot: 'off', + video: 'off', + }, + expect: { + // Remove visual regression expectations for a11y tests + toMatchSnapshot: undefined, + }, +}; + +export default config; diff --git a/e2e/scripts/start.sh b/e2e/scripts/start.sh index 071cdfafc4..b04fb73691 100755 --- a/e2e/scripts/start.sh +++ b/e2e/scripts/start.sh @@ -4,6 +4,22 @@ set -e ### starts up a playwright docker container to run e2e tests +# Parse command line options, if --a11y is passed, set A11Y_MODE to true +A11Y_MODE=false +SCRIPT_ARGS=() +while [[ $# -gt 0 ]]; do + case $1 in + --a11y) + A11Y_MODE=true + shift + ;; + *) + SCRIPT_ARGS+=("$1") + shift + ;; + esac +done + # Get correct playwright image - must match installed version of @playwright/test regex="@playwright/test@(.+)" result="$(yarn list --pattern "@playwright/test" --depth=0 | grep playwright/test)" @@ -16,13 +32,22 @@ else exit 1 fi +# Set container name and command based on mode +if [ "$A11Y_MODE" = true ]; then + CONTAINER_NAME="e2e-playwright-a11y-tests" + TEST_COMMAND="yarn test:playwright:a11y" +else + CONTAINER_NAME="e2e-playwright-tests" + TEST_COMMAND="yarn test:playwright" +fi + # Run e2e playwright tests inside container docker run \ --ipc host `# recommended by playwright, see https://playwright.dev/docs/docker#end-to-end-tests` \ --platform linux/arm64 `# explicitly set platform` \ --rm `# removes named container on every run` \ --init `# handles terminating signals like SIGTERM` \ - --name e2e-playwright-tests `# reusable name of container` \ + --name ${CONTAINER_NAME} `# reusable name of container` \ -e PORT=${PORT} `# port of local web server ` \ -e ENV_URL=${ENV_URL} `# url of web server, overrides hostname and PORT ` \ -e PLAYWRIGHT_HTML_REPORT=${PLAYWRIGHT_HTML_REPORT} `# where to save the playwright html report ` \ @@ -30,4 +55,4 @@ docker run \ -v $(pwd)/:/app/e2e `# mount local e2e/ directory in app/e2e directory in container` \ -v $(pwd)/../e2e_server/tmp/:/app/e2e_server/tmp `# mount required example.json file in container` \ ${pw_image} `# playwright docker image derived above from @playwright/test version used ` \ - yarn test:playwright "$@" # runs test.sh forwarding any additional passed args + ${TEST_COMMAND} "${SCRIPT_ARGS[@]}" # runs test script forwarding any additional passed args diff --git a/e2e/scripts/test.sh b/e2e/scripts/test.sh index cfd8757c95..6ee9805a7f 100755 --- a/e2e/scripts/test.sh +++ b/e2e/scripts/test.sh @@ -8,18 +8,41 @@ attempt_counter=0 retries=5 interval=2 +# Parse command line options +A11Y_MODE=false +while [[ $# -gt 0 ]]; do + case $1 in + --a11y) + A11Y_MODE=true + shift + ;; + *) + break + ;; + esac +done + export PORT="${PORT:-9002}" if [ -f /.dockerenv ]; then hostname=host.docker.internal else hostname=localhost - echo " + if [ "$A11Y_MODE" = true ]; then + echo " + !!! Warning: you are running e2e tests outside of docker !!! + + Please run 'yarn test:e2e:a11y' from the root package.json + + " + else + echo " !!! Warning: you are running e2e tests outside of docker !!! Please run 'yarn test' from e2e/package.json " + fi fi export ENV_URL="${ENV_URL:-"http://${hostname}:${PORT}"}" @@ -41,6 +64,15 @@ echo "Connected to e2e server at ${ENV_URL}" # Install dependencies only e2e modules for testing yarn install --frozen-lockfile +# This setting is used to "seed" randomized data of the charts, +# so while it says VRT, we enabled it for A11y tests as well. export VRT=true -# Run playwright tests with passed args -playwright test "$@" + +# Set up environment and run tests based on mode +if [ "$A11Y_MODE" = true ]; then + # Run playwright accessibility tests with passed args + playwright test --config=playwright.a11y.config.ts "$@" +else + # Run playwright tests with passed args + playwright test "$@" +fi diff --git a/e2e/tests_a11y/annotations_chart_a11y.test.ts b/e2e/tests_a11y/annotations_chart_a11y.test.ts new file mode 100644 index 0000000000..6e5c97fe27 --- /dev/null +++ b/e2e/tests_a11y/annotations_chart_a11y.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Annotations Chart Accessibility', () => { + test('should generate correct a11y summary for line annotation', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/annotations-lines--single-bar-histogram', + 'Chart type:bar chart', + ); + }); + + test('should generate correct a11y summary for rect annotation', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/annotations-rects--styling&knob-showLineAnnotations=true&knob-chartRotation=0', + 'Chart type:line chart', + ); + }); + + test('should generate correct a11y summary for advanced markers', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/annotations-lines--advanced-markers', + 'Chart type:bar chart', + ); + }); + + test('should generate correct a11y summary for outside annotations', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/annotations-rects--outside', + 'Chart type:line chart', + ); + }); +}); diff --git a/e2e/tests_a11y/area_chart_a11y.test.ts b/e2e/tests_a11y/area_chart_a11y.test.ts new file mode 100644 index 0000000000..54055f6e19 --- /dev/null +++ b/e2e/tests_a11y/area_chart_a11y.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Area Chart Accessibility', () => { + test('should generate correct a11y summary for basic area chart', async ({ page }) => { + await common.testA11ySummary(page)('http://localhost:9001/?path=/story/area-chart--basic', 'Chart type:area chart'); + }); + + test('should generate correct a11y summary for stacked area chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/area-chart--stacked', + 'Chart type:area chart', + ); + }); + + test('should generate correct a11y summary for stacked percentage area chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/area-chart--stacked-percentage', + 'Chart type:area chart', + ); + }); + + test('should generate correct a11y summary for band area chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/area-chart--band-area', + 'Chart type:Mixed chart: area and line chart', + ); + }); + + test('should generate correct a11y summary for discontinuous area chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/line-chart--discontinuous-data-points', + 'Chart type:line chart', + ); + }); +}); diff --git a/e2e/tests_a11y/axis_chart_a11y.test.ts b/e2e/tests_a11y/axis_chart_a11y.test.ts new file mode 100644 index 0000000000..e8a3e7df2b --- /dev/null +++ b/e2e/tests_a11y/axis_chart_a11y.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Axis Chart Accessibility', () => { + test('should generate correct a11y summary for basic axis chart', async ({ page }) => { + await common.testA11ySummary(page)('http://localhost:9001/?path=/story/axes--basic', 'Chart type:area chart'); + }); + + test('should generate correct a11y summary for tick label rotation', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/axes--tick-label-rotation', + 'Chart type:area chart', + ); + }); + + test('should generate correct a11y summary for many tick labels', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/axes--many-tick-labels', + 'Chart type:area chart', + ); + }); + + test('should generate correct a11y summary for custom mixed axes', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/axes--custom-mixed', + 'Chart type:Mixed chart: bar and line chart', + ); + }); + + test('should generate correct a11y summary for duplicate ticks', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/axes--duplicate-ticks', + 'Chart type:line chart', + ); + }); + + test('should generate correct a11y summary for fit domain', async ({ page }) => { + await common.testA11ySummary(page)('http://localhost:9001/?path=/story/axes--fit-domain', 'Chart type:line chart'); + }); + + test('should generate correct a11y summary for timeslip with different locale', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/area-chart--timeslip', + 'Chart type:area chart', + ); + }); + + test('should generate correct a11y summary for small multiples of sunburst charts', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/small-multiples-alpha--sunbursts', + 'Chart type:sunburst chart The table represents only 20 of the 179 data pointsSmall multiple titleDepthLabelParentValuePercentageAfrica1Machinery and transport equipmentnone$12 Bn48%Africa2LiberiaMachinery and transport equipment$12 Bn48%Africa1Mineral fuels, lubricants and related materialsnone$13 Bn52%Africa2South AfricaMineral fuels, lubricants and related materials$13 Bn52%Asia1Mineral fuels, lubricants and related materialsnone$717 Bn26%Asia2JapanMineral fuels, lubricants and related materials$177 Bn6%Asia2ChinaMineral fuels, lubricants and related materials$168 Bn6%Asia2South KoreaMineral fuels, lubricants and related materials$108 Bn4%Asia2IndiaMineral fuels, lubricants and related materials$97 Bn4%Asia2SingaporeMineral fuels, lubricants and related materials$67 Bn2%Asia2ThailandMineral fuels, lubricants and related materials$27 Bn1%Asia2IndonesiaMineral fuels, lubricants and related materials$24 Bn1%Asia2TurkeyMineral fuels, lubricants and related materials$22 Bn1%Asia2MalaysiaMineral fuels, lubricants and related materials$13 Bn0%Asia2Hong KongMineral fuels, lubricants and related materials$13 Bn0%Asia1Manufactured goods classified chiefly by materialnone$238 Bn9%Asia2ChinaManufactured goods classified chiefly by material$79 Bn3%Asia2South KoreaManufactured goods classified chiefly by material$33 Bn1%Asia2JapanManufactured goods classified chiefly by material$32 Bn1%Asia2IndiaManufactured goods classified chiefly by material$31 Bn1%Click to show more data', + ); + }); +}); diff --git a/e2e/tests_a11y/bar_chart_a11y.test.ts b/e2e/tests_a11y/bar_chart_a11y.test.ts new file mode 100644 index 0000000000..f9aec140ca --- /dev/null +++ b/e2e/tests_a11y/bar_chart_a11y.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Bar Chart Accessibility', () => { + test('should generate correct a11y summary for basic bar chart', async ({ page }) => { + await common.testA11ySummary(page)('http://localhost:9001/?path=/story/bar-chart--basic', 'Chart type:bar chart'); + }); + + test('should generate correct a11y summary for horizontal bar chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bar-chart--with-axis-and-legend', + 'Chart type:bar chart', + ); + }); + + test('should include axis descriptions when provided', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bar-chart--with-axis-and-legend', + 'Chart type:bar chart', + ); + }); + + test('should generate correct a11y summary for bar chart with ordinal axis', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bar-chart--test-switch-ordinal-linear-axis&knob-scaleType=ordinal', + 'Chart type:bar chart', + ); + }); + + test('should generate correct a11y summary for bar chart with discover configuration', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bar-chart--test-discover&knob-use custom minInterval of 30s=', + 'Chart type:bar chart', + ); + }); + + test('should generate correct a11y summary for histogram mode bar chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bar-chart--test-histogram-mode-linear&knob-chartRotation=0&knob-stacked=false&knob-bars padding=0.25&knob-histogram padding=0.05&knob-other series=line&knob-point series alignment=center&knob-hasHistogramBarSeries=&knob-debug=false&knob-bars-1 enableHistogramMode=true&knob-bars-2 enableHistogramMode=', + 'Chart type:Mixed chart: bar and line chart', + ); + }); + + test('should generate correct a11y summary for bar chart with value labels', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bar-chart--with-value-label', + 'Chart type:bar chart', + ); + }); + + test('should generate correct a11y summary for bar chart with functional accessors', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bar-chart--functional-accessors', + 'Chart type:bar chart', + ); + }); + + test('should generate correct a11y summary for basic stacked bar chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bar-chart--stacked-as-percentage', + 'Chart type:bar chart', + ); + }); +}); diff --git a/e2e/tests_a11y/bullet_chart_a11y.test.ts b/e2e/tests_a11y/bullet_chart_a11y.test.ts new file mode 100644 index 0000000000..037e07665b --- /dev/null +++ b/e2e/tests_a11y/bullet_chart_a11y.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Bullet Chart Accessibility', () => { + test('should generate correct a11y summary for single bullet chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bullet-graph--single', + 'Chart type:Bullet chart', + ); + }); + + test('should generate correct a11y summary for angular bullet chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bullet-graph--angular', + 'Chart type:Bullet chart', + ); + }); + + test('should generate correct a11y summary for single row bullet chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bullet-graph--single-row', + 'Chart type:Bullet chart', + ); + }); + + test('should generate correct a11y summary for single column bullet chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bullet-graph--single-column', + 'Chart type:Bullet chart', + ); + }); + + test('should generate correct a11y summary for grid bullet chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bullet-graph--grid', + 'Chart type:Bullet chart', + ); + }); + + test('should generate correct a11y summary for color bands bullet chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/bullet-graph--color-bands', + 'Chart type:Bullet chart', + ); + }); +}); diff --git a/e2e/tests_a11y/edge_cases_a11y.test.ts b/e2e/tests_a11y/edge_cases_a11y.test.ts new file mode 100644 index 0000000000..6fc812b306 --- /dev/null +++ b/e2e/tests_a11y/edge_cases_a11y.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test, expect } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Edge Cases Accessibility', () => { + test('no screen reader summary for empty charts', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/test-cases--no-series'; + await common.loadElementFromURL(page)(url, '.echChart'); + + // For empty charts, accessibility content may not exist, so we check if the chart element exists + const chartElement = page.locator('.echChart').first(); + await expect(chartElement).toBeVisible(); + const chartText = await chartElement.textContent(); + expect(chartText).toBe('No Results'); + const a11yExists = await page.locator('.echScreenReaderOnly').count(); + expect(a11yExists).toBe(0); + }); + + test('should handle bar chart with empty data points', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/bar-chart--with-linear-x-axis'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:bar chart'); + }); + + test('should generate correct a11y summary for error boundary', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/test-cases--error-boundary'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + // TODO This doesn't throw the error yet, the default storybook pages first + // loads without error, the error needs then to be triggered manually. + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:bar chart'); + }); + + test('should generate correct a11y summary for RTL text', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/test-cases--rtl-text'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe( + 'Chart type:treemap chart The table fully represents the dataset of 10 data pointsLabelValuePercentageמכונות וציוד הובלה3.1 t36%דלקים מינרליים, חומרי סיכה וחומרים נלווים1.9 t22%כימיקלים ומוצרים נלווים848.2 b10%מוצרים מיוצרים שונים816.8 b9%מוצרים מיוצרים המסווגים בעיקר לפי חומר745.2 b9%סחורות ועסקאות שאינן מסווגות במקום אחר450.5 b5%חומרים גולמיים, בלתי אכילים, למעט דלקים393.9 b5%מזון וחיות חיות353.3 b4%משקאות וטבק54.5 b1%שמנים, שומנים ושעווה מהחי וצומח36.0 b0%', + ); + }); + + test('should generate correct a11y summary for point style override', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/test-cases--point-style-overrides'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:line chart'); + }); +}); diff --git a/e2e/tests_a11y/flame_chart_a11y.test.ts b/e2e/tests_a11y/flame_chart_a11y.test.ts new file mode 100644 index 0000000000..9a475a8a7b --- /dev/null +++ b/e2e/tests_a11y/flame_chart_a11y.test.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test, expect } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe.skip('Flame Chart Accessibility', () => { + test('should generate correct a11y summary for CPU profile flame chart', async ({ page }) => { + const url = 'http://localhost:9002/?path=/story/flame-alpha--cpu-profile-g-l-flame-chart'; + await common.loadElementFromURL(page)(url, '.echChart'); + + // TODO Flame Chart does not have a11y content yet + const a11yContent = await page.locator('.echScreenReaderOnly').count(); + expect(a11yContent).toBe(0); + }); + + test('should generate correct a11y summary for flame chart with search', async ({ page }) => { + const url = + 'http://localhost:9001/?path=/story/flame-alpha--cpu-profile-g-l-flame-chart&knob-Text%20to%20search=gotype'; + await common.loadElementFromURL(page)(url, '.echChart'); + + // TODO Flame Chart does not have a11y content yet + const a11yContent = await page.locator('.echScreenReaderOnly').count(); + expect(a11yContent).toBe(0); + }); +}); diff --git a/e2e/tests_a11y/goal_chart_a11y.test.ts b/e2e/tests_a11y/goal_chart_a11y.test.ts new file mode 100644 index 0000000000..f1609fa83f --- /dev/null +++ b/e2e/tests_a11y/goal_chart_a11y.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Goal Chart Accessibility', () => { + test('should generate correct a11y summary for goal chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/goal-alpha--minimal-goal', + 'Revenue 2020 YTD (thousand USD) Chart type:goal chartMinimum:0Maximum:300Target:260Value:280', + ); + }); + + test('should generate correct a11y summary for gauge chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/goal-alpha--gauge-with-target', + 'Revenue 2020 YTD (thousand USD) Chart type:goal chartMinimum:0Maximum:300Target:260Value:170', + ); + }); + + test('should generate correct a11y summary for goal chart without target', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/goal-alpha--gaps', + 'Revenue 2020 YTD (thousand USD) Chart type:goal chartMinimum:0Maximum:300Target:260Value:280', + ); + }); + + test('should generate correct a11y summary for full circle goal chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/goal-alpha--full-circle', + 'Chart type:goal chartMinimum:0Maximum:300Target:260Value:280', + ); + }); +}); diff --git a/e2e/tests_a11y/grid_chart_a11y.test.ts b/e2e/tests_a11y/grid_chart_a11y.test.ts new file mode 100644 index 0000000000..dacf03c8ff --- /dev/null +++ b/e2e/tests_a11y/grid_chart_a11y.test.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Grid Chart Accessibility', () => { + test('should generate correct a11y summary for grid lines chart', async ({ page }) => { + await common.testA11ySummary(page)('http://localhost:9001/?path=/story/grids--lines', 'Chart type:line chart'); + }); +}); diff --git a/e2e/tests_a11y/heatmap_chart_a11y.test.ts b/e2e/tests_a11y/heatmap_chart_a11y.test.ts new file mode 100644 index 0000000000..ab84e0c27d --- /dev/null +++ b/e2e/tests_a11y/heatmap_chart_a11y.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test, expect } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Heatmap Chart Accessibility', () => { + test('should generate correct a11y summary for heatmap chart', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/heatmap-alpha--basic'; + await common.loadElementFromURL(page)(url, '.echChart'); + + // Wait for the chart to load + await page.waitForSelector('.echChart', { timeout: 5000 }); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:Heatmap chart'); + }); + + test('should generate correct a11y summary for time heatmap chart', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/heatmap-alpha--time'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:Heatmap chart'); + }); + + test('should generate correct a11y summary for small multiples heatmap', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/small-multiples-alpha--heatmap'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:Heatmap chart'); + }); +}); diff --git a/e2e/tests_a11y/legend_chart_a11y.test.ts b/e2e/tests_a11y/legend_chart_a11y.test.ts new file mode 100644 index 0000000000..9590fd58a0 --- /dev/null +++ b/e2e/tests_a11y/legend_chart_a11y.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test, expect } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Legend Chart Accessibility', () => { + test('should generate correct a11y summary for legend positioning', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/legend--positioning'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:bar chart'); + }); + + test('should generate correct a11y summary for legend actions', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/legend--actions'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:bar chart'); + }); + + test('should generate correct a11y summary for legend color picker', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/legend--color-picker'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:bar chart'); + }); + + test('should generate correct a11y summary for legend inside chart', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/legend--inside-chart'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:area chart'); + }); + + test('should generate correct a11y summary for legend spacing buffer', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/legend--legend-spacing-buffer'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:bar chart'); + }); + + test('should generate correct a11y summary for legend with pie chart', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/legend--piechart'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe( + 'Chart type:sunburst chart The table represents only 20 of the 31 data pointsDepthLabelParentValuePercentage1Mineral fuels, lubricants and related materialsnone$779 Bn24%2AsiaMineral fuels, lubricants and related materials$454 Bn14%3JapanAsia$177 Bn5%3ChinaAsia$168 Bn5%3South KoreaAsia$108 Bn3%2North AmericaMineral fuels, lubricants and related materials$325 Bn10%3United StatesNorth America$325 Bn10%1Miscellaneous manufactured articlesnone$227 Bn7%2North AmericaMiscellaneous manufactured articles$227 Bn7%3United StatesNorth America$227 Bn7%1Crude materials, inedible, except fuelsnone$174 Bn5%2AsiaCrude materials, inedible, except fuels$174 Bn5%3ChinaAsia$174 Bn5%1Manufactured goods classified chiefly by materialnone$130 Bn4%2North AmericaManufactured goods classified chiefly by material$130 Bn4%3United StatesNorth America$130 Bn4%1Chemicals and related productsnone$128 Bn4%2North AmericaChemicals and related products$128 Bn4%3United StatesNorth America$128 Bn4%1Machinery and transport equipmentnone$1,854 Bn56%Click to show more data', + ); + }); + + test('should generate correct a11y summary for legend tabular data', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/legend--tabular-data'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('Chart type:bar chart'); + }); +}); diff --git a/e2e/tests_a11y/line_chart_a11y.test.ts b/e2e/tests_a11y/line_chart_a11y.test.ts new file mode 100644 index 0000000000..44b20f8f07 --- /dev/null +++ b/e2e/tests_a11y/line_chart_a11y.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Line Chart Accessibility', () => { + test('should generate correct a11y summary for basic line chart', async ({ page }) => { + await common.testA11ySummary(page)('http://localhost:9001/?path=/story/line-chart--basic', 'Chart type:line chart'); + }); + + test('should generate correct a11y summary for multi-series line chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/line-chart--multiple-with-axis-and-legend', + 'Chart type:line chart', + ); + }); + + test('should generate correct a11y summary for stacked line chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/line-chart--stacked-with-axis-and-legend', + 'Chart type:line chart', + ); + }); + + test('should generate correct a11y summary for line chart with ordinal axis', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/line-chart--ordinal-with-axis', + 'Chart type:line chart', + ); + }); + + test('should generate correct a11y summary for line chart with path ordering', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/line-chart--test-path-ordering', + 'Chart type:line chart', + ); + }); + + test('should generate correct a11y summary for line chart with discontinuous data', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/line-chart--discontinuous-data-points', + 'Chart type:line chart', + ); + }); +}); diff --git a/e2e/tests_a11y/metric_chart_a11y.test.ts b/e2e/tests_a11y/metric_chart_a11y.test.ts new file mode 100644 index 0000000000..6f25af0beb --- /dev/null +++ b/e2e/tests_a11y/metric_chart_a11y.test.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test, expect } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Metric Chart Accessibility', () => { + test('should generate correct a11y summary for metric chart', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/metric-alpha--basic'; + await common.loadElementFromURL(page)(url, '.echChart'); + + const sparklines = await page.locator('.echSingleMetricSparkline').elementHandles(); + expect(sparklines.length).toBe(1); + + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe('The Cluster CPU Usage trend The trend shows a peak of CPU usage in the last 5 minutes'); + }); +}); diff --git a/e2e/tests_a11y/mixed_chart_a11y.test.ts b/e2e/tests_a11y/mixed_chart_a11y.test.ts new file mode 100644 index 0000000000..83930c241a --- /dev/null +++ b/e2e/tests_a11y/mixed_chart_a11y.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Mixed Chart Accessibility', () => { + test('should generate correct a11y summary for fitting functions non-stacked', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/mixed-charts--fitting-functions-non-stacked-series', + 'Chart type:area chart', + ); + }); + + test('should generate correct a11y summary for fitting functions stacked', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/mixed-charts--fitting-functions-stacked-series', + 'Chart type:area chart', + ); + }); + + test('should generate correct a11y summary for polarized stacked charts', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/mixed-charts--polarized-stacked', + 'Chart type:bar chart', + ); + }); +}); diff --git a/e2e/tests_a11y/pie_chart_a11y.test.ts b/e2e/tests_a11y/pie_chart_a11y.test.ts new file mode 100644 index 0000000000..a78619cec2 --- /dev/null +++ b/e2e/tests_a11y/pie_chart_a11y.test.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test, expect } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Pie Chart Accessibility', () => { + test('should generate correct a11y summary for basic pie chart', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/sunburst--most-basic'; + await common.loadElementFromURL(page)(url, '.echChart'); + + // Wait for the chart to load + await page.waitForSelector('.echChart', { timeout: 5000 }); + + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe( + 'Chart type:sunburst chart The table fully represents the dataset of 10 data pointsLabelValuePercentageMineral fuels, lubricants and related materials$1,930 Bn22%Chemicals and related products$848 Bn10%Miscellaneous manufactured articles$817 Bn9%Manufactured goods classified chiefly by material$745 Bn9%Commodities and transactions not classified elsewhere$451 Bn5%Crude materials, inedible, except fuels$394 Bn5%Food and live animals$353 Bn4%Beverages and tobacco$54 Bn1%Animal and vegetable oils, fats and waxes$36 Bn0%Machinery and transport equipment$3,110 Bn36%', + ); + }); + + test('should generate correct a11y summary for donut chart', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/sunburst--donut-chart-with-fill-labels'; + await common.loadElementFromURL(page)(url, '.echChart'); + + // Wait for the chart to load + await page.waitForSelector('.echChart', { timeout: 5000 }); + + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe( + 'Chart type:sunburst chart The table fully represents the dataset of 10 data pointsLabelValuePercentageMineral fuels, lubricants and related materials$1,930 Bn22%Chemicals and related products$848 Bn10%Miscellaneous manufactured articles$817 Bn9%Manufactured goods classified chiefly by material$745 Bn9%Commodities and transactions not classified elsewhere$451 Bn5%Crude materials, inedible, except fuels$394 Bn5%Food and live animals$353 Bn4%Beverages and tobacco$54 Bn1%Animal and vegetable oils, fats and waxes$36 Bn0%Machinery and transport equipment$3,110 Bn36%', + ); + }); + + test('should generate correct a11y summary for mosaic chart', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/mosaic-alpha--other-slices'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe( + 'Chart type:mosaic chart The table fully represents the dataset of 14 data pointsDepthLabelParentValuePercentage1AMERICASnone1,30740%1ASIAnone1,02532%1EUROPEnone59418%1AFRICAnone3059%2United StatesAMERICAS55317%2OtherAMERICAS75323%2South KoreaASIA1775%2JapanASIA1775%2ChinaASIA39312%2OtherASIA2779%2San MarinoEUROPE1354%2GermanyEUROPE2538%2OtherEUROPE2056%2OtherAFRICA3059%', + ); + }); + + test('should generate correct a11y summary for sunburst with linked labels', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/sunburst--linked-labels-only'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe( + 'Chart type:sunburst chart The table fully represents the dataset of 10 data pointsLabelValuePercentageMineral fuels, lubricants and related materials$1,930 Bn22%Chemicals and related products$848 Bn10%Miscellaneous manufactured articles$817 Bn9%Manufactured goods classified chiefly by material$745 Bn9%Commodities and transactions not classified elsewhere$451 Bn5%Crude materials, inedible, except fuels$394 Bn5%Food and live animals$353 Bn4%Beverages and tobacco$54 Bn1%Animal and vegetable oils, fats and waxes$36 Bn0%Machinery and transport equipment$3,110 Bn36%', + ); + }); + + test('should generate correct a11y summary for treemap with fill labels', async ({ page }) => { + const url = 'http://localhost:9001/?path=/story/treemap--one-layer2'; + await common.loadElementFromURL(page)(url, '.echChart'); + await common.waitForA11yContent(page)(); + + const summaryText = await common.getA11ySummaryText(page)(); + expect(summaryText).toBe( + 'Chart type:treemap chart The table fully represents the dataset of 10 data pointsLabelValuePercentageMachinery and transport equipment$3,110 Bn36%Mineral fuels, lubricants and related materials$1,930 Bn22%Chemicals and related products$848 Bn10%Miscellaneous manufactured articles$817 Bn9%Manufactured goods classified chiefly by material$745 Bn9%Commodities and transactions not classified elsewhere$451 Bn5%Crude materials, inedible, except fuels$394 Bn5%Food and live animals$353 Bn4%Beverages and tobacco$54 Bn1%Animal and vegetable oils, fats and waxes$36 Bn0%', + ); + }); +}); diff --git a/e2e/tests_a11y/scales_chart_a11y.test.ts b/e2e/tests_a11y/scales_chart_a11y.test.ts new file mode 100644 index 0000000000..a6f46c74f0 --- /dev/null +++ b/e2e/tests_a11y/scales_chart_a11y.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Scales Chart Accessibility', () => { + test('should generate correct a11y summary for log scale options', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/scales--log-scale-options', + 'Chart type:line chart', + ); + }); + + test('should generate correct a11y summary for linear binary scale', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/scales--linear-binary', + 'Chart type:line chart', + ); + }); +}); diff --git a/e2e/tests_a11y/stylings_chart_a11y.test.ts b/e2e/tests_a11y/stylings_chart_a11y.test.ts new file mode 100644 index 0000000000..d10ea94eb5 --- /dev/null +++ b/e2e/tests_a11y/stylings_chart_a11y.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Stylings Chart Accessibility', () => { + test('should generate correct a11y summary for charts with texture', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/stylings--with-texture', + 'Chart type:area chart', + ); + }); + + test('should generate correct a11y summary for texture multiple series', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/stylings--texture-multiple-series', + 'Chart type:area chart', + ); + }); + + test('should generate correct a11y summary for highlighter style', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/stylings--highlighter-style', + 'Chart type:line chart', + ); + }); +}); diff --git a/e2e/tests_a11y/waffle_chart_a11y.test.ts b/e2e/tests_a11y/waffle_chart_a11y.test.ts new file mode 100644 index 0000000000..f0030d89ca --- /dev/null +++ b/e2e/tests_a11y/waffle_chart_a11y.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Waffle Chart Accessibility', () => { + test('should generate correct a11y summary for simple waffle chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/waffle-alpha--simple', + 'Chart type:waffle chart The table represents only 20 of the 100 data pointsLabelValuePercentageAl$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Al$3,110 Bn46%Click to show more data', + ); + }); + + test('should generate correct a11y summary for waffle test chart', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/waffle-alpha--test', + 'Chart type:waffle chart The table represents only 20 of the 100 data pointsLabelValuePercentage01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%01348%Click to show more data', + ); + }); +}); diff --git a/e2e/tests_a11y/wordcloud_chart_a11y.test.ts b/e2e/tests_a11y/wordcloud_chart_a11y.test.ts new file mode 100644 index 0000000000..2b072e33ce --- /dev/null +++ b/e2e/tests_a11y/wordcloud_chart_a11y.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { test } from '@playwright/test'; + +import { common } from '../page_objects/common'; + +test.describe('Word Cloud Chart Accessibility', () => { + test('should generate correct a11y summary for simple wordcloud', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/wordcloud-alpha--simple-wordcloud', + 'Chart type:Word cloud chart', + ); + }); + + test('should generate correct a11y summary for single template wordcloud', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/wordcloud-alpha--simple-wordcloud&knob-template=single', + 'Chart type:Word cloud chart', + ); + }); + + test('should generate correct a11y summary for right angled template wordcloud', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/wordcloud-alpha--simple-wordcloud&knob-template=rightAngled', + 'Chart type:Word cloud chart', + ); + }); + + test('should generate correct a11y summary for multiple template wordcloud', async ({ page }) => { + await common.testA11ySummary(page)( + 'http://localhost:9001/?path=/story/wordcloud-alpha--simple-wordcloud&knob-template=multiple', + 'Chart type:Word cloud chart', + ); + }); +}); diff --git a/github_bot/src/build.ts b/github_bot/src/build.ts index bda89037d7..f1930867f4 100644 --- a/github_bot/src/build.ts +++ b/github_bot/src/build.ts @@ -35,6 +35,7 @@ export const getBuildConfig = (isMainBranch: boolean): BuildConfig => ({ { name: 'Deploy - firebase', id: 'deploy_fb' }, ...(isMainBranch ? [{ name: 'Deploy - GitHub Pages', id: 'deploy_ghp' }] : []), { name: 'Jest', id: 'jest' }, - { name: 'Playwright e2e', id: 'playwright' }, + { name: 'Playwright e2e VRT', id: 'playwright_vrt' }, + { name: 'Playwright e2e A11Y', id: 'playwright_a11y' }, ], }); diff --git a/package.json b/package.json index 4e77b87a8d..acf783e5ab 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "test:tz-ny": "TZ=America/New_York jest --verbose --config=jest.tz.config.js", "test:tz-jp": "TZ=Asia/Tokyo jest --verbose --config=jest.tz.config.js", "test:e2e": "cd e2e && yarn test", + "test:e2e:a11y": "cd e2e && yarn test:a11y", "test:e2e:generate": "yarn test:e2e:generate:examples && yarn test:e2e:generate:page", "test:e2e:generate:examples": "./e2e_server/scripts/extract_examples.sh", "test:e2e:generate:page": "./e2e_server/scripts/compile_vrt_page.sh",