Sets up E2E tests with playwright and adds tests (#17917)

* init playwright set up

* Remove unneeded files

* Configure playwright & clean up tests

* Correct gulp set up

* Fix playwright config

* Sets up playwright test boilerplate

* Clean up playwright config file

* Fix timeout issue with init setup

* Add first test

* Rename first test file

* Clean up test

* Clean up

* Adds connection test

* Changes directory name to e2e

* Disconnect from SQL server

* Clean up

* Minor clean up

* Minor adjustment to test hooks

* Add e2e test readme

* Fix formatting in README

* Rename utility file

* Capture screenshots on error

* Read vsCodeVersionName from .env

* Uses env variables to load test conn values

* Adds instructions to .env file

* Adds query execution tests

* Corrects instructions in example .env

* Update E2E test readme.

* Add user name, and password env vars

Author: lewis-sanchez

.gitignore

@@ -19,4 +19,9 @@ test-reports
 .vscode-test
 localization/i18n/
 src/constants/localizedConstants.ts
-package.nls.*.json
+package.nls.*.json
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+.env

gulpfile.js

@@ -74,7 +74,7 @@ gulp.task('ext:lint', () => {
 	return gulp.src([
 		config.paths.project.root + '/src/**/*.ts',
 		'!' + config.paths.project.root + '/src/views/htmlcontent/**/*',
-		config.paths.project.root + '/test/unit/**/*.ts'
+		config.paths.project.root + '/test/**/*.ts'
 	])
 		.pipe(gulpESLintNew({
 			quiet: true
@@ -359,7 +359,7 @@ gulp.task('ext:copy-dependencies', (done) => {
 // Compile tests
 gulp.task('ext:compile-tests', (done) => {
 	return gulp.src([
-		config.paths.project.root + '/test/unit/**/*.ts',
+		config.paths.project.root + '/test/**/*.ts',
 		config.paths.project.root + '/typings/**/*.ts'])
 		.pipe(srcmap.init())
 		.pipe(tsProject())
@@ -369,8 +369,8 @@ gulp.task('ext:compile-tests', (done) => {
 				process.exit(1);
 			}
 		})
-		.pipe(srcmap.write('.', { includeContent: false, sourceRoot: '../test/unit' }))
-		.pipe(gulp.dest('out/test/unit/'));
+		.pipe(srcmap.write('.', { includeContent: false, sourceRoot: '../test' }))
+		.pipe(gulp.dest('out/test/'));
 
 });
 
@@ -432,7 +432,7 @@ gulp.task('watch-src', function () {
 });
 
 gulp.task('watch-tests', function () {
-	return gulp.watch('./test/unit/**/*.ts', gulp.series('ext:compile-tests'))
+	return gulp.watch('./test/**/*.ts', gulp.series('ext:compile-tests'))
 });
 
 gulp.task('watch-reactviews', function () {

images/editor-view-with-tests.png

@@ -66,6 +66,7 @@
     "@angular/upgrade": "~2.1.2",
     "@eslint/compat": "^1.1.0",
     "@eslint/js": "^9.5.0",
+    "@playwright/test": "^1.45.0",
     "@jgoz/esbuild-plugin-typecheck": "^4.0.0",
     "@types/azdata": "^1.44.0",
     "@types/ejs": "^3.1.0",
@@ -143,6 +144,7 @@
     "axios": "^1.6.0",
     "core-js": "^2.4.1",
     "decompress-zip": "^0.2.2",
+    "dotenv": "^16.4.5",
     "ejs": "^3.1.7",
     "error-ex": "^1.3.0",
     "figures": "^1.4.0",

images/test-explorer-view.png

@@ -0,0 +1,43 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import dotenv from 'dotenv';
+import { defineConfig } from '@playwright/test';
+
+dotenv.config({ path: './test/e2e/.env' })
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './test/e2e',
+  /* Run tests in files in parallel */
+  fullyParallel: false,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: 1,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Set timeout to 5 minutes */
+  timeout: 5 * 60 * 1000,
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+});

package.json

@@ -0,0 +1,32 @@
+# To run tests locally, create a .env file next to this file and set the environment variables
+# as desired to establish a connection with a database.
+
+# The name of the VS Code version name.
+# Options are: "stable", "insiders"
+VS_CODE_VERSION_NAME=stable
+
+# The name of the database server to connect to.
+# This value is the name of the server that will be used by the tests.
+SERVER_NAME=(localdb)\MSSqlLocalDb
+
+# The name of the database to connect to.
+# This value is optional and can be left empty.
+DATABASE_NAME=
+
+# The authentication type to use to connect to the server.
+# Options are: "SQL Login", "Integrated", "Microsoft Entra Id - Universal w/ MFA Support"
+AUTHENTICATION_TYPE=SQL Login
+
+# The username to log in to the database.
+USER_NAME=test-user
+
+# The password to log in to the database.
+PASSWORD=
+
+# Indicates whether to save the login password for future logins.
+# Options are: "Yes", "No"
+SAVE_PASSWORD=No
+
+# The profile name is the name that will appear for the connection profile.
+# This value is optional and can be left empty.
+PROFILE_NAME=test-server

playwright.config.ts

@@ -0,0 +1,73 @@
+# MSSQL for VSCode Extension - E2E Tests
+
+## Get started
+
+### Limitations
+
+If running extension tests from CLI, the VS Code version the tests run with cannot be running already. As a workaround, VS Code Insiders should be used for test development, so that tests may run in VS Code Stable.
+
+For more information regarding this limitation, refer to [using insiders version for extension development.](https://code.visualstudio.com/api/working-with-extensions/testing-extension#using-insiders-version-for-extension-development)
+
+### Prerequisites
+
+- [Node.js](https://nodejs.org/en/) v18 or higher
+- [yarn](https://yarnpkg.com/) v1.22.0 or higher, `npm install -g yarn`
+
+---
+
+### Setup
+
+Tests are currently only setup to run locally.
+
+1. Install root dependencies
+
+From the root of the repo, install all of the build dependencies:
+
+```shell
+[sudo] yarn
+```
+
+2. Compile the extension and tests by running:
+
+```shell
+gulp build
+```
+
+or watch for changes in the tests by running:
+
+```shell
+gulp watch-tests
+```
+
+### Running tests
+
+To run tests, the following options can be used:
+
+1. Using Playwright Test for VSCode:
+    - Install the `Playwright Test for VSCode` extension in VS code.
+    - [Run tests with a single click](https://github.com/microsoft/playwright-vscode/blob/main/README.md#run-tests-with-a-single-click)
+
+    > Note: If you don't see any tests appearing in the **Test Explorer** view, like in the image, then you'll need to run them from the terminal first to get them to appear. Please refer to option 2.
+
+![Playwright Test for VSCode Test Explorer](../../images/test-explorer-view.png).
+
+2. Setup environment variables in the `test\e2e` folder
+    - Create a `.env` file
+    - Add the variables that you want based on the .env.example
+    - Example:
+        ```env
+        VS_CODE_VERSION_NAME=stable
+        SERVER_NAME=(localdb)\MSSqlLocalDb
+        AUTHENTICATION_TYPE=Integrated
+        PROFILE_NAME=test-server
+        ```
+
+3. To run tests from the command line execute the following command from the root:
+
+    ```shell
+    npx playwright test
+    ```
+
+The tests will automatically appear in the **Test Explorer** view after running them once, and green run icons will appear to the left of line numbers in the editor, like this:
+
+![Run buttons to the left of line numbers in editor](../../images/editor-view-with-tests.png)

test/e2e/.env.example

@@ -0,0 +1,33 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { ElectronApplication, Page, test, expect } from '@playwright/test';
+import { launchVsCodeWithMssqlExtension } from './utils/launchVscodeWithMsSqlExt';
+import { screenshotOnFailure } from './utils/screenshotOnError';
+
+test.describe('MSSQL Extension - Activity Bar', async () => {
+	let vsCodeApp: ElectronApplication;
+	let vsCodePage: Page;
+
+	test.beforeAll(async () => {
+		const { electronApp, page } = await launchVsCodeWithMssqlExtension();
+		vsCodeApp = electronApp;
+		vsCodePage = page;
+	});
+
+	test('MSSQL button is present in activity bar', async () => {
+		await vsCodePage.click('a[aria-label="SQL Server (Ctrl+Alt+D)"]');
+		const count = await vsCodePage.locator('a[aria-label="SQL Server (Ctrl+Alt+D)"]').count();
+		expect(count).toEqual(1);
+	});
+
+	test.afterEach(async ({ }, testInfo) => {
+		await screenshotOnFailure(vsCodePage, testInfo);
+	});
+
+	test.afterAll(async () => {
+		await vsCodeApp.close();
+	});
+});

test/e2e/README.md

@@ -0,0 +1,61 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { ElectronApplication, Page, Locator, test, expect } from '@playwright/test';
+import { launchVsCodeWithMssqlExtension } from './utils/launchVscodeWithMsSqlExt';
+import { screenshotOnFailure } from './utils/screenshotOnError';
+import { getServerName, getDatabaseName, getAuthenticationType, getUserName, getPassword, getProfileName, getSavePassword } from './utils/envConfigReader';
+import { addDatabaseConnection } from './utils/testHelpers';
+
+test.describe('MSSQL Extension - Database Connection', async () => {
+	let vsCodeApp: ElectronApplication;
+	let vsCodePage: Page;
+
+	test.beforeAll(async () => {
+		const { electronApp, page } = await launchVsCodeWithMssqlExtension();
+		vsCodeApp = electronApp;
+		vsCodePage = page;
+	});
+
+	test('Connect to local SQL Database, and disconnect', async () => {
+		const serverName = getServerName();
+		const databaseName = getDatabaseName();
+		const authType = getAuthenticationType();
+		const userName = getUserName();
+		const password = getPassword();
+		const savePassword = getSavePassword();
+		const profileName = getProfileName();
+		await addDatabaseConnection(vsCodePage, serverName, databaseName, authType, userName, password, savePassword, profileName);
+
+		let addedSqlConnection: Locator;
+		if (profileName) {
+			addedSqlConnection = await vsCodePage.locator(`div[aria-label="${profileName}"]`);
+		}
+		else {
+			addedSqlConnection = await vsCodePage.getByText(`${serverName}`);
+		}
+
+		await expect(addedSqlConnection).toBeVisible({ timeout: 20 * 1000 });
+
+		await addedSqlConnection.click({ button: 'right' });
+		const disconnectOption = await vsCodePage.locator('span[aria-label="Disconnect"]');
+		await disconnectOption.click();
+		const isDiconnectOptionVisible = await disconnectOption.isVisible()
+		if (isDiconnectOptionVisible) {
+			await disconnectOption.click();
+		}
+
+		await addedSqlConnection.click({ button: 'right' });
+		await expect(disconnectOption).toBeHidden({ timeout: 10000 });
+	});
+
+	test.afterEach(async ({ }, testInfo) => {
+		await screenshotOnFailure(vsCodePage, testInfo);
+	});
+
+	test.afterAll(async () => {
+		await vsCodeApp.close();
+	});
+});