Skip to content

Commit 277e715

Browse files
authored
Fixing test coverage by adding more report types and updating to nyc from Istanbul (#18309)
* Fixing test coverage by adding more report types and updating to nyc from istanbul * Fixing typo * Fixing typo 2 * Adding back test reporter, removing console stmts * Fixing test task * remove code coverage from pr build * Fixing updated pr check
1 parent 32c1c45 commit 277e715

File tree

9 files changed

+1005
-320
lines changed

9 files changed

+1005
-320
lines changed

.github/workflows/daily-build-and-test.yml

+1-29
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ jobs:
3737
3838
- name: Run tests
3939
run: |
40-
DISPLAY=:10 yarn test --coverage --log
40+
DISPLAY=:10 yarn test
4141
4242
- name: Unit Test Report
4343
uses: dorny/test-reporter@v1
@@ -48,34 +48,6 @@ jobs:
4848
reporter: jest-junit
4949
badge-title: 'unit-tests'
5050

51-
- name: Generate Code Coverage Report
52-
id: testcoverage
53-
run: |
54-
yarn gulp cover
55-
56-
- name: Code Coverage Report
57-
uses: irongut/[email protected]
58-
with:
59-
filename: ./coverage/cobertura-coverage.xml
60-
badge: true
61-
fail_below_min: false
62-
format: markdown
63-
hide_branch_rate: false
64-
hide_complexity: true
65-
indicators: true
66-
output: both
67-
68-
- name: Create a check run for code coverage
69-
uses: LouisBrunner/[email protected]
70-
if: always()
71-
with:
72-
token: ${{ secrets.GITHUB_TOKEN }}
73-
name: Unit Test Coverage Report
74-
conclusion: ${{ steps.testcoverage.conclusion }}
75-
output: |
76-
{"summary":"${{ steps.testcoverage.summary }}"}
77-
output_text_description_file: ./code-coverage-results.md
78-
7951
- name: Setup environment for smoke tests
8052
run: |
8153
echo "Setting up environment for smoke tests"

.github/workflows/pr-checks.yml

+2-4
Original file line numberDiff line numberDiff line change
@@ -62,15 +62,13 @@ jobs:
6262
- name: Code coverage
6363
run: |
6464
cd main
65-
DISPLAY=:10 yarn test --coverage --log
66-
yarn gulp cover
65+
DISPLAY=:10 yarn test
6766
xml_coverage_main=$(find ./coverage -name 'cobertura-coverage.xml')
6867
line_rate_main=$(grep -m 1 -o 'line-rate="[0-9.]\+"' "$xml_coverage_main" | sed 's/line-rate="\([0-9.]*\)"/\1/')
6968
line_rate_main=$(printf "%.2f" $(echo "$line_rate_main * 100" | bc))
7069
echo "line_rate_main=$line_rate_main" >> $GITHUB_ENV
7170
cd ../pr
72-
DISPLAY=:10 yarn test --coverage --log
73-
yarn gulp cover
71+
DISPLAY=:10 yarn test
7472
xml_coverage_pr=$(find ./coverage -name 'cobertura-coverage.xml')
7573
line_rate_pr=$(grep -m 1 -o 'line-rate="[0-9.]\+"' "$xml_coverage_pr" | sed 's/line-rate="\([0-9.]*\)"/\1/')
7674
line_rate_pr=$(printf "%.2f" $(echo "$line_rate_pr * 100" | bc))

build/templates/build.yml

+8-9
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,10 @@ steps:
4343
workingDirectory: "$(Build.SourcesDirectory)"
4444
4545
46-
- task: gulp@1
47-
displayName: Run tests and compute coverage
48-
inputs:
49-
targets: test:cover --log
50-
gulpFile: ${{ parameters.gulpfile }}
46+
- pwsh: |
47+
yarn test
48+
displayName: Run tests
49+
workingDirectory: "$(Build.SourcesDirectory)"
5150
5251
- task: PublishTestResults@2
5352
displayName: Publish test results
@@ -56,11 +55,11 @@ steps:
5655
testResultsFiles: '$(Build.SourcesDirectory)/test-reports/test-results-ext.xml'
5756
condition: succeededOrFailed()
5857

59-
- task: gulp@1
60-
displayName: Process test coverage
58+
- task: PublishCodeCoverageResults@2
59+
displayName: Publish code coverage
6160
inputs:
62-
targets: cover
63-
gulpFile: ${{ parameters.gulpfile }}
61+
summaryFileLocation: '$(Build.SourcesDirectory)/coverage/cobertura-coverage.xml'
62+
condition: succeededOrFailed()
6463

6564
- task: gulp@1
6665
displayName: Package (online)

gulpfile.js

+1-5
Original file line numberDiff line numberDiff line change
@@ -407,8 +407,6 @@ gulp.task('ext:smoke', run('npx playwright test'));
407407

408408
gulp.task('test', gulp.series('ext:test'));
409409

410-
require('./tasks/covertasks');
411-
412410
gulp.task('clean', function (done) {
413411
return del('out', done);
414412
});
@@ -428,6 +426,4 @@ gulp.task('watch-reactviews', function () {
428426
});
429427

430428
// Do a full build first so we have the latest compiled files before we start watching for more changes
431-
gulp.task('watch', gulp.series('build', gulp.parallel('watch-src', 'watch-tests', 'watch-reactviews')));
432-
433-
gulp.task('cover', gulp.series('remap-coverage', 'cover:combine-json'));
429+
gulp.task('watch', gulp.series('build', gulp.parallel('watch-src', 'watch-tests', 'watch-reactviews')));

package.json

+7-3
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@
5656
"lint": "eslint --quiet --cache",
5757
"localization": "gulp ext:extract-localization-strings",
5858
"smoketest": "gulp ext:smoke",
59-
"test": "gulp test:cover --log",
59+
"test": "node ./out/test/unit/runTest.js",
6060
"package": "vsce package",
6161
"prepare": "husky",
6262
"lint-staged": "lint-staged --quiet",
@@ -83,6 +83,7 @@
8383
"@eslint/js": "^9.5.0",
8484
"@fluentui/react-components": "^9.54.13",
8585
"@fluentui/react-list-preview": "^0.3.6",
86+
"@istanbuljs/nyc-config-typescript": "^1.0.2",
8687
"@jgoz/esbuild-plugin-typecheck": "^4.0.0",
8788
"@monaco-editor/react": "^4.6.0",
8889
"@playwright/test": "^1.45.0",
@@ -146,8 +147,9 @@
146147
"husky": "^9.0.11",
147148
"istanbul": "^0.4.5",
148149
"lint-staged": "^15.2.10",
150+
"mocha-junit-reporter": "^2.2.1",
149151
"npm-run-all": "^4.1.5",
150-
"pm-mocha-jenkins-reporter": "^0.2.6",
152+
"nyc": "^17.1.0",
151153
"prettier": "^3.3.3",
152154
"react": "^18.3.1",
153155
"react-dom": "^18.3.1",
@@ -158,9 +160,11 @@
158160
"rxjs": "5.0.0-beta.12",
159161
"sinon": "^14.0.0",
160162
"slickgrid": "github:Microsoft/SlickGrid.ADS#2.3.46",
163+
"source-map-support": "^0.5.21",
161164
"systemjs": "0.19.40",
162165
"systemjs-builder": "^0.15.32",
163166
"systemjs-plugin-json": "^0.2.0",
167+
"ts-node": "^10.9.2",
164168
"typemoq": "^1.7.0",
165169
"typescript": "^5.6.2",
166170
"typescript-eslint": "^8.7.0",
@@ -889,7 +893,7 @@
889893
"type": "array",
890894
"description": "%mssql.selectedSubscriptions%",
891895
"items": {
892-
"type": "string"
896+
"type": "string"
893897
},
894898
"$comment": "This setting must be registered in case the user does not have the Azure Resource Groups extension installed. All extensions using this configuration must register this setting."
895899
},

tasks/covertasks.js

-42
This file was deleted.

test/unit/index.ts

+113-34
Original file line numberDiff line numberDiff line change
@@ -3,39 +3,118 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
//
7-
// PLEASE DO NOT MODIFY / DELETE UNLESS YOU KNOW WHAT YOU ARE DOING
8-
//
9-
// This file is providing the test runner to use when running extension tests.
10-
// By default the test runner in use is Mocha based.
11-
//
12-
// You can provide your own test runner if you want to override it by exporting
13-
// a function run(testRoot: string, clb: (error:Error) => void) that the extension
14-
// host can call to run the tests. The test runner is expected to use console.log
15-
// to report the results back to the caller. When the tests are finished, return
16-
// a possible error to the callback or null if none.
17-
18-
import * as testRunner from "./istanbultestrunner";
19-
20-
// You can directly control Mocha options by uncommenting the following lines
21-
// See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info
22-
testRunner.configure(
23-
// Mocha Options
24-
{
25-
ui: "tdd", // the TDD UI is being used in extension.test.ts (suite, test, etc.)
26-
reporter: "pm-mocha-jenkins-reporter",
6+
"use strict";
7+
8+
// Recommended modules, loading them here to speed up NYC init
9+
// and minimize risk of race condition
10+
import "ts-node/register";
11+
import "source-map-support/register";
12+
13+
import * as Mocha from "mocha";
14+
// Simulates the recommended config option
15+
// extends: "@istanbuljs/nyc-config-typescript",
16+
import * as baseConfig from "@istanbuljs/nyc-config-typescript";
17+
import * as glob from "glob";
18+
import * as path from "path";
19+
20+
const NYC = require("nyc");
21+
22+
// Linux: prevent a weird NPE when mocha on Linux requires the window size from the TTY
23+
// Since we are not running in a tty environment, we just implementt he method statically
24+
const tty = require("tty");
25+
if (!tty.getWindowSize) {
26+
tty.getWindowSize = (): number[] => {
27+
return [80, 75];
28+
};
29+
}
30+
31+
export async function run(): Promise<void> {
32+
const testsRoot = path.resolve(__dirname, "..");
33+
34+
process.env.JUNIT_REPORT_PATH =
35+
path.join(__dirname, "..", "..") + "/test-reports/test-results-ext.xml";
36+
37+
// Setup coverage pre-test, including post-test hook to report
38+
const nyc = new NYC({
39+
...baseConfig,
40+
cwd: path.join(__dirname, "..", "..", ".."),
41+
reporter: ["text-summary", "html", "lcov", "cobertura"],
42+
all: true,
43+
silent: false,
44+
instrument: true,
45+
hookRequire: true,
46+
hookRunInContext: true,
47+
hookRunInThisContext: true,
48+
include: ["out/**/*.js"],
49+
exclude: [
50+
"out/test/**",
51+
"**/node_modules/**",
52+
"**/libs/**",
53+
"**/lib/**",
54+
"**/htmlcontent/**/*.js",
55+
"**/reactviews/**/*.js",
56+
"**/*.bundle.js",
57+
],
58+
tempDir: "./coverage/.nyc_output",
59+
});
60+
await nyc.reset();
61+
await nyc.wrap();
62+
63+
// Print a warning for any module that should be instrumented and is already loaded,
64+
// delete its cache entry and re-require
65+
// NOTE: This would not be a good practice for production code (possible memory leaks), but can be accepted for unit tests
66+
Object.keys(require.cache)
67+
.filter((f) => nyc.exclude.shouldInstrument(f))
68+
.forEach((m) => {
69+
console.warn("Module loaded before NYC, invalidating:", m);
70+
delete require.cache[m];
71+
require(m);
72+
});
73+
74+
// Debug which files will be included/excluded
75+
// console.log('Glob verification', await nyc.exclude.glob(nyc.cwd));
76+
77+
// Create the mocha test
78+
const mocha = new Mocha({
79+
ui: "tdd",
80+
timeout: 10 * 1000,
81+
reporter: "mocha-junit-reporter",
2782
reporterOptions: {
28-
junit_report_name: "Extension Tests",
29-
junit_report_path:
30-
__dirname + "../../test-reports/extension_tests.xml",
31-
junit_report_stack: 1,
83+
mochaFile: path.join(
84+
__dirname,
85+
"..",
86+
"..",
87+
"..",
88+
"test-reports",
89+
"test-results-ext.xml",
90+
),
3291
},
33-
useColors: true, // colored output from test results
34-
},
35-
// Coverage configuration options
36-
{
37-
coverConfig: "../../../coverconfig.json",
38-
},
39-
);
40-
41-
module.exports = testRunner;
92+
});
93+
(mocha.options as any).color = true;
94+
95+
// Add all files to the test suite
96+
const files = glob.sync("**/*.test.js", { cwd: testsRoot });
97+
files.forEach((f) => mocha.addFile(path.resolve(testsRoot, f)));
98+
99+
const failures: number = await new Promise((resolve) => mocha.run(resolve));
100+
await nyc.writeCoverageFile();
101+
102+
// Capture text-summary reporter's output and log it in console
103+
console.log(await captureStdout(nyc.report.bind(nyc)));
104+
105+
if (failures > 0) {
106+
throw new Error(`${failures} tests failed.`);
107+
}
108+
}
109+
110+
async function captureStdout(fn) {
111+
let w = process.stdout.write,
112+
buffer = "";
113+
process.stdout.write = (s) => {
114+
buffer = buffer + s;
115+
return true;
116+
};
117+
await fn();
118+
process.stdout.write = w;
119+
return buffer;
120+
}

test/unit/runTest.ts

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import * as path from "path";
7+
8+
import { runTests } from "@vscode/test-electron";
9+
10+
async function main() {
11+
try {
12+
// The folder containing the Extension Manifest package.json
13+
// Passed to `--extensionDevelopmentPath`
14+
const extensionDevelopmentPath = path.resolve(__dirname, "../../../");
15+
16+
// The path to test runner
17+
// Passed to --extensionTestsPath
18+
const extensionTestsPath = path.resolve(__dirname, "./index");
19+
20+
// Download VS Code, unzip it and run the integration test
21+
await runTests({
22+
extensionDevelopmentPath,
23+
extensionTestsPath,
24+
launchArgs: [],
25+
});
26+
} catch (err) {
27+
console.error("Failed to run tests", err);
28+
process.exit(1);
29+
}
30+
}
31+
32+
void main();

0 commit comments

Comments
 (0)