Skip to content

Commit ec046ec

Browse files
fs-eirefs-eire
committed
Merge remote-tracking branch 'origin/main' into fix-softmax
2 parents 7e141d0 + f1d790c commit ec046ec

File tree

15 files changed

+858
-46
lines changed

15 files changed

+858
-46
lines changed

‎.github/actions/linux-web-init-and-check/action.yml

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
name: "Web Build Initialize and Check on Linux"
2+
description: "Initializes and checks the ONNX Runtime Web build on Linux."
3+
runs:
4+
using: "composite"
5+
steps:
6+
- name: Setup Node.js
7+
uses: actions/setup-node@v3
8+
with:
9+
node-version: "22.x"
10+
11+
- name: npm ci /js/
12+
run: npm ci
13+
shell: bash
14+
working-directory: ${{ github.workspace }}/js
15+
16+
- name: npm ci /js/common/
17+
run: npm ci
18+
shell: bash
19+
working-directory: ${{ github.workspace }}/js/common
20+
21+
- name: run onnxruntime-common tests
22+
run: npm test
23+
shell: bash
24+
working-directory: ${{ github.workspace }}/js/common
25+
26+
- name: run onnxruntime-common tests (enable Float16Array)
27+
run: npm run test:f16
28+
shell: bash
29+
working-directory: ${{ github.workspace }}/js/common
30+
31+
- name: npm ci /js/web/
32+
run: npm ci
33+
shell: bash
34+
working-directory: ${{ github.workspace }}/js/web
35+
36+
- name: run TypeScript type check in /js/web/
37+
run: npm run prebuild
38+
shell: bash
39+
working-directory: ${{ github.workspace }}/js/web
40+
41+
- name: run ESLint
42+
run: npm run lint
43+
shell: bash
44+
working-directory: ${{ github.workspace }}/js
45+
46+
- name: Format code
47+
run: npm run format
48+
shell: bash
49+
working-directory: ${{ github.workspace }}/js
50+
51+
- name: Check unformatted files
52+
run: |
53+
node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)"
54+
shell: bash
55+
working-directory: ${{ github.workspace }}/js
56+
57+
- name: TypeDoc Validation
58+
run: npx typedoc --emit none --treatWarningsAsErrors
59+
shell: bash
60+
working-directory: ${{ github.workspace }}/js/common
61+
62+
- name: Generating documents
63+
run: npm run build:doc
64+
shell: bash
65+
working-directory: ${{ github.workspace }}/js/web
66+
67+
- name: Check out of dated documents
68+
run: |
69+
node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)"
70+
shell: bash
71+
working-directory: ${{ github.workspace }}/js/web

‎.github/actions/webgpu-validate-shader-key/action.yml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: "WebGPU Validate Shader Key"
2+
description: "Validate if the shader key is consistent for WebGPU shaders."
3+
4+
inputs:
5+
log_file_path:
6+
required: true
7+
type: string
8+
is_chromium_log:
9+
required: false
10+
type: boolean
11+
default: false
12+
13+
runs:
14+
using: "composite"
15+
steps:
16+
- name: Validate shader keys (chromium log)
17+
if: ${{ inputs.is_chromium_log }}
18+
shell: cmd
19+
run: |
20+
node parse-chromium-debug-log.js < "${{ inputs.log_file_path }}" | node validate-shader-key.js
21+
working-directory: ${{ github.action_path }}
22+
23+
- name: Validate shader keys (native log)
24+
if: ${{ !inputs.is_chromium_log }}
25+
shell: cmd
26+
run: |
27+
node validate-shader-key.js < "${{ inputs.log_file_path }}"
28+
working-directory: ${{ github.action_path }}

‎.github/actions/webgpu-validate-shader-key/parse-chromium-debug-log.js

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
"use strict";
5+
6+
const { EOL } = require("os");
7+
const readline = require("readline");
8+
9+
// This script is used to parse the Chromium debug log and extract the raw log data.
10+
11+
async function processChromiumDebugLog() {
12+
const rl = readline.createInterface({
13+
input: process.stdin,
14+
crlfDelay: Infinity,
15+
});
16+
17+
for await (const line of rl) {
18+
const result =
19+
/^\[.+INFO:CONSOLE\(\d+\)]\ "(?<raw_log_data>.+)",\ source:\ [^"]+?\(\d+\)$/.exec(
20+
line
21+
);
22+
if (!result) {
23+
continue;
24+
}
25+
const rawLogData = result.groups.raw_log_data;
26+
process.stdout.write(`${rawLogData}${EOL}`);
27+
}
28+
}
29+
30+
processChromiumDebugLog();

‎.github/actions/webgpu-validate-shader-key/validate-shader-key.js

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
"use strict";
5+
6+
const { EOL } = require("os");
7+
const readline = require("readline");
8+
9+
// This script is used to parse the raw log data and check if there are inconsistent shader.
10+
// When the shader key is the same, the shader code should be the same.
11+
12+
const shaderMap = new Map();
13+
14+
const regexShaderStart =
15+
/^===\ WebGPU\ Shader\ code\ \[.+?Key=\"(?<key>.+)\"]\ Start\ ===$/;
16+
const regexShaderEnd =
17+
/^===\ WebGPU\ Shader\ code\ \[.+?Key=\"(?<key>.+)\"]\ End\ ===$/;
18+
19+
async function processVerboseLog() {
20+
const rl = readline.createInterface({
21+
input: process.stdin,
22+
crlfDelay: Infinity,
23+
});
24+
25+
let currentShaderKey = null;
26+
let currentShaderCode = null;
27+
28+
for await (const line of rl) {
29+
const resultStart = regexShaderStart.exec(line);
30+
if (resultStart) {
31+
if (currentShaderKey) {
32+
throw new Error(
33+
`Found incomplete shader code for key "${currentShaderKey}".`
34+
);
35+
}
36+
37+
currentShaderKey = resultStart.groups.key;
38+
currentShaderCode = "";
39+
continue;
40+
}
41+
42+
const resultEnd = regexShaderEnd.exec(line);
43+
if (resultEnd) {
44+
if (!currentShaderKey) {
45+
throw new Error(
46+
`Found unexpected shader end for key "${resultEnd.groups.key}".`
47+
);
48+
} else if (currentShaderKey !== resultEnd.groups.key) {
49+
throw new Error(
50+
`Found inconsistent shader key. Expected "${currentShaderKey}", but got "${resultEnd.groups.key}".`
51+
);
52+
}
53+
54+
if (shaderMap.has(currentShaderKey)) {
55+
if (shaderMap.get(currentShaderKey) !== currentShaderCode) {
56+
throw new Error(`Found inconsistent shader code for key "${currentShaderKey}".
57+
=== Previous Shader Start ===
58+
${shaderMap.get(currentShaderKey)}
59+
=== Previous Shader End ===
60+
61+
=== Current Shader Start ===
62+
${currentShaderCode}
63+
=== Current Shader End ===`);
64+
}
65+
} else {
66+
shaderMap.set(currentShaderKey, currentShaderCode);
67+
}
68+
69+
currentShaderKey = null;
70+
currentShaderCode = null;
71+
continue;
72+
}
73+
74+
if (currentShaderKey) {
75+
currentShaderCode += line + EOL;
76+
}
77+
}
78+
79+
if (currentShaderKey) {
80+
throw new Error(
81+
`Found incomplete shader code for key "${currentShaderKey}".`
82+
);
83+
}
84+
85+
if (shaderMap.size === 0) {
86+
throw new Error("No shader code found.");
87+
}
88+
89+
console.log(
90+
`All shader code is consistent. Total ${shaderMap.size} shader code found.`
91+
);
92+
}
93+
94+
processVerboseLog();

‎.github/workflows/linux-dnnl.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# This file is very similar to linux_ci.yml, but much simpler
55

66

7-
name: Linux CI
7+
name: Linux DNNL CI
88

99
on:
1010
push:

‎.github/workflows/linux-wasm-ci-build-and-test-workflow.yml

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
name: "Linux WASM CI Reusable Workflow for build and test"
2+
description: "This is a reusable workflow for Linux WASM CI pipelines to build and test"
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
build_config:
8+
required: true
9+
type: string
10+
extra_build_args:
11+
required: false
12+
type: string
13+
default: ""
14+
skip_publish:
15+
required: false
16+
type: boolean
17+
default: false
18+
build_jsep:
19+
required: false
20+
type: boolean
21+
default: false
22+
build_webgpu:
23+
required: false
24+
type: boolean
25+
default: false
26+
27+
jobs:
28+
build-wasm:
29+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
30+
env:
31+
buildArch: x64
32+
common_build_args: --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --config ${{ inputs.build_config }} --skip_submodule_sync --build_wasm --enable_wasm_simd --enable_wasm_threads ${{ inputs.extra_build_args }}
33+
34+
steps:
35+
- name: Checkout code
36+
uses: actions/checkout@v4
37+
with:
38+
submodules: recursive
39+
40+
- name: Set up Python
41+
uses: actions/setup-python@v4
42+
with:
43+
python-version: "3.12"
44+
architecture: ${{ env.buildArch }}
45+
46+
- name: Export GitHub Actions cache environment variables
47+
uses: actions/github-script@v7
48+
with:
49+
script: |
50+
core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
51+
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
52+
53+
- name: Install EMSDK
54+
run: |
55+
set -ex
56+
cd ${{ github.workspace }}/cmake/external/emsdk
57+
./emsdk install 4.0.4
58+
./emsdk activate 4.0.4
59+
60+
- name: Build and test (browser) (simd + threads)
61+
run: |
62+
set -e -x
63+
source ${{ github.workspace }}/cmake/external/emsdk/emsdk_env.sh
64+
cd '${{ github.workspace }}'
65+
python ./tools/ci_build/build.py \
66+
${{ env.common_build_args }} \
67+
--build_dir ${{ github.workspace }}/build/wasm_inferencing \
68+
--wasm_run_tests_in_browser
69+
70+
- name: Build (simd + threads + JSEP)
71+
if: ${{ inputs.build_jsep == true }}
72+
run: |
73+
set -e -x
74+
source ${{ github.workspace }}/cmake/external/emsdk/emsdk_env.sh
75+
cd '${{ github.workspace }}'
76+
python ./tools/ci_build/build.py \
77+
${{ env.common_build_args }} \
78+
--build_dir ${{ github.workspace }}/build/wasm_inferencing_jsep \
79+
--use_jsep \
80+
--use_webnn \
81+
--target onnxruntime_webassembly \
82+
--skip_tests
83+
84+
- name: Build (simd + threads + WebGPU experimental)
85+
if: ${{ inputs.build_webgpu == true }}
86+
run: |
87+
set -e -x
88+
source ${{ github.workspace }}/cmake/external/emsdk/emsdk_env.sh
89+
cd '${{ github.workspace }}'
90+
python ./tools/ci_build/build.py \
91+
${{ env.common_build_args }} \
92+
--build_dir ${{ github.workspace }}/build/wasm_inferencing_webgpu \
93+
--use_webgpu \
94+
--use_jsep \
95+
--use_webnn \
96+
--target onnxruntime_webassembly \
97+
--skip_tests
98+
99+
- name: Create Artifacts
100+
if: ${{ inputs.skip_publish != true }}
101+
run: |
102+
mkdir -p ${{ github.workspace }}/artifacts/wasm/
103+
cp ${{ github.workspace }}/build/wasm_inferencing/${{ inputs.build_config }}/ort-wasm-simd-threaded.wasm ${{ github.workspace }}/artifacts/wasm/
104+
cp ${{ github.workspace }}/build/wasm_inferencing/${{ inputs.build_config }}/ort-wasm-simd-threaded.mjs ${{ github.workspace }}/artifacts/wasm/
105+
if [ -d ${{ github.workspace }}/build/wasm_inferencing_jsep ]; then
106+
cp ${{ github.workspace }}/build/wasm_inferencing_jsep/${{ inputs.build_config }}/ort-wasm-simd-threaded.jsep.wasm ${{ github.workspace }}/artifacts/wasm/
107+
cp ${{ github.workspace }}/build/wasm_inferencing_jsep/${{ inputs.build_config }}/ort-wasm-simd-threaded.jsep.mjs ${{ github.workspace }}/artifacts/wasm/
108+
fi
109+
110+
- name: Create WebGPU Artifacts
111+
if: ${{ inputs.skip_publish != true && inputs.build_webgpu == true }}
112+
run: |
113+
mkdir -p ${{ github.workspace }}/artifacts/wasm_webgpu/
114+
cp ${{ github.workspace }}/build/wasm_inferencing_webgpu/${{ inputs.build_config }}/ort-wasm-simd-threaded.jsep.wasm ${{ github.workspace }}/artifacts/wasm_webgpu/
115+
cp ${{ github.workspace }}/build/wasm_inferencing_webgpu/${{ inputs.build_config }}/ort-wasm-simd-threaded.jsep.mjs ${{ github.workspace }}/artifacts/wasm_webgpu/
116+
117+
- name: Upload WASM artifacts
118+
if: ${{ inputs.skip_publish != true }}
119+
uses: actions/upload-artifact@v4
120+
with:
121+
name: ${{ inputs.build_config }}_wasm
122+
path: ${{ github.workspace }}/artifacts/wasm
123+
124+
- name: Upload WebGPU artifacts
125+
if: ${{ inputs.skip_publish != true && inputs.build_webgpu == true }}
126+
uses: actions/upload-artifact@v4
127+
with:
128+
name: ${{ inputs.build_config }}_wasm_webgpu
129+
path: ${{ github.workspace }}/artifacts/wasm_webgpu
130+
131+
- name: Publish test results
132+
if: ${{ always() && inputs.build_config == 'Debug' }}
133+
uses: actions/upload-artifact@v4
134+
with:
135+
name: test-results
136+
path: ${{ github.workspace }}/build/**/*.results.xml

0 commit comments

Comments
 (0)