Skip to content

Commit 0ac47ba

Browse files
authored
Merge pull request #686 from coldav/colin/add_xfail_city_runner
Added Xfail and Mayfail attributes to city runner
2 parents e3858ab + 9e3a667 commit 0ac47ba

17 files changed

+175
-86
lines changed

.github/actions/run_opencl_cts/action.yml

+11-23
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,16 @@ runs:
3434
fi
3535
echo QEMU SETTING: $QEMU_SETTING
3636
set -x
37-
echo > expect_fail.csv
37+
# Build override file, all is done first, then the target specific.
38+
# The last file can overwrite previous overrides.
39+
for csv in .github/opencl_cts/override_all.csv .github/opencl_cts/override_${{ inputs.target }}.csv
40+
do
41+
[ -f $csv ] && cat $csv
42+
done > override.csv
43+
echo override file:
44+
cat override.csv
45+
3846
# $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
39-
[ -f .github/opencl_cts/expect_fail_all.csv ] && cat .github/opencl_cts/expect_fail_all.csv >> expect_fail.csv
40-
[ -f .github/opencl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/opencl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
41-
cat expect_fail.csv "$GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER" > disable.csv
4247
# Note: use 'eval' built-in to handle quoting/escaping/splitting reqs
4348
RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
4449
--color=always --timeout $CTS_TIMEOUT \
@@ -49,23 +54,6 @@ runs:
4954
-e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
5055
-e CL_PLATFORM_INDEX=0 \
5156
-s $GITHUB_WORKSPACE/test_conformance/$CTS_CSV_FILE \
52-
-i disable.csv"
57+
-i $GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER \
58+
-o override.csv"
5359
eval $RUN_CITIES
54-
55-
- name: Run expected failed opencl cts
56-
shell: bash
57-
env:
58-
CTS_TIMEOUT: 18:00:00
59-
run: |
60-
echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER"
61-
set -x
62-
RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
63-
--color=always --timeout $CTS_TIMEOUT \
64-
$QEMU_SETTING \
65-
-b $GITHUB_WORKSPACE/test_conformance \
66-
-L $GITHUB_WORKSPACE/install_icd/lib \
67-
-e CLC_EXECUTABLE=$GITHUB_WORKSPACE/install_ock/bin/clc \
68-
-e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
69-
-e CL_PLATFORM_INDEX=0 \
70-
-s expect_fail.csv"
71-
eval $RUN_CITIES || echo failed as expected

.github/actions/run_sycl_cts/action.yml

+15-27
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,19 @@ runs:
6767
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
6868
export ONEAPI_DEVICE_SELECTOR=opencl:0
6969
export CTS_CSV_FILE=$GITHUB_WORKSPACE/.github/scripts/sycl-cts.csv
70-
echo > expect_fail.csv
70+
7171
# $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
72-
[ -f .github/sycl_cts/expect_fail_all.csv ] && cat .github/sycl_cts/expect_fail_all.csv >> expect_fail.csv
73-
[ -f .github/sycl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/sycl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
74-
cp expect_fail.csv disable.csv
75-
[ -f "$SYCL_CTS_FILTER" ] && cat "$SYCL_CTS_FILTER" >> disable.csv
72+
73+
# Build override file, all is done first, then the target specific. The last file can overwrite prevous overrides.
74+
for csv in .github/sycl_cts/override_all.csv ..github/sycl_cts/override_${{ inputs.target }}.csv
75+
do
76+
[ -f $csv ] && cat $csv
77+
done > override.csv
78+
79+
echo override file:
80+
cat override.csv
81+
82+
exitcode=0
7683
python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
7784
--color=always \
7885
--timeout $SYCL_CTS_TIMEOUT \
@@ -85,30 +92,11 @@ runs:
8592
-l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
8693
-r SYCL-CTS/cts.xml \
8794
-v \
88-
-i disable.csv || exitcode=$?
95+
-o override.csv \
96+
$SYCL_CTS_FILTER || exitcode=$?
97+
8998
export OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so
9099
$GITHUB_WORKSPACE/.github/scripts/create_sycl_cts_test_lists.sh $PREPEND_PATH SYCL-CTS $CTS_CSV_FILE csv.txt cts_all.txt
91100
# output a diff of the generated list csv.txt and cts_all.txt
92101
diff csv.txt cts_all.txt || echo "WARNING - Missing some tests from sycl cts file based on test_all --list-tests - see > above"
93102
exit $exitcode
94-
95-
- name: run sycl cts expected fails
96-
shell: bash
97-
env:
98-
PREPEND_PATH: '' # TODO: have qemu as input and set up this
99-
SYCL_CTS_TIMEOUT: '02:00:00'
100-
run: |
101-
echo running sycl cts
102-
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
103-
export ONEAPI_DEVICE_SELECTOR=opencl:0
104-
python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
105-
--color=always \
106-
--timeout $SYCL_CTS_TIMEOUT \
107-
$PREPEND_PATH \
108-
-p sycl_cts \
109-
-b SYCL-CTS/bin \
110-
-L SYCL-CTS/lib \
111-
-e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
112-
-l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
113-
-r SYCL-CTS/cts.xml \
114-
-s expect_fail.csv || echo failed as expected

.github/opencl_cts/expect_fail_all.csv

-1
This file was deleted.

.github/opencl_cts/expect_fail_host_riscv64_linux.csv

-1
This file was deleted.

.github/opencl_cts/override_all.csv

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
API,api/test_api,Xfail
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Math,math_brute_force/test_bruteforce -w,Xfail

.github/sycl_cts/expect_fail_all.csv

-2
This file was deleted.

.github/sycl_cts/override_all.csv

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
SYCL_CTS,test_math_builtin_api "math_builtin_float_base_*",Xfail
2+
SYCL_CTS,test_math_builtin_api "math_builtin_float_double_*",Xfail
3+
SYCL_CTS,test_event "event::wait does not report asynchronous errors",Mayfail

.github/workflows/planned_testing_caller.yml

+11-9
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,14 @@ jobs:
2323
if: github.repository == 'uxlfoundation/oneapi-construction-kit' || github.event_name != 'schedule'
2424
uses: ./.github/workflows/planned_testing.yml
2525
with:
26-
target_list: '[ "host_x86_64_linux",
27-
"host_aarch64_linux",
28-
"host_riscv64_linux",
29-
"host_i686_linux",
30-
"host_refsi_linux",
31-
"host_x86_64_windows" ]'
26+
# target_list: '[ "host_x86_64_linux",
27+
# "host_aarch64_linux",
28+
# "host_riscv64_linux",
29+
# "host_i686_linux",
30+
# "host_refsi_linux",
31+
# "host_x86_64_windows" ]'
32+
target_list: '[ "host_x86_64_linux", "host_riscv64_linux"]'
33+
#', "host_aarch64_linux", "host_riscv64_linux", "host_i686_linux", "host_refsi_linux", "host_x86_64_windows" ]'
3234
ock: true
3335
test_tornado: true
3436
test_sycl_cts: true
@@ -37,7 +39,7 @@ jobs:
3739
# in a reasonable time
3840

3941
# The following can be used to download from a previous workflow run (change id)
40-
# download_ock_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445
41-
# download_dpcpp_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445
42-
# download_sycl_cts_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445
42+
download_ock_artefact: host_x86_64_linux=13505807383;host_aarch64_linux=13505807383
43+
download_dpcpp_artefact: host_x86_64_linux=13505807383;host_aarch64_linux=13505807383
44+
download_sycl_cts_artefact: host_x86_64_linux=13505807383;host_aarch64_linux=13505807383
4345
pull_request: ${{ github.event_name == 'pull_request' }}

doc/scripts/city_runner.rst

+20
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,16 @@ Other Options
612612
test list. Tests marked as ``Ignored`` are not run and not counted.
613613
This is only supported for the CTS and GTest profiles at the moment.
614614

615+
``--override-source``, ``-o``
616+
A csv file containing a list of tests which will override tests in the test
617+
csv provided by `-s`. These will override if they match the first two values
618+
of an entry in the csv file. This is useful for updating the attribute or pool
619+
values (the 3rd and 4th optional values). For example if we have an
620+
expected fail, we can add an `Xfail` element.
621+
622+
Note this works in order so if there are more than one line in the override
623+
matching the test csv file it will choose the last one.
624+
615625
CSV File Format
616626
---------------
617627

@@ -654,6 +664,16 @@ Unimplemented
654664
associated with tests which have been left unimplemented by the CTS, or test
655665
things that aren't in CL 1.x.
656666

667+
Xfail
668+
Tests marked with the ``Xfail`` attribute are run and expected to fail. Fails
669+
are counted in the pass rate and showed up in number of expected fails. It will only
670+
count towards a failing exit code if it unexpectedly passes.
671+
Mayfail
672+
Tests marked with the ``Mayfail`` attribute are run and are allowed to fail.
673+
This should usually be reserved for those tests that intermittently fail.
674+
Fails are counted in the pass rate and show up in may fails. It will
675+
not count towards a failing exit code.
676+
657677
.. _pools:
658678

659679
Pools

scripts/testing/city_runner/profile.py

+9-2
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,11 @@ def add_options(self, parser):
121121
"--ignored-source",
122122
default="",
123123
help="File containing a list of ignored tests to skip.")
124+
parser.add_argument(
125+
"-o",
126+
"--override-source",
127+
default="",
128+
help="File containing a list of tests which if match the first 2 columns will override, in order.")
124129
parser.add_argument(
125130
"-l", "--log-file", type=str, help="File to log test output to")
126131
parser.add_argument(
@@ -225,16 +230,18 @@ def create_run(self, test, worker_state=None):
225230
""" Create a new test run from a test description. """
226231
raise NotImplementedError()
227232

228-
def load_tests(self, csv_paths, disabled_path, ignored_path):
233+
def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
229234
""" Create the list of tests to run from a CSV. """
230235
if not csv_paths or any(not csv_path or not os.path.exists(csv_path) for csv_path in csv_paths):
231236
raise Exception("Test list file not specified or does not exist")
232237
if disabled_path and not os.path.exists(disabled_path):
233238
raise Exception("Disabled test list file does not exist")
234239
if ignored_path and not os.path.exists(ignored_path):
235240
raise Exception("Ignored test list file does not exist")
241+
if override_path and not os.path.exists(override_path):
242+
raise Exception("Override test list file does not exist")
236243
tests = (TestList
237-
.from_file(csv_paths, disabled_path, ignored_path, self.args.test_prefix)
244+
.from_file(csv_paths, disabled_path, ignored_path, override_path, self.args.test_prefix)
238245
.filter(self.args.patterns))
239246
return tests
240247

scripts/testing/city_runner/profiles/basic.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -100,14 +100,17 @@ def build_environment_vars(self):
100100

101101
return env
102102

103-
def load_tests(self, csv_paths, disabled_path, ignored_path):
103+
def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
104104
""" Find the list of tests from CSV. """
105105
if disabled_path:
106106
print("Warning: disabled list not supported for basic profile")
107107

108108
if ignored_path:
109109
print("Warning: ignored list not supported for basic profile")
110110

111+
if override_path:
112+
print("Warning: override list not supported for basic profile")
113+
111114
parsed_tests = []
112115
# Load tests from CSV if any were provided
113116
if csv_paths:

scripts/testing/city_runner/profiles/gtest.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -195,10 +195,12 @@ def parse_gtest_csv(self, csv_file):
195195

196196
return test_names
197197

198-
def load_tests(self, csv_paths, disabled_path, ignored_path):
198+
def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
199199
""" Find the list of tests from CSV or fallback to gtest binary. """
200200
if disabled_path:
201201
print("Warning: disabled list not supported for gtest profile")
202+
if override_path:
203+
print("Warning: override list not supported for gtest profile")
202204
executable = TestExecutable(self.args.binary_name,
203205
self.args.binary_name)
204206

scripts/testing/city_runner/profiles/tensorflow.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ def parse_options(self, argv):
118118

119119
return args
120120

121-
def load_tests(self, csv_paths, disabled_path, ignored_path):
121+
def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
122122
"""
123123
Find the list of tests from CSV or fallback to Tensorflow binary.
124124
"""
@@ -128,6 +128,9 @@ def load_tests(self, csv_paths, disabled_path, ignored_path):
128128
if ignored_path:
129129
print("Warning: ignored list not supported for tensorflow profile")
130130

131+
if override_path:
132+
print("Warning: override list not supported for tensorflow profile")
133+
131134
# Find path to Tensorflow executable
132135
Tensorflow_exe_path = os.path.abspath(self.args.binary_path)
133136
Tensorflow_dir = os.path.dirname(Tensorflow_exe_path)

scripts/testing/city_runner/runner.py

+8-3
Original file line numberDiff line numberDiff line change
@@ -121,9 +121,11 @@ def execute(self):
121121
test_source = self.args.test_source # CSV
122122
disabled_source = self.args.disabled_source # Disabled CSV
123123
ignored_source = self.args.ignored_source # Ignored CSV
124+
override_source = self.args.override_source # Override CSV
124125
self.tests = self.profile.load_tests(test_source,
125126
disabled_source,
126-
ignored_source)
127+
ignored_source,
128+
override_source)
127129
if self.args.repeat > 1:
128130
self.tests *= self.args.repeat
129131
self.num_tests = len(self.tests)
@@ -260,8 +262,11 @@ def process_results(self):
260262
# Return the city runner exit code.
261263
if self.aborted:
262264
return 130
263-
if self.results.num_fails and not self.args.relaxed:
264-
return 1
265+
if not self.args.relaxed:
266+
if self.results.num_fails > 0:
267+
return 1
268+
if self.results.num_xpasses > 0:
269+
return 1
265270
return 0
266271

267272
def process_output(self, run):

0 commit comments

Comments
 (0)