Skip to content

Commit 7029900

Browse files
committed
Added Xfail and Mayfail attributes to city runner
Support attribute Xfail and Mayfail for city runner. These can be set as additional elements in the csv file. Xfail means we expect it to fail and will give a failing return code if it unexpectedly pass, otherwise it does not contribute to an overall fail. Mayfail means it sometimes fails and will not contribute to a failing return code. Unlike some of the other attributes, these cannot be set with additional csv files, as it was felt this leads to overcomplexity. Instead it is expected that some form of csv combination script could be used alongside this.
1 parent ebed2d4 commit 7029900

16 files changed

+167
-77
lines changed

.github/actions/run_opencl_cts/action.yml

+11-23
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,16 @@ runs:
3434
fi
3535
echo QEMU SETTING: $QEMU_SETTING
3636
set -x
37-
echo > expect_fail.csv
37+
# Build override file, all is done first, then the target specific.
38+
# The last file can overwrite previous overrides.
39+
for csv in .github/opencl_cts/override_all.csv .github/opencl_cts/override_${{ inputs.target }}.csv
40+
do
41+
[ -f $csv ] && cat $csv >> override.csv
42+
done > override.csv
43+
echo override file:
44+
cat override.csv
45+
3846
# $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
39-
[ -f .github/opencl_cts/expect_fail_all.csv ] && cat .github/opencl_cts/expect_fail_all.csv >> expect_fail.csv
40-
[ -f .github/opencl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/opencl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
41-
cat expect_fail.csv "$GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER" > disable.csv
4247
# Note: use 'eval' built-in to handle quoting/escaping/splitting reqs
4348
RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
4449
--color=always --timeout $CTS_TIMEOUT \
@@ -49,23 +54,6 @@ runs:
4954
-e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
5055
-e CL_PLATFORM_INDEX=0 \
5156
-s $GITHUB_WORKSPACE/test_conformance/$CTS_CSV_FILE \
52-
-i disable.csv"
57+
-i $GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER \
58+
-o override.csv"
5359
eval $RUN_CITIES
54-
55-
- name: Run expected failed opencl cts
56-
shell: bash
57-
env:
58-
CTS_TIMEOUT: 18:00:00
59-
run: |
60-
echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER"
61-
set -x
62-
RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
63-
--color=always --timeout $CTS_TIMEOUT \
64-
$QEMU_SETTING \
65-
-b $GITHUB_WORKSPACE/test_conformance \
66-
-L $GITHUB_WORKSPACE/install_icd/lib \
67-
-e CLC_EXECUTABLE=$GITHUB_WORKSPACE/install_ock/bin/clc \
68-
-e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
69-
-e CL_PLATFORM_INDEX=0 \
70-
-s expect_fail.csv"
71-
eval $RUN_CITIES || echo failed as expected

.github/actions/run_sycl_cts/action.yml

+14-27
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,18 @@ runs:
6767
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
6868
export ONEAPI_DEVICE_SELECTOR=opencl:0
6969
export CTS_CSV_FILE=$GITHUB_WORKSPACE/.github/scripts/sycl-cts.csv
70-
echo > expect_fail.csv
70+
echo > override.csv
7171
# $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
72-
[ -f .github/sycl_cts/expect_fail_all.csv ] && cat .github/sycl_cts/expect_fail_all.csv >> expect_fail.csv
73-
[ -f .github/sycl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/sycl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
74-
cp expect_fail.csv disable.csv
75-
[ -f "$SYCL_CTS_FILTER" ] && cat "$SYCL_CTS_FILTER" >> disable.csv
72+
73+
# Build override file, all is done first, then the target specific. The last file can overwrite prevous overrides.
74+
for csv in .github/sycl_cts/override_all.csv ..github/sycl_cts/override_${{ inputs.target }}.csv
75+
do
76+
[ -f $csv ] && cat $csv
77+
done > override.csv
78+
79+
echo override file:
80+
cat override.csv
81+
7682
python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
7783
--color=always \
7884
--timeout $SYCL_CTS_TIMEOUT \
@@ -85,30 +91,11 @@ runs:
8591
-l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
8692
-r SYCL-CTS/cts.xml \
8793
-v \
88-
-i disable.csv || exitcode=$?
94+
-o override.csv \
95+
$SYCL_CTS_FILTER
96+
8997
export OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so
9098
$GITHUB_WORKSPACE/.github/scripts/create_sycl_cts_test_lists.sh $PREPEND_PATH SYCL-CTS $CTS_CSV_FILE csv.txt cts_all.txt
9199
# output a diff of the generated list csv.txt and cts_all.txt
92100
diff csv.txt cts_all.txt || echo "WARNING - Missing some tests from sycl cts file based on test_all --list-tests - see > above"
93101
exit $exitcode
94-
95-
- name: run sycl cts expected fails
96-
shell: bash
97-
env:
98-
PREPEND_PATH: '' # TODO: have qemu as input and set up this
99-
SYCL_CTS_TIMEOUT: '02:00:00'
100-
run: |
101-
echo running sycl cts
102-
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
103-
export ONEAPI_DEVICE_SELECTOR=opencl:0
104-
python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
105-
--color=always \
106-
--timeout $SYCL_CTS_TIMEOUT \
107-
$PREPEND_PATH \
108-
-p sycl_cts \
109-
-b SYCL-CTS/bin \
110-
-L SYCL-CTS/lib \
111-
-e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
112-
-l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
113-
-r SYCL-CTS/cts.xml \
114-
-s expect_fail.csv || echo failed as expected

.github/opencl_cts/expect_fail_all.csv

-1
This file was deleted.

.github/opencl_cts/expect_fail_host_riscv64_linux.csv

-1
This file was deleted.

.github/opencl_cts/override_all.csv

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
API,api/test_api,Xfail
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Math,math_brute_force/test_bruteforce -w,Xfail

.github/sycl_cts/expect_fail_all.csv

-2
This file was deleted.

.github/sycl_cts/override_all.csv

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
SYCL_CTS,test_math_builtin_api "math_builtin_float_base_*",Xfail
2+
SYCL_CTS,test_math_builtin_api "math_builtin_float_double_*",Xfail
3+
SYCL_CTS,test_event "event::wait does not report asynchronous errors",Mayfail

doc/scripts/city_runner.rst

+20
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,16 @@ Other Options
612612
test list. Tests marked as ``Ignored`` are not run and not counted.
613613
This is only supported for the CTS and GTest profiles at the moment.
614614

615+
``--override-source``, ``-o``
616+
A csv file containing a list of tests which will override tests in the test
617+
csv provided by `-s`. These will override if they match the first two values
618+
of an entry in the csv file. This is useful for updating the attribute or pool
619+
values (the 3rd and 4th optional values). For example if we have an
620+
expected fail, we can add an `Xfail` element.
621+
622+
Note this works in order so if there are more than one line in the override
623+
matching the test csv file it will choose the last one.
624+
615625
CSV File Format
616626
---------------
617627

@@ -654,6 +664,16 @@ Unimplemented
654664
associated with tests which have been left unimplemented by the CTS, or test
655665
things that aren't in CL 1.x.
656666

667+
Xfail
668+
Tests marked with the ``Xfail`` attribute are run and expected to fail. Fails
669+
are counted in the pass rate and showed up in number of expected fails. It will only
670+
count towards a failing exit code if it unexpectedly passes.
671+
Mayfail
672+
Tests marked with the ``Mayfail`` attribute are run and are allowed to fail.
673+
This should usually be reserved for those tests that intermittently fail.
674+
Fails are counted in the pass rate and show up in may fails. It will
675+
not count towards a failing exit code.
676+
657677
.. _pools:
658678

659679
Pools

scripts/testing/city_runner/profile.py

+9-2
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,11 @@ def add_options(self, parser):
121121
"--ignored-source",
122122
default="",
123123
help="File containing a list of ignored tests to skip.")
124+
parser.add_argument(
125+
"-o",
126+
"--override-source",
127+
default="",
128+
help="File containing a list of tests which if match the first 2 columns will override, in order.")
124129
parser.add_argument(
125130
"-l", "--log-file", type=str, help="File to log test output to")
126131
parser.add_argument(
@@ -225,16 +230,18 @@ def create_run(self, test, worker_state=None):
225230
""" Create a new test run from a test description. """
226231
raise NotImplementedError()
227232

228-
def load_tests(self, csv_paths, disabled_path, ignored_path):
233+
def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
229234
""" Create the list of tests to run from a CSV. """
230235
if not csv_paths or any(not csv_path or not os.path.exists(csv_path) for csv_path in csv_paths):
231236
raise Exception("Test list file not specified or does not exist")
232237
if disabled_path and not os.path.exists(disabled_path):
233238
raise Exception("Disabled test list file does not exist")
234239
if ignored_path and not os.path.exists(ignored_path):
235240
raise Exception("Ignored test list file does not exist")
241+
if override_path and not os.path.exists(override_path):
242+
raise Exception("Override test list file does not exist")
236243
tests = (TestList
237-
.from_file(csv_paths, disabled_path, ignored_path, self.args.test_prefix)
244+
.from_file(csv_paths, disabled_path, ignored_path, override_path, self.args.test_prefix)
238245
.filter(self.args.patterns))
239246
return tests
240247

scripts/testing/city_runner/profiles/basic.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -100,14 +100,17 @@ def build_environment_vars(self):
100100

101101
return env
102102

103-
def load_tests(self, csv_paths, disabled_path, ignored_path):
103+
def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
104104
""" Find the list of tests from CSV. """
105105
if disabled_path:
106106
print("Warning: disabled list not supported for basic profile")
107107

108108
if ignored_path:
109109
print("Warning: ignored list not supported for basic profile")
110110

111+
if override_path:
112+
print("Warning: override list not supported for basic profile")
113+
111114
parsed_tests = []
112115
# Load tests from CSV if any were provided
113116
if csv_paths:

scripts/testing/city_runner/profiles/gtest.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -195,10 +195,12 @@ def parse_gtest_csv(self, csv_file):
195195

196196
return test_names
197197

198-
def load_tests(self, csv_paths, disabled_path, ignored_path):
198+
def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
199199
""" Find the list of tests from CSV or fallback to gtest binary. """
200200
if disabled_path:
201201
print("Warning: disabled list not supported for gtest profile")
202+
if override_path:
203+
print("Warning: override list not supported for gtest profile")
202204
executable = TestExecutable(self.args.binary_name,
203205
self.args.binary_name)
204206

scripts/testing/city_runner/profiles/tensorflow.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ def parse_options(self, argv):
118118

119119
return args
120120

121-
def load_tests(self, csv_paths, disabled_path, ignored_path):
121+
def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
122122
"""
123123
Find the list of tests from CSV or fallback to Tensorflow binary.
124124
"""
@@ -128,6 +128,9 @@ def load_tests(self, csv_paths, disabled_path, ignored_path):
128128
if ignored_path:
129129
print("Warning: ignored list not supported for tensorflow profile")
130130

131+
if override_path:
132+
print("Warning: override list not supported for tensorflow profile")
133+
131134
# Find path to Tensorflow executable
132135
Tensorflow_exe_path = os.path.abspath(self.args.binary_path)
133136
Tensorflow_dir = os.path.dirname(Tensorflow_exe_path)

scripts/testing/city_runner/runner.py

+8-3
Original file line numberDiff line numberDiff line change
@@ -121,9 +121,11 @@ def execute(self):
121121
test_source = self.args.test_source # CSV
122122
disabled_source = self.args.disabled_source # Disabled CSV
123123
ignored_source = self.args.ignored_source # Ignored CSV
124+
override_source = self.args.override_source # Override CSV
124125
self.tests = self.profile.load_tests(test_source,
125126
disabled_source,
126-
ignored_source)
127+
ignored_source,
128+
override_source)
127129
if self.args.repeat > 1:
128130
self.tests *= self.args.repeat
129131
self.num_tests = len(self.tests)
@@ -260,8 +262,11 @@ def process_results(self):
260262
# Return the city runner exit code.
261263
if self.aborted:
262264
return 130
263-
if self.results.num_fails and not self.args.relaxed:
264-
return 1
265+
if not self.args.relaxed:
266+
if self.results.num_fails > 0:
267+
return 1
268+
if self.results.num_xfail_unexpectedly_passed > 0:
269+
return 1
265270
return 0
266271

267272
def process_output(self, run):

0 commit comments

Comments
 (0)