SYCL CTS CI for aarch64 graviton (#681)

Author: alan-forbes-cp

.github/actions/do_build_dpcpp/action.yml

@@ -131,4 +131,4 @@ runs:
       with:
         name: dpcpp_${{inputs.target}}
         path: llvm/build/install/dpcpp.tar
-        retention-days: 1
+        retention-days: 7

.github/actions/do_build_icd/action.yml

@@ -42,7 +42,7 @@ runs:
       with:
         name: header_${{inputs.target}}
         path: headers_install_${{steps.calc_vars.outputs.arch}}
-        retention-days: 1
+        retention-days: 7
 
     - name: clone icd 
       uses: actions/checkout@v4
@@ -71,5 +71,5 @@ runs:
       with:
         name: icd_${{inputs.target}}
         path: install_icd_${{steps.calc_vars.outputs.arch}}
-        retention-days: 1
+        retention-days: 7
 

.github/actions/do_build_ock_artefact/action.yml

@@ -36,21 +36,24 @@ runs:
         with:
           llvm_version: ${{ inputs.llvm_version }}
           llvm_build_type: RelAssert
-          cross_arch: ${{ steps.calc_vars.outputs.arch == 'x86_64' && 'none' || steps.calc_vars.outputs.arch }}
+          cross_arch: ${{ ( steps.calc_vars.outputs.arch == 'x86_64' || steps.calc_vars.outputs.arch == 'aarch64' )
+                       && 'none' || steps.calc_vars.outputs.arch }}
           os: ${{ contains(inputs.target, 'windows') && 'windows' || 'ubuntu' }}
           ubuntu_version: ${{ contains(inputs.target, 'riscv64') && '24.04' || '22.04' }}
 
-      - name: build ock x86
-        if: steps.calc_vars.outputs.arch  == 'x86_64' && contains(inputs.download_ock_artefact, inputs.target) != true
+      - name: build native ock - x86_64 and aarch64
+        if: ( steps.calc_vars.outputs.arch == 'x86_64' || steps.calc_vars.outputs.arch == 'aarch64' )
+            && contains(inputs.download_ock_artefact, inputs.target) != true
         uses: ./.github/actions/do_build_ock
         with:
           build_targets: install
           offline_kernel_tests: OFF
           extra_flags: -DCA_ENABLE_TESTS=OFF -DCA_ENABLE_EXAMPLES=OFF -DCA_ENABLE_DOCUMENTATION=OFF
           shell_to_use: ${{ contains(inputs.target, 'windows') && 'pwsh' || 'bash' }}
 
-      - name: build ock other ${{ inputs.target }}
-        if: steps.calc_vars.outputs.arch  != 'x86_64' && contains(inputs.download_ock_artefact, inputs.target) != true
+      - name: build ock - other ${{ inputs.target }}
+        if: ( steps.calc_vars.outputs.arch != 'x86_64' && steps.calc_vars.outputs.arch != 'aarch64' )
+            && contains(inputs.download_ock_artefact, inputs.target) != true
         uses: ./.github/actions/do_build_ock
         with:
           build_targets: install
@@ -88,4 +91,4 @@ runs:
         with:
           name: ock_${{ inputs.target }}
           path: install
-          retention-days: 1
+          retention-days: 7

.github/actions/do_build_opencl_cts/action.yml

@@ -70,5 +70,5 @@ runs:
           !test_conformance/**/*.ninja
           !test_conformance/test_common
           !test_conformance/test_common/**
-        retention-days: 1
+        retention-days: 7
 

.github/actions/do_build_sycl_cts/action.yml

@@ -67,6 +67,10 @@ runs:
         # We build SYCL-CTS without installing it, so build directly in the top level.
         git -C SYCL-CTS.src log -1
         git -C SYCL-CTS.src apply $GITHUB_WORKSPACE/.github/patches/SYCL-CTS-0002-Permit-building-for-unknown-architectures.patch
+        export JOPT="4"
+        if [[ "${{inputs.target}}" =~ .*aarch64.* ]] ; then # needed for cp-graviton runner
+          JOPT="2"
+        fi
         cmake -S SYCL-CTS.src \
             -GNinja \
             -B SYCL-CTS \
@@ -77,7 +81,7 @@ runs:
             -DCMAKE_CXX_COMPILER="$GITHUB_WORKSPACE/install_dpcpp/bin/clang++" \
             -DCMAKE_CXX_FLAGS="--target=${{steps.calc_vars.outputs.arch}}-linux-gnu" \
             -DCMAKE_CXX_LINK_FLAGS="-fuse-ld=lld"
-        ninja -C SYCL-CTS -v -j4 -k 0 || :
+        ninja -C SYCL-CTS -v -j$JOPT -k 0 || :
 
     - name: package artefacts  # package/unpackage avoids known 'permissions loss' issue
       if: contains(inputs.download_sycl_cts_artefact, inputs.target) != true    
@@ -107,5 +111,5 @@ runs:
       with:
         name: sycl_cts_${{inputs.target}}
         path: SYCL-CTS/sycl-cts.tar
-        retention-days: 2
+        retention-days: 7
 

.github/actions/do_build_tornado/action.yml

@@ -66,4 +66,4 @@ runs:
       with:
         name: tornado_${{inputs.target}}
         path: TornadoVM_SDK
-        retention-days: 1
+        retention-days: 7

.github/actions/run_sycl_cts/action.yml

@@ -62,7 +62,6 @@ runs:
         SYCL_CTS_TIMEOUT: '02:00:00'
         SYCL_CTS_FILTER: ''
       run: |
-        set -x
         echo running sycl cts
         export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
         export ONEAPI_DEVICE_SELECTOR=opencl:0
@@ -71,15 +70,16 @@ runs:
         # $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
 
         # Build override file, all is done first, then the target specific. The last file can overwrite prevous overrides.
-        for csv in .github/sycl_cts/override_all.csv ..github/sycl_cts/override_${{ inputs.target }}.csv
+        for csv in .github/sycl_cts/override_all.csv .github/sycl_cts/override_${{ inputs.target }}.csv
         do
            [ -f $csv ] && cat $csv
         done > override.csv
 
-        echo override file:
+        echo OVERRIDE FILE:
         cat override.csv
 
         exitcode=0
+        set -x
         python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
           --color=always \
           --timeout $SYCL_CTS_TIMEOUT \

.github/scripts/create_sycl_cts_test_lists.sh

@@ -51,11 +51,14 @@ while IFS= read -r line; do
   # Process the line to work out how to run the test - use csv reader to select test and args to run
   test_to_run=`echo $line | python3 -c 'import sys, csv
 for line in csv.reader(sys.stdin): print(line[1])'`
-  # Actually run the test with --list-tests
-  eval "$prepend_path ./bin/$test_to_run --list-tests -v quiet" >> $temp_file
-  if [ $? -ne 0 ]; then
-    echo "Error : Unable to process $line"
-    exit 1
+  # Actually run the test, if binary exists, with --list-tests
+  binary=$(echo ./bin/$test_to_run | sed 's/\ .*//')
+  if [[ -f $binary ]]; then
+    eval "$prepend_path ./bin/$test_to_run --list-tests -v quiet" >> $temp_file
+    if [ $? -ne 0 ]; then
+      echo "Error : Unable to process $line"
+      exit 1
+    fi
   fi
 done <<< $sycl_cts_file
 

.github/scripts/sycl-cts.csv

@@ -228,7 +228,7 @@ SYCL_CTS,test_kernel_bundle "device image api"
 SYCL_CTS,test_kernel_bundle "device_image common reference semantics"
 SYCL_CTS,test_kernel_bundle get_kernel_bundle_builtin_kernels
 SYCL_CTS,test_kernel_bundle get_kernel_bundle_dev_no_linker_or_compiler_aspect
-SYCL_CTS,test_kernel_bundle get_kernel_bundle_dev_not_associated_with_context --allow-running-no-tests 
+SYCL_CTS,test_kernel_bundle get_kernel_bundle_dev_not_associated_with_context --allow-running-no-tests
 SYCL_CTS,test_kernel_bundle get_kernel_bundle_with_incompatible_kernels
 SYCL_CTS,test_kernel_bundle get_kernel_bundle_without_kernel_attr_verify_that_exception_is_not_thrown
 SYCL_CTS,test_kernel_bundle get_kernel_bundle_zero_device

.github/sycl_cts/override_host_aarch64_linux.csv

@@ -0,0 +1,11 @@
+SYCL_CTS,test_atomic_ref_stress "sycl::atomic_ref atomicity *",Mayfail
+SYCL_CTS,test_atomic_ref "sycl::atomic_ref fetch_add()/fetch_sub() test. *",Mayfail
+SYCL_CTS,test_vector_operators vector_OPERATORS_bool,Xfail
+SYCL_CTS,test_vector_operators vector_OPERATORS_byte,Xfail
+SYCL_CTS,test_vector_operators vector_OPERATORS_char,Xfail
+SYCL_CTS,test_vector_operators vector_OPERATORS_double,Xfail
+SYCL_CTS,test_vector_operators vector_OPERATORS_float,Xfail
+SYCL_CTS,test_vector_operators vector_OPERATORS_half --allow-running-no-tests,Xfail
+SYCL_CTS,test_vector_operators vector_OPERATORS_int,Xfail
+SYCL_CTS,test_vector_operators vector_OPERATORS_int32_t,Xfail
+SYCL_CTS,test_vector_operators vector_OPERATORS_int8_t,Xfail

.github/workflows/planned_testing.yml

@@ -54,7 +54,10 @@ jobs:
     runs-on: ubuntu-22.04
     outputs:
       matrix_only_linux_x86_64_aarch64_riscv64: ${{ steps.vars.outputs.matrix_only_linux_x86_64_aarch64_riscv64 }}
-      matrix_only_linux_x86_64: ${{ steps.vars.outputs.matrix_only_linux_x86_64 }}
+      matrix_only_linux_x86_64_aarch64:         ${{ steps.vars.outputs.matrix_only_linux_x86_64_aarch64 }}
+      matrix_only_linux_x86_64:                 ${{ steps.vars.outputs.matrix_only_linux_x86_64 }}
+      matrix_only_linux_aarch64:                ${{ steps.vars.outputs.matrix_only_linux_aarch64 }}
+      matrix_only_linux_riscv64:                ${{ steps.vars.outputs.matrix_only_linux_riscv64 }}
     steps:
     - id: vars
       # TODO: If we expand on this, come up with a more programmatical way of doing only certain targets.
@@ -66,21 +69,41 @@ jobs:
            {\"target\": \"host_refsi_linux\"}, \
            {\"target\": \"host_i686_linux\"}, \
            {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
+        echo matrix_only_linux_x86_64_aarch64="[ \
+           {\"target\": \"host_arm_linux\"}, \
+           {\"target\": \"host_riscv64_linux\"}, \
+           {\"target\": \"host_refsi_linux\"}, \
+           {\"target\": \"host_i686_linux\"}, \
+           {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
         echo matrix_only_linux_x86_64="[ \
            {\"target\": \"host_aarch64_linux\"}, \
            {\"target\": \"host_riscv64_linux\"}, \
            {\"target\": \"host_arm_linux\"}, \
            {\"target\": \"host_refsi_linux\"}, \
            {\"target\": \"host_i686_linux\"}, \
            {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
+        echo matrix_only_linux_aarch64="[ \
+           {\"target\": \"host_x86_64_linux\"}, \
+           {\"target\": \"host_riscv64_linux\"}, \
+           {\"target\": \"host_arm_linux\"}, \
+           {\"target\": \"host_refsi_linux\"}, \
+           {\"target\": \"host_i686_linux\"}, \
+           {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
+        echo matrix_only_linux_riscv64="[ \
+           {\"target\": \"host_x86_64_linux\"}, \
+           {\"target\": \"host_aarch64_linux\"}, \
+           {\"target\": \"host_arm_linux\"}, \
+           {\"target\": \"host_refsi_linux\"}, \
+           {\"target\": \"host_i686_linux\"}, \
+           {\"target\": \"host_x86_64_windows\"}]" >> $GITHUB_OUTPUT
         cat $GITHUB_OUTPUT    
 
   create_ock_artefacts_windows:
     needs: [workflow_vars]
     # Currently there is only one windows target so we don't bother with the
     # matrix, just check if it's enabled
     runs-on: windows-2019
-    if : inputs.ock && contains(inputs.target_list, 'host_x86_64_windows')
+    if: inputs.ock && contains(inputs.target_list, 'host_x86_64_windows')
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -105,13 +128,14 @@ jobs:
 
     # risc-v needs ubuntu 24.04 so we get the latest qemu as well as how we
     # build llvm. Otherwise we choose ubuntu-22.04 (use a container for both for consistency).
-    runs-on: cp-ubuntu-24.04
+    runs-on: ${{ contains(matrix.target, 'host_aarch64') && 'cp-graviton' || 'cp-ubuntu-24.04' }}
     container:
-      image: ${{ contains(matrix.target, 'host_riscv') && 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
-             || 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' }}
+      image: ${{  contains(matrix.target, 'host_riscv')   && 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
+             || ( contains(matrix.target, 'host_aarch64') && 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
+             ||                                              'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' ) }}
       volumes:
         - ${{github.workspace}}:${{github.workspace}}
-    if : inputs.ock && contains(inputs.target_list, 'linux')
+    if: inputs.ock && contains(inputs.target_list, 'linux')
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -136,10 +160,11 @@ jobs:
         target: ${{ fromJson(inputs.target_list) }}
         exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64_aarch64_riscv64) }}
 
-    runs-on: cp-ubuntu-24.04
+    runs-on: ${{ contains(matrix.target, 'host_aarch64') && 'cp-graviton' || 'cp-ubuntu-24.04' }}
     container:
-      image: ${{ contains(matrix.target, 'host_riscv') && 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
-             || 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' }}
+      image: ${{  contains(matrix.target, 'host_riscv')   && 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
+             || ( contains(matrix.target, 'host_aarch64') && 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
+             ||                                              'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' ) }}
       volumes:
         - ${{github.workspace}}:${{github.workspace}}
     steps:
@@ -196,7 +221,7 @@ jobs:
     container:
       image: ${{  contains(matrix.target, 'host_riscv')   && 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
              || ( contains(matrix.target, 'host_aarch64') && 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
-             || 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' ) }}
+             ||                                              'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' ) }}
       volumes:
         - ${{github.workspace}}:${{github.workspace}}
     steps:
@@ -221,21 +246,50 @@ jobs:
         with:
           target: ${{ matrix.target }}
 
-  build_dpcpp_native_host:
+  build_dpcpp_native:
     needs: [workflow_vars]
     strategy:
       matrix:
         target: ${{ fromJson(inputs.target_list) }}
-        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64) }}
+        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64_aarch64) }}
+
+    runs-on: ${{ contains(matrix.target, 'host_aarch64') && 'cp-graviton' || 'cp-ubuntu-24.04' }}
+    container:
+      image: ${{ contains(matrix.target, 'host_aarch64') && 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
+             ||                                             'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' }}
+      volumes:
+        - ${{github.workspace}}:${{github.workspace}}
+
+    if: inputs.test_sycl_cts
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: set up gh
+        uses: ./.github/actions/setup_gh
+        with:
+          os: ${{ contains( matrix.target, 'windows') && 'windows' || 'ubuntu' }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: build dpc++ artefact
+        uses: ./.github/actions/do_build_dpcpp
+        with:
+          target: ${{ matrix.target }}
+          download_dpcpp_artefact: ${{ inputs.download_dpcpp_artefact }}
+
+  build_dpcpp_cross:
+    needs: [workflow_vars, build_dpcpp_native]  # cross builds need pre-built native artefact
+    strategy:
+      fail-fast: false  # let all matrix jobs complete
+      matrix:
+        target: ${{ fromJson(inputs.target_list) }}
+        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_riscv64) }}
 
-    # TODO: Extend if we decide to enable for windows or build natively on another target
     runs-on: cp-ubuntu-24.04
     container:
-      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest'
+      image: 'ghcr.io/uxlfoundation/ock_ubuntu_24.04-x86-64:latest'
       volumes:
         - ${{github.workspace}}:${{github.workspace}}
 
-    if : inputs.test_sycl_cts
+    if: inputs.test_sycl_cts && contains(inputs.target_list, 'host_riscv64_linux')
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -251,49 +305,56 @@ jobs:
           download_dpcpp_artefact: ${{ inputs.download_dpcpp_artefact }}
 
   build_sycl_cts:
-    needs: [workflow_vars, build_icd, build_dpcpp_native_host]
+    needs: [workflow_vars, build_icd, build_dpcpp_native]
     strategy:
+      fail-fast: false  # let all matrix jobs complete
       matrix:
         target: ${{ fromJson(inputs.target_list) }}
-        # TODO: For now just linux x86_64
-        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64) }}
+        # TODO: For now just linux x86_64 & aarch64
+        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64_aarch64) }}
 
-    # TODO: Extend if we decide to enable for windows or build natively on another target
-    runs-on: cp-ubuntu-24.04
+    runs-on: ${{ contains(matrix.target, 'host_aarch64') && 'cp-graviton' || 'cp-ubuntu-24.04' }}
     container:
-      image: 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest'
+      image: ${{ contains(matrix.target, 'host_aarch64') && 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
+             ||                                             'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' }}
       volumes:
         - ${{github.workspace}}:${{github.workspace}}
 
-    if : inputs.test_sycl_cts
+    if: inputs.test_sycl_cts
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
       - name: set up gh
         uses: ./.github/actions/setup_gh
         with:
           os: ${{ contains( matrix.target, 'windows') && 'windows' || 'ubuntu' }}
-          token: ${{ secrets.GITHUB_TOKEN }}        
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: build sycl cts artefact
         uses: ./.github/actions/do_build_sycl_cts
         with:
           target: ${{ matrix.target }}
           download_sycl_cts_artefact: ${{ inputs.download_sycl_cts_artefact }}
 
   run_sycl_cts:
-    needs: [workflow_vars, create_ock_artefacts_ubuntu, build_dpcpp_native_host, build_sycl_cts]
+    needs: [workflow_vars, create_ock_artefacts_ubuntu, build_dpcpp_native, build_sycl_cts]
     strategy:
+      fail-fast: false  # let all matrix jobs complete
       matrix:
         target: ${{ fromJson(inputs.target_list) }}
-        # TODO: For now just linux x86_64
-        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64) }}
+        # TODO: For now just linux x86_64 and aarch64
+        exclude: ${{ fromJson(needs.workflow_vars.outputs.matrix_only_linux_x86_64_aarch64) }}
 
-    runs-on: 'ubuntu-22.04'
-    if : inputs.test_sycl_cts
+    runs-on: ${{ contains(matrix.target, 'host_aarch64') && 'cp-graviton' || 'cp-ubuntu-24.04' }}
+    container:
+      image: ${{ contains(matrix.target, 'host_aarch64') && 'ghcr.io/uxlfoundation/ock_ubuntu_22.04-aarch64:latest'
+             ||                                             'ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest' }}
+      volumes:
+        - ${{github.workspace}}:${{github.workspace}}
+    if: inputs.test_sycl_cts
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
-      - name: build dpc++ artefact
+      - name: run sycl cts
         uses: ./.github/actions/run_sycl_cts
         with:
           target: ${{ matrix.target }}