[hipSYCL] Add hipSYCL GitHub CI

.github/scripts/domain-check.js

@@ -0,0 +1,156 @@
+//
+// This script is used by pr.yml to determine if a domain must be tested based
+// on the files modified in the pull request.
+//
+
+// Given a domain name and set of files, return true if the domain should be
+// tested
+function matchesPattern(domain, filePaths) {
+  // filter files that end in .md
+  filePaths = filePaths.filter(
+    (filePath) =>
+      !filePath.endsWith(".md") &&
+      !filePath.startsWith("docs/") &&
+      !filePath.startsWith("third-party-programs/"),
+  );
+  // These directories contain domain specific code
+  const dirs = "(tests/unit_tests|examples|src|include/oneapi/mkl)";
+  const domains = "(blas|lapack|rng|dft)";
+  // matches changes to the domain of interest or non domain-specific code
+  const re = new RegExp(`^(${dirs}/${domain}|(?!${dirs}/${domains}))`);
+  const match = filePaths.some((filePath) => re.test(filePath));
+  return match;
+}
+
+// Return the list of files modified in the pull request
+async function prFiles(github, context) {
+  let allFiles = [];
+  let page = 0;
+  let filesPerPage = 100; // GitHub's maximum per page
+
+  while (true) {
+    page++;
+    const response = await github.rest.pulls.listFiles({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: context.payload.pull_request.number,
+      per_page: filesPerPage,
+      page: page,
+    });
+
+    if (response.data.length === 0) {
+      break; // Exit the loop if no more files are returned
+    }
+
+    allFiles = allFiles.concat(response.data.map((file) => file.filename));
+
+    if (response.data.length < filesPerPage) {
+      break; // Exit the loop if last page
+    }
+  }
+
+  return allFiles;
+}
+
+// Called by pr.yml. See:
+// https://github.com/actions/github-script/blob/main/README.md for more
+// information on the github and context parameters
+module.exports = async ({ github, context, domain }) => {
+  if (!context.payload.pull_request) {
+    console.log("Not a pull request. Testing all domains.");
+    return true;
+  }
+  const files = await prFiles(github, context);
+  const match = matchesPattern(domain, files);
+  console.log("Domain: ", domain);
+  console.log("PR files: ", files);
+  console.log("Match: ", match);
+  return match;
+};
+
+//
+// Test the matchesPattern function
+//
+// Run this script with `node domain-check.js` It should exit with code 0 if
+// all tests pass.
+//
+// If you need to change the set of files that are ignored, add a test pattern
+// below with positive and negative examples. It is also possible to test by
+// setting up a fork and then submitting pull requests that modify files, but
+// it requires a lot of manual work.
+//
+test_patterns = [
+  {
+    domain: "blas",
+    files: ["tests/unit_tests/blas/test_blas.cpp"],
+    expected: true,
+  },
+  {
+    domain: "rng",
+    files: ["examples/rng/example_rng.cpp"],
+    expected: true,
+  },
+  {
+    domain: "lapack",
+    files: ["include/oneapi/mkl/lapack/lapack.hpp"],
+    expected: true,
+  },
+  {
+    domain: "dft",
+    files: ["src/dft/lapack.hpp"],
+    expected: true,
+  },
+  {
+    domain: "dft",
+    files: ["src/dft/lapack.md"],
+    expected: false,
+  },
+  {
+    domain: "blas",
+    files: ["tests/unit_tests/dft/test_blas.cpp"],
+    expected: false,
+  },
+  {
+    domain: "rng",
+    files: ["examples/blas/example_rng.cpp"],
+    expected: false,
+  },
+  {
+    domain: "lapack",
+    files: ["include/oneapi/mkl/rng/lapack.hpp"],
+    expected: false,
+  },
+  {
+    domain: "dft",
+    files: ["src/lapack/lapack.hpp"],
+    expected: false,
+  },
+  {
+    domain: "dft",
+    files: ["docs/dft/dft.rst"],
+    expected: false,
+  },
+  {
+    domain: "dft",
+    files: ["third-party-programs/dft/dft.rst"],
+    expected: false,
+  },
+];
+
+function testPattern(test) {
+  const result = matchesPattern(test.domain, test.files);
+  if (result !== test.expected) {
+    console.log("Fail:");
+    console.log("  domain:", test.domain);
+    console.log("  files:", test.files);
+    console.log("  expected:", test.expected);
+    console.log("  result:", result);
+    process.exit(1);
+  }
+}
+
+if (require.main === module) {
+  // invoke test for each test pattern
+  test_patterns.forEach(testPattern);
+  console.log("All tests pass");
+}

.github/workflows/documentation.yml

@@ -0,0 +1,60 @@
+name: Documentation
+permissions: read-all
+
+# Trigger for PR or merge to develop branch
+on:
+  push:
+    branches: develop
+    paths:
+      - 'docs/**'
+  pull_request:
+    paths:
+      - 'docs/**'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+    - name: Install Dependencies
+      run: pip install -r docs/requirements.txt
+    - name: Configure & Build
+      run: |
+        cmake -DCMAKE_VERBOSE_MAKEFILE=on -B build docs
+        cmake --build build
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+      with:
+        name: docs
+        path: build/Documentation/html
+
+  publish:
+    needs: build
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' && github.ref == 'refs/heads/develop'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+    - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+      with:
+        ref: gh-pages
+        path: gh-pages
+    - name: Remove old site
+      run: rm -rf gh-pages/*
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+      with:
+        name: docs
+        path: gh-pages
+    - name: Push to GitHub Pages
+      run: |
+        cd gh-pages
+        touch .nojekyll
+        git add .
+        git config --global user.name "GitHub Actions"
+        git config --global user.email [email protected]
+        git commit -m "Update documentation"
+        git push --force origin gh-pages

.github/workflows/linux.yml

@@ -0,0 +1,159 @@
+name: Build tests with Open SYCL
+
+on: [push, pull_request]
+
+jobs:
+  test-with-hipsycl:
+    name: Run tests with Open SYCL
+    strategy:
+      matrix:
+        clang_version: [13]
+        rocm_version: ['5.1.1']
+        cuda_version: ['11.6']
+        os: [ubuntu-20.04, ubuntu-18.04]
+    runs-on: ${{matrix.os}}
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+    - name: install ROCm
+      run: |
+        sudo apt install libnuma-dev cmake unzip
+        wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add -
+        echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{matrix.rocm_version}}/ ubuntu main' | sudo tee /etc/apt/sources.list.d/rocm.list
+        sudo apt update
+        sudo apt install rocm-dev rocblas
+    - name: install LLVM
+      run: |
+        wget https://apt.llvm.org/llvm.sh
+        chmod +x llvm.sh
+        sudo ./llvm.sh ${{matrix.clang_version}}
+        sudo apt install libclang-${{matrix.clang_version}}-dev clang-tools-${{matrix.clang_version}} libomp-${{matrix.clang_version}}-dev
+    - name: install boost from apt
+      if: matrix.os == 'ubuntu-20.04'
+      run: |
+        sudo apt update
+        sudo apt install libboost-all-dev
+    - name: install boost from source
+      if: matrix.os == 'ubuntu-18.04'
+      run: |
+        cd
+        wget -q https://boostorg.jfrog.io/artifactory/main/release/1.70.0/source/boost_1_70_0.zip
+        unzip boost_1_70_0.zip
+        cd boost_1_70_0
+        ./bootstrap.sh --prefix=/usr
+        ./b2
+        sudo ./b2 install -j4
+    - name: install CUDA
+      run: |
+        cd
+        wget https://developer.download.nvidia.com/compute/cuda/11.6.0/local_installers/cuda_11.6.0_510.39.01_linux.run
+        sudo sh cuda_11.6.0_510.39.01_linux.run --override --silent --toolkit
+        cd
+    - name: test CUDA and cuBLAS version and fix path by symlink
+      run: |
+         /usr/local/cuda/bin/nvcc --version
+         echo $(ls /usr/local/cuda-11.6/targets/x86_64-linux/lib/stubs/libcuda*)
+         echo $LD_LIBRARY_PATH
+         sudo ln -s /usr/local/cuda-11.6/targets/x86_64-linux/lib/stubs/libcuda.so /usr/local/cuda-11.6/targets/x86_64-linux/lib/stubs/libcuda.so.1
+    - name: build Open SYCL
+      run: |
+        cd
+        git clone https://github.com/OpenSYCL/OpenSYCL.git
+        cd OpenSYCL
+        mkdir build && cd build
+        cmake -DCMAKE_CXX_COMPILER=/usr/bin/clang++-${{matrix.clang_version}} \
+              -DCLANG_EXECUTABLE_PATH=/usr/bin/clang++-${{matrix.clang_version}} \
+              -DLLVM_DIR=/usr/lib/llvm-${{matrix.clang_version}}/cmake \
+              -DCUDA_CUDA_LIBRARY=/usr/local/cuda-${{matrix.cuda_version}}/lib64/stubs/libcuda.so \
+              -DWITH_CUDA_BACKEND=ON \
+              -DWITH_ROCM_BACKEND=OFF \
+              -DWITH_LEVEL_ZERO_BACKEND=OFF \
+              -DCMAKE_INSTALL_PREFIX=/opt/OpenSYCL \
+              -DROCM_PATH=/opt/rocm-${{matrix.rocm_version}} \
+              -DCUDA_PATH=/usr/local/cuda-${{matrix.cuda_version}} ..
+        make -j2 install
+    - name: Build Open SYCL CPU Tests
+      run: |
+        cd $HOME/OpenSYCL/build
+        mkdir tests-omp && cd tests-omp
+        cmake \
+              -DOPENSYCL_TARGETS="omp" \
+              -DOpenSYCL_DIR=/opt/OpenSYCL/lib/cmake/OpenSYCL ../../tests
+        make -j 2 VERBOSE=ON
+    - name: Open SYCL tests for CPU
+      run: |
+        cd $HOME/OpenSYCL/build/tests-omp
+        LD_LIBRARY_PATH=$HOME/OpenSYCL/build/install/lib ./sycl_tests
+    - name: install LAPACK (for CBLAS)
+      run: |
+        cd
+        git clone https://github.com/Reference-LAPACK/lapack.git
+        cd lapack
+        mkdir build && cd build
+        cmake -DCMAKE_INSTALL_LIBDIR=/opt/lapack -DBUILD_SHARED_LIBS=ON -DCBLAS=ON ..
+        sudo cmake --build . -j --target install
+    - name: clone and build oneMKL with cuBLAS backend
+      env:
+        rocblas_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/rocblas/
+        hip_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/hip/
+        AMDDeviceLibs_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/AMDDeviceLibs/
+        amd_comgr_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/amd_comgr/
+        hsa-runtime64_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/hsa-runtime64/
+      run: |
+        cd
+        git clone https://github.com/oneapi-src/oneMKL.git
+        cd oneMKL
+        mkdir build
+        cd build
+        cmake \
+              -DCMAKE_CXX_COMPILER=/usr/bin/clang++-${{matrix.clang_version}} \
+              -DCMAKE_C_COMPILER=/usr/bin/clang-${{matrix.clang_version}} \
+              -DCUDA_CUDA_LIBRARY=/usr/local/cuda-${{matrix.cuda_version}}/lib64/stubs/libcuda.so \
+              -DENABLE_CUBLAS_BACKEND=True \
+              -DENABLE_CURAND_BACKEND=False \
+              -DENABLE_CUSOLVER_BACKEND=False \
+              -DENABLE_MKLGPU_BACKEND=False \
+              -DENABLE_MKLCPU_BACKEND=False \
+              -DENABLE_NETLIB_BACKEND=False \
+              -DENABLE_ROCBLAS_BACKEND=False \
+              -DBUILD_SHARED_LIBS=ON \
+              -DTARGET_DOMAINS=blas \
+              -DREF_BLAS_ROOT=/opt/lapack/ \
+              -DhipSYCL_DIR=/opt/OpenSYCL/lib/cmake/hipSYCL \
+              -DHIPSYCL_TARGETS="omp;cuda:sm_60" \
+              -DONEMKL_SYCL_IMPLEMENTATION=hipsycl ..
+        cmake --build . -j2
+        cmake --install . --prefix /opt/oneMKL
+    - name: clone and build oneMKL with rocBLAS backend
+      env:
+        rocblas_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/rocblas/
+        hip_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/hip/
+        AMDDeviceLibs_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/AMDDeviceLibs/
+        amd_comgr_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/amd_comgr/
+        hsa-runtime64_DIR: /opt/rocm-${{matrix.rocm_version}}/lib/cmake/hsa-runtime64/
+      run: |
+        cd
+        git clone https://github.com/oneapi-src/oneMKL.git oneMKL-rocBLAS
+        cd oneMKL-rocBLAS
+        mkdir build
+        cd build
+        cmake \
+              -DCMAKE_CXX_COMPILER=/usr/bin/clang++-${{matrix.clang_version}} \
+              -DCMAKE_C_COMPILER=/usr/bin/clang-${{matrix.clang_version}} \
+              -DROCBLAS_LIBRARIES=/opt/rocm-${{matrix.rocm_version}}/lib/cmake/rocblas/ \
+              -DENABLE_CUBLAS_BACKEND=False \
+              -DENABLE_CURAND_BACKEND=False \
+              -DENABLE_CUSOLVER_BACKEND=False \
+              -DENABLE_MKLGPU_BACKEND=False \
+              -DENABLE_MKLCPU_BACKEND=False \
+              -DENABLE_NETLIB_BACKEND=False \
+              -DENABLE_ROCBLAS_BACKEND=True \
+              -DBUILD_SHARED_LIBS=ON \
+              -DTARGET_DOMAINS=blas \
+              -DREF_BLAS_ROOT=/opt/lapack/ \
+              -DhipSYCL_DIR=/opt/OpenSYCL/lib/cmake/hipSYCL \
+              -DHIPSYCL_TARGETS="omp;hip:gfx906" \
+              -DONEMKL_SYCL_IMPLEMENTATION=hipsycl ..
+        cmake --build . -j2
+        cmake --install . --prefix /opt/oneMKL-rocBLAS