Add velox-cudf support for AssignUniqueId

.github/workflows/benchmark.yml

@@ -47,14 +47,14 @@ jobs:
       INSTALL_PREFIX: ${{ github.workspace }}/dependencies
     steps:
 
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           path: velox
 
           persist-credentials: false
 
       - name: Restore Dependencies
-        uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         id: restore-deps
         with:
           path: ${{ env.INSTALL_PREFIX }}
@@ -84,7 +84,7 @@ jobs:
 
       - name: Save Dependencies
         if: ${{ steps.restore-deps.outputs.cache-hit != 'true' }}
-        uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: ${{ env.INSTALL_PREFIX }}
           key: dependencies-benchmark-${{ hashFiles('velox/scripts/setup-ubuntu.sh') }}
@@ -151,12 +151,12 @@ jobs:
           merge-multiple: true
           path: /tmp/artifacts/
 
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           path: velox
           persist-credentials: false
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: '3.10'
           cache: pip
@@ -187,7 +187,7 @@ jobs:
         run: echo "failed=true" >> $GITHUB_OUTPUT
 
       - name: Create a GitHub Status on the contender commit (whether the upload was successful)
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         if: ${{ !cancelled() && steps.extract.conclusion != 'failure' }}
         with:
           script: |

.github/workflows/breeze.yml

@@ -54,13 +54,13 @@ jobs:
         working-directory: velox
     steps:
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5.0.0
         with:
           path: velox
           persist-credentials: false
 
       - name: Install uv
-        uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v6.6.0
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
 
       - name: Install Dependencies
         run: |
@@ -95,13 +95,13 @@ jobs:
         working-directory: velox
     steps:
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5.0.0
         with:
           path: velox
           persist-credentials: false
 
       - name: Install uv
-        uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v6.6.0
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
 
       - name: Install Dependencies
         run: |

.github/workflows/build-metrics.yml

@@ -49,7 +49,7 @@ jobs:
       run:
         shell: bash
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5.0.0
         with:
           ref: ${{ inputs.ref || github.sha }}
           persist-credentials: false
@@ -145,7 +145,7 @@ jobs:
     needs: metrics
     steps:
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
           persist-credentials: true

.github/workflows/build_pyvelox.yml

@@ -50,14 +50,14 @@ jobs:
       matrix:
         os: [8-core-ubuntu, macos-13, macos-14]
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           ref: ${{ inputs.ref || github.ref }}
           fetch-depth: 0
 
           persist-credentials: false
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: '3.10'
 
@@ -162,12 +162,12 @@ jobs:
 
       - run: ls wheelhouse
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: '3.10'
 
       - name: Publish a Python distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
           packages_dir: wheelhouse

.github/workflows/docker.yml

@@ -55,16 +55,27 @@ jobs:
     steps:
       - name: Free Disk Space
         run: |
-          # 15G
-          sudo rm -rf /usr/local/lib/android || :
-          # 5.3GB
-          sudo rm -rf /opt/hostedtoolcache/CodeQL || :
+          # Re-used from free-disk-space github action.
+          getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); }
+          # Show before
+          echo "Original available disk space: " $(getAvailableSpace)
+          # Remove DotNet.
+          sudo rm -rf /usr/share/dotnet || true
+          # Remove android
+          sudo rm -rf /usr/local/lib/android || true
+          # Remove CodeQL
+          sudo rm -rf /opt/hostedtoolcache/CodeQL || true
+          # Remove Haskell
+          sudo rm -rf /opt/ghc || true
+          sudo rm -rf /usr/local/.ghcup || true
+          # Show after
+          echo "New available disk space: " $(getAvailableSpace)
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -130,7 +141,7 @@ jobs:
       packages: write
     steps:
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -168,7 +179,7 @@ jobs:
         target: [java]
     steps:
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

.github/workflows/docs.yml

@@ -51,7 +51,7 @@ jobs:
           key: ccache-docs-8-core-ubuntu
 
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
           persist-credentials: true

.github/workflows/linux-build-base.yml

@@ -36,13 +36,12 @@ jobs:
     env:
       CCACHE_DIR: ${{ github.workspace }}/ccache
       VELOX_DEPENDENCY_SOURCE: SYSTEM
-      GTest_SOURCE: BUNDLED
       cudf_SOURCE: BUNDLED
       CUDA_VERSION: '12.8'
       faiss_SOURCE: BUNDLED
       USE_CLANG: "${{ inputs.use-clang && 'true' || 'false' }}"
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5.0.0
         with:
           fetch-depth: 2
           persist-credentials: false
@@ -109,10 +108,10 @@ jobs:
           else
             # cuDF (unsupported for Clang) and Faiss (link issue when using Clang)
             # are excluded for Clang compilation and need to be added back when using GCC.
-            EXTRA_CMAKE_FLAGS+="-DVELOX_ENABLE_CUDF=ON"
-            EXTRA_CMAKE_FLAGS+="-DVELOX_ENABLE_FAISS=ON"
+            EXTRA_CMAKE_FLAGS+=("-DVELOX_ENABLE_CUDF=ON")
+            EXTRA_CMAKE_FLAGS+=("-DVELOX_ENABLE_FAISS=ON")
             # Investigate issues with remote function service: Issue #13897
-            EXTRA_CMAKE_FLAGS+="-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON"
+            EXTRA_CMAKE_FLAGS+=("-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON")
           fi
           make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}"
 
@@ -134,6 +133,8 @@ jobs:
             source "/opt/miniforge/etc/profile.d/conda.sh"
             conda activate adapters
           fi
+          # Needed for HADOOP 3.3.6 minicluster. Can remove after updating to 3.4.2.
+          wget https://repo1.maven.org/maven2/org/mockito/mockito-core/2.23.4/mockito-core-2.23.4.jar -O /usr/local/hadoop/share/hadoop/mapreduce/mockito-core-2.23.4.jar
 
           export CLASSPATH=`/usr/local/hadoop/bin/hdfs classpath --glob`
           ctest -j 8 --label-exclude cuda_driver --output-on-failure --no-tests=error
@@ -163,7 +164,7 @@ jobs:
         run: |
           mkdir -p "$CCACHE_DIR"
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5.0.0
         with:
           path: velox
           persist-credentials: false
@@ -206,3 +207,63 @@ jobs:
       - name: Run Tests
         run: |
           cd _build/debug && ctest -j 8 --output-on-failure --no-tests=error
+
+  fedora-debug:
+    runs-on: 8-core-ubuntu-22.04
+    container: ghcr.io/facebookincubator/velox-dev:fedora
+    # prevent errors when forks ff their main branch
+    if: ${{ github.repository == 'facebookincubator/velox' }}
+    name: Fedora debug
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/ccache
+    defaults:
+      run:
+        shell: bash
+        working-directory: velox
+    steps:
+      - name: Get Ccache Stash
+        uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-fedora-debug-default-gcc
+
+      - name: Ensure Stash Dirs Exists
+        working-directory: ${{ github.workspace }}
+        run: |
+          mkdir -p "$CCACHE_DIR"
+
+      - uses: actions/checkout@v4
+        with:
+          path: velox
+          persist-credentials: false
+
+      - name: Clear CCache Statistics
+        run: |
+          ccache -sz
+
+      - name: Make Debug Build
+        env:
+          VELOX_DEPENDENCY_SOURCE: SYSTEM
+          faiss_SOURCE: BUNDLED
+          fmt_SOURCE: BUNDLED
+          simdjson_SOURCE: BUNDLED
+          gRPC_SOURCE: SYSTEM
+          MAKEFLAGS: NUM_THREADS=4 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3
+          EXTRA_CMAKE_FLAGS: >-
+            -DVELOX_ENABLE_PARQUET=ON
+            -DARROW_THRIFT_USE_SHARED=ON
+            -DVELOX_ENABLE_EXAMPLES=ON
+        run: |
+          uv tool install --force [email protected]
+          dnf install -y -q --setopt=install_weak_deps=False grpc-devel grpc-plugins
+          export EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DVELOX_ENABLE_FAISS=ON"
+          make debug
+
+      - name: CCache after
+        run: |
+          ccache -vs
+
+      - uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-fedora-debug-default-gcc

.github/workflows/macos.yml

@@ -22,6 +22,9 @@ on:
       - CMakeLists.txt
       - CMake/**
       - scripts/setup-macos.sh
+      - scripts/setup-common.sh
+      - scripts/setup-versions.sh
+      - scripts/setup-helper-functions.sh
       - .github/workflows/macos.yml
 
   pull_request:
@@ -31,6 +34,9 @@ on:
       - CMakeLists.txt
       - CMake/**
       - scripts/setup-macos.sh
+      - scripts/setup-common.sh
+      - scripts/setup-versions.sh
+      - scripts/setup-helper-functions.sh
       - .github/workflows/macos.yml
 
 permissions:
@@ -58,7 +64,7 @@ jobs:
       INSTALL_PREFIX: /tmp/deps-install
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5.0.0
         with:
           persist-credentials: false
 

.github/workflows/preliminary_checks.yml

@@ -32,10 +32,10 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
-      - uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 # v3.1.4
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
       - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
 
   title-check:
@@ -49,7 +49,7 @@ jobs:
           import re
           import os
           title = os.environ["title"]
-          title_re = r"^(feat|fix|build|test|docs|refactor|misc)(\(.+\))?!?: ([A-Z].+)[^.]$"
+          title_re = r"^(feat|fix|perf|build|test|docs|refactor|misc)(\(.+\))?!?: ([A-Z].+)[^.]$"
           match = re.search(title_re, title)
 
           if match is None: