Merge branch 'develop' into fg/onpair-improvements

develop added benchmarks/string-bench (#9060), which uses vortex_onpair::DEFAULT_DICT12_CONFIG.
This branch removes that constant in favour of re-exporting upstream onpair's DEFAULT_CONFIG,
which holds the same values (dict-12, threshold 0.15, seed 42), so string-bench is updated to
the new name. No textual conflicts.

Signed-off-by: Francesco Gargiulo <gargiulo.fr@gmail.com>

Author: gargiulofrancesco

.github/workflows/bench-pr.yml

@@ -23,7 +23,7 @@ jobs:
     timeout-minutes: 120
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
+          && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
           || 'ubuntu-latest' }}
     strategy:
       matrix:
@@ -33,6 +33,10 @@ jobs:
             build_args: "--features lance"
           - id: compress-bench
             name: Compression
+          # No run_args: the default suite emits the three tracked metrics
+          # (size, write, read). The codec microbenchmark is a local diagnostic.
+          - id: string-bench
+            name: String Encoding
     steps:
       - uses: runs-on/action@v2
         if: github.event.pull_request.head.repo.fork == false
@@ -104,7 +108,8 @@ jobs:
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json
+          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} \
+            ${{ matrix.benchmark.run_args }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         if: github.event.pull_request.head.repo.fork == false

.github/workflows/bench.yml

@@ -35,7 +35,7 @@ jobs:
     timeout-minutes: 120
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=bench-dedicated/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id)
+          && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id)
           || 'ubuntu-latest' }}
     strategy:
       fail-fast: false
@@ -44,11 +44,17 @@ jobs:
           - id: random-access-bench
             name: Random Access
             build_args: "--features lance"
-            formats: "parquet,lance,vortex"
+            v4_ingest: true
           - id: compress-bench
             name: Compression
             build_args: "--features lance"
-            formats: "parquet,lance,vortex"
+            run_args: "--formats parquet,lance,vortex --ingest-jsonl results.ingest.jsonl"
+            v4_ingest: true
+          # No run_args: the default suite emits the three tracked metrics
+          # (size, write, read). The codec microbenchmark is a local diagnostic.
+          - id: string-bench
+            name: String Encoding
+            v4_ingest: false
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'
@@ -116,7 +122,8 @@ jobs:
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} --formats ${{ matrix.benchmark.formats }} -d gh-json -o results.json --ingest-jsonl results.ingest.jsonl
+          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} \
+            ${{ matrix.benchmark.run_args }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
@@ -130,28 +137,30 @@ jobs:
           bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json
 
       # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live
-      # benchmarks website; a failure here fails the job. Gated on the ingest-role ARN var
-      # (the assume-role input that MUST exist for OIDC to succeed). post-ingest.py mints
-      # the RDS IAM token internally (boto3) from the assumed GitHubBenchmarkIngestRole;
-      # sslmode=verify-full validates the cert.
+      # benchmarks website for suites that emit v4 records; a failure here fails the job.
+      # All suites also upload S3 results above. string-bench does not emit
+      # results.ingest.jsonl, so these steps are disabled for it.
+      # Also gated on the ingest-role ARN var (the assume-role input that MUST exist for
+      # OIDC to succeed). post-ingest.py mints the RDS IAM token internally (boto3) from
+      # the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert.
       #
       # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only
       # the uv binary, never the synced workspace. A full sync would rebuild
       # vortex-python via sccache->S3, which fails under the ingest-role creds
       # (rds-db:connect only) and is pure waste here.
       - name: Install uv for v4 ingest
-        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+        if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest
         uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
         with:
           sync: false
       - name: Configure AWS credentials for v4 ingest (OIDC)
-        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+        if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest
         uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
         with:
           role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }}
           aws-region: ${{ vars.RDS_BENCH_REGION }}
       - name: Ingest results to v4 Postgres
-        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+        if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest
         shell: bash
         env:
           RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}

.github/workflows/ci.yml

@@ -126,14 +126,6 @@ jobs:
             pytest --benchmark-disable test/test_hf_datasets.py
         working-directory: vortex-python/
 
-      - name: Setup benchmark environment
-        run: sudo bash scripts/setup-benchmark.sh
-
-      - name: Pytest Benchmarks - Vortex
-        run: |
-          bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/
-        working-directory: vortex-python/
-
       - name: Doctest - PyVortex
         run: |
           uv run --all-packages make doctest
@@ -144,6 +136,16 @@ jobs:
           uv run --all-packages make html
         working-directory: docs/
 
+      # Keep this last: setup-benchmark.sh confines everything except the wrapped
+      # benchmark to the housekeeping CPUs, so any step after it runs on few cores.
+      - name: Setup benchmark environment
+        run: sudo bash scripts/setup-benchmark.sh
+
+      - name: Pytest Benchmarks - Vortex
+        run: |
+          bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/
+        working-directory: vortex-python/
+
   # Pins the measurement_id hash that scripts/post-ingest.py uses as the primary key of every
   # benchmarks-database fact row. The golden vectors are a frozen artifact (see the note inside
   # the JSON); this job fails if a change to scripts/_measurement_id.py drifts the hash, which

.github/workflows/codspeed.yml

@@ -84,12 +84,10 @@ jobs:
         env:
           RUSTFLAGS: "-C target-feature=+avx2"
         run: cargo codspeed build ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) --profile bench
-      - name: Setup benchmark environment
-        run: sudo bash scripts/setup-benchmark.sh
       - name: Run benchmarks
         uses: CodSpeedHQ/action@d872884a306dd4853acf0f584f4b706cf0cc72a2
         with:
-          run: bash scripts/bench-taskset.sh cargo codspeed run
+          run: cargo codspeed run
           token: ${{ secrets.CODSPEED_TOKEN }}
           mode: "simulation"
 

.github/workflows/nightly-bench.yml

@@ -30,4 +30,4 @@ jobs:
       matrix:
         machine_type:
           - id: x86
-            instance_name: i7i.metal-24xl
+            instance_name: c6id.metal

.github/workflows/rust-instrumented.yml

@@ -67,6 +67,8 @@ jobs:
           sccache: s3
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
       - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: "true"
       - name: Ensure llvm-tools are installed
         run: |
           rustup component add llvm-tools || \
@@ -186,6 +188,8 @@ jobs:
           sccache: s3
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
       - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: "true"
       - name: Install Rust nightly toolchain
         run: |
           rustup toolchain install $NIGHTLY_TOOLCHAIN
@@ -253,6 +257,8 @@ jobs:
           sccache: s3
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
       - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: "true"
       - name: Install Rust nightly toolchain
         run: |
           rustup toolchain install $NIGHTLY_TOOLCHAIN
@@ -303,6 +309,8 @@ jobs:
           sccache: s3
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
       - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: "true"
       - name: Install nightly with miri
         run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri
       - name: Run Miri

.github/workflows/sql-benchmarks.yml

@@ -13,7 +13,7 @@ on:
       machine_type:
         required: false
         type: string
-        default: i7i.metal-24xl
+        default: c6id.metal
 
 jobs:
   resolve-matrix:
@@ -58,7 +58,7 @@ jobs:
 
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=bench-dedicated/instance-type={1}/tag={2}{3}', github.run_id, inputs.machine_type, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '')
+          && format('runs-on={0}/runner=bench-dedicated/family={1}/tag={2}{3}', github.run_id, inputs.machine_type, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '')
           || 'ubuntu-latest' }}
     steps:
       - uses: runs-on/action@v2
@@ -166,7 +166,6 @@ jobs:
             --output results.json \
             --ingest-jsonl results.ingest.jsonl \
             --no-build \
-            --runner "ec2_${{ inputs.machine_type }}" \
             ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \
             ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }}
 
@@ -187,7 +186,6 @@ jobs:
             --output results.json \
             --ingest-jsonl results.ingest.jsonl \
             --no-build \
-            --runner "ec2_${{ inputs.machine_type }}" \
             ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \
             --opt remote-data-dir="$REMOTE_STORAGE" \
             ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }}

Cargo.lock

@@ -68,6 +68,7 @@ members = [
     "benchmarks/datafusion-bench",
     "benchmarks/duckdb-bench",
     "benchmarks/random-access-bench",
+    "benchmarks/string-bench",
     "vortex-geo",
 ]
 exclude = ["java/testfiles", "wasm-test"]

Cargo.toml

@@ -4,6 +4,7 @@
 """CLI for benchmark orchestration."""
 
 import json
+import os
 import subprocess
 from contextlib import contextmanager
 from datetime import datetime, timedelta
@@ -87,6 +88,17 @@ def run_ref_auto_complete() -> list[str]:
     return list(map(lambda x: x.run_id, ResultStore().list_runs(limit=None)))
 
 
+def default_runner() -> str | None:
+    """Derive the runner ID from the machine actually provisioned.
+
+    runs-on exposes the real EC2 instance type in RUNS_ON_INSTANCE_TYPE, which can
+    differ from what the workflow requested, so records must never be labeled from
+    workflow inputs.
+    """
+    instance_type = os.environ.get("RUNS_ON_INSTANCE_TYPE")
+    return f"ec2_{instance_type}" if instance_type else None
+
+
 def targets_from_axes(
     engine: str, format: str, benchmark: Benchmark | None = None
 ) -> tuple[list[BenchmarkTarget], list[str]]:
@@ -265,7 +277,10 @@ def run(
     ] = None,
     runner: Annotated[
         str | None,
-        typer.Option("--runner", help="Benchmark runner ID (e.g., ec2_c6id.8xlarge)"),
+        typer.Option(
+            "--runner",
+            help="Benchmark runner ID (e.g., ec2_c6id.metal); defaults to the actual EC2 instance type when available",
+        ),
     ] = None,
     output: Annotated[
         Path | None,
@@ -281,6 +296,7 @@ def run(
     query_list = parse_queries(queries)
     exclude_list = parse_queries(exclude_queries)
     strict_failures = targets_json is not None
+    runner = runner or default_runner()
 
     try:
         bench_opts = parse_options(options)