benchmarks: add string compression benchmark

.github/workflows/bench-pr.yml

@@ -33,6 +33,10 @@ jobs:
             build_args: "--features lance"
           - id: compress-bench
             name: Compression
+          # No run_args: the default suite emits the three tracked metrics
+          # (size, write, read). The codec microbenchmark is a local diagnostic.
+          - id: string-bench
+            name: String Encoding
     steps:
       - uses: runs-on/action@v2
         if: github.event.pull_request.head.repo.fork == false
@@ -104,7 +108,8 @@ jobs:
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json
+          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} \
+            ${{ matrix.benchmark.run_args }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         if: github.event.pull_request.head.repo.fork == false

.github/workflows/bench.yml

@@ -44,11 +44,17 @@ jobs:
           - id: random-access-bench
             name: Random Access
             build_args: "--features lance"
-            formats: "parquet,lance,vortex"
+            v4_ingest: true
           - id: compress-bench
             name: Compression
             build_args: "--features lance"
-            formats: "parquet,lance,vortex"
+            run_args: "--formats parquet,lance,vortex --ingest-jsonl results.ingest.jsonl"
+            v4_ingest: true
+          # No run_args: the default suite emits the three tracked metrics
+          # (size, write, read). The codec microbenchmark is a local diagnostic.
+          - id: string-bench
+            name: String Encoding
+            v4_ingest: false
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'
@@ -116,7 +122,8 @@ jobs:
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} --formats ${{ matrix.benchmark.formats }} -d gh-json -o results.json --ingest-jsonl results.ingest.jsonl
+          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} \
+            ${{ matrix.benchmark.run_args }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
@@ -130,28 +137,30 @@ jobs:
           bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json
 
       # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live
-      # benchmarks website; a failure here fails the job. Gated on the ingest-role ARN var
-      # (the assume-role input that MUST exist for OIDC to succeed). post-ingest.py mints
-      # the RDS IAM token internally (boto3) from the assumed GitHubBenchmarkIngestRole;
-      # sslmode=verify-full validates the cert.
+      # benchmarks website for suites that emit v4 records; a failure here fails the job.
+      # All suites also upload S3 results above. string-bench does not emit
+      # results.ingest.jsonl, so these steps are disabled for it.
+      # Also gated on the ingest-role ARN var (the assume-role input that MUST exist for
+      # OIDC to succeed). post-ingest.py mints the RDS IAM token internally (boto3) from
+      # the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert.
       #
       # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only
       # the uv binary, never the synced workspace. A full sync would rebuild
       # vortex-python via sccache->S3, which fails under the ingest-role creds
       # (rds-db:connect only) and is pure waste here.
       - name: Install uv for v4 ingest
-        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+        if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest
         uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
         with:
           sync: false
       - name: Configure AWS credentials for v4 ingest (OIDC)
-        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+        if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest
         uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6
         with:
           role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }}
           aws-region: ${{ vars.RDS_BENCH_REGION }}
       - name: Ingest results to v4 Postgres
-        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+        if: vars.GH_BENCH_INGEST_ROLE_ARN != '' && matrix.benchmark.v4_ingest
         shell: bash
         env:
           RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}

Cargo.toml

@@ -68,6 +68,7 @@ members = [
     "benchmarks/datafusion-bench",
     "benchmarks/duckdb-bench",
     "benchmarks/random-access-bench",
+    "benchmarks/string-bench",
     "vortex-geo",
 ]
 exclude = ["java/testfiles", "wasm-test"]

benchmarks/string-bench/Cargo.toml

@@ -0,0 +1,46 @@
+[package]
+name = "string-bench"
+description = "Vortex string benchmarks for direct compression and serialized write/read round trips"
+authors.workspace = true
+categories.workspace = true
+edition.workspace = true
+homepage.workspace = true
+include.workspace = true
+keywords.workspace = true
+license.workspace = true
+readme.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+publish = false
+
+[dependencies]
+anyhow = { workspace = true }
+bytes = { workspace = true }
+clap = { workspace = true, features = ["derive"] }
+futures = { workspace = true }
+indicatif = { workspace = true }
+parquet = { workspace = true }
+regex = { workspace = true }
+tabled = { workspace = true, features = ["std"] }
+tokio = { workspace = true, features = ["full"] }
+tracing = { workspace = true }
+vortex = { workspace = true }
+vortex-arrow = { workspace = true }
+vortex-bench = { workspace = true }
+vortex-btrblocks = { workspace = true }
+vortex-fsst = { workspace = true }
+vortex-onpair = { workspace = true }
+
+[features]
+unstable_encodings = [
+    "vortex/unstable_encodings",
+    "vortex-btrblocks/unstable_encodings",
+]
+
+[[bin]]
+name = "string-bench"
+required-features = ["unstable_encodings"]
+
+[lints]
+workspace = true

benchmarks/string-bench/README.md

@@ -0,0 +1,112 @@
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+<!-- SPDX-FileCopyrightText: Copyright the Vortex contributors -->
+
+# string-bench
+
+Compares Vortex string encoders on configured real-world Utf8 columns.
+
+## What it measures
+
+Three metrics per (column, encoder), from the default `vortex` suite:
+
+| Metric | Meaning | Table unit | `gh-json` unit |
+| --- | --- | --- | --- |
+| `size` | Serialized file bytes over canonical uncompressed bytes | % | % |
+| `write` | Repartition + zone stats + compress (string scheme and children) + layout + serialize | MB/s | ms |
+| `read` | Open + scan, decoding each row split to canonical form | MB/s | ms |
+
+Values are the median of `--iterations` runs. The `gh-json` output reports
+durations in milliseconds so that lower is better for every emitted metric; the
+human table reports MB/s over canonical uncompressed bytes.
+
+### The canonical baseline
+
+Every metric normalizes against canonical uncompressed bytes: one 16-byte view
+per row plus the bytes of the strings too long to inline, so a string of 12
+bytes or fewer costs only its view. The codec suite uses the same baseline.
+
+### How faithful the timings are
+
+Write and read run the real Vortex writer and scan on a current-thread runtime
+with no worker pool, so both timings are single-threaded CPU costs.
+
+`read` fuses the canonical decode into each row split's scan task — the shape
+production uses, where `into_record_batch_stream` fuses the Arrow conversion —
+and drops each decoded chunk before the next split runs, so steady-state memory
+is one chunk, not the whole column. Splits are awaited one at a time rather than
+through the scan's own stream, which spawns `concurrency *
+available_parallelism()` of them at once; on one thread that read-ahead buys no
+parallelism, it only holds more chunks in memory and ties the result to the
+host's core count.
+
+Excluded by design:
+
+- **Physical I/O** — the file lives in a `Vec<u8>` and `open_buffer` slices it,
+  so there is no read-driver request coalescing, no segment cache, and none of
+  the copying a real file or object-store read does.
+- **Parallel scan throughput** — production spreads splits across workers, so
+  these per-thread costs do not translate directly into query time.
+- **Filters and projections** — the whole column is read with no predicate, so
+  `read` is the full-decode cost, not decode-with-mask or compute pushdown.
+
+`size` covers encoded children, metadata, padding, and file markers.
+
+### Codec diagnostic
+
+`--suite codec` runs a separate microbenchmark: one whole-column encoder call,
+with no Vortex layout, child compression, serialization, or I/O.
+
+It is not part of the tracked set. Its size metric is also not interchangeable
+with the file suite's: it trains one dictionary or symbol table for the entire
+column and leaves the encoded array's children uncompressed, whereas Vortex
+writes in chunks and compresses those children.
+
+## Inputs and local data
+
+The current input catalog is:
+
+| Output name | Source |
+| --- | --- |
+| `clickbench/URL/shard-N` | ClickBench `hits_N.parquet`, `URL` column |
+| `tpch/l_comment` | TPC-H SF1 `lineitem`, `l_comment` column |
+
+On first use, the selected ClickBench shard is downloaded through
+`vortex-bench`'s shared idempotent downloader. TPC-H SF1 is generated
+deterministically through the shared TPC-H dataset helper. Both are stored
+under `vortex-bench/data/`, which is gitignored, and reused on later runs.
+Input preparation is outside benchmark timing.
+
+## Running locally
+
+```bash
+# Tracked metrics: size, write, read for every configured column and encoder.
+cargo run -p string-bench --profile release_debug --features unstable_encodings
+
+# Focus on selected columns or encoders.
+cargo run -p string-bench --profile release_debug --features unstable_encodings -- \
+  --columns URL --encoders onpair
+
+# Add the direct codec microbenchmark.
+cargo run -p string-bench --profile release_debug --features unstable_encodings -- \
+  --suite both
+
+# Emit benchmark-comparator JSONL.
+cargo run -p string-bench --profile release_debug --features unstable_encodings -- \
+  --display-format gh-json --output-path results.json
+```
+
+Run `cargo run -p string-bench --features unstable_encodings -- --help` for all
+filters and tuning options.
+
+Before timing, the benchmark checks that each requested encoding was produced
+and, unless `--no-verify` is set, compares decoded output with the input.
+
+## CI
+
+The develop benchmark workflow runs the default suite after each merge to
+`develop` and publishes the results to the shared benchmark history.
+
+Metric names are `<metric>/<input>/<encoder>`, for example
+`read/clickbench/URL/shard-0/onpair-12` and `size/tpch/l_comment/fsst`. The unit
+is reported separately in the JSON output and the CI table, which groups rows by
+unit — so metric-first names keep the rows you compare adjacent.