ci: handle missing benchmark baselines

Signed-off-by: Francesco Gargiulo <gargiulo.fr@gmail.com>

Author: gargiulofrancesco

.github/workflows/bench-pr.yml

@@ -33,6 +33,9 @@ jobs:
             build_args: "--features lance"
           - id: compress-bench
             name: Compression
+          - id: string-bench
+            name: String Encoding
+            run_args: "--suite both"
     steps:
       - uses: runs-on/action@v2
         if: github.event.pull_request.head.repo.fork == false
@@ -104,7 +107,8 @@ jobs:
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json
+          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} \
+            ${{ matrix.benchmark.run_args }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         if: github.event.pull_request.head.repo.fork == false

scripts/compare-benchmark-jsons.py

@@ -184,7 +184,12 @@ def read_jsonl_rows_for_commit(path: str, commit_id: str) -> pd.DataFrame:
 
 
 def read_latest_baseline_rows(path: str, pr: pd.DataFrame) -> pd.DataFrame:
-    """Read rows from the latest history commit matching the PR benchmark."""
+    """Read rows from the latest history commit matching the PR benchmark.
+
+    A benchmark can be new to the PR workflow and therefore have no baseline
+    yet. Return an empty frame with the PR schema in that case so the report
+    can show the measurements without comparison.
+    """
 
     pr_identities = set(benchmark_identity_rows(pr)["benchmark_identity"])
     if not pr_identities:
@@ -202,7 +207,7 @@ def read_latest_baseline_rows(path: str, pr: pd.DataFrame) -> pd.DataFrame:
                     baseline_commit_id = commit_id
 
     if baseline_commit_id is None:
-        raise ValueError("No baseline rows found for the benchmark under test")
+        return pr.iloc[0:0].copy()
 
     return read_jsonl_rows_for_commit(path, baseline_commit_id)
 
@@ -230,7 +235,7 @@ def select_latest_baseline_rows(base: pd.DataFrame, pr: pd.DataFrame) -> pd.Data
     matches = base_identities[base_identities["benchmark_identity"].isin(pr_identities)]
     matches = matches[matches["commit_id"].notna()]
     if matches.empty:
-        raise ValueError("No baseline rows found for the benchmark under test")
+        return base.iloc[0:0].copy()
 
     baseline_commit_id = matches["commit_id"].iloc[-1]
     return base[base["commit_id"] == baseline_commit_id].copy()
@@ -256,6 +261,7 @@ def normalize_measurement_rows(df: pd.DataFrame) -> pd.DataFrame:
         columns=["engine", "file_format", "query"],
         index=df.index,
     )
+    df["query"] = pd.array(df["query"], dtype="Int64")
     return df
 
 
@@ -914,11 +920,11 @@ def main() -> None:
     title = format_title(benchmark_name, pr)
     base = read_latest_baseline_rows(sys.argv[1], pr)
 
-    base_commit_id = set(base["commit_id"].unique())
+    base_commit_ids = set(base["commit_id"].unique())
     pr_commit_id = set(pr["commit_id"].unique())
-    assert len(base_commit_id) == 1, base_commit_id
+    assert len(base_commit_ids) <= 1, base_commit_ids
     assert len(pr_commit_id) == 1, pr_commit_id
-    base_commit_id = next(iter(base_commit_id))
+    base_commit_id = next(iter(base_commit_ids), None)
     pr_commit_id = next(iter(pr_commit_id))
 
     base_file_sizes, base = split_file_size_rows(base)
@@ -990,13 +996,17 @@ def main() -> None:
     if summary_fields:
         print("<br>".join(summary_fields))
         print("")
+    if base_commit_id is None:
+        print("_No baseline is available for this benchmark yet; PR measurements are shown without comparison._")
+        print("")
     if verdict is not None or engine_summary is not None:
         print(format_report_help())
         print("")
     print("---")
     print("")
 
     grouped_tables = df3.groupby(["engine", "file_format", "unit"], dropna=False, sort=False)
+    base_label = str(base_commit_id)[:8] if base_commit_id is not None else "none"
     for engine, file_format, unit in sorted(grouped_tables.groups.keys(), key=group_sort_key):
         group_df = grouped_tables.get_group((engine, file_format, unit)).sort_values("name")
         group_performance = format_performance(
@@ -1014,7 +1024,7 @@ def main() -> None:
                     for name, ratio in zip(group_df["name"], group_df["ratio"])
                 ],
                 f"PR {pr_commit_id[:8]} ({unit})": group_df["value_pr"].map(format_measurement_value),
-                f"base {base_commit_id[:8]} ({unit})": group_df["value_base"].map(format_measurement_value),
+                f"base {base_label} ({unit})": group_df["value_base"].map(format_measurement_value),
                 "ratio (PR/base)": group_df["ratio"].map(format_comparison_ratio),
             }
         )

scripts/tests/test_benchmark_reporting.py

@@ -254,6 +254,39 @@ def test_comparison_report_groups_by_target_and_unit(tmp_path: Path) -> None:
     ]
 
 
+def test_comparison_report_handles_missing_benchmark_baseline(tmp_path: Path) -> None:
+    base_rows = [stored_custom_row("base-sha", "other timing/fixture", "ms", 12.5)]
+    pr_rows = [stored_custom_row("pr-sha", "new timing/fixture", "ms", 3.125)]
+
+    report = render_report(tmp_path, base_rows, pr_rows, "New benchmark")
+
+    assert "No baseline is available for this benchmark yet" in report
+    assert "base none (ms)" in report
+    assert markdown_row(report, "new timing/fixture") == [
+        "new timing/fixture",
+        "3.125",
+        "—",
+        "no baseline",
+    ]
+
+
+def test_comparison_report_handles_mixed_query_types_in_baseline(tmp_path: Path) -> None:
+    base_rows = [
+        stored_custom_row("base-sha", "random-access/fixture", "ms", 12.5),
+        stored_timing_row("base-sha", "tpch_q01/datafusion:parquet", 100),
+    ]
+    pr_rows = [stored_custom_row("pr-sha", "random-access/fixture", "ms", 13.0)]
+
+    report = render_report(tmp_path, base_rows, pr_rows, "Random Access")
+
+    assert markdown_row(report, "random-access/fixture") == [
+        "random-access/fixture",
+        "13",
+        "12.5",
+        "1.04",
+    ]
+
+
 def test_comparison_report_retains_sql_analysis(tmp_path: Path) -> None:
     targets = [
         ("parquet", "parquet", 100, 105),