Merge pull request #765 from ClickHouse/gizmosql-debugging

Gizmosql benchmark fixed to correctly clear cache in between queries and updated with results from a true benchmark run.

Author: george-larionov

gizmosql/benchmark.sh

@@ -1,8 +1,11 @@
 #!/bin/bash
 
+# needed by DuckDB
+export HOME=/home/ubuntu
+
 # Install requirements
-sudo apt-get update -y
-sudo apt install openjdk-17-jre-headless unzip netcat-openbsd -y
+apt-get update -y
+apt install openjdk-17-jre-headless unzip netcat-openbsd -y
 
 # Detect architecture (maps x86_64->amd64, aarch64->arm64)
 ARCH=$(uname -m)
@@ -15,13 +18,13 @@ fi
 # Server setup Install
 curl -L -o gizmosql.zip "https://github.com/gizmodata/gizmosql/releases/latest/download/gizmosql_cli_linux_${ARCH}.zip"
 unzip gizmosql.zip
-sudo mv gizmosql_server gizmosql_client /usr/local/bin/
+mv gizmosql_server gizmosql_client /usr/local/bin/
 
 # Install Java and the GizmoSQLLine CLI client
 pushd /tmp
 curl -L -o gizmosqlline https://github.com/gizmodata/gizmosqlline/releases/latest/download/gizmosqlline
 chmod +x gizmosqlline
-sudo mv gizmosqlline /usr/local/bin/
+mv gizmosqlline /usr/local/bin/
 popd
 
 # Source our env vars and utility functions for starting/stopping gizmosql server
@@ -50,7 +53,7 @@ time gizmosqlline \
 stop_gizmosql
 
 # Run the queries
-script --quiet --command="./run.sh" log.txt
+./run.sh 2>&1 | tee log.txt
 
 # Remove carriage returns from the log
 sed -i 's/\r$//' log.txt

gizmosql/results/c6a.4xlarge.json

@@ -1,57 +1,57 @@
 {
     "system": "GizmoSQL",
-    "date": "2026-01-24",
+    "date": "2026-01-28",
     "machine": "c6a.4xlarge",
     "cluster_size": 1,
     "proprietary": "no",
     "hardware": "cpu",
     "tuned": "no",
     "tags": ["C++","column-oriented"],
-    "load_time": 131.966,
-    "data_size": 26924298240,
+    "load_time": 142.531,
+    "data_size": 26960736256,
     "result": [
-        [0.053, 0.008, 0.007],
-        [0.129, 0.012, 0.01],
-        [0.175, 0.029, 0.027],
-        [0.321, 0.039, 0.039],
-        [0.363, 0.261, 0.26],
-        [0.838, 0.423, 0.416],
-        [0.096, 0.015, 0.014],
-        [0.091, 0.016, 0.015],
-        [0.529, 0.358, 0.361],
-        [0.679, 0.521, 0.524],
-        [0.299, 0.14, 0.137],
-        [0.295, 0.147, 0.144],
-        [0.554, 0.404, 0.403],
-        [0.915, 0.696, 0.702],
-        [0.62, 0.43, 0.426],
-        [0.451, 0.324, 0.33],
-        [1.023, 0.832, 0.84],
-        [0.783, 0.562, 0.576],
-        [1.993, 1.517, 1.539],
-        [0.074, 0.014, 0.012],
-        [17.562, 0.483, 0.479],
-        [0.953, 0.493, 0.482],
-        [11.484, 0.531, 0.541],
-        [0.54, 0.125, 0.117],
-        [0.108, 0.04, 0.041],
-        [0.227, 0.149, 0.149],
-        [0.112, 0.037, 0.036],
-        [0.862, 0.374, 0.375],
-        [12.601, 7.034, 7.025],
-        [0.14, 0.053, 0.046],
-        [0.629, 0.345, 0.346],
-        [2.031, 0.424, 0.43],
-        [2.052, 1.713, 1.772],
-        [2.309, 1.704, 1.716],
-        [2.438, 1.841, 1.816],
-        [0.526, 0.416, 0.426],
-        [0.097, 0.047, 0.044],
-        [0.079, 0.021, 0.018],
-        [0.08, 0.023, 0.021],
-        [0.152, 0.074, 0.066],
-        [0.079, 0.02, 0.015],
-        [0.077, 0.02, 0.016],
-        [0.08, 0.022, 0.021]
+        [0.055, 0.008, 0.007],
+        [0.13, 0.012, 0.01],
+        [0.691, 0.029, 0.028],
+        [0.436, 0.043, 0.042],
+        [0.499, 0.258, 0.257],
+        [1.237, 0.431, 0.425],
+        [0.117, 0.014, 0.014],
+        [0.147, 0.017, 0.017],
+        [1.268, 0.356, 0.358],
+        [2.798, 0.503, 0.507],
+        [1.344, 0.138, 0.131],
+        [1.657, 0.146, 0.139],
+        [1.252, 0.394, 0.389],
+        [2.838, 0.694, 0.687],
+        [1.839, 0.424, 0.421],
+        [0.549, 0.32, 0.321],
+        [2.791, 0.852, 0.861],
+        [2.536, 0.57, 0.577],
+        [5.571, 1.525, 1.532],
+        [0.17, 0.013, 0.012],
+        [17.947, 0.474, 0.469],
+        [19.808, 0.488, 0.483],
+        [25.45, 1.293, 1.014],
+        [2.064, 0.127, 0.126],
+        [0.165, 0.038, 0.04],
+        [1.156, 0.145, 0.143],
+        [0.164, 0.036, 0.035],
+        [18.303, 0.36, 0.359],
+        [13.179, 7.927, 7.987],
+        [0.203, 0.051, 0.044],
+        [4.466, 0.338, 0.342],
+        [7.257, 0.417, 0.41],
+        [6.178, 1.684, 1.724],
+        [18.424, 1.679, 1.668],
+        [18.451, 1.782, 1.783],
+        [0.605, 0.417, 0.418],
+        [0.131, 0.04, 0.042],
+        [0.094, 0.02, 0.02],
+        [0.105, 0.022, 0.022],
+        [0.213, 0.068, 0.073],
+        [0.112, 0.019, 0.016],
+        [0.104, 0.018, 0.015],
+        [0.098, 0.023, 0.023]
     ]
 }

gizmosql/run.sh

@@ -9,10 +9,6 @@ TEMP_SQL_FILE="/tmp/benchmark_queries_$$.sql"
 # Ensure server is stopped on script exit
 trap stop_gizmosql EXIT
 
-echo "Clear Linux memory caches to ensure fair benchmark comparisons"
-sync
-echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null
-
 # Read queries from file
 mapfile -t queries < queries.sql
 
@@ -21,6 +17,10 @@ echo "Running benchmark with ${#queries[@]} queries, ${TRIES} tries each..."
 for query in "${queries[@]}"; do
     > "${TEMP_SQL_FILE}"
 
+    # Clear Linux memory caches to ensure fair benchmark comparisons
+    sync
+    echo 3 | tee /proc/sys/vm/drop_caches > /dev/null
+
     # Start the GizmoSQL server
     start_gizmosql