Firebolt: keep the three entries in the no-cold bucket

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 Changes in the benchmark methodology or presentation, as well as major news.
 
+### 2026-09-01
+The three Firebolt entries (`firebolt`, `firebolt-parquet`, `firebolt-parquet-partitioned`) are back to `BENCH_RESTARTABLE=no` and the `no-cold` tag: as of Sept 2026 Firebolt Core does not handle the start / stop cycle that [#1666](https://github.com/ClickHouse/ClickBench/pull/1666) introduced gracefully, so those runs did not lead to meaningful measurements. Caches are still flushed before the first run of each query, as for every other `BENCH_RESTARTABLE=no` system. The result files #1666 produced (`results/20260829`) keep no tag.
+
 ### 2026-08-29
 Firebolt now does a true cold run. The three self-hosted firebolt-core entries (`firebolt`, `firebolt-parquet`, `firebolt-parquet-partitioned`) set `BENCH_RESTARTABLE=no` and therefore only got their page cache flushed before each first run; they now go through the common runner's full `./stop` → `drop_caches` → `./start` cycle like every other daemon, and lost the `no-cold` tag. The engine handles `SIGTERM` and shuts down cleanly, and the database lives on a bind-mounted volume, so it is still there when the container is started back up. The result files produced before this change keep the tag. The managed-cloud Firebolt results from 2025-06-07 also keep it: those ran against the hosted service, which cannot be restarted.
 

firebolt-parquet-partitioned/benchmark.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
+# Firebolt Core does not handle start/stop cycles gracefully as of Sept 2026, so we stay no-cold.
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh

firebolt-parquet-partitioned/start

@@ -15,6 +15,9 @@ mkdir -p data fb-volume
 # ownership accordingly so the bind-mounted dir is usable.
 sudo chown 1111:1111 fb-volume
 
+# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026
+# Firebolt Core does not handle it gracefully. Caches are still
+# flushed as usual.
 # If the container exists (stopped by the cold cycle before this
 # query, or by a prior agent pre-snapshot cycle), just start it back —
 # the data lives on the bind-mounted fb-volume below, so the

firebolt-parquet-partitioned/stop

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -e
 
+# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026
+# Firebolt Core does not handle it gracefully. Caches are still
+# flushed as usual.
 # Full engine restart is what the cold cycle needs, so stop the process
 # but leave the container in place: the database lives on the
 # bind-mounted fb-volume and has to still be there when ./start brings

firebolt-parquet-partitioned/template.json

@@ -8,6 +8,7 @@
     "column-oriented",
     "PostgreSQL compatible",
     "ClickHouse derivative",
-    "stateless"
+    "stateless",
+    "no-cold"
   ]
 }

firebolt-parquet/benchmark.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
+# Firebolt Core does not handle start/stop cycles gracefully as of Sept 2026, so we stay no-cold.
+export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh

firebolt-parquet/start

@@ -15,6 +15,9 @@ mkdir -p data fb-volume
 # ownership accordingly so the bind-mounted dir is usable.
 sudo chown 1111:1111 fb-volume
 
+# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026
+# Firebolt Core does not handle it gracefully. Caches are still
+# flushed as usual.
 # If the container exists (stopped by the cold cycle before this
 # query, or by a prior agent pre-snapshot cycle), just start it back —
 # the data lives on the bind-mounted fb-volume below, so the

firebolt-parquet/stop

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -e
 
+# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026
+# Firebolt Core does not handle it gracefully. Caches are still
+# flushed as usual.
 # Full engine restart is what the cold cycle needs, so stop the process
 # but leave the container in place: the database lives on the
 # bind-mounted fb-volume and has to still be there when ./start brings

firebolt-parquet/template.json

@@ -8,6 +8,7 @@
     "column-oriented",
     "PostgreSQL compatible",
     "ClickHouse derivative",
-    "stateless"
+    "stateless",
+    "no-cold"
   ]
 }

firebolt/benchmark.sh

@@ -2,5 +2,7 @@
 # Thin shim — actual flow is in lib/benchmark-common.sh.
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
+# Firebolt Core does not handle start/stop cycles gracefully as of Sept 2026, so we stay no-cold.
+export BENCH_RESTARTABLE=no
 ./cleanup  # a previous run may have left its container and data behind
 exec ../lib/benchmark-common.sh

firebolt/install

@@ -6,4 +6,6 @@ if ! command -v docker >/dev/null || ! command -v jq >/dev/null; then
     sudo apt-get update -y
     sudo apt-get install -y docker.io jq
 fi
-sudo docker pull ghcr.io/firebolt-db/engine:dev
+# dev_mv1 rather than dev: a build we verify and re-tag by hand, so that mainline
+# regressions cannot walk into the benchmark unnoticed.
+sudo docker pull ghcr.io/firebolt-db/engine:dev_mv1

firebolt/results/20260901/c8g.metal-48xl.json

@@ -0,0 +1,59 @@
+{
+    "system": "Firebolt",
+    "date": "2026-09-01",
+    "machine": "c8g.metal-48xl",
+    "cluster_size": 1,
+    "proprietary": "yes",
+    "hardware": "cpu",
+    "tuned": "no",
+    "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative","no-cold"],
+    "load_time": 90,
+    "data_size": 15830695398,
+    "concurrent_qps": 42.43,
+    "concurrent_error_ratio": 0,
+    "result": [
+        [0.003, 0.046, 0.002],
+        [0.575, 0.318, 0.184],
+        [0.18, 0.036, 0.12],
+        [0.152, 0.033, 0.029],
+        [0.079, 0.061, 0.075],
+        [1.412, 0.186, 0.361],
+        [0.165, 0.027, 0.026],
+        [0.12, 0.046, 0.114],
+        [0.63, 0.111, 0.115],
+        [0.173, 0.133, 0.242],
+        [0.098, 0.101, 0.045],
+        [0.239, 0.123, 0.045],
+        [0.084, 0.058, 0.06],
+        [0.099, 0.103, 0.093],
+        [0.088, 0.061, 0.068],
+        [0.06, 0.059, 0.06],
+        [0.112, 0.098, 0.104],
+        [0.101, 0.097, 0.093],
+        [4.921, 0.172, 0.158],
+        [0.029, 0.008, 0.008],
+        [11.857, 0.045, 0.03],
+        [0.03, 0.012, 0.015],
+        [10.091, 0.062, 0.041],
+        [0.271, 0.18, 0.178],
+        [0.03, 0.029, 0.028],
+        [0.027, 0.027, 0.026],
+        [0.029, 0.028, 0.028],
+        [0.099, 0.074, 0.082],
+        [9.559, 0.202, 0.185],
+        [0.044, 0.042, 0.042],
+        [0.682, 0.085, 0.051],
+        [4.947, 0.121, 0.061],
+        [0.204, 0.197, 0.19],
+        [0.196, 0.196, 0.229],
+        [0.225, 0.2, 0.199],
+        [0.068, 0.065, 0.064],
+        [0.049, 0.028, 0.025],
+        [0.034, 0.021, 0.018],
+        [0.049, 0.015, 0.014],
+        [0.131, 0.052, 0.049],
+        [0.055, 0.014, 0.016],
+        [0.057, 0.018, 0.013],
+        [0.035, 0.018, 0.015]
+]
+}

firebolt/start

@@ -13,6 +13,9 @@ mkdir -p data fb-volume
 # what the official installer at get.firebolt.io does.
 sudo chmod 0777 fb-volume
 
+# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026
+# Firebolt Core does not handle it gracefully. Caches are still
+# flushed as usual.
 # If the container exists (stopped by the cold cycle before this
 # query, or by a prior agent pre-snapshot cycle), just start it back —
 # the data lives on the bind-mounted fb-volume below, so the
@@ -35,7 +38,7 @@ else
         -p 127.0.0.1:3473:3473 \
         -v "$(pwd)/fb-volume:/var/lib/firebolt" \
         -v "$(pwd)/data:/firebolt-core/clickbench" \
-        "${FB_IMAGE:-ghcr.io/firebolt-db/engine:dev}" >/dev/null
+        "${FB_IMAGE:-ghcr.io/firebolt-db/engine:dev_mv1}" >/dev/null
 fi
 
 # Wait for the cluster to be "actually" ready. firebolt-core's HTTP

firebolt/stop

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -e
 
+# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026
+# Firebolt Core does not handle it gracefully. Caches are still
+# flushed as usual.
 # Full engine restart is what the cold cycle needs, so stop the process
 # but leave the container in place: the database lives on the
 # bind-mounted fb-volume and has to still be there when ./start brings

firebolt/template.json

@@ -7,6 +7,7 @@
     "C++",
     "column-oriented",
     "PostgreSQL compatible",
-    "ClickHouse derivative"
+    "ClickHouse derivative",
+    "no-cold"
   ]
 }