diff --git a/CHANGELOG.md b/CHANGELOG.md index 215110814a..2afb5ea2bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ Changes in the benchmark methodology or presentation, as well as major news. +### 2026-09-01 +The three Firebolt entries (`firebolt`, `firebolt-parquet`, `firebolt-parquet-partitioned`) are back to `BENCH_RESTARTABLE=no` and the `no-cold` tag: as of Sept 2026 Firebolt Core does not handle the start / stop cycle that [#1666](https://github.com/ClickHouse/ClickBench/pull/1666) introduced gracefully, so those runs did not lead to meaningful measurements. Caches are still flushed before the first run of each query, as for every other `BENCH_RESTARTABLE=no` system. The result files #1666 produced (`results/20260829`) keep no tag. + ### 2026-08-29 Firebolt now does a true cold run. The three self-hosted firebolt-core entries (`firebolt`, `firebolt-parquet`, `firebolt-parquet-partitioned`) set `BENCH_RESTARTABLE=no` and therefore only got their page cache flushed before each first run; they now go through the common runner's full `./stop` → `drop_caches` → `./start` cycle like every other daemon, and lost the `no-cold` tag. The engine handles `SIGTERM` and shuts down cleanly, and the database lives on a bind-mounted volume, so it is still there when the container is started back up. The result files produced before this change keep the tag. The managed-cloud Firebolt results from 2025-06-07 also keep it: those ran against the hosted service, which cannot be restarted. diff --git a/firebolt-parquet-partitioned/benchmark.sh b/firebolt-parquet-partitioned/benchmark.sh index 12a00b626c..fa258673ee 100755 --- a/firebolt-parquet-partitioned/benchmark.sh +++ b/firebolt-parquet-partitioned/benchmark.sh @@ -1,4 +1,6 @@ #!/bin/bash # Thin shim — actual flow is in lib/benchmark-common.sh. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" +# Firebolt Core does not handle start/stop cycles gracefully as of Sept 2026, so we stay no-cold. +export BENCH_RESTARTABLE=no exec ../lib/benchmark-common.sh diff --git a/firebolt-parquet-partitioned/start b/firebolt-parquet-partitioned/start index fa5390cdd7..a337ae0bb2 100755 --- a/firebolt-parquet-partitioned/start +++ b/firebolt-parquet-partitioned/start @@ -15,6 +15,9 @@ mkdir -p data fb-volume # ownership accordingly so the bind-mounted dir is usable. sudo chown 1111:1111 fb-volume +# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026 +# Firebolt Core does not handle it gracefully. Caches are still +# flushed as usual. # If the container exists (stopped by the cold cycle before this # query, or by a prior agent pre-snapshot cycle), just start it back — # the data lives on the bind-mounted fb-volume below, so the diff --git a/firebolt-parquet-partitioned/stop b/firebolt-parquet-partitioned/stop index 1a97dc1ef7..8fc5ca6a29 100755 --- a/firebolt-parquet-partitioned/stop +++ b/firebolt-parquet-partitioned/stop @@ -1,6 +1,9 @@ #!/bin/bash set -e +# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026 +# Firebolt Core does not handle it gracefully. Caches are still +# flushed as usual. # Full engine restart is what the cold cycle needs, so stop the process # but leave the container in place: the database lives on the # bind-mounted fb-volume and has to still be there when ./start brings diff --git a/firebolt-parquet-partitioned/template.json b/firebolt-parquet-partitioned/template.json index beefe5faf2..139eeb30f4 100644 --- a/firebolt-parquet-partitioned/template.json +++ b/firebolt-parquet-partitioned/template.json @@ -8,6 +8,7 @@ "column-oriented", "PostgreSQL compatible", "ClickHouse derivative", - "stateless" + "stateless", + "no-cold" ] } diff --git a/firebolt-parquet/benchmark.sh b/firebolt-parquet/benchmark.sh index ddd7f28318..54fb04b57d 100755 --- a/firebolt-parquet/benchmark.sh +++ b/firebolt-parquet/benchmark.sh @@ -1,4 +1,6 @@ #!/bin/bash # Thin shim — actual flow is in lib/benchmark-common.sh. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +# Firebolt Core does not handle start/stop cycles gracefully as of Sept 2026, so we stay no-cold. +export BENCH_RESTARTABLE=no exec ../lib/benchmark-common.sh diff --git a/firebolt-parquet/start b/firebolt-parquet/start index 8c52455181..46f880ff0a 100755 --- a/firebolt-parquet/start +++ b/firebolt-parquet/start @@ -15,6 +15,9 @@ mkdir -p data fb-volume # ownership accordingly so the bind-mounted dir is usable. sudo chown 1111:1111 fb-volume +# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026 +# Firebolt Core does not handle it gracefully. Caches are still +# flushed as usual. # If the container exists (stopped by the cold cycle before this # query, or by a prior agent pre-snapshot cycle), just start it back — # the data lives on the bind-mounted fb-volume below, so the diff --git a/firebolt-parquet/stop b/firebolt-parquet/stop index 1a97dc1ef7..8fc5ca6a29 100755 --- a/firebolt-parquet/stop +++ b/firebolt-parquet/stop @@ -1,6 +1,9 @@ #!/bin/bash set -e +# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026 +# Firebolt Core does not handle it gracefully. Caches are still +# flushed as usual. # Full engine restart is what the cold cycle needs, so stop the process # but leave the container in place: the database lives on the # bind-mounted fb-volume and has to still be there when ./start brings diff --git a/firebolt-parquet/template.json b/firebolt-parquet/template.json index 6ba2563637..a1b48e08a2 100644 --- a/firebolt-parquet/template.json +++ b/firebolt-parquet/template.json @@ -8,6 +8,7 @@ "column-oriented", "PostgreSQL compatible", "ClickHouse derivative", - "stateless" + "stateless", + "no-cold" ] } diff --git a/firebolt/benchmark.sh b/firebolt/benchmark.sh index 4d44261401..0ca9d14a5f 100755 --- a/firebolt/benchmark.sh +++ b/firebolt/benchmark.sh @@ -2,5 +2,7 @@ # Thin shim — actual flow is in lib/benchmark-common.sh. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes +# Firebolt Core does not handle start/stop cycles gracefully as of Sept 2026, so we stay no-cold. +export BENCH_RESTARTABLE=no ./cleanup # a previous run may have left its container and data behind exec ../lib/benchmark-common.sh diff --git a/firebolt/install b/firebolt/install index 9f61da7b5c..45ebc9f586 100755 --- a/firebolt/install +++ b/firebolt/install @@ -6,4 +6,6 @@ if ! command -v docker >/dev/null || ! command -v jq >/dev/null; then sudo apt-get update -y sudo apt-get install -y docker.io jq fi -sudo docker pull ghcr.io/firebolt-db/engine:dev +# dev_mv1 rather than dev: a build we verify and re-tag by hand, so that mainline +# regressions cannot walk into the benchmark unnoticed. +sudo docker pull ghcr.io/firebolt-db/engine:dev_mv1 diff --git a/firebolt/results/20260901/c8g.metal-48xl.json b/firebolt/results/20260901/c8g.metal-48xl.json new file mode 100644 index 0000000000..965017930c --- /dev/null +++ b/firebolt/results/20260901/c8g.metal-48xl.json @@ -0,0 +1,59 @@ +{ + "system": "Firebolt", + "date": "2026-09-01", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative","no-cold"], + "load_time": 90, + "data_size": 15830695398, + "concurrent_qps": 42.43, + "concurrent_error_ratio": 0, + "result": [ + [0.003, 0.046, 0.002], + [0.575, 0.318, 0.184], + [0.18, 0.036, 0.12], + [0.152, 0.033, 0.029], + [0.079, 0.061, 0.075], + [1.412, 0.186, 0.361], + [0.165, 0.027, 0.026], + [0.12, 0.046, 0.114], + [0.63, 0.111, 0.115], + [0.173, 0.133, 0.242], + [0.098, 0.101, 0.045], + [0.239, 0.123, 0.045], + [0.084, 0.058, 0.06], + [0.099, 0.103, 0.093], + [0.088, 0.061, 0.068], + [0.06, 0.059, 0.06], + [0.112, 0.098, 0.104], + [0.101, 0.097, 0.093], + [4.921, 0.172, 0.158], + [0.029, 0.008, 0.008], + [11.857, 0.045, 0.03], + [0.03, 0.012, 0.015], + [10.091, 0.062, 0.041], + [0.271, 0.18, 0.178], + [0.03, 0.029, 0.028], + [0.027, 0.027, 0.026], + [0.029, 0.028, 0.028], + [0.099, 0.074, 0.082], + [9.559, 0.202, 0.185], + [0.044, 0.042, 0.042], + [0.682, 0.085, 0.051], + [4.947, 0.121, 0.061], + [0.204, 0.197, 0.19], + [0.196, 0.196, 0.229], + [0.225, 0.2, 0.199], + [0.068, 0.065, 0.064], + [0.049, 0.028, 0.025], + [0.034, 0.021, 0.018], + [0.049, 0.015, 0.014], + [0.131, 0.052, 0.049], + [0.055, 0.014, 0.016], + [0.057, 0.018, 0.013], + [0.035, 0.018, 0.015] +] +} diff --git a/firebolt/start b/firebolt/start index 90fe378381..2f6dea704a 100755 --- a/firebolt/start +++ b/firebolt/start @@ -13,6 +13,9 @@ mkdir -p data fb-volume # what the official installer at get.firebolt.io does. sudo chmod 0777 fb-volume +# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026 +# Firebolt Core does not handle it gracefully. Caches are still +# flushed as usual. # If the container exists (stopped by the cold cycle before this # query, or by a prior agent pre-snapshot cycle), just start it back — # the data lives on the bind-mounted fb-volume below, so the @@ -35,7 +38,7 @@ else -p 127.0.0.1:3473:3473 \ -v "$(pwd)/fb-volume:/var/lib/firebolt" \ -v "$(pwd)/data:/firebolt-core/clickbench" \ - "${FB_IMAGE:-ghcr.io/firebolt-db/engine:dev}" >/dev/null + "${FB_IMAGE:-ghcr.io/firebolt-db/engine:dev_mv1}" >/dev/null fi # Wait for the cluster to be "actually" ready. firebolt-core's HTTP diff --git a/firebolt/stop b/firebolt/stop index 505bb4a3f6..dedf7e4e09 100755 --- a/firebolt/stop +++ b/firebolt/stop @@ -1,6 +1,9 @@ #!/bin/bash set -e +# The start/stop cycle is off (BENCH_RESTARTABLE=no): as of Sept 2026 +# Firebolt Core does not handle it gracefully. Caches are still +# flushed as usual. # Full engine restart is what the cold cycle needs, so stop the process # but leave the container in place: the database lives on the # bind-mounted fb-volume and has to still be there when ./start brings diff --git a/firebolt/template.json b/firebolt/template.json index b48735638d..584ebd9ec3 100644 --- a/firebolt/template.json +++ b/firebolt/template.json @@ -7,6 +7,7 @@ "C++", "column-oriented", "PostgreSQL compatible", - "ClickHouse derivative" + "ClickHouse derivative", + "no-cold" ] }