diff --git a/doris-parquet-single/README.md b/doris-parquet-single/README.md new file mode 100644 index 0000000000..7e65b4c0b8 --- /dev/null +++ b/doris-parquet-single/README.md @@ -0,0 +1,11 @@ +# Doris Parquet (single) + +Like [doris-parquet](../doris-parquet/), but over the single `hits.parquet` +file instead of the 100 partitioned files: a local Doris cluster with 1 FE and +1 BE queries the file in place through the `local()` table-valued function. + +## References + +- [doris-parquet](../doris-parquet/) +- [clickhouse-parquet](../clickhouse-parquet/) +- [duckdb-parquet](../duckdb-parquet/) diff --git a/doris-parquet-single/benchmark.sh b/doris-parquet-single/benchmark.sh new file mode 100755 index 0000000000..a33527ee30 --- /dev/null +++ b/doris-parquet-single/benchmark.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +exec ../lib/benchmark-common.sh diff --git a/doris-parquet-single/check b/doris-parquet-single/check new file mode 100755 index 0000000000..6248536714 --- /dev/null +++ b/doris-parquet-single/check @@ -0,0 +1,44 @@ +#!/bin/bash +set -e + +# Print state that is otherwise invisible in the benchmark log (the harness +# shows only the last failing ./check stderr): the FE's view of the BE, the +# daemon launchers' own output (captured by ./start), and the BE's logs. +diagnose() { + mysql -h127.0.0.1 -P9030 -uroot -e 'SHOW BACKENDS\G' 2>/dev/null \ + | grep -E 'Alive|ErrMsg|Version|LastHeartbeat|HeartbeatFailureCounter' >&2 || true + if pgrep -f doris_be >/dev/null 2>&1; then + echo "doris: BE process is running" >&2 + else + echo "doris: BE process is NOT running" >&2 + fi + echo "--- start_be.sh output:" >&2 + cat start_be.out >&2 2>/dev/null || true + local doris_home + doris_home=$(cat .doris_home 2>/dev/null) || return 0 + echo "--- be/log contents:" >&2 + ls -la "$doris_home"/be/log/ >&2 2>/dev/null || true + echo "--- be.out tail:" >&2 + tail -n 30 "$doris_home"/be/log/be.out >&2 2>/dev/null || true + echo "--- be.INFO tail:" >&2 + tail -n 15 "$doris_home"/be/log/be.INFO >&2 2>/dev/null || true + echo "--- dmesg:" >&2 + dmesg 2>/dev/null | grep -iE 'doris|segfault|oom|killed process' | tail -n 5 >&2 || true +} + +# On Doris 4.x even `SELECT 1` is dispatched to a backend, so this fails +# with "No backend available as scan node" until a BE is alive — which is +# exactly the readiness this benchmark needs. +if ! err=$(mysql -h127.0.0.1 -P9030 -uroot -e 'SELECT 1' 2>&1 >/dev/null); then + echo "doris: SELECT 1 failed: ${err}" >&2 + diagnose + exit 1 +fi + +if ! mysql -h127.0.0.1 -P9030 -uroot -B -e 'SHOW BACKENDS' 2>/dev/null \ + | awk -F'\t' 'NR==1 { for (i = 1; i <= NF; i++) if ($i == "Alive") c = i; next } + c && tolower($c) == "true" { found = 1 } END { exit !found }'; then + echo "doris: FE is up but no BE is alive" >&2 + diagnose + exit 1 +fi diff --git a/doris-parquet-single/create.sql b/doris-parquet-single/create.sql new file mode 100644 index 0000000000..227dc358ab --- /dev/null +++ b/doris-parquet-single/create.sql @@ -0,0 +1,8 @@ +CREATE DATABASE IF NOT EXISTS hits; +CREATE VIEW IF NOT EXISTS hits.hits +AS +SELECT * FROM local( + "file_path" = "hits.parquet", + "shared_storage" = "true", -- omit "backend_id" = "be_id", + "format" = "parquet" +); diff --git a/doris-parquet-single/data-size b/doris-parquet-single/data-size new file mode 100755 index 0000000000..0365e76bcc --- /dev/null +++ b/doris-parquet-single/data-size @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +DORIS_HOME=$(cat .doris_home) +find "$DORIS_HOME/be/" -name 'hits.parquet' -printf '%s\n' \ + | awk '{ s += $1 } END { print s+0 }' diff --git a/doris-parquet-single/get-result-json.sh b/doris-parquet-single/get-result-json.sh new file mode 100755 index 0000000000..c36c083a2a --- /dev/null +++ b/doris-parquet-single/get-result-json.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# set -x +DATE=$(date -u +%Y-%m-%d) +YYYYMMDD=${DATE//-/} +MACHINE=$(sudo dmidecode -s system-product-name) +mkdir -p "results/${YYYYMMDD}" + +echo -e "{ + \"system\": \"Doris (Parquet, single)\", + \"date\": \"${DATE}\", + \"machine\": \"${MACHINE}\", + \"cluster_size\": 1, + \"comment\": \"\", + \"tags\": [\"C++\", \"column-oriented\", \"MySQL compatible\", \"ClickHouse derivative\"], + \"load_time\": 0, + \"data_size\": 14779976446, + \"result\": [ +$( + r=$(sed -r -e 's/query[0-9]+,/[/; s/$/],/' result.csv) + echo "${r%?}" +) + ] +} +" | tee "results/${YYYYMMDD}/${MACHINE}.json" diff --git a/doris-parquet-single/install b/doris-parquet-single/install new file mode 100755 index 0000000000..dca153ff58 --- /dev/null +++ b/doris-parquet-single/install @@ -0,0 +1,62 @@ +#!/bin/bash +set -e + +# Apache Doris only ships x64 release tarballs (apache-doris-*-bin-x64). +# No arm64 build exists on the public mirror, so c8g.* runs would +# download the x64 tarball, the BE process would refuse to come up +# ("No backend available"), and the bench would fail late after the +# 14 GB download. Fail fast on arm64 instead. +if [ "$(uname -m)" != "x86_64" ] && [ "$(uname -m)" != "amd64" ]; then + echo "doris-parquet-single: Apache Doris has no $(uname -m) release tarball; skipping" >&2 + exit 1 +fi + +# This benchmark runs on Ubuntu 22.04+. Same 4.1.0-rc01 as the doris entry: +# the 3.0.5 BE that doris-parquet pins never becomes alive on the current +# Ubuntu 24.04 benchmark image. +ROOT=$(pwd) +URL='https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-4.1.0-rc01-bin-x64.tar.gz' + +file_name="$(basename "$URL")" +dir_name="${file_name/.tar.gz/}" +DORIS_HOME="$ROOT/$dir_name/$dir_name" + +# Idempotent: skip if already extracted. +if [ ! -d "$DORIS_HOME" ]; then + if [ ! -f "$file_name" ]; then + wget --continue --progress=dot:giga "$URL" + fi + mkdir -p "$dir_name" + tar zxf "$file_name" -C "$dir_name" + + # Disable internal caches so cold runs are actually cold. + printf "\ndisable_storage_page_cache = true\n" >> "$DORIS_HOME"/be/conf/be.conf + printf "\nsegment_cache_capacity = 0\n" >> "$DORIS_HOME"/be/conf/be.conf + + # Pin both daemons' self-detected address to loopback so the FE's saved + # BE address (ADD BACKEND '127.0.0.1:9050') matches what the BE reports + # in heartbeats. Without it the BE picks the instance's private IP and + # can stay Alive=false forever — same single-node pin the starrocks + # entry needs. + printf "\npriority_networks = 127.0.0.1/32\n" >> "$DORIS_HOME"/be/conf/be.conf + printf "\npriority_networks = 127.0.0.1/32\n" >> "$DORIS_HOME"/fe/conf/fe.conf +fi + +# Install dependencies (idempotent — apt-get is fine to re-run). +sudo apt-get update -y +sudo apt-get install -y openjdk-17-jdk mysql-client bc + +sudo systemctl disable unattended-upgrades 2>/dev/null || true +sudo systemctl stop unattended-upgrades 2>/dev/null || true + +sudo sysctl -w vm.max_map_count=2000000 + +# cloud-init adds a 16 GB swapfile to every benchmark instance (f236e7ba2, +# 2026-05-09) for systems that load the dataset into RAM. Doris's +# start_be.sh hard-refuses to launch while swap is active ("Disable swap +# memory before starting be") — this is why no doris entry has produced +# results since that date. Parquet-in-place needs no swap headroom, so +# turn it off (persists until reboot; instances never reboot mid-run). +sudo swapoff -a + +echo "$DORIS_HOME" > .doris_home diff --git a/doris-parquet-single/load b/doris-parquet-single/load new file mode 100755 index 0000000000..612f83f6a2 --- /dev/null +++ b/doris-parquet-single/load @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +ROOT=$(pwd) +DORIS_HOME=$(cat .doris_home) +export DORIS_HOME + +# The dataset must be visible to the BE process (the local() TVF reads files +# relative to the BE working dir). benchmark.sh already downloaded +# hits.parquet into this directory, so move it into the BE dir instead of +# downloading a second copy. +if [ -f "hits.parquet" ]; then + mv -f hits.parquet "$DORIS_HOME/be/" +elif [ ! -f "$DORIS_HOME/be/hits.parquet" ]; then + "$ROOT/../lib/download-hits-parquet-single" "$DORIS_HOME/be" +fi + +# Create the view that wraps a local() TVF over the parquet file. Idempotent +# (CREATE OR REPLACE / IF NOT EXISTS). +mysql -h127.0.0.1 -P9030 -uroot < "$ROOT/create.sql" + +# Pre-set parquet flags to match original benchmark. +mysql -h127.0.0.1 -P9030 -uroot \ + -e 'set global enable_parquet_filter_by_min_max=true; set global enable_parquet_lazy_materialization=true;' + +# Note: data files remain — for parquet-as-storage there's no separate copy, +# the TVF reads them directly. +sync diff --git a/doris-parquet-single/queries.sql b/doris-parquet-single/queries.sql new file mode 100644 index 0000000000..77ed25865d --- /dev/null +++ b/doris-parquet-single/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +SELECT AVG(UserID) FROM hits; +SELECT COUNT(DISTINCT UserID) FROM hits; +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_FORMAT(EventTime, '%Y-%m-%d %H:%i:00') AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_FORMAT(EventTime, '%Y-%m-%d %H:%i:00') ORDER BY DATE_FORMAT(EventTime, '%Y-%m-%d %H:%i:00') LIMIT 10 OFFSET 1000; diff --git a/doris-parquet-single/query b/doris-parquet-single/query new file mode 100755 index 0000000000..67aebec9ca --- /dev/null +++ b/doris-parquet-single/query @@ -0,0 +1,33 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it via mysql client against Doris's `hits` DB. +# Stdout: query result. +# Stderr: query runtime in fractional seconds on the last line. +# Exit non-zero on error. +set -e + +query=$(cat) + +# Clear the FE/BE caches before each query (parquet path). +curl -sS http://127.0.0.1:8040/api/clear_cache/all >/dev/null 2>&1 || true + +out=$(mysql -vvv -h127.0.0.1 -P9030 -uroot hits -e "$query" 2>&1) || status=$? +status=${status:-0} + +printf '%s\n' "$out" | grep -vP '^\([0-9.]+\s+sec\)$|rows? in set|Empty set' + +if [ "$status" -ne 0 ] || printf '%s\n' "$out" | grep -qE '^ERROR'; then + printf '%s\n' "$out" >&2 + exit 1 +fi + +secs=$(printf '%s\n' "$out" \ + | grep -oP '\((?:([0-9.]+)\s+min\s+)?([0-9.]+)\s+sec\)' \ + | tail -n1 \ + | sed -r 's/\((([0-9.]+) min )?([0-9.]+) sec\)/\2 \3/' \ + | awk '{ if ($2 != "") print $1*60 + $2; else print $1 }') + +if [ -z "$secs" ]; then + echo "no timing in mysql output" >&2 + exit 1 +fi +printf '%s\n' "$secs" >&2 diff --git a/doris-parquet-single/results/20260819/c6a.2xlarge.json b/doris-parquet-single/results/20260819/c6a.2xlarge.json new file mode 100644 index 0000000000..5fb340d4bc --- /dev/null +++ b/doris-parquet-single/results/20260819/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Doris (Parquet, single)", + "date": "2026-08-19", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative","stateless"], + "load_time": 4, + "data_size": 14779976446, + "concurrent_qps": 0.293, + "concurrent_error_ratio": 0.997, + "result": [ + [0.9, 0.67, 0.6], + [0.89, 0.59, 0.55], + [1, 0.67, 0.55], + [1.21, 0.65, 0.65], + [1.89, 1.24, 1.3], + [2.36, 1.92, 1.91], + [1.08, 0.59, 0.57], + [1.02, 0.58, 0.55], + [1.97, 1.45, 1.44], + [2.24, 1.75, 1.79], + [1.28, 0.85, 0.81], + [1.22, 0.88, 0.86], + [2.23, 1.65, 1.57], + [3.29, 2.4, 2.59], + [2.56, 1.92, 1.92], + [2.24, 1.98, 2], + [4.69, 4.31, 4.26], + [2.7, 1.11, 1.08], + [6.22, 5.16, 5.06], + [0.81, 0.32, 0.29], + [10.22, 2.04, 1.93], + [11.94, 2.24, 2.12], + [22.84, 3.85, 3.8], + [13.63, 2.59, 2.44], + [3.35, 1.35, 1.25], + [1.52, 1.24, 1.23], + [3.19, 1.37, 1.36], + [10.29, 2.66, 2.57], + [18.91, 18.38, 18.23], + [1.18, 0.67, 0.63], + [3.59, 1.65, 1.66], + [7.07, 2.15, 2.04], + [8.69, 7.93, 8.28], + [10.95, 7.76, 7.65], + [10.92, 7.98, 7.82], + [2.16, 1.59, 1.57], + [0.3, 0.09, 0.08], + [0.29, 0.09, 0.09], + [0.29, 0.08, 0.08], + [0.32, 0.07, 0.08], + [0.29, 0.07, 0.09], + [0.3, 0.1, 0.08], + [0.29, 0.09, 0.09] +] + } + \ No newline at end of file diff --git a/doris-parquet-single/results/20260819/c6a.4xlarge.json b/doris-parquet-single/results/20260819/c6a.4xlarge.json new file mode 100644 index 0000000000..600af5b1ec --- /dev/null +++ b/doris-parquet-single/results/20260819/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Doris (Parquet, single)", + "date": "2026-08-19", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative","stateless"], + "load_time": 10, + "data_size": 14779976446, + "concurrent_qps": 0.827, + "concurrent_error_ratio": 0.116, + "result": [ + [0.65, 0.54, 0.74], + [0.78, 0.64, 0.7], + [0.78, 0.53, 0.62], + [0.87, 0.88, 0.82], + [1.21, 0.85, 1.16], + [1.7, 1.62, 2.02], + [0.75, 0.63, 0.74], + [0.8, 0.51, 0.69], + [1.4, 1.1, 0.98], + [1.61, 1.16, 1.35], + [1.08, 0.88, 0.61], + [1.08, 0.82, 0.73], + [1.52, 1.1, 1.41], + [2.95, 1.6, 1.8], + [1.79, 1.39, 1.62], + [1.48, 1.24, 1.4], + [3.61, 3.16, 3.18], + [2.63, 0.8, 1.06], + [5.72, 3.59, 3.38], + [0.71, 0.26, 0.21], + [10.15, 1.51, 1.57], + [11.86, 1.99, 1.96], + [22.72, 3.66, 3.71], + [13.59, 2.14, 1.95], + [3.21, 1.04, 1.18], + [1.33, 0.78, 1.08], + [3.21, 0.94, 1.2], + [10.27, 2.41, 2.27], + [12.43, 9.58, 9.95], + [1.02, 0.68, 0.74], + [3.38, 1.03, 1.41], + [6.92, 1.77, 1.84], + [6.85, 5.34, 5.24], + [11.04, 6.27, 6.27], + [10.7, 6.33, 6.16], + [1.4, 1.11, 1.29], + [0.27, 0.09, 0.08], + [0.27, 0.09, 0.08], + [0.29, 0.08, 0.07], + [0.28, 0.09, 0.07], + [0.29, 0.1, 0.09], + [0.29, 0.09, 0.1], + [0.31, 0.1, 0.09] +] + } + \ No newline at end of file diff --git a/doris-parquet-single/results/20260819/c6a.metal.json b/doris-parquet-single/results/20260819/c6a.metal.json new file mode 100644 index 0000000000..86af6f73f5 --- /dev/null +++ b/doris-parquet-single/results/20260819/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "Doris (Parquet, single)", + "date": "2026-08-19", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative","stateless"], + "load_time": 4, + "data_size": 14779976446, + "concurrent_qps": 5.918, + "concurrent_error_ratio": 0, + "result": [ + [0.62, 0.3, 0.31], + [0.78, 0.31, 0.3], + [0.75, 0.3, 0.29], + [0.81, 0.3, 0.31], + [1, 0.36, 0.34], + [1.18, 0.37, 0.41], + [0.74, 0.3, 0.31], + [0.81, 0.32, 0.3], + [1.07, 0.51, 0.49], + [1.34, 0.47, 0.48], + [1.01, 0.37, 0.33], + [0.93, 0.35, 0.34], + [1.45, 0.41, 0.41], + [2.56, 0.5, 0.48], + [1.47, 0.42, 0.43], + [0.96, 0.44, 0.42], + [2.57, 0.71, 0.76], + [2.4, 0.34, 0.37], + [4.9, 0.81, 0.71], + [0.74, 0.16, 0.16], + [9.97, 0.41, 0.41], + [11.59, 0.47, 0.47], + [22.24, 0.71, 0.64], + [13.3, 0.74, 0.7], + [2.92, 0.36, 0.35], + [1.16, 0.3, 0.33], + [2.98, 0.33, 0.34], + [10.13, 0.61, 0.62], + [8.92, 1.83, 1.77], + [0.91, 0.35, 0.33], + [3.09, 0.37, 0.39], + [6.43, 0.46, 0.44], + [4.59, 1.11, 1.15], + [10.14, 1.37, 1.29], + [10.17, 1.44, 1.32], + [0.93, 0.39, 0.39], + [0.31, 0.09, 0.06], + [0.31, 0.08, 0.07], + [0.29, 0.07, 0.07], + [0.33, 0.08, 0.07], + [0.33, 0.07, 0.07], + [0.28, 0.08, 0.06], + [0.33, 0.08, 0.06] +] + } + \ No newline at end of file diff --git a/doris-parquet-single/results/20260819/c7a.metal-48xl.json b/doris-parquet-single/results/20260819/c7a.metal-48xl.json new file mode 100644 index 0000000000..db1308dd2e --- /dev/null +++ b/doris-parquet-single/results/20260819/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Doris (Parquet, single)", + "date": "2026-08-19", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative","stateless"], + "load_time": 8, + "data_size": 14779976446, + "concurrent_qps": 10.617, + "concurrent_error_ratio": 0, + "result": [ + [0.86, 0.25, 0.2], + [0.9, 0.28, 0.19], + [0.94, 0.27, 0.21], + [0.74, 0.25, 0.19], + [0.87, 0.3, 0.25], + [1.1, 0.29, 0.28], + [0.68, 0.24, 0.19], + [1.01, 0.23, 0.19], + [0.98, 0.39, 0.37], + [1.26, 0.42, 0.36], + [1.56, 0.27, 0.23], + [0.92, 0.28, 0.24], + [1.29, 0.3, 0.29], + [2.51, 0.34, 0.32], + [1.34, 0.33, 0.3], + [0.89, 0.32, 0.26], + [2.42, 0.54, 0.45], + [2.34, 0.25, 0.24], + [4.73, 0.52, 0.52], + [0.66, 0.16, 0.13], + [9.98, 0.39, 0.36], + [11.74, 0.39, 0.38], + [22.14, 0.54, 0.44], + [13.21, 0.64, 0.59], + [2.89, 0.25, 0.25], + [1.46, 0.25, 0.24], + [2.88, 0.29, 0.25], + [10.03, 0.48, 0.44], + [8.67, 1.49, 1.44], + [0.8, 0.26, 0.23], + [3, 0.31, 0.28], + [6.39, 0.35, 0.32], + [4.48, 0.93, 0.81], + [9.95, 0.91, 0.79], + [10, 0.9, 0.73], + [1.08, 0.32, 0.27], + [0.26, 0.06, 0.07], + [0.26, 0.08, 0.07], + [0.27, 0.08, 0.08], + [0.29, 0.07, 0.09], + [0.31, 0.08, 0.08], + [0.3, 0.1, 0.07], + [0.27, 0.09, 0.09] +] + } + \ No newline at end of file diff --git a/doris-parquet-single/start b/doris-parquet-single/start new file mode 100755 index 0000000000..afb51d10a2 --- /dev/null +++ b/doris-parquet-single/start @@ -0,0 +1,55 @@ +#!/bin/bash +set -e + +DORIS_HOME=$(cat .doris_home) +export DORIS_HOME +export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-$(dpkg --print-architecture)/" +export PATH=$JAVA_HOME/bin:$PATH + +# `SHOW BACKENDS` reports Alive per BE; find the column by header name — +# positions shift between Doris versions. +backend_alive() { + mysql -h127.0.0.1 -P9030 -uroot -B -e 'SHOW BACKENDS' 2>/dev/null \ + | awk -F'\t' 'NR==1 { for (i = 1; i <= NF; i++) if ($i == "Alive") c = i; next } + c && tolower($c) == "true" { found = 1 } END { exit !found }' +} + +# Idempotent: done only when the FE answers AND a BE is alive. +if mysql -h127.0.0.1 -P9030 -uroot -e 'SELECT 1' >/dev/null 2>&1 && backend_alive; then + exit 0 +fi + +ulimit -n 65535 + +# Tolerate "already running" from either daemon: we may get here with the FE +# up but the BE not alive yet. Capture the launchers' output — the harness +# silences ./start entirely, and if start_be.sh refuses to launch (a failed +# env check prints to its stderr and no be.out is ever created), this file +# is the only evidence; ./check dumps it on failure. +"$DORIS_HOME"/fe/bin/start_fe.sh --daemon > start_fe.out 2>&1 || true +"$DORIS_HOME"/be/bin/start_be.sh --daemon > start_be.out 2>&1 || true + +# Wait for the FE to accept connections (up to 10 min). SHOW FRONTENDS is +# FE-only metadata: on Doris 4.x, `SELECT 1` is dispatched to a backend and +# fails until a BE is alive, which would stall this loop for its full +# timeout before ADD BACKEND below ever runs. +for _ in $(seq 1 300); do + mysql -h127.0.0.1 -P9030 -uroot -e 'SHOW FRONTENDS' >/dev/null 2>&1 && break + sleep 2 +done + +mysql -h127.0.0.1 -P9030 -uroot \ + -e "ALTER SYSTEM ADD BACKEND '127.0.0.1:9050'" 2>/dev/null || true + +# Wait for the BE to become ALIVE, not merely registered: an earlier version +# waited for a non-empty SHOW BACKENDS column by position, which passes as +# soon as the row exists — ./load then created the local() view against a +# still-initializing BE and the FE answered +# "Can not build local(): No available backend". +for _ in $(seq 1 300); do + backend_alive && exit 0 + sleep 2 +done +echo "doris BE did not become alive within 10 min" >&2 +mysql -h127.0.0.1 -P9030 -uroot -e 'SHOW BACKENDS' 2>&1 | tail -5 >&2 || true +exit 1 diff --git a/doris-parquet-single/stop b/doris-parquet-single/stop new file mode 100755 index 0000000000..d8d0385b7a --- /dev/null +++ b/doris-parquet-single/stop @@ -0,0 +1,6 @@ +#!/bin/bash + +DORIS_HOME=$(cat .doris_home 2>/dev/null) || exit 0 +"$DORIS_HOME"/fe/bin/stop_fe.sh 2>/dev/null || true +"$DORIS_HOME"/be/bin/stop_be.sh 2>/dev/null || true +exit 0 diff --git a/doris-parquet-single/template.json b/doris-parquet-single/template.json new file mode 100644 index 0000000000..d23f311df2 --- /dev/null +++ b/doris-parquet-single/template.json @@ -0,0 +1,13 @@ +{ + "system": "Doris (Parquet, single)", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "C++", + "column-oriented", + "MySQL compatible", + "ClickHouse derivative", + "stateless" + ] +} diff --git a/starrocks-parquet-partitioned/README.md b/starrocks-parquet-partitioned/README.md new file mode 100644 index 0000000000..83c6f94fbd --- /dev/null +++ b/starrocks-parquet-partitioned/README.md @@ -0,0 +1,11 @@ +# StarRocks (Parquet, partitioned) + +Runs a local StarRocks cluster with 1 FE and 1 BE and queries the 100 +`hits_{0..99}.parquet` files in place through a view over the `FILES()` table +function (`file://` protocol), without loading them into StarRocks storage. + +## References + +- [starrocks](../starrocks/) +- [doris-parquet](../doris-parquet/) +- [clickhouse-parquet-partitioned](../clickhouse-parquet-partitioned/) diff --git a/starrocks-parquet-partitioned/benchmark.sh b/starrocks-parquet-partitioned/benchmark.sh new file mode 100755 index 0000000000..d5cf12db78 --- /dev/null +++ b/starrocks-parquet-partitioned/benchmark.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" +exec ../lib/benchmark-common.sh diff --git a/starrocks-parquet-partitioned/check b/starrocks-parquet-partitioned/check new file mode 100755 index 0000000000..c6e836c8c1 --- /dev/null +++ b/starrocks-parquet-partitioned/check @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +mysql -h127.0.0.1 -P9030 -uroot -e 'SELECT 1' >/dev/null diff --git a/starrocks-parquet-partitioned/create.sql b/starrocks-parquet-partitioned/create.sql new file mode 100644 index 0000000000..51059b7d75 --- /dev/null +++ b/starrocks-parquet-partitioned/create.sql @@ -0,0 +1,121 @@ +DROP DATABASE IF EXISTS hits; +CREATE DATABASE hits; +-- The parquet files store EventTime as raw INT64 epoch seconds and EventDate +-- as raw INT32 epoch days (no parquet logical types), so the view converts +-- them to DATETIME/DATE — otherwise predicates like EventDate >= '2013-07-14' +-- silently match nothing. Same fixup as duckdb-parquet's make_date/toDateTime. +-- Unlike the single hits.parquet, the partitioned files also lack the String +-- annotation on their BYTE_ARRAY columns, so those infer as VARBINARY and +-- string queries fail ("Error statistic type : varbinary") — cast them back +-- to VARCHAR (duckdb-parquet's binary_as_string equivalent). +CREATE VIEW hits.hits AS +SELECT + WatchID, + JavaEnable, + CAST(Title AS VARCHAR) AS Title, + GoodEvent, + CAST(from_unixtime(EventTime) AS DATETIME) AS EventTime, + CAST(days_add('1970-01-01', EventDate) AS DATE) AS EventDate, + CounterID, + ClientIP, + RegionID, + UserID, + CounterClass, + OS, + UserAgent, + CAST(URL AS VARCHAR) AS URL, + CAST(Referer AS VARCHAR) AS Referer, + IsRefresh, + RefererCategoryID, + RefererRegionID, + URLCategoryID, + URLRegionID, + ResolutionWidth, + ResolutionHeight, + ResolutionDepth, + FlashMajor, + FlashMinor, + CAST(FlashMinor2 AS VARCHAR) AS FlashMinor2, + NetMajor, + NetMinor, + UserAgentMajor, + CAST(UserAgentMinor AS VARCHAR) AS UserAgentMinor, + CookieEnable, + JavascriptEnable, + IsMobile, + MobilePhone, + CAST(MobilePhoneModel AS VARCHAR) AS MobilePhoneModel, + CAST(Params AS VARCHAR) AS Params, + IPNetworkID, + TraficSourceID, + SearchEngineID, + CAST(SearchPhrase AS VARCHAR) AS SearchPhrase, + AdvEngineID, + IsArtifical, + WindowClientWidth, + WindowClientHeight, + ClientTimeZone, + ClientEventTime, + SilverlightVersion1, + SilverlightVersion2, + SilverlightVersion3, + SilverlightVersion4, + CAST(PageCharset AS VARCHAR) AS PageCharset, + CodeVersion, + IsLink, + IsDownload, + IsNotBounce, + FUniqID, + CAST(OriginalURL AS VARCHAR) AS OriginalURL, + HID, + IsOldCounter, + IsEvent, + IsParameter, + DontCountHits, + WithHash, + CAST(HitColor AS VARCHAR) AS HitColor, + LocalEventTime, + Age, + Sex, + Income, + Interests, + Robotness, + RemoteIP, + WindowName, + OpenerName, + HistoryLength, + CAST(BrowserLanguage AS VARCHAR) AS BrowserLanguage, + CAST(BrowserCountry AS VARCHAR) AS BrowserCountry, + CAST(SocialNetwork AS VARCHAR) AS SocialNetwork, + CAST(SocialAction AS VARCHAR) AS SocialAction, + HTTPError, + SendTiming, + DNSTiming, + ConnectTiming, + ResponseStartTiming, + ResponseEndTiming, + FetchTiming, + SocialSourceNetworkID, + CAST(SocialSourcePage AS VARCHAR) AS SocialSourcePage, + ParamPrice, + CAST(ParamOrderID AS VARCHAR) AS ParamOrderID, + CAST(ParamCurrency AS VARCHAR) AS ParamCurrency, + ParamCurrencyID, + CAST(OpenstatServiceName AS VARCHAR) AS OpenstatServiceName, + CAST(OpenstatCampaignID AS VARCHAR) AS OpenstatCampaignID, + CAST(OpenstatAdID AS VARCHAR) AS OpenstatAdID, + CAST(OpenstatSourceID AS VARCHAR) AS OpenstatSourceID, + CAST(UTMSource AS VARCHAR) AS UTMSource, + CAST(UTMMedium AS VARCHAR) AS UTMMedium, + CAST(UTMCampaign AS VARCHAR) AS UTMCampaign, + CAST(UTMContent AS VARCHAR) AS UTMContent, + CAST(UTMTerm AS VARCHAR) AS UTMTerm, + CAST(FromTag AS VARCHAR) AS FromTag, + HasGCLID, + RefererHash, + URLHash, + CLID +FROM FILES( + "path" = "file://__DATA_DIR__/hits_*.parquet", + "format" = "parquet" +); diff --git a/starrocks-parquet-partitioned/data-size b/starrocks-parquet-partitioned/data-size new file mode 100755 index 0000000000..09d3960c8c --- /dev/null +++ b/starrocks-parquet-partitioned/data-size @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +find . -maxdepth 1 -name 'hits_*.parquet' -printf '%s\n' \ + | awk '{ s += $1 } END { print s+0 }' diff --git a/starrocks-parquet-partitioned/install b/starrocks-parquet-partitioned/install new file mode 100755 index 0000000000..a197358dcb --- /dev/null +++ b/starrocks-parquet-partitioned/install @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +VERSION="4.0.2-ubuntu-$(dpkg --print-architecture)" +SR_DIR="StarRocks-$VERSION" + +if [ ! -d "$SR_DIR" ]; then + if [ ! -f "$SR_DIR.tar.gz" ]; then + wget --continue --progress=dot:giga \ + "https://releases.starrocks.io/starrocks/$SR_DIR.tar.gz" \ + -O "$SR_DIR.tar.gz" + fi + tar zxf "$SR_DIR.tar.gz" + + # Configure FE/BE. + mkdir -p "$SR_DIR/meta" "$SR_DIR/storage" + printf "\nmeta_dir = $PWD/$SR_DIR/meta \n" >> "$SR_DIR/fe/conf/fe.conf" + printf "\nstorage_root_path = $PWD/$SR_DIR/storage\n" >> "$SR_DIR/be/conf/be.conf" + # Disable internal caches so the cold run is actually cold. + printf "\ndisable_storage_page_cache = true\n" >> "$SR_DIR/be/conf/be.conf" + printf "\ndatacache_enable = false\n" >> "$SR_DIR/be/conf/be.conf" +fi + +sudo apt-get update -y +sudo apt-get install -y openjdk-17-jre mariadb-client bc + +echo "$SR_DIR" > .sr_dir diff --git a/starrocks-parquet-partitioned/load b/starrocks-parquet-partitioned/load new file mode 100755 index 0000000000..4e8d566b2f --- /dev/null +++ b/starrocks-parquet-partitioned/load @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +# The parquet files stay in this directory (downloaded by benchmark.sh via +# lib/download-hits-parquet-partitioned); the view reads them in place through +# the FILES() table function, so there is no separate ingest step. The file:// +# protocol needs an absolute path, so substitute it at load time. +sed "s|__DATA_DIR__|$PWD|" create.sql | mysql -h127.0.0.1 -P9030 -uroot + +sync diff --git a/starrocks-parquet-partitioned/queries.sql b/starrocks-parquet-partitioned/queries.sql new file mode 100644 index 0000000000..199d008bfb --- /dev/null +++ b/starrocks-parquet-partitioned/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +SELECT AVG(UserID) FROM hits; +SELECT COUNT(DISTINCT UserID) FROM hits; +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; diff --git a/starrocks-parquet-partitioned/query b/starrocks-parquet-partitioned/query new file mode 100755 index 0000000000..025bb4f1e1 --- /dev/null +++ b/starrocks-parquet-partitioned/query @@ -0,0 +1,30 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it via mysql client against StarRocks. +# Stdout: query result. +# Stderr: query runtime in fractional seconds on the last line. +# Exit non-zero on error. +set -e + +query=$(cat) + +out=$(mysql -vvv -h127.0.0.1 -P9030 -uroot hits -e "$query" 2>&1) || status=$? +status=${status:-0} + +printf '%s\n' "$out" | grep -vP '^\([0-9.]+\s+sec\)$|rows? in set|Empty set' + +if [ "$status" -ne 0 ] || printf '%s\n' "$out" | grep -qE '^ERROR'; then + printf '%s\n' "$out" >&2 + exit 1 +fi + +secs=$(printf '%s\n' "$out" \ + | grep -oP '\((?:([0-9.]+)\s+min\s+)?([0-9.]+)\s+sec\)' \ + | tail -n1 \ + | sed -r 's/\((([0-9.]+) min )?([0-9.]+) sec\)/\2 \3/' \ + | awk '{ if ($2 != "") print $1*60 + $2; else print $1 }') + +if [ -z "$secs" ]; then + echo "no timing in mysql output" >&2 + exit 1 +fi +printf '%s\n' "$secs" >&2 diff --git a/starrocks-parquet-partitioned/results/20260818/c6a.4xlarge.json b/starrocks-parquet-partitioned/results/20260818/c6a.4xlarge.json new file mode 100644 index 0000000000..dcca64b849 --- /dev/null +++ b/starrocks-parquet-partitioned/results/20260818/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "date": "2026-08-18", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 20, + "data_size": 14737666736, + "concurrent_qps": 0.243, + "concurrent_error_ratio": 0.136, + "result": [ + [3.744, 0.342, 0.325], + [2.988, 0.244, 0.217], + [3.481, 0.525, 0.494], + [3.347, 0.438, 0.381], + [3.643, 0.767, 0.705], + [4.607, 1.455, 1.431], + [3.09, 0.323, 0.289], + [3.02, 0.241, 0.219], + [4.004, 1.057, 1.035], + [4.549, 1.289, 1.259], + [3.909, 0.81, 0.767], + [4.034, 0.896, 0.882], + [4.282, 1.304, 1.243], + [5.318, 1.677, 1.765], + [4.631, 1.539, 1.515], + [3.902, 1.009, 0.961], + [6.643, 3.646, 3.658], + [5.446, 2.473, 2.356], + [10.034, 7.02, 6.79], + [3.175, 0.296, 0.273], + [12.334, 2.714, 2.669], + [14.189, 3.343, 3.285], + [25.154, 6.248, 6.412], + [58.78, 33.412, 33.915], + [5.562, 1.358, 1.298], + [4, 0.984, 0.942], + [5.622, 1.314, 1.313], + [12.575, 3.318, 3.246], + [15.665, 12.389, 12.962], + [3.263, 0.403, 0.391], + [6.042, 1.809, 1.722], + [9.872, 2.128, 2.207], + [8.286, 5.677, 5.764], + [15.274, 7.497, 7.556], + [15.286, 7.516, 7.462], + [3.74, 0.827, 0.798], + [12.662, 2.91, 2.836], + [11.902, 2.817, 2.805], + [12.778, 2.952, 2.87], + [22.08, 5.339, 5.396], + [5.654, 0.964, 0.968], + [5.072, 0.974, 0.986], + [4.12, 0.765, 0.765] +] + } + \ No newline at end of file diff --git a/starrocks-parquet-partitioned/results/20260819/c6a.2xlarge.json b/starrocks-parquet-partitioned/results/20260819/c6a.2xlarge.json new file mode 100644 index 0000000000..f540032575 --- /dev/null +++ b/starrocks-parquet-partitioned/results/20260819/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "date": "2026-08-19", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 8, + "data_size": 14737666736, + "concurrent_qps": 0.238, + "concurrent_error_ratio": 0.197, + "result": [ + [4.217, 0.757, 0.729], + [3.318, 0.389, 0.381], + [4.203, 1.079, 1.055], + [4.209, 0.853, 0.823], + [4.726, 1.281, 1.281], + [6.329, 2.757, 2.723], + [3.528, 0.653, 0.627], + [3.272, 0.395, 0.415], + [5.294, 1.824, 1.752], + [6.374, 2.535, 2.509], + [5.334, 1.705, 1.686], + [5.58, 1.95, 1.948], + [6, 2.574, 2.4], + [7.773, 3.44, 3.45], + [6.515, 2.87, 2.869], + [5.004, 1.634, 1.612], + [10.143, 6.034, 6.017], + [8.069, 4.124, 4.015], + [16.34, 12.573, 12.461], + [3.933, 0.543, 0.517], + [12.945, 6.003, 6.257], + [15.05, 7.367, 7.305], + [25.569, 12.318, 12.413], + [89.327, 83.168, 82.858], + [7.174, 2.775, 2.762], + [5.593, 2.019, 1.963], + [7.247, 2.78, 2.731], + [13.436, 6.488, 6.367], + [25.437, 22.789, 23.04], + [3.807, 0.832, 0.793], + [8.368, 3.669, 3.662], + [10.182, 4.089, 4.099], + [null, null, null], + [16.862, 10.12, 10.035], + [17.187, 10.058, 10.08], + [4.635, 1.28, 1.263], + [13.407, 6.015, 5.986], + [12.854, 5.783, 5.849], + [13.587, 6.143, 5.978], + [22.427, 10.614, 10.448], + [6.84, 2.002, 1.987], + [6.705, 2.194, 2.09], + [5.649, 1.645, 1.629] +] + } + \ No newline at end of file diff --git a/starrocks-parquet-partitioned/results/20260819/c6a.4xlarge.json b/starrocks-parquet-partitioned/results/20260819/c6a.4xlarge.json new file mode 100644 index 0000000000..7283855c92 --- /dev/null +++ b/starrocks-parquet-partitioned/results/20260819/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "date": "2026-08-19", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 17, + "data_size": 14737666736, + "concurrent_qps": 0.338, + "concurrent_error_ratio": 0.09, + "result": [ + [3.781, 0.347, 0.318], + [2.984, 0.244, 0.201], + [3.38, 0.513, 0.48], + [3.333, 0.403, 0.368], + [3.645, 0.707, 0.649], + [4.501, 1.398, 1.372], + [3.136, 0.324, 0.289], + [2.928, 0.258, 0.213], + [3.877, 0.993, 0.982], + [4.521, 1.258, 1.204], + [3.953, 0.874, 0.778], + [4.052, 0.912, 0.901], + [4.336, 1.223, 1.2], + [5.305, 1.741, 1.625], + [4.587, 1.445, 1.431], + [3.87, 0.935, 0.878], + [6.412, 3.299, 3.25], + [5.323, 2.257, 2.251], + [9.815, 5.871, 6.384], + [3.245, 0.306, 0.27], + [12.32, 2.531, 2.546], + [14.248, 3.194, 3.237], + [25.086, 5.871, 5.92], + [59.223, 31.302, 31.6], + [5.622, 1.319, 1.295], + [4.054, 0.976, 0.915], + [5.562, 1.315, 1.29], + [12.593, 3.043, 3.061], + [15.334, 12.56, 12.528], + [3.316, 0.409, 0.359], + [5.946, 1.775, 1.703], + [9.829, 2.079, 1.964], + [8.241, 4.944, 5.022], + [14.841, 6.586, 6.483], + [14.88, 6.47, 6.445], + [3.67, 0.74, 0.706], + [12.71, 2.731, 2.757], + [11.727, 2.748, 2.662], + [12.746, 2.764, 2.764], + [22.169, 5.134, 5.073], + [5.72, 0.954, 0.892], + [5.034, 1.058, 0.93], + [4.025, 0.758, 0.76] +] + } + \ No newline at end of file diff --git a/starrocks-parquet-partitioned/results/20260819/c6a.large.json b/starrocks-parquet-partitioned/results/20260819/c6a.large.json new file mode 100644 index 0000000000..a7af7c4ac4 --- /dev/null +++ b/starrocks-parquet-partitioned/results/20260819/c6a.large.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "date": "2026-08-19", + "machine": "c6a.large", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 4, + "data_size": 14737666736, + "concurrent_qps": 0.082, + "concurrent_error_ratio": 0.617, + "result": [ + [7.785, 1.724, 1.605], + [6.705, 0.955, 0.797], + [8.116, 2.338, 2.332], + [8.477, 1.902, 1.77], + [10.966, 4.158, 4.034], + [14.321, 6.537, 6.499], + [7.353, 1.351, 1.156], + [6.312, 0.984, 0.805], + [11.776, 6.129, 5.836], + [14.253, 7.258, 7.136], + [11.531, 3.46, 3.553], + [10.843, 4.019, 4.098], + [14.307, 7.48, 7.353], + [22.537, 13.989, 13.05], + [16.559, 9.864, 9.641], + [11.939, 6.141, 6.045], + [null, null, null], + [22.645, 16.302, 37.627], + [null, null, null], + [8.623, 1.232, 1.053], + [25.849, 19.533, 20.129], + [31.171, 24.24, 25.329], + [50.048, 44.734, 44.088], + [180.08, 171.65, 172.958], + [15.628, 5.961, 5.728], + [11.024, 4.09, 3.983], + [15.707, 5.97, 5.787], + [28.398, 21.449, 22.682], + [94.613, 86.46, 85.022], + [8.896, 1.864, 1.869], + [18.062, 7.975, 7.914], + [22.326, 15.915, 15.771], + [null, null, null], + [null, null, null], + [null, null, null], + [9.872, 4.513, 4.159], + [27.478, 21.191, 22.355], + [26.946, 20.527, 21.764], + [28.203, 21.373, 22.508], + [44.317, 38.574, 37.432], + [14.492, 4.031, 4.093], + [14.044, 4.353, 4.152], + [12.151, 3.345, 3.302] +] + } + \ No newline at end of file diff --git a/starrocks-parquet-partitioned/results/20260819/c6a.metal.json b/starrocks-parquet-partitioned/results/20260819/c6a.metal.json new file mode 100644 index 0000000000..3d0f2c786c --- /dev/null +++ b/starrocks-parquet-partitioned/results/20260819/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "date": "2026-08-19", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 56, + "data_size": 14737666736, + "concurrent_qps": 2.42, + "concurrent_error_ratio": 0, + "result": [ + [3.53, 0.181, 0.164], + [2.799, 0.147, 0.124], + [2.871, 0.411, 0.21], + [2.942, 0.296, 0.216], + [3.039, 0.362, 0.26], + [3.435, 0.547, 0.48], + [2.813, 0.227, 0.143], + [2.889, 0.165, 0.137], + [3.309, 0.423, 0.402], + [3.941, 0.574, 0.498], + [3.194, 0.477, 0.384], + [3.08, 0.359, 0.368], + [3.406, 0.47, 0.461], + [4.758, 0.576, 0.469], + [3.381, 0.461, 0.482], + [3.084, 0.331, 0.285], + [4.882, 0.834, 0.756], + [4.664, 0.66, 0.751], + [6.645, 1.009, 1.013], + [2.764, 0.125, 0.119], + [12.022, 0.811, 0.634], + [13.707, 0.841, 0.831], + [24.417, 1.62, 1.583], + [58.485, 8.073, 6.954], + [4.978, 0.355, 0.348], + [3.288, 0.341, 0.276], + [5.086, 0.439, 0.351], + [12.285, 1.208, 1.217], + [10.823, 1.923, 1.877], + [2.86, 0.262, 0.167], + [5.474, 0.635, 0.629], + [9.15, 0.62, 0.678], + [7.189, 1.21, 1.178], + [12.725, 1.88, 1.86], + [12.714, 1.85, 1.877], + [2.822, 0.26, 0.246], + [12.377, 0.897, 0.818], + [11.406, 0.831, 0.858], + [12.466, 0.903, 0.895], + [21.513, 1.429, 1.417], + [5.366, 0.372, 0.347], + [4.553, 0.433, 0.437], + [3.52, 0.313, 0.313] +] + } + \ No newline at end of file diff --git a/starrocks-parquet-partitioned/results/20260819/c6a.xlarge.json b/starrocks-parquet-partitioned/results/20260819/c6a.xlarge.json new file mode 100644 index 0000000000..c5c0aec2d4 --- /dev/null +++ b/starrocks-parquet-partitioned/results/20260819/c6a.xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "date": "2026-08-19", + "machine": "c6a.xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 3, + "data_size": 14737666736, + "concurrent_qps": 0.18, + "concurrent_error_ratio": 0.26, + "result": [ + [5.016, 1.467, 1.408], + [4.015, 0.732, 0.687], + [5.881, 1.948, 1.908], + [6.011, 1.592, 1.519], + [7.177, 2.579, 2.582], + [10.109, 5.416, 5.34], + [4.671, 1.372, 1.301], + [4.064, 0.745, 0.675], + [8.05, 3.402, 3.292], + [10.336, 4.903, 4.827], + [8.06, 3.276, 3.197], + [8.565, 3.82, 3.775], + [10.104, 5.385, 5.311], + [13.548, 7.119, 7.044], + [10.272, 5.545, 5.464], + [7.57, 3.164, 3.241], + [17.729, 12.105, 12.032], + [13.661, 8.181, 8.211], + [null, null, null], + [5.362, 1.045, 0.984], + [22.244, 10.644, 10.722], + [26.794, 14.033, 14.429], + [45.288, 34.139, 33.337], + [171.971, 160.75, 162.627], + [12.259, 5.415, 5.346], + [8.749, 3.894, 3.782], + [12.314, 5.412, 5.305], + [23.419, 11.889, 11.866], + [46.795, 44.791, 43.488], + [5.296, 1.556, 1.465], + [14.141, 7.036, 6.984], + [18.182, 7.889, 7.914], + [null, null, null], + [null, null, null], + [null, null, null], + [6.967, 2.449, 2.416], + [23.536, 11.265, 11.453], + [22.764, 11.249, 11.398], + [23.708, 11.996, 11.365], + [39.41, 21.045, 20.621], + [11.149, 3.839, 3.964], + [11.048, 4.149, 4.003], + [9.019, 3.147, 3.125] +] + } + \ No newline at end of file diff --git a/starrocks-parquet-partitioned/results/20260819/c7a.metal-48xl.json b/starrocks-parquet-partitioned/results/20260819/c7a.metal-48xl.json new file mode 100644 index 0000000000..f63309d726 --- /dev/null +++ b/starrocks-parquet-partitioned/results/20260819/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "date": "2026-08-19", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 52, + "data_size": 14737666736, + "concurrent_qps": 4.285, + "concurrent_error_ratio": 0, + "result": [ + [3.368, 0.159, 0.153], + [2.751, 0.139, 0.104], + [2.753, 0.375, 0.181], + [2.955, 0.142, 0.139], + [2.976, 0.203, 0.193], + [3.816, 0.4, 0.478], + [2.884, 0.126, 0.138], + [2.887, 0.116, 0.12], + [3.186, 0.528, 0.338], + [3.8, 0.418, 0.421], + [3.147, 0.455, 0.25], + [3.118, 0.293, 0.266], + [3.342, 0.36, 0.393], + [4.721, 0.421, 0.417], + [3.456, 0.394, 0.399], + [3.087, 0.22, 0.229], + [4.789, 0.743, 0.549], + [4.704, 0.505, 0.487], + [6.672, 0.818, 0.711], + [2.867, 0.254, 0.108], + [11.929, 0.621, 0.523], + [13.622, 0.655, 0.631], + [24.306, 0.995, 0.959], + [58.463, 5.556, 5.44], + [5.048, 0.315, 0.31], + [3.154, 0.266, 0.269], + [5.078, 0.537, 0.327], + [12.191, 0.871, 1.029], + [10.709, 1.467, 1.47], + [2.877, 0.163, 0.146], + [5.337, 0.612, 0.474], + [8.947, 0.525, 0.477], + [7.089, 1.081, 0.961], + [12.425, 1.384, 1.324], + [12.436, 1.347, 1.275], + [2.932, 0.214, 0.181], + [12.362, 0.848, 0.628], + [11.395, 0.842, 0.557], + [12.338, 0.807, 0.612], + [21.591, 0.931, 0.878], + [5.435, 0.47, 0.28], + [4.579, 0.335, 0.639], + [5.476, 0.245, 0.266] +] + } + \ No newline at end of file diff --git a/starrocks-parquet-partitioned/results/20260819/c8g.4xlarge.json b/starrocks-parquet-partitioned/results/20260819/c8g.4xlarge.json new file mode 100644 index 0000000000..e1faff28bb --- /dev/null +++ b/starrocks-parquet-partitioned/results/20260819/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "date": "2026-08-19", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 20, + "data_size": 14737666736, + "concurrent_qps": 0.238, + "concurrent_error_ratio": 0.139, + "result": [ + [3.162, 0.27, 0.235], + [2.59, 0.203, 0.197], + [2.975, 0.402, 0.379], + [2.97, 0.303, 0.268], + [3.138, 0.476, 0.435], + [3.71, 0.927, 0.893], + [2.692, 0.269, 0.226], + [2.675, 0.213, 0.205], + [3.519, 0.67, 0.631], + [3.647, 0.931, 0.895], + [3.522, 0.762, 0.738], + [3.672, 0.882, 0.876], + [3.794, 0.974, 0.946], + [4.763, 1.184, 1.272], + [3.923, 1.155, 1.127], + [3.242, 0.545, 0.515], + [5.596, 1.941, 1.963], + [4.769, 1.351, 1.339], + [8.578, 2.837, 3.264], + [2.69, 0.253, 0.242], + [12.046, 2.208, 2.193], + [13.688, 2.78, 2.771], + [24.736, 5.04, 4.991], + [58.243, 26.028, 26.035], + [5.254, 1.185, 1.178], + [3.544, 0.912, 0.9], + [6.768, 1.179, 1.173], + [12.154, 2.233, 2.252], + [10.81, 8.106, 8.089], + [2.707, 0.304, 0.279], + [5.633, 1.533, 1.517], + [9.278, 1.627, 1.62], + [7.596, 1.923, 1.92], + [13.109, 3.257, 3.166], + [13.227, 3.318, 3.301], + [2.985, 0.465, 0.423], + [12.406, 2.344, 2.291], + [11.42, 2.381, 2.353], + [12.258, 2.372, 2.375], + [21.604, 4.054, 4.014], + [5.26, 0.888, 0.874], + [4.516, 0.977, 0.971], + [3.447, 0.764, 0.752] +] + } + \ No newline at end of file diff --git a/starrocks-parquet-partitioned/results/20260819/c8g.metal-48xl.json b/starrocks-parquet-partitioned/results/20260819/c8g.metal-48xl.json new file mode 100644 index 0000000000..9953aa65bd --- /dev/null +++ b/starrocks-parquet-partitioned/results/20260819/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "date": "2026-08-19", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 64, + "data_size": 14737666736, + "concurrent_qps": 4.59, + "concurrent_error_ratio": 0, + "result": [ + [2.964, 0.114, 0.13], + [2.375, 0.091, 0.089], + [2.594, 0.128, 0.123], + [2.561, 0.108, 0.1], + [2.989, 0.155, 0.144], + [3.201, 0.299, 0.304], + [2.393, 0.109, 0.093], + [2.4, 0.099, 0.099], + [3.033, 0.271, 0.271], + [3.958, 0.323, 0.372], + [2.532, 0.241, 0.238], + [3.736, 0.263, 0.26], + [3.073, 0.29, 0.287], + [4.505, 0.36, 0.339], + [3.161, 0.343, 0.326], + [2.679, 0.179, 0.175], + [4.481, 0.495, 0.51], + [4.387, 0.406, 0.41], + [6.404, 0.614, 0.603], + [2.531, 0.09, 0.084], + [11.713, 0.603, 0.538], + [13.38, 0.688, 0.664], + [24.055, 1.157, 1.152], + [58.417, 5.997, 6.111], + [4.595, 0.294, 0.278], + [2.944, 0.245, 0.233], + [4.804, 0.287, 0.275], + [11.899, 0.6, 0.617], + [10.383, 1.264, 1.267], + [2.621, 0.123, 0.117], + [5.139, 0.428, 0.401], + [8.71, 0.45, 0.434], + [6.783, 0.769, 0.779], + [12.096, 1.01, 0.994], + [12.105, 0.996, 0.995], + [2.593, 0.156, 0.152], + [12.026, 0.579, 0.559], + [11.128, 0.601, 0.59], + [12.077, 0.597, 0.608], + [21.466, 0.992, 0.961], + [5.094, 0.28, 0.258], + [4.297, 0.294, 0.285], + [3.32, 0.228, 0.216] +] + } + \ No newline at end of file diff --git a/starrocks-parquet-partitioned/start b/starrocks-parquet-partitioned/start new file mode 100755 index 0000000000..b4cbeb0bb6 --- /dev/null +++ b/starrocks-parquet-partitioned/start @@ -0,0 +1,72 @@ +#!/bin/bash +set -e + +SR_DIR=$(cat .sr_dir) +export STARROCKS_HOME="$PWD/$SR_DIR" +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-$(dpkg --print-architecture) +export PATH=$JAVA_HOME/bin:$PATH + +# Idempotent: if FE is up AND at least one BE is alive, we're done. +# +# `SHOW BACKENDS` TSV columns are: +# 1 BackendId 2 IP 3 HeartbeatPort 4 BePort 5 HttpPort +# 6 BrpcPort 7 LastStartTime 8 LastHeartbeat 9 Alive ... +# An earlier version of this script checked column 10 +# (SystemDecommissioned) instead of 9 (Alive), so the wait loop +# timed out even when SHOW BACKENDS reported the BE as alive. +backend_alive() { + mysql -h127.0.0.1 -P9030 -uroot -B -N -e 'SHOW BACKENDS' 2>/dev/null \ + | awk -F'\t' 'tolower($9)=="true" { found=1 } END { exit !found }' +} + +if mysql -h127.0.0.1 -P9030 -uroot -e 'SELECT 1' >/dev/null 2>&1 \ + && backend_alive; then + exit 0 +fi + +# Use 127.0.0.1 everywhere so the FE/BE rendezvous is stable across +# snapshot+restore. `hostname -i` can return different addresses +# depending on /etc/hosts vs DNS, and starrocks's FE persists the BE +# address — on restore the new BE registers as 127.0.0.1 but FE +# remembers something else, so SHOW BACKENDS reports +# "FE saved address not match backend address" +# and the BE stays Alive=false forever. +IPADDR=127.0.0.1 + +# Pin the BE's externally-visible address as well, so it reports back +# to FE as 127.0.0.1 too. +BE_CONF="$STARROCKS_HOME/be/conf/be.conf" +if ! grep -q '^priority_networks' "$BE_CONF"; then + echo "priority_networks = 127.0.0.1/32" >> "$BE_CONF" +fi +FE_CONF="$STARROCKS_HOME/fe/conf/fe.conf" +if ! grep -q '^priority_networks' "$FE_CONF"; then + echo "priority_networks = 127.0.0.1/32" >> "$FE_CONF" +fi + +"$STARROCKS_HOME/fe/bin/start_fe.sh" --daemon +"$STARROCKS_HOME/be/bin/start_be.sh" --daemon + +# Wait for FE to accept connections (up to 5 min). +for _ in $(seq 1 60); do + mysql -h127.0.0.1 -P9030 -uroot -e 'SELECT 1' >/dev/null 2>&1 && break + sleep 5 +done + +# On restore, FE may have a stale BE entry pointing at the prior +# address. Drop it so the re-registered BE wins. ALTER ... DROP +# BACKEND is idempotent on "not found". +mysql -h127.0.0.1 -P9030 -uroot \ + -e "ALTER SYSTEM DROP BACKEND '${IPADDR}:9050'" 2>/dev/null || true + +# Register BE; idempotent on "already exists". +mysql -h127.0.0.1 -P9030 -uroot \ + -e "ALTER SYSTEM ADD BACKEND '${IPADDR}:9050'" 2>/dev/null || true + +for _ in $(seq 1 60); do + backend_alive && exit 0 + sleep 5 +done +echo "starrocks BE did not become alive within 5 min" >&2 +mysql -h127.0.0.1 -P9030 -uroot -e 'SHOW BACKENDS' 2>&1 | tail -5 >&2 || true +exit 1 diff --git a/starrocks-parquet-partitioned/stop b/starrocks-parquet-partitioned/stop new file mode 100755 index 0000000000..d34da09e35 --- /dev/null +++ b/starrocks-parquet-partitioned/stop @@ -0,0 +1,6 @@ +#!/bin/bash + +SR_DIR=$(cat .sr_dir 2>/dev/null) || exit 0 +"$SR_DIR/fe/bin/stop_fe.sh" 2>/dev/null || true +"$SR_DIR/be/bin/stop_be.sh" 2>/dev/null || true +exit 0 diff --git a/starrocks-parquet-partitioned/template.json b/starrocks-parquet-partitioned/template.json new file mode 100644 index 0000000000..fa0d3b48ac --- /dev/null +++ b/starrocks-parquet-partitioned/template.json @@ -0,0 +1,12 @@ +{ + "system": "StarRocks (Parquet, partitioned)", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "C++", + "column-oriented", + "MySQL compatible", + "stateless" + ] +} diff --git a/starrocks-parquet/README.md b/starrocks-parquet/README.md new file mode 100644 index 0000000000..e925a44c5d --- /dev/null +++ b/starrocks-parquet/README.md @@ -0,0 +1,11 @@ +# StarRocks (Parquet, single) + +Runs a local StarRocks cluster with 1 FE and 1 BE and queries the single +`hits.parquet` file in place through a view over the `FILES()` table function +(`file://` protocol), without loading it into StarRocks storage. + +## References + +- [starrocks](../starrocks/) +- [doris-parquet](../doris-parquet/) +- [clickhouse-parquet](../clickhouse-parquet/) diff --git a/starrocks-parquet/benchmark.sh b/starrocks-parquet/benchmark.sh new file mode 100755 index 0000000000..a33527ee30 --- /dev/null +++ b/starrocks-parquet/benchmark.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +exec ../lib/benchmark-common.sh diff --git a/starrocks-parquet/check b/starrocks-parquet/check new file mode 100755 index 0000000000..c6e836c8c1 --- /dev/null +++ b/starrocks-parquet/check @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +mysql -h127.0.0.1 -P9030 -uroot -e 'SELECT 1' >/dev/null diff --git a/starrocks-parquet/create.sql b/starrocks-parquet/create.sql new file mode 100644 index 0000000000..03d6ebdeff --- /dev/null +++ b/starrocks-parquet/create.sql @@ -0,0 +1,117 @@ +DROP DATABASE IF EXISTS hits; +CREATE DATABASE hits; +-- The parquet files store EventTime as raw INT64 epoch seconds and EventDate +-- as raw INT32 epoch days (no parquet logical types), so the view converts +-- them to DATETIME/DATE — otherwise predicates like EventDate >= '2013-07-14' +-- silently match nothing. Same fixup as duckdb-parquet's make_date/toDateTime. +CREATE VIEW hits.hits AS +SELECT + WatchID, + JavaEnable, + Title, + GoodEvent, + CAST(from_unixtime(EventTime) AS DATETIME) AS EventTime, + CAST(days_add('1970-01-01', EventDate) AS DATE) AS EventDate, + CounterID, + ClientIP, + RegionID, + UserID, + CounterClass, + OS, + UserAgent, + URL, + Referer, + IsRefresh, + RefererCategoryID, + RefererRegionID, + URLCategoryID, + URLRegionID, + ResolutionWidth, + ResolutionHeight, + ResolutionDepth, + FlashMajor, + FlashMinor, + FlashMinor2, + NetMajor, + NetMinor, + UserAgentMajor, + UserAgentMinor, + CookieEnable, + JavascriptEnable, + IsMobile, + MobilePhone, + MobilePhoneModel, + Params, + IPNetworkID, + TraficSourceID, + SearchEngineID, + SearchPhrase, + AdvEngineID, + IsArtifical, + WindowClientWidth, + WindowClientHeight, + ClientTimeZone, + ClientEventTime, + SilverlightVersion1, + SilverlightVersion2, + SilverlightVersion3, + SilverlightVersion4, + PageCharset, + CodeVersion, + IsLink, + IsDownload, + IsNotBounce, + FUniqID, + OriginalURL, + HID, + IsOldCounter, + IsEvent, + IsParameter, + DontCountHits, + WithHash, + HitColor, + LocalEventTime, + Age, + Sex, + Income, + Interests, + Robotness, + RemoteIP, + WindowName, + OpenerName, + HistoryLength, + BrowserLanguage, + BrowserCountry, + SocialNetwork, + SocialAction, + HTTPError, + SendTiming, + DNSTiming, + ConnectTiming, + ResponseStartTiming, + ResponseEndTiming, + FetchTiming, + SocialSourceNetworkID, + SocialSourcePage, + ParamPrice, + ParamOrderID, + ParamCurrency, + ParamCurrencyID, + OpenstatServiceName, + OpenstatCampaignID, + OpenstatAdID, + OpenstatSourceID, + UTMSource, + UTMMedium, + UTMCampaign, + UTMContent, + UTMTerm, + FromTag, + HasGCLID, + RefererHash, + URLHash, + CLID +FROM FILES( + "path" = "file://__DATA_DIR__/hits.parquet", + "format" = "parquet" +); diff --git a/starrocks-parquet/data-size b/starrocks-parquet/data-size new file mode 100755 index 0000000000..f7b1b9e0b8 --- /dev/null +++ b/starrocks-parquet/data-size @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +find . -maxdepth 1 -name 'hits.parquet' -printf '%s\n' \ + | awk '{ s += $1 } END { print s+0 }' diff --git a/starrocks-parquet/install b/starrocks-parquet/install new file mode 100755 index 0000000000..a197358dcb --- /dev/null +++ b/starrocks-parquet/install @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +VERSION="4.0.2-ubuntu-$(dpkg --print-architecture)" +SR_DIR="StarRocks-$VERSION" + +if [ ! -d "$SR_DIR" ]; then + if [ ! -f "$SR_DIR.tar.gz" ]; then + wget --continue --progress=dot:giga \ + "https://releases.starrocks.io/starrocks/$SR_DIR.tar.gz" \ + -O "$SR_DIR.tar.gz" + fi + tar zxf "$SR_DIR.tar.gz" + + # Configure FE/BE. + mkdir -p "$SR_DIR/meta" "$SR_DIR/storage" + printf "\nmeta_dir = $PWD/$SR_DIR/meta \n" >> "$SR_DIR/fe/conf/fe.conf" + printf "\nstorage_root_path = $PWD/$SR_DIR/storage\n" >> "$SR_DIR/be/conf/be.conf" + # Disable internal caches so the cold run is actually cold. + printf "\ndisable_storage_page_cache = true\n" >> "$SR_DIR/be/conf/be.conf" + printf "\ndatacache_enable = false\n" >> "$SR_DIR/be/conf/be.conf" +fi + +sudo apt-get update -y +sudo apt-get install -y openjdk-17-jre mariadb-client bc + +echo "$SR_DIR" > .sr_dir diff --git a/starrocks-parquet/load b/starrocks-parquet/load new file mode 100755 index 0000000000..ac28064f0c --- /dev/null +++ b/starrocks-parquet/load @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +# The parquet file stays in this directory (downloaded by benchmark.sh via +# lib/download-hits-parquet-single); the view reads it in place through the +# FILES() table function, so there is no separate ingest step. The file:// +# protocol needs an absolute path, so substitute it at load time. +sed "s|__DATA_DIR__|$PWD|" create.sql | mysql -h127.0.0.1 -P9030 -uroot + +sync diff --git a/starrocks-parquet/queries.sql b/starrocks-parquet/queries.sql new file mode 100644 index 0000000000..199d008bfb --- /dev/null +++ b/starrocks-parquet/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +SELECT AVG(UserID) FROM hits; +SELECT COUNT(DISTINCT UserID) FROM hits; +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; diff --git a/starrocks-parquet/query b/starrocks-parquet/query new file mode 100755 index 0000000000..025bb4f1e1 --- /dev/null +++ b/starrocks-parquet/query @@ -0,0 +1,30 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it via mysql client against StarRocks. +# Stdout: query result. +# Stderr: query runtime in fractional seconds on the last line. +# Exit non-zero on error. +set -e + +query=$(cat) + +out=$(mysql -vvv -h127.0.0.1 -P9030 -uroot hits -e "$query" 2>&1) || status=$? +status=${status:-0} + +printf '%s\n' "$out" | grep -vP '^\([0-9.]+\s+sec\)$|rows? in set|Empty set' + +if [ "$status" -ne 0 ] || printf '%s\n' "$out" | grep -qE '^ERROR'; then + printf '%s\n' "$out" >&2 + exit 1 +fi + +secs=$(printf '%s\n' "$out" \ + | grep -oP '\((?:([0-9.]+)\s+min\s+)?([0-9.]+)\s+sec\)' \ + | tail -n1 \ + | sed -r 's/\((([0-9.]+) min )?([0-9.]+) sec\)/\2 \3/' \ + | awk '{ if ($2 != "") print $1*60 + $2; else print $1 }') + +if [ -z "$secs" ]; then + echo "no timing in mysql output" >&2 + exit 1 +fi +printf '%s\n' "$secs" >&2 diff --git a/starrocks-parquet/results/20260818/c6a.4xlarge.json b/starrocks-parquet/results/20260818/c6a.4xlarge.json new file mode 100644 index 0000000000..b5f3abeab8 --- /dev/null +++ b/starrocks-parquet/results/20260818/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-18", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 2, + "data_size": 14779976446, + "concurrent_qps": 0.262, + "concurrent_error_ratio": 0.113, + "result": [ + [3.017, 0.31, 0.295], + [2.86, 0.192, 0.197], + [3.269, 0.487, 0.46], + [3.312, 0.374, 0.355], + [3.561, 0.674, 0.673], + [4.39, 1.387, 1.375], + [2.972, 0.297, 0.277], + [2.941, 0.191, 0.194], + [4.037, 0.963, 0.96], + [4.24, 1.207, 1.179], + [3.794, 0.83, 0.811], + [3.914, 0.915, 0.902], + [4.149, 1.162, 1.209], + [5.088, 1.505, 1.449], + [4.449, 1.429, 1.454], + [3.79, 0.887, 0.905], + [6.521, 3.295, 3.329], + [5.345, 2.311, 2.195], + [7.362, 4.511, 4.676], + [3.141, 0.296, 0.263], + [12.862, 2.646, 2.654], + [14.64, 3.256, 3.241], + [26.325, 6.22, 6.085], + [62.456, 31.632, 31.719], + [5.484, 1.325, 1.249], + [3.879, 0.921, 0.892], + [5.34, 1.296, 1.269], + [12.541, 2.862, 2.819], + [15.151, 12.512, 12.426], + [3.15, 0.378, 0.371], + [5.564, 1.694, 1.664], + [9.164, 1.921, 1.948], + [7.855, 4.834, 4.99], + [15.045, 6.276, 6.378], + [14.99, 6.206, 6.24], + [3.617, 0.725, 0.718], + [12.65, 2.717, 2.691], + [12.119, 3.144, 3.097], + [12.592, 2.907, 2.77], + [21.707, 4.876, 4.827], + [5.558, 0.886, 0.891], + [4.768, 0.885, 0.892], + [3.892, 0.709, 0.684] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/results/20260819/c6a.2xlarge.json b/starrocks-parquet/results/20260819/c6a.2xlarge.json new file mode 100644 index 0000000000..589f2063f4 --- /dev/null +++ b/starrocks-parquet/results/20260819/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-19", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 6, + "data_size": 14779976446, + "concurrent_qps": 0.123, + "concurrent_error_ratio": 0.288, + "result": [ + [3.403, 0.649, 0.622], + [3.163, 0.327, 0.318], + [3.922, 0.97, 0.941], + [4.051, 0.752, 0.741], + [4.533, 1.177, 1.214], + [6.235, 2.757, 2.81], + [3.338, 0.54, 0.531], + [3.112, 0.33, 0.313], + [5.057, 1.666, 1.674], + [5.912, 2.261, 2.222], + [5.096, 1.662, 1.638], + [5.33, 1.919, 1.912], + [5.849, 2.378, 2.378], + [7.023, 3.004, 2.959], + [6.375, 2.83, 2.787], + [4.771, 1.544, 1.558], + [9.865, 6.039, 6.06], + [7.795, 4.125, 4.017], + [11.126, 8.048, 8.013], + [3.771, 0.484, 0.466], + [12.684, 5.701, 5.531], + [14.779, 7.07, 7.312], + [24.959, 12.513, 12.304], + [88.444, 82.191, 81.468], + [6.91, 2.773, 2.722], + [5.356, 1.986, 1.996], + [6.958, 2.726, 2.749], + [12.958, 5.885, 5.804], + [25.716, 22.197, 23.203], + [3.75, 0.772, 0.753], + [7.962, 3.516, 3.537], + [9.742, 3.987, 3.928], + [null, null, null], + [16.924, 9.903, 10.078], + [16.822, 10.073, 10.014], + [4.452, 1.231, 1.223], + [12.907, 5.822, 5.642], + [12.628, 5.931, 6.154], + [13.025, 5.703, 5.589], + [21.664, 10.063, 10.094], + [6.216, 1.804, 1.78], + [6.106, 1.833, 1.789], + [5.147, 1.425, 1.408] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/results/20260819/c6a.4xlarge.json b/starrocks-parquet/results/20260819/c6a.4xlarge.json new file mode 100644 index 0000000000..9061778211 --- /dev/null +++ b/starrocks-parquet/results/20260819/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-19", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 11, + "data_size": 14779976446, + "concurrent_qps": 0.362, + "concurrent_error_ratio": 0.977, + "result": [ + [3.045, 0.312, 0.293], + [2.898, 0.205, 0.184], + [3.319, 0.492, 0.489], + [3.311, 0.369, 0.357], + [3.543, 0.666, 0.654], + [4.524, 1.377, 1.361], + [3.025, 0.28, 0.272], + [3.036, 0.196, 0.185], + [3.915, 0.957, 0.917], + [4.361, 1.228, 1.155], + [3.907, 0.813, 0.823], + [4.035, 0.936, 0.932], + [4.241, 1.157, 1.162], + [4.957, 1.531, 1.436], + [4.399, 1.397, 1.39], + [3.804, 0.884, 0.854], + [6.387, 3.253, 3.225], + [5.232, 2.234, 2.14], + [7.398, 4.441, 4.506], + [3.186, 0.277, 0.268], + [12.451, 2.612, 2.612], + [14.239, 3.22, 3.363], + [25.191, 5.99, 5.848], + [59.314, 30.995, 30.634], + [5.404, 1.277, 1.225], + [3.942, 0.91, 0.913], + [5.35, 1.311, 1.246], + [12.508, 2.865, 2.826], + [15.049, 12.655, 12.432], + [3.234, 0.418, 0.372], + [5.593, 1.702, 1.692], + [9.155, 1.955, 1.952], + [7.972, 4.894, 4.956], + [15.06, 6.052, 6.112], + [14.91, 6.143, 6.063], + [3.553, 0.713, 0.673], + [12.662, 2.745, 2.642], + [12.067, 3.063, 3.022], + [12.584, 2.827, 2.82], + [21.727, 4.787, 4.722], + [5.503, 0.875, 0.887], + [4.841, 0.887, 0.882], + [3.879, 0.685, 0.683] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/results/20260819/c6a.large.json b/starrocks-parquet/results/20260819/c6a.large.json new file mode 100644 index 0000000000..f7760db734 --- /dev/null +++ b/starrocks-parquet/results/20260819/c6a.large.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-19", + "machine": "c6a.large", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 3, + "data_size": 14779976446, + "concurrent_qps": 0.09, + "concurrent_error_ratio": 0.481, + "result": [ + [7.3, 1.23, 1.248], + [6.029, 0.605, 0.636], + [7.663, 1.966, 1.951], + [8.42, 1.47, 1.472], + [10.02, 3.859, 4.005], + [13.225, 6.412, 6.597], + [6.61, 0.964, 0.993], + [6.313, 0.606, 0.629], + [11.863, 5.822, 5.829], + [12.524, 7.085, 6.853], + [10.309, 3.335, 3.237], + [10.29, 3.799, 3.682], + [12.01, 5.247, 5.367], + [15.08, 7.833, 7.635], + [13.41, 6.502, 6.482], + [11.625, 5.683, 5.986], + [null, null, null], + [22.64, 16.307, 31.603], + [null, null, null], + [8.535, 0.881, 0.905], + [25.298, 19.129, 19.542], + [30.589, 23.678, 24.787], + [49.241, 43.933, 43.046], + [169.803, 165.247, 165.974], + [14.897, 5.627, 5.572], + [10.261, 3.877, 3.957], + [15.022, 5.713, 5.563], + [27.416, 20.555, 21.606], + [95.106, 86.328, 84.838], + [7.861, 1.607, 1.582], + [16.812, 7.423, 7.366], + [21.68, 14.493, 14.432], + [null, null, null], + [null, null, null], + [null, null, null], + [10.413, 3.953, 4.211], + [26.9, 20.306, 21.214], + [25.893, 19.869, 20.968], + [26.187, 20.201, 21.012], + [43.12, 37.557, 36.53], + [13.175, 3.363, 3.297], + [12.948, 3.492, 3.553], + [9.89, 2.843, 2.704] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/results/20260819/c6a.metal.json b/starrocks-parquet/results/20260819/c6a.metal.json new file mode 100644 index 0000000000..adbc4d933c --- /dev/null +++ b/starrocks-parquet/results/20260819/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-19", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 9, + "data_size": 14779976446, + "concurrent_qps": 2.468, + "concurrent_error_ratio": 0, + "result": [ + [2.775, 0.234, 0.243], + [2.743, 0.187, 0.156], + [2.859, 0.242, 0.24], + [2.88, 0.229, 0.217], + [3.028, 0.319, 0.336], + [3.392, 0.476, 0.466], + [2.752, 0.224, 0.185], + [2.753, 0.18, 0.162], + [3.266, 0.464, 0.466], + [3.757, 0.567, 0.555], + [3.122, 0.404, 0.396], + [3.109, 0.44, 0.382], + [3.358, 0.491, 0.449], + [4.634, 0.528, 0.519], + [3.462, 0.471, 0.472], + [3.005, 0.366, 0.346], + [4.836, 0.742, 0.743], + [4.708, 0.654, 0.641], + [6.456, 1.017, 1.023], + [3.118, 0.19, 0.162], + [12.027, 0.932, 0.997], + [13.712, 1.186, 0.881], + [24.451, 2.232, 1.527], + [58.45, 8.278, 8.465], + [4.903, 0.418, 0.373], + [3.166, 0.326, 0.318], + [4.915, 0.424, 0.351], + [12.099, 0.936, 0.922], + [10.807, 1.866, 1.815], + [2.839, 0.259, 0.235], + [5.211, 0.585, 0.598], + [8.696, 0.621, 0.662], + [6.955, 1.281, 1.214], + [12.754, 1.801, 1.881], + [12.732, 1.82, 1.785], + [2.954, 0.313, 0.282], + [12.198, 1.058, 0.988], + [11.353, 0.936, 0.837], + [12.202, 1.018, 0.87], + [21.447, 1.451, 1.473], + [5.257, 0.377, 0.343], + [4.338, 0.382, 0.41], + [3.545, 0.328, 0.32] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/results/20260819/c6a.xlarge.json b/starrocks-parquet/results/20260819/c6a.xlarge.json new file mode 100644 index 0000000000..2149714e5d --- /dev/null +++ b/starrocks-parquet/results/20260819/c6a.xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-19", + "machine": "c6a.xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 2, + "data_size": 14779976446, + "concurrent_qps": 0.162, + "concurrent_error_ratio": 0.416, + "result": [ + [4.236, 1.185, 1.15], + [3.612, 0.543, 0.535], + [5.099, 1.64, 1.613], + [5.651, 1.403, 1.339], + [6.277, 2.285, 2.174], + [9.624, 5.093, 5.048], + [4.101, 1.045, 1.014], + [3.532, 0.54, 0.553], + [7.148, 3.069, 3.073], + [9, 4.137, 4.027], + [7.344, 2.998, 2.913], + [7.849, 3.423, 3.366], + [8.784, 4.31, 4.332], + [11.03, 5.455, 5.375], + [9.577, 5.186, 5.133], + [6.862, 2.843, 2.906], + [16.696, 12.2, 12.203], + [12.777, 8.154, 8.243], + [null, null, null], + [4.886, 0.823, 0.821], + [21.473, 10.393, 11.306], + [25.576, 13.791, 13.468], + [45.478, 33.922, 35.15], + [163.767, 153.355, 154.43], + [11.136, 5.079, 5.022], + [8.094, 3.667, 3.672], + [11.209, 5.084, 5.016], + [23.15, 12.024, 11.826], + [46.051, 43.816, 43.006], + [4.747, 1.386, 1.331], + [12.85, 6.423, 6.425], + [16.443, 7.351, 7.376], + [null, null, null], + [null, null, null], + [null, null, null], + [5.98, 2.099, 2.199], + [22.959, 11.184, 11.547], + [22.543, 11.936, 11.598], + [22.934, 10.889, 11.201], + [38.299, 19.286, 20.52], + [9.806, 3.226, 3.122], + [9.474, 3.297, 3.297], + [7.769, 2.61, 2.537] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/results/20260819/c7a.metal-48xl.json b/starrocks-parquet/results/20260819/c7a.metal-48xl.json new file mode 100644 index 0000000000..d04ca787e8 --- /dev/null +++ b/starrocks-parquet/results/20260819/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-19", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 7, + "data_size": 14779976446, + "concurrent_qps": 4.337, + "concurrent_error_ratio": 0, + "result": [ + [2.719, 0.221, 0.186], + [2.798, 0.141, 0.144], + [2.746, 0.221, 0.198], + [2.91, 0.227, 0.197], + [2.882, 0.253, 0.22], + [3.306, 0.4, 0.394], + [2.797, 0.175, 0.153], + [2.691, 0.156, 0.135], + [3.155, 0.384, 0.39], + [3.646, 0.466, 0.443], + [2.983, 0.334, 0.316], + [2.997, 0.442, 0.332], + [3.251, 0.384, 0.426], + [4.435, 0.395, 0.404], + [3.301, 0.408, 0.371], + [2.895, 0.316, 0.258], + [4.622, 0.673, 0.535], + [4.559, 0.521, 0.499], + [6.328, 0.772, 0.686], + [2.733, 0.161, 0.142], + [11.914, 0.664, 0.664], + [13.562, 0.728, 0.679], + [24.305, 1.093, 1.019], + [58.429, 6.297, 6.015], + [4.916, 0.345, 0.355], + [3.169, 0.279, 0.306], + [4.772, 0.422, 0.289], + [12.079, 0.699, 0.712], + [10.711, 1.424, 1.424], + [2.859, 0.208, 0.188], + [5.01, 0.526, 0.483], + [8.47, 0.616, 0.527], + [6.873, 0.94, 0.919], + [12.404, 1.294, 1.36], + [12.458, 1.349, 1.343], + [2.863, 0.242, 0.369], + [12.052, 0.773, 0.644], + [11.29, 0.612, 0.641], + [12.114, 0.712, 0.627], + [21.292, 0.968, 0.92], + [5.115, 0.441, 0.301], + [4.318, 0.336, 0.344], + [3.35, 0.263, 0.259] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/results/20260819/c8g.4xlarge.json b/starrocks-parquet/results/20260819/c8g.4xlarge.json new file mode 100644 index 0000000000..8632c09be3 --- /dev/null +++ b/starrocks-parquet/results/20260819/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-19", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 2, + "data_size": 14779976446, + "concurrent_qps": 0.452, + "concurrent_error_ratio": 0.98, + "result": [ + [2.724, 0.231, 0.231], + [2.665, 0.195, 0.186], + [2.889, 0.353, 0.337], + [2.97, 0.268, 0.263], + [3.114, 0.441, 0.433], + [3.707, 0.916, 0.91], + [2.708, 0.226, 0.218], + [2.701, 0.197, 0.193], + [3.345, 0.621, 0.603], + [3.682, 0.852, 0.823], + [3.521, 0.769, 0.758], + [3.623, 0.864, 0.845], + [3.733, 0.931, 0.915], + [4.658, 1.127, 1.097], + [3.898, 1.081, 1.069], + [3.198, 0.503, 0.514], + [5.541, 2.036, 1.988], + [4.838, 1.353, 1.311], + [6.857, 2.457, 2.453], + [2.892, 0.243, 0.231], + [12.198, 2.305, 2.248], + [13.877, 2.775, 2.775], + [24.89, 5.09, 5.115], + [59.251, 24.907, 24.843], + [5.106, 1.121, 1.113], + [3.616, 0.873, 0.865], + [5.106, 1.114, 1.105], + [12.279, 2.203, 2.131], + [10.806, 8.109, 8.077], + [2.84, 0.282, 0.268], + [5.347, 1.395, 1.385], + [8.897, 1.48, 1.47], + [7.369, 1.947, 1.951], + [13.43, 3.398, 3.341], + [13.457, 3.382, 3.326], + [3.087, 0.448, 0.428], + [12.379, 2.242, 2.201], + [11.863, 2.767, 2.718], + [12.291, 2.241, 2.264], + [21.437, 3.812, 3.837], + [5.267, 0.754, 0.741], + [4.494, 0.801, 0.791], + [3.5, 0.619, 0.611] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/results/20260819/c8g.metal-48xl.json b/starrocks-parquet/results/20260819/c8g.metal-48xl.json new file mode 100644 index 0000000000..8b6574c68d --- /dev/null +++ b/starrocks-parquet/results/20260819/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-19", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 8, + "data_size": 14779976446, + "concurrent_qps": 4.597, + "concurrent_error_ratio": 0, + "result": [ + [2.474, 0.14, 0.134], + [2.494, 0.112, 0.108], + [2.583, 0.155, 0.146], + [2.804, 0.152, 0.137], + [2.819, 0.18, 0.197], + [3.11, 0.287, 0.286], + [2.609, 0.127, 0.119], + [2.542, 0.119, 0.109], + [3.054, 0.309, 0.298], + [3.427, 0.363, 0.357], + [2.954, 0.296, 0.287], + [3.109, 0.326, 0.311], + [3.134, 0.284, 0.28], + [4.371, 0.342, 0.344], + [3.197, 0.324, 0.316], + [3.035, 0.199, 0.205], + [4.634, 0.503, 0.506], + [4.502, 0.409, 0.418], + [6.326, 0.642, 0.615], + [2.66, 0.135, 0.12], + [11.943, 0.67, 0.648], + [13.629, 0.781, 0.735], + [24.177, 1.203, 1.154], + [58.225, 5.99, 5.976], + [4.683, 0.302, 0.286], + [3, 0.231, 0.227], + [4.693, 0.295, 0.282], + [12.042, 0.64, 0.644], + [10.425, 1.325, 1.29], + [2.594, 0.159, 0.141], + [4.912, 0.449, 0.415], + [8.288, 0.452, 0.426], + [6.798, 0.828, 0.765], + [12.21, 1.08, 1.059], + [12.253, 1.07, 1.07], + [2.673, 0.19, 0.172], + [11.989, 0.644, 0.684], + [11.203, 0.652, 0.64], + [11.974, 0.675, 0.638], + [21.095, 1.063, 0.982], + [5.095, 0.275, 0.275], + [4.078, 0.294, 0.292], + [3.236, 0.241, 0.229] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/results/20260819/t3a.small.json b/starrocks-parquet/results/20260819/t3a.small.json new file mode 100644 index 0000000000..0eaca37fd3 --- /dev/null +++ b/starrocks-parquet/results/20260819/t3a.small.json @@ -0,0 +1,60 @@ +{ + "system": "StarRocks (Parquet, single)", + "date": "2026-08-19", + "machine": "t3a.small", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","MySQL compatible","stateless"], + "load_time": 5, + "data_size": 14779976446, + "concurrent_qps": 0.047, + "concurrent_error_ratio": 0.692, + "result": [ + [12.167, 2.433, 2.423], + [11.561, 1.273, 1.169], + [13.342, 3.619, 3.428], + [13.397, 3.918, 3.93], + [19.313, 8.265, 7.147], + [32.34, 23.754, 21.631], + [12.042, 1.63, 1.644], + [12.009, 1.348, 1.234], + [19.998, 10.272, 11.992], + [23.309, 12.3, 14.049], + [16.937, 7.916, 7.737], + [19.054, 8.998, 8.725], + [23.725, 46.449, 12.444], + [null, null, null], + [26.977, 15.405, 41.048], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [12.561, 2.986, 2.966], + [37.67, 30.17, 28.074], + [45.797, 37.729, 35.537], + [75.429, 64.069, 63.703], + [null, 292.901, 291.835], + [21.811, 13.491, 13.124], + [19.003, 8.777, 8.776], + [22.288, 13.463, 13.217], + [39.256, 33.857, 29.25], + [null, null, null], + [13.777, 2.91, 2.883], + [26.461, 17.321, 38.84], + [null, null, null], + [null, null, null], + [null, null, null], + [null, null, null], + [20.042, 8.869, 10.058], + [40.344, 35.455, 29.999], + [38.448, 31.184, 29.17], + [38.97, 32.314, 29.858], + [64.952, 55.627, 55.876], + [19.875, 10.529, 10.984], + [17.123, 9.609, 9.805], + [17.279, 7.315, 7.122] +] + } + \ No newline at end of file diff --git a/starrocks-parquet/start b/starrocks-parquet/start new file mode 100755 index 0000000000..b4cbeb0bb6 --- /dev/null +++ b/starrocks-parquet/start @@ -0,0 +1,72 @@ +#!/bin/bash +set -e + +SR_DIR=$(cat .sr_dir) +export STARROCKS_HOME="$PWD/$SR_DIR" +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-$(dpkg --print-architecture) +export PATH=$JAVA_HOME/bin:$PATH + +# Idempotent: if FE is up AND at least one BE is alive, we're done. +# +# `SHOW BACKENDS` TSV columns are: +# 1 BackendId 2 IP 3 HeartbeatPort 4 BePort 5 HttpPort +# 6 BrpcPort 7 LastStartTime 8 LastHeartbeat 9 Alive ... +# An earlier version of this script checked column 10 +# (SystemDecommissioned) instead of 9 (Alive), so the wait loop +# timed out even when SHOW BACKENDS reported the BE as alive. +backend_alive() { + mysql -h127.0.0.1 -P9030 -uroot -B -N -e 'SHOW BACKENDS' 2>/dev/null \ + | awk -F'\t' 'tolower($9)=="true" { found=1 } END { exit !found }' +} + +if mysql -h127.0.0.1 -P9030 -uroot -e 'SELECT 1' >/dev/null 2>&1 \ + && backend_alive; then + exit 0 +fi + +# Use 127.0.0.1 everywhere so the FE/BE rendezvous is stable across +# snapshot+restore. `hostname -i` can return different addresses +# depending on /etc/hosts vs DNS, and starrocks's FE persists the BE +# address — on restore the new BE registers as 127.0.0.1 but FE +# remembers something else, so SHOW BACKENDS reports +# "FE saved address not match backend address" +# and the BE stays Alive=false forever. +IPADDR=127.0.0.1 + +# Pin the BE's externally-visible address as well, so it reports back +# to FE as 127.0.0.1 too. +BE_CONF="$STARROCKS_HOME/be/conf/be.conf" +if ! grep -q '^priority_networks' "$BE_CONF"; then + echo "priority_networks = 127.0.0.1/32" >> "$BE_CONF" +fi +FE_CONF="$STARROCKS_HOME/fe/conf/fe.conf" +if ! grep -q '^priority_networks' "$FE_CONF"; then + echo "priority_networks = 127.0.0.1/32" >> "$FE_CONF" +fi + +"$STARROCKS_HOME/fe/bin/start_fe.sh" --daemon +"$STARROCKS_HOME/be/bin/start_be.sh" --daemon + +# Wait for FE to accept connections (up to 5 min). +for _ in $(seq 1 60); do + mysql -h127.0.0.1 -P9030 -uroot -e 'SELECT 1' >/dev/null 2>&1 && break + sleep 5 +done + +# On restore, FE may have a stale BE entry pointing at the prior +# address. Drop it so the re-registered BE wins. ALTER ... DROP +# BACKEND is idempotent on "not found". +mysql -h127.0.0.1 -P9030 -uroot \ + -e "ALTER SYSTEM DROP BACKEND '${IPADDR}:9050'" 2>/dev/null || true + +# Register BE; idempotent on "already exists". +mysql -h127.0.0.1 -P9030 -uroot \ + -e "ALTER SYSTEM ADD BACKEND '${IPADDR}:9050'" 2>/dev/null || true + +for _ in $(seq 1 60); do + backend_alive && exit 0 + sleep 5 +done +echo "starrocks BE did not become alive within 5 min" >&2 +mysql -h127.0.0.1 -P9030 -uroot -e 'SHOW BACKENDS' 2>&1 | tail -5 >&2 || true +exit 1 diff --git a/starrocks-parquet/stop b/starrocks-parquet/stop new file mode 100755 index 0000000000..d34da09e35 --- /dev/null +++ b/starrocks-parquet/stop @@ -0,0 +1,6 @@ +#!/bin/bash + +SR_DIR=$(cat .sr_dir 2>/dev/null) || exit 0 +"$SR_DIR/fe/bin/stop_fe.sh" 2>/dev/null || true +"$SR_DIR/be/bin/stop_be.sh" 2>/dev/null || true +exit 0 diff --git a/starrocks-parquet/template.json b/starrocks-parquet/template.json new file mode 100644 index 0000000000..2461f34a0e --- /dev/null +++ b/starrocks-parquet/template.json @@ -0,0 +1,12 @@ +{ + "system": "StarRocks (Parquet, single)", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "C++", + "column-oriented", + "MySQL compatible", + "stateless" + ] +}