diff --git a/scramdb/README.md b/scramdb/README.md new file mode 100644 index 0000000000..a08a7d792b --- /dev/null +++ b/scramdb/README.md @@ -0,0 +1,5 @@ +# ScramDB + +PostgreSQL-wire-compatible UTAP database, installed natively from the official +installer (`curl -fsSL https://scramdb.com/install | bash`), pinned to one +SHA256-verified build for reproducibility. Run with `bash benchmark.sh`. diff --git a/scramdb/benchmark.sh b/scramdb/benchmark.sh new file mode 100755 index 0000000000..a19d43829a --- /dev/null +++ b/scramdb/benchmark.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" + +export PGHOST="${PGHOST:-localhost}" +export PGPORT="${PGPORT:-5432}" +export PGUSER="${PGUSER:-scramdb}" +export PGDATABASE="${PGDATABASE:-scramdb}" +export PGCONNECT_TIMEOUT="${PGCONNECT_TIMEOUT:-10}" + +exec ../lib/benchmark-common.sh diff --git a/scramdb/check b/scramdb/check new file mode 100755 index 0000000000..e226837132 --- /dev/null +++ b/scramdb/check @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +psql -t -c 'SELECT 1' >/dev/null diff --git a/scramdb/create.sql b/scramdb/create.sql new file mode 100644 index 0000000000..41c961c00f --- /dev/null +++ b/scramdb/create.sql @@ -0,0 +1,108 @@ +CREATE TABLE hits +( + WatchID BIGINT NOT NULL, + JavaEnable SMALLINT NOT NULL, + Title TEXT NOT NULL, + GoodEvent SMALLINT NOT NULL, + EventTime TIMESTAMP NOT NULL, + EventDate Date NOT NULL, + CounterID INTEGER NOT NULL, + ClientIP INTEGER NOT NULL, + RegionID INTEGER NOT NULL, + UserID BIGINT NOT NULL, + CounterClass SMALLINT NOT NULL, + OS SMALLINT NOT NULL, + UserAgent SMALLINT NOT NULL, + URL TEXT NOT NULL, + Referer TEXT NOT NULL, + IsRefresh SMALLINT NOT NULL, + RefererCategoryID SMALLINT NOT NULL, + RefererRegionID INTEGER NOT NULL, + URLCategoryID SMALLINT NOT NULL, + URLRegionID INTEGER NOT NULL, + ResolutionWidth SMALLINT NOT NULL, + ResolutionHeight SMALLINT NOT NULL, + ResolutionDepth SMALLINT NOT NULL, + FlashMajor SMALLINT NOT NULL, + FlashMinor SMALLINT NOT NULL, + FlashMinor2 TEXT NOT NULL, + NetMajor SMALLINT NOT NULL, + NetMinor SMALLINT NOT NULL, + UserAgentMajor SMALLINT NOT NULL, + UserAgentMinor VARCHAR(255) NOT NULL, + CookieEnable SMALLINT NOT NULL, + JavascriptEnable SMALLINT NOT NULL, + IsMobile SMALLINT NOT NULL, + MobilePhone SMALLINT NOT NULL, + MobilePhoneModel TEXT NOT NULL, + Params TEXT NOT NULL, + IPNetworkID INTEGER NOT NULL, + TraficSourceID SMALLINT NOT NULL, + SearchEngineID SMALLINT NOT NULL, + SearchPhrase TEXT NOT NULL, + AdvEngineID SMALLINT NOT NULL, + IsArtifical SMALLINT NOT NULL, + WindowClientWidth SMALLINT NOT NULL, + WindowClientHeight SMALLINT NOT NULL, + ClientTimeZone SMALLINT NOT NULL, + ClientEventTime TIMESTAMP NOT NULL, + SilverlightVersion1 SMALLINT NOT NULL, + SilverlightVersion2 SMALLINT NOT NULL, + SilverlightVersion3 INTEGER NOT NULL, + SilverlightVersion4 SMALLINT NOT NULL, + PageCharset TEXT NOT NULL, + CodeVersion INTEGER NOT NULL, + IsLink SMALLINT NOT NULL, + IsDownload SMALLINT NOT NULL, + IsNotBounce SMALLINT NOT NULL, + FUniqID BIGINT NOT NULL, + OriginalURL TEXT NOT NULL, + HID INTEGER NOT NULL, + IsOldCounter SMALLINT NOT NULL, + IsEvent SMALLINT NOT NULL, + IsParameter SMALLINT NOT NULL, + DontCountHits SMALLINT NOT NULL, + WithHash SMALLINT NOT NULL, + HitColor CHAR NOT NULL, + LocalEventTime TIMESTAMP NOT NULL, + Age SMALLINT NOT NULL, + Sex SMALLINT NOT NULL, + Income SMALLINT NOT NULL, + Interests SMALLINT NOT NULL, + Robotness SMALLINT NOT NULL, + RemoteIP INTEGER NOT NULL, + WindowName INTEGER NOT NULL, + OpenerName INTEGER NOT NULL, + HistoryLength SMALLINT NOT NULL, + BrowserLanguage TEXT NOT NULL, + BrowserCountry TEXT NOT NULL, + SocialNetwork TEXT NOT NULL, + SocialAction TEXT NOT NULL, + HTTPError SMALLINT NOT NULL, + SendTiming INTEGER NOT NULL, + DNSTiming INTEGER NOT NULL, + ConnectTiming INTEGER NOT NULL, + ResponseStartTiming INTEGER NOT NULL, + ResponseEndTiming INTEGER NOT NULL, + FetchTiming INTEGER NOT NULL, + SocialSourceNetworkID SMALLINT NOT NULL, + SocialSourcePage TEXT NOT NULL, + ParamPrice BIGINT NOT NULL, + ParamOrderID TEXT NOT NULL, + ParamCurrency TEXT NOT NULL, + ParamCurrencyID SMALLINT NOT NULL, + OpenstatServiceName TEXT NOT NULL, + OpenstatCampaignID TEXT NOT NULL, + OpenstatAdID TEXT NOT NULL, + OpenstatSourceID TEXT NOT NULL, + UTMSource TEXT NOT NULL, + UTMMedium TEXT NOT NULL, + UTMCampaign TEXT NOT NULL, + UTMContent TEXT NOT NULL, + UTMTerm TEXT NOT NULL, + FromTag TEXT NOT NULL, + HasGCLID SMALLINT NOT NULL, + RefererHash BIGINT NOT NULL, + URLHash BIGINT NOT NULL, + CLID INTEGER NOT NULL +); diff --git a/scramdb/create.sql.with-pk b/scramdb/create.sql.with-pk new file mode 100644 index 0000000000..1850bffedc --- /dev/null +++ b/scramdb/create.sql.with-pk @@ -0,0 +1,109 @@ +CREATE TABLE hits +( + WatchID BIGINT NOT NULL, + JavaEnable SMALLINT NOT NULL, + Title TEXT NOT NULL, + GoodEvent SMALLINT NOT NULL, + EventTime TIMESTAMP NOT NULL, + EventDate Date NOT NULL, + CounterID INTEGER NOT NULL, + ClientIP INTEGER NOT NULL, + RegionID INTEGER NOT NULL, + UserID BIGINT NOT NULL, + CounterClass SMALLINT NOT NULL, + OS SMALLINT NOT NULL, + UserAgent SMALLINT NOT NULL, + URL TEXT NOT NULL, + Referer TEXT NOT NULL, + IsRefresh SMALLINT NOT NULL, + RefererCategoryID SMALLINT NOT NULL, + RefererRegionID INTEGER NOT NULL, + URLCategoryID SMALLINT NOT NULL, + URLRegionID INTEGER NOT NULL, + ResolutionWidth SMALLINT NOT NULL, + ResolutionHeight SMALLINT NOT NULL, + ResolutionDepth SMALLINT NOT NULL, + FlashMajor SMALLINT NOT NULL, + FlashMinor SMALLINT NOT NULL, + FlashMinor2 TEXT NOT NULL, + NetMajor SMALLINT NOT NULL, + NetMinor SMALLINT NOT NULL, + UserAgentMajor SMALLINT NOT NULL, + UserAgentMinor VARCHAR(255) NOT NULL, + CookieEnable SMALLINT NOT NULL, + JavascriptEnable SMALLINT NOT NULL, + IsMobile SMALLINT NOT NULL, + MobilePhone SMALLINT NOT NULL, + MobilePhoneModel TEXT NOT NULL, + Params TEXT NOT NULL, + IPNetworkID INTEGER NOT NULL, + TraficSourceID SMALLINT NOT NULL, + SearchEngineID SMALLINT NOT NULL, + SearchPhrase TEXT NOT NULL, + AdvEngineID SMALLINT NOT NULL, + IsArtifical SMALLINT NOT NULL, + WindowClientWidth SMALLINT NOT NULL, + WindowClientHeight SMALLINT NOT NULL, + ClientTimeZone SMALLINT NOT NULL, + ClientEventTime TIMESTAMP NOT NULL, + SilverlightVersion1 SMALLINT NOT NULL, + SilverlightVersion2 SMALLINT NOT NULL, + SilverlightVersion3 INTEGER NOT NULL, + SilverlightVersion4 SMALLINT NOT NULL, + PageCharset TEXT NOT NULL, + CodeVersion INTEGER NOT NULL, + IsLink SMALLINT NOT NULL, + IsDownload SMALLINT NOT NULL, + IsNotBounce SMALLINT NOT NULL, + FUniqID BIGINT NOT NULL, + OriginalURL TEXT NOT NULL, + HID INTEGER NOT NULL, + IsOldCounter SMALLINT NOT NULL, + IsEvent SMALLINT NOT NULL, + IsParameter SMALLINT NOT NULL, + DontCountHits SMALLINT NOT NULL, + WithHash SMALLINT NOT NULL, + HitColor CHAR NOT NULL, + LocalEventTime TIMESTAMP NOT NULL, + Age SMALLINT NOT NULL, + Sex SMALLINT NOT NULL, + Income SMALLINT NOT NULL, + Interests SMALLINT NOT NULL, + Robotness SMALLINT NOT NULL, + RemoteIP INTEGER NOT NULL, + WindowName INTEGER NOT NULL, + OpenerName INTEGER NOT NULL, + HistoryLength SMALLINT NOT NULL, + BrowserLanguage TEXT NOT NULL, + BrowserCountry TEXT NOT NULL, + SocialNetwork TEXT NOT NULL, + SocialAction TEXT NOT NULL, + HTTPError SMALLINT NOT NULL, + SendTiming INTEGER NOT NULL, + DNSTiming INTEGER NOT NULL, + ConnectTiming INTEGER NOT NULL, + ResponseStartTiming INTEGER NOT NULL, + ResponseEndTiming INTEGER NOT NULL, + FetchTiming INTEGER NOT NULL, + SocialSourceNetworkID SMALLINT NOT NULL, + SocialSourcePage TEXT NOT NULL, + ParamPrice BIGINT NOT NULL, + ParamOrderID TEXT NOT NULL, + ParamCurrency TEXT NOT NULL, + ParamCurrencyID SMALLINT NOT NULL, + OpenstatServiceName TEXT NOT NULL, + OpenstatCampaignID TEXT NOT NULL, + OpenstatAdID TEXT NOT NULL, + OpenstatSourceID TEXT NOT NULL, + UTMSource TEXT NOT NULL, + UTMMedium TEXT NOT NULL, + UTMCampaign TEXT NOT NULL, + UTMContent TEXT NOT NULL, + UTMTerm TEXT NOT NULL, + FromTag TEXT NOT NULL, + HasGCLID SMALLINT NOT NULL, + RefererHash BIGINT NOT NULL, + URLHash BIGINT NOT NULL, + CLID INTEGER NOT NULL, + PRIMARY KEY (CounterID, EventDate, UserID, EventTime, WatchID) +); diff --git a/scramdb/data-size b/scramdb/data-size new file mode 100755 index 0000000000..28eda049c4 --- /dev/null +++ b/scramdb/data-size @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +BENCH_DIR="$(cd "$(dirname "$0")" && pwd)" + +# The compiled-query cache sits inside the data directory but is derived from +# the queries, not from the data. +du -sb --exclude=scramvm_cache "$BENCH_DIR/db" | cut -f1 diff --git a/scramdb/install b/scramdb/install new file mode 100755 index 0000000000..ed2ed54213 --- /dev/null +++ b/scramdb/install @@ -0,0 +1,134 @@ +#!/bin/bash +set -euo pipefail + +# Native install: the official one-command installer, pinned to a single build +# so the run is reproducible. The installer resolves the host architecture +# itself (linux/amd64 and linux/arm64 tarballs are published for every build) +# and verifies every download against that release's SHA256SUMS before it +# writes anything, so nothing here is architecture-specific. + +: "${HOME:?HOME must be set: the install directory lives under it}" + +BENCH_DIR="$(cd "$(dirname "$0")" && pwd)" +VERSION="${SCRAMDB_VERSION:-c664e19}" +INSTALL_DIR="${SCRAMDB_INSTALL_DIR:-$HOME/.scramdb}" +BIN="$INSTALL_DIR/bin/scramdb" + +# psql drives load/query/check; bc converts psql's millisecond timings. +sudo apt-get update -y +sudo apt-get install -y postgresql-client bc + +# A server left behind by a previous run still holds the data directory. +"$BENCH_DIR/stop" + +ram_kb=$(awk '/MemTotal/ { print $2 }' /proc/meminfo) +ram_gb=$(( ram_kb / 1024 / 1024 )) + +if [ "$ram_gb" -lt 16 ]; then + echo "WARNING: ScramDB targets machines with at least 16 GB of RAM; this one has ${ram_gb} GB." >&2 + echo "WARNING: proceeding anyway, but this result is not representative of the product." >&2 +fi + +curl -fsSL https://scramdb.com/install \ + | SCRAMDB_YES=1 SCRAMDB_MODIFY_PATH=0 SCRAMDB_INSTALL_DIR="$INSTALL_DIR" \ + bash -s -- --version "$VERSION" + +if [ ! -x "$BIN" ]; then + echo "install: the installer produced no executable at $BIN" >&2 + exit 1 +fi + +rm -rf "$BENCH_DIR/db" +mkdir -p "$BENCH_DIR/db" +rm -f "$BENCH_DIR/scramdb.log" "$BENCH_DIR/scramdb.pid" + +# The analytics configuration ScramDB ships and documents for a dedicated +# analytics box. Deviations from the installed default: WAL archiving off (a +# benchmark box does no point-in-time recovery, and archived segments would +# otherwise count into data_size), and the UDF and GPU lanes off (nothing +# beyond the benchmark runs). +# +# The 65 percent execution share is the documented setting for machines at or +# above the 16 GB target. Below it, 0 keeps the engine's automatic derivation, +# which sizes every budget to the RAM actually present. +if [ "$ram_gb" -ge 16 ]; then EXEC_PCT=65; else EXEC_PCT=0; fi + +# Below 8 GB the ingest pipeline is sized to the machine as well: the default +# batch times pipeline depth is an admission working set a 4 GB box does not +# have. copy_consumers (load lane, rank 4): N lane-owned writers add N-1 +# batches to that same working set, so the < 8 GB tier keeps the single +# consumer (today's path) and only the >= 8 GB tier auto-sizes from the +# box's own executor width. +if [ "$ram_gb" -ge 8 ]; then COPY_BATCH=131072; COPY_DEPTH=8; COPY_CONSUMERS=0; else COPY_BATCH=16384; COPY_DEPTH=2; COPY_CONSUMERS=1; fi + +cat > "$BENCH_DIR/scramdb-config.toml" <&2 +psql -v ON_ERROR_STOP=1 -c "ANALYZE hits" + +# The dataset is exactly 99,997,497 rows; refuse to benchmark a partial load. +rows=$(psql -t -A -c "SELECT count(*) FROM hits") +if [ "$rows" != "99997497" ]; then + echo "load: hits has $rows rows, expected 99997497 (partial load)" >&2 + exit 1 +fi + +rm -f "$BENCH_DIR/hits.csv" +sync diff --git a/scramdb/queries.sql b/scramdb/queries.sql new file mode 100644 index 0000000000..31f65fc898 --- /dev/null +++ b/scramdb/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +SELECT AVG(UserID) FROM hits; +SELECT COUNT(DISTINCT UserID) FROM hits; +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000; diff --git a/scramdb/query b/scramdb/query new file mode 100755 index 0000000000..25f5fc45c0 --- /dev/null +++ b/scramdb/query @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +query=$(cat) + +raw=$(psql -t -c '\timing' -c "$query" 2>&1) && exit_code=0 || exit_code=$? + +if [ "$exit_code" -ne 0 ] || printf '%s\n' "$raw" | grep -qE '^ERROR|psql: error'; then + printf '%s\n' "$raw" >&2 + exit 1 +fi + +printf '%s\n' "$raw" | grep -v '^Time:' || true + +ms=$(printf '%s\n' "$raw" | grep -oP 'Time:\s*\K[0-9.]+' | tail -n1) +if [ -z "$ms" ]; then + echo "no Time: in psql output" >&2 + exit 1 +fi + +LC_ALL=C printf '%.3f\n' "$(echo "$ms / 1000" | bc -l)" >&2 diff --git a/scramdb/results/20260906/c6a.2xlarge.json b/scramdb/results/20260906/c6a.2xlarge.json new file mode 100644 index 0000000000..d97647a66d --- /dev/null +++ b/scramdb/results/20260906/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "ScramDB", + "date": "2026-09-06", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","UTAP","JIT"], + "load_time": 754, + "data_size": 16079225186, + "concurrent_qps": 0.142, + "concurrent_error_ratio": 0.012, + "result": [ + [0.036, 0.001, 0.001], + [1.581, 0.027, 0.027], + [0.042, 0.001, 0.001], + [4.084, 0.085, 0.1], + [5.138, 1.352, 1.345], + [5.573, 2.81, 2.84], + [0.035, 0.001, 0.001], + [1.582, 0.03, 0.036], + [4.432, 0.957, 0.937], + [7.347, 1.147, 1.137], + [8.952, 0.294, 0.308], + [8.934, 0.347, 0.351], + [4.256, 0.657, 0.645], + [8.272, 2.061, 2.073], + [4.269, 0.752, 0.742], + [4.402, 0.556, 0.615], + [7.493, 1.132, 1.189], + [9.033, 2.692, 2.621], + [7.052, 2.133, 2.125], + [0.522, 0.022, 0.02], + [5.903, 1.938, 1.936], + [7.413, 1.871, 1.869], + [14.154, 3.369, 3.367], + [0.835, 0.281, 0.283], + [0.444, 0.052, 0.053], + [4.053, 0.888, 0.884], + [0.432, 0.054, 0.05], + [7.347, 1.341, 1.339], + [6.606, 4.244, 4.257], + [0.038, 0.003, 0.003], + [6.317, 0.969, 0.959], + [8.067, 1.039, 1.031], + [317.365, 342.318, 359.436], + [55.915, 51.942, 53.096], + [56.854, 52.756, 50.099], + [4.45, 0.576, 0.583], + [0.536, 0.046, 0.045], + [0.591, 0.023, 0.021], + [0.538, 0.038, 0.03], + [0.851, 0.635, 0.517], + [0.561, 0.017, 0.014], + [0.534, 0.015, 0.015], + [0.576, 0.018, 0.017] +] + } + \ No newline at end of file diff --git a/scramdb/results/20260906/c6a.4xlarge.json b/scramdb/results/20260906/c6a.4xlarge.json new file mode 100644 index 0000000000..78149a365c --- /dev/null +++ b/scramdb/results/20260906/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "ScramDB", + "date": "2026-09-06", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","UTAP","JIT"], + "load_time": 732, + "data_size": 16089800584, + "concurrent_qps": 1.295, + "concurrent_error_ratio": 0, + "result": [ + [0.042, 0.001, 0.001], + [1.586, 0.02, 0.015], + [0.042, 0.001, 0.001], + [4.031, 0.081, 0.091], + [5.076, 1.195, 1.179], + [5.585, 2.184, 2.2], + [0.044, 0.001, 0.001], + [1.591, 0.022, 0.039], + [4.331, 0.612, 0.603], + [8.173, 0.721, 0.774], + [8.867, 0.195, 0.202], + [8.855, 0.22, 0.222], + [4.234, 0.505, 0.443], + [8.599, 1.39, 1.387], + [4.219, 0.493, 0.504], + [4.304, 0.417, 0.426], + [8.062, 0.821, 0.816], + [9.519, 2.279, 2.279], + [8.08, 1.6, 1.602], + [0.641, 0.022, 0.016], + [5.937, 0.992, 0.989], + [8.306, 0.967, 0.965], + [14.213, 1.805, 1.801], + [0.956, 0.171, 0.165], + [0.478, 0.041, 0.046], + [4.058, 0.468, 0.474], + [0.458, 0.043, 0.049], + [7.298, 0.9, 0.898], + [7.703, 2.558, 2.57], + [0.053, 0.004, 0.003], + [7.32, 0.638, 0.628], + [10.162, 0.767, 0.762], + [10.541, 2.493, 2.389], + [8.614, 2.561, 2.535], + [6.587, 2.547, 2.52], + [4.348, 0.416, 0.424], + [0.485, 0.05, 0.037], + [0.493, 0.024, 0.025], + [0.498, 0.035, 0.032], + [0.692, 0.486, 0.487], + [0.505, 0.015, 0.022], + [0.475, 0.016, 0.02], + [0.538, 0.018, 0.023] +] + } + \ No newline at end of file diff --git a/scramdb/results/20260906/c6a.large.json b/scramdb/results/20260906/c6a.large.json new file mode 100644 index 0000000000..91fdba380f --- /dev/null +++ b/scramdb/results/20260906/c6a.large.json @@ -0,0 +1,60 @@ +{ + "system": "ScramDB", + "date": "2026-09-06", + "machine": "c6a.large", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","UTAP","JIT"], + "load_time": 929, + "data_size": 18541099582, + "concurrent_qps": 0.023, + "concurrent_error_ratio": 0.125, + "result": [ + [0.043, 0, 0.001], + [52.05, 43.818, 43.712], + [0.033, 0.001, 0.001], + [52.291, 44.249, 44.238], + [54.047, 46.853, 46.958], + [55.368, 54.799, 54.208], + [0.032, 0.001, 0.001], + [52.075, 43.793, 43.796], + [53.671, 53.263, 52.806], + [54.082, 53.823, 53.363], + [53.714, 45.201, 53.197], + [53.945, 44.669, 47.63], + [63.132, 64.13, 65.126], + [85.403, 82.33, 85.601], + [66.937, 65.247, 75.534], + [52.794, 51.401, 51.321], + [null, null, null], + [null, null, null], + [null, null, null], + [51.465, 43.259, 43.234], + [53.661, 46.201, 46.202], + [53.779, 46.708, 46.747], + [56.542, 50.579, 50.561], + [52.283, 44.006, 44.005], + [51.604, 43.114, 43.101], + [52.513, 44.1, 44.114], + [51.733, 43.227, 43.208], + [53.132, 44.3, 44.305], + [null, null, null], + [0.047, 0.003, 0.003], + [53.093, 51.962, 51.859], + [56.948, 56.479, 54.629], + [null, null, null], + [null, null, null], + [null, null, null], + [52.657, 50.761, 50.804], + [51.495, 43.584, 43.68], + [51.452, 43.377, 43.34], + [51.377, 43.304, 43.238], + [51.76, 50.198, 50.459], + [51.405, 43.358, 43.324], + [51.427, 43.206, 43.189], + [51.426, 43.183, 43.166] +] + } + \ No newline at end of file diff --git a/scramdb/results/20260906/c6a.metal.json b/scramdb/results/20260906/c6a.metal.json new file mode 100644 index 0000000000..8213c2e399 --- /dev/null +++ b/scramdb/results/20260906/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "ScramDB", + "date": "2026-09-06", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","UTAP","JIT"], + "load_time": 718, + "data_size": 16122448826, + "concurrent_qps": 3.013, + "concurrent_error_ratio": 0, + "result": [ + [0.042, 0.001, 0.001], + [1.595, 0.022, 0.013], + [0.032, 0.001, 0.001], + [4.054, 0.028, 0.03], + [5.449, 1.25, 1.531], + [6.003, 1.864, 1.753], + [0.033, 0.002, 0.001], + [1.598, 0.019, 0.013], + [4.22, 0.295, 0.28], + [8.812, 0.331, 0.332], + [8.882, 0.105, 0.094], + [8.856, 0.109, 0.099], + [4.134, 0.11, 0.122], + [8.609, 0.686, 0.695], + [4.125, 0.115, 0.122], + [4.137, 0.197, 0.162], + [8.16, 0.201, 0.226], + [9.241, 1.205, 1.235], + [9.768, 0.456, 0.457], + [0.275, 0.019, 0.009], + [9.172, 0.136, 0.181], + [13.693, 0.147, 0.181], + [22.908, 0.352, 0.317], + [0.518, 0.433, 0.092], + [0.171, 0.048, 0.018], + [4.03, 0.343, 0.298], + [0.161, 0.041, 0.018], + [11.304, 0.175, 0.17], + [9.006, 1.648, 1.659], + [0.038, 0.004, 0.004], + [7.818, 0.141, 0.185], + [12.1, 0.2, 0.25], + [12.013, 0.754, 0.834], + [9.49, 0.604, 0.583], + [9.487, 0.592, 0.61], + [4.134, 0.14, 0.159], + [0.221, 0.044, 0.033], + [0.201, 0.025, 0.024], + [0.247, 0.042, 0.024], + [0.788, 0.491, 0.402], + [0.238, 0.022, 0.019], + [0.225, 0.02, 0.019], + [0.232, 0.023, 0.02] +] + } + \ No newline at end of file diff --git a/scramdb/results/20260906/c6a.xlarge.json b/scramdb/results/20260906/c6a.xlarge.json new file mode 100644 index 0000000000..1c20919aa3 --- /dev/null +++ b/scramdb/results/20260906/c6a.xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "ScramDB", + "date": "2026-09-06", + "machine": "c6a.xlarge", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","UTAP","JIT"], + "load_time": 906, + "data_size": 18828906227, + "concurrent_qps": 0.048, + "concurrent_error_ratio": 0.094, + "result": [ + [0.032, 0.001, 0.001], + [29.324, 18.065, 18.027], + [0.033, 0.001, 0.001], + [31.402, 19.148, 18.759], + [32.536, 20.265, 20.199], + [33.165, 21.523, 21.595], + [0.034, 0.001, 0.001], + [29.305, 18.313, 18.018], + [31.627, 19.734, 19.707], + [33.335, 22.063, 22.016], + [34.152, 21.227, 21.227], + [34.488, 21.5, 21.498], + [31.281, 19.352, 19.349], + [35.37, 26.503, 30.688], + [31.358, 19.658, 17.753], + [31.573, 19.745, 19.717], + [33.794, 23.932, 23.963], + [null, null, null], + [null, null, null], + [28.707, 17.7, 17.716], + [32.558, 18.575, 18.508], + [33.56, 23.102, 23.11], + [37.911, 26.701, 26.697], + [29.385, 18.478, 18.426], + [28.653, 17.691, 17.767], + [30.794, 18.579, 18.573], + [28.63, 16.15, 17.734], + [32.942, 22.322, 22.32], + [35.071, 27.71, 28.26], + [0.037, 0.003, 0.003], + [32.554, 21.493, 21.483], + [34.316, 25.595, 25.521], + [null, null, null], + [null, null, null], + [null, null, null], + [34.03, 16.222, 15.633], + [28.281, 17.527, 17.372], + [28.699, 17.999, 17.965], + [28.257, 17.521, 17.471], + [29.016, 17.837, 17.933], + [28.57, 17.989, 17.833], + [28.523, 17.974, 17.759], + [28.252, 17.663, 17.465] +] + } + \ No newline at end of file diff --git a/scramdb/results/20260906/c7a.metal-48xl.json b/scramdb/results/20260906/c7a.metal-48xl.json new file mode 100644 index 0000000000..ae1e038c5f --- /dev/null +++ b/scramdb/results/20260906/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "ScramDB", + "date": "2026-09-06", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","UTAP","JIT"], + "load_time": 668, + "data_size": 16119309270, + "concurrent_qps": 3.617, + "concurrent_error_ratio": 0, + "result": [ + [0.042, 0.001, 0.002], + [1.6, 0.016, 0.016], + [0.04, 0.002, 0.001], + [4.045, 0.021, 0.026], + [4.982, 0.985, 0.988], + [6.097, 1.984, 1.8], + [0.031, 0.002, 0.001], + [1.598, 0.015, 0.015], + [4.219, 0.266, 0.282], + [8.698, 0.264, 0.255], + [8.866, 0.079, 0.077], + [8.851, 0.084, 0.074], + [4.085, 0.057, 0.055], + [8.624, 0.822, 0.849], + [4.083, 0.061, 0.059], + [4.052, 0.117, 0.064], + [8.133, 0.089, 0.087], + [8.822, 0.746, 0.743], + [9.653, 0.171, 0.199], + [0.171, 0.013, 0.008], + [9.2, 0.09, 0.086], + [13.977, 0.093, 0.085], + [23.233, 0.155, 0.193], + [0.352, 0.327, 0.087], + [0.139, 0.043, 0.017], + [4.007, 0.433, 0.406], + [0.157, 0.023, 0.013], + [11.411, 0.081, 0.07], + [8.747, 2.177, 2.164], + [0.032, 0.005, 0.004], + [7.856, 0.081, 0.074], + [12.613, 0.077, 0.082], + [12.221, 0.294, 0.347], + [9.347, 0.25, 0.27], + [9.341, 0.247, 0.264], + [4.08, 0.066, 0.068], + [0.189, 0.027, 0.029], + [0.212, 0.022, 0.021], + [0.235, 0.023, 0.024], + [0.561, 0.376, 0.383], + [0.226, 0.018, 0.017], + [0.223, 0.017, 0.01], + [0.226, 0.02, 0.018] +] + } + \ No newline at end of file diff --git a/scramdb/results/20260906/c8g.4xlarge.json b/scramdb/results/20260906/c8g.4xlarge.json new file mode 100644 index 0000000000..79b66220e1 --- /dev/null +++ b/scramdb/results/20260906/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "ScramDB", + "date": "2026-09-06", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","UTAP","JIT"], + "load_time": 636, + "data_size": 16112914607, + "concurrent_qps": 2.187, + "concurrent_error_ratio": 0, + "result": [ + [0.027, 0.001, 0.001], + [1.516, 0.013, 0.012], + [0.03, 0.001, 0.001], + [4.01, 0.034, 0.039], + [5.096, 1.207, 1.211], + [5.813, 2.154, 2.151], + [0.03, 0.001, 0.001], + [1.523, 0.012, 0.01], + [4.261, 0.443, 0.44], + [8.167, 0.481, 0.485], + [8.811, 0.126, 0.129], + [8.799, 0.26, 0.26], + [4.101, 0.204, 0.205], + [8.413, 1.101, 1.075], + [4.1, 0.235, 0.236], + [4.097, 0.163, 0.162], + [7.836, 0.303, 0.303], + [8.518, 1.021, 0.975], + [7.762, 0.533, 0.54], + [0.439, 0.009, 0.007], + [5.864, 0.647, 0.646], + [7.35, 0.627, 0.627], + [14.108, 1.096, 1.093], + [0.781, 0.115, 0.11], + [0.403, 0.032, 0.023], + [4.022, 0.262, 0.262], + [0.406, 0.029, 0.024], + [7.096, 0.416, 0.416], + [6.155, 2.984, 2.956], + [0.033, 0.003, 0.003], + [7.161, 0.283, 0.283], + [9.126, 0.269, 0.261], + [9.48, 18.8, 0.713], + [6.245, 1.011, 0.888], + [6.25, 0.963, 0.963], + [4.157, 0.21, 0.213], + [0.399, 0.027, 0.015], + [0.395, 0.012, 0.009], + [0.417, 0.019, 0.014], + [0.534, 0.364, 0.33], + [0.403, 0.007, 0.005], + [0.41, 0.008, 0.008], + [0.395, 0.012, 0.012] +] + } + \ No newline at end of file diff --git a/scramdb/results/20260906/c8g.metal-48xl.json b/scramdb/results/20260906/c8g.metal-48xl.json new file mode 100644 index 0000000000..7b8a3c7858 --- /dev/null +++ b/scramdb/results/20260906/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "ScramDB", + "date": "2026-09-06", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","UTAP","JIT"], + "load_time": 657, + "data_size": 16123516657, + "concurrent_qps": 3.422, + "concurrent_error_ratio": 0, + "result": [ + [0.023, 0.001, 0], + [1.528, 0.014, 0.017], + [0.019, 0.001, 0.001], + [3.992, 0.018, 0.017], + [4.899, 0.945, 0.933], + [5.55, 1.487, 1.454], + [0.018, 0.001, 0.001], + [1.537, 0.013, 0.013], + [4.157, 0.283, 0.284], + [8.793, 0.278, 0.257], + [8.828, 0.074, 0.068], + [8.821, 0.077, 0.072], + [4.054, 0.052, 0.051], + [8.645, 1.02, 1.021], + [4.032, 0.058, 0.055], + [4.018, 0.065, 0.065], + [8.076, 0.082, 0.106], + [8.739, 0.799, 0.809], + [9.619, 0.164, 0.184], + [0.16, 0.011, 0.015], + [9.14, 0.079, 0.074], + [14.004, 0.079, 0.074], + [23.271, 0.137, 0.132], + [0.304, 0.458, 0.303], + [0.125, 0.024, 0.011], + [3.996, 0.389, 0.45], + [0.11, 0.027, 0.011], + [11.329, 0.069, 0.066], + [8.818, 2.638, 2.684], + [0.022, 0.003, 0.003], + [7.773, 0.106, 0.065], + [12.687, 0.068, 0.07], + [12.375, 0.292, 0.358], + [9.277, 0.276, 0.309], + [9.27, 0.275, 0.292], + [4.044, 0.079, 0.079], + [0.152, 0.037, 0.028], + [0.144, 0.024, 0.027], + [0.168, 0.032, 0.031], + [0.575, 0.363, 0.317], + [0.148, 0.011, 0.014], + [0.155, 0.013, 0.009], + [0.153, 0.014, 0.015] +] + } + \ No newline at end of file diff --git a/scramdb/start b/scramdb/start new file mode 100755 index 0000000000..0f843f0827 --- /dev/null +++ b/scramdb/start @@ -0,0 +1,75 @@ +#!/bin/bash +set -euo pipefail + +: "${HOME:?HOME must be set: the install directory lives under it}" + +BENCH_DIR="$(cd "$(dirname "$0")" && pwd)" +INSTALL_DIR="${SCRAMDB_INSTALL_DIR:-$HOME/.scramdb}" +BIN="$INSTALL_DIR/bin/scramdb" +CONFIG="${SCRAMDB_CONFIG:-$BENCH_DIR/scramdb-config.toml}" +PIDFILE="$BENCH_DIR/scramdb.pid" +LOGFILE="$BENCH_DIR/scramdb.log" +READY_TIMEOUT="${SCRAMDB_START_TIMEOUT:-120}" + +# The driver calls ./start on a server that may already be up. +if "$BENCH_DIR/check" >/dev/null 2>&1; then + exit 0 +fi + +if [ ! -x "$BIN" ]; then + echo "start: $BIN is missing; run ./install first" >&2 + exit 1 +fi +if [ ! -f "$CONFIG" ]; then + echo "start: $CONFIG is missing; run ./install first" >&2 + exit 1 +fi + +# A stock Ubuntu gives a shell 1024 descriptors; the server holds one per +# segment file plus one per connection. Raising the soft limit to this user's +# hard limit is always permitted, and the server inherits it. +ulimit -n "$(ulimit -Hn)" + +# The log is appended across the cold cycle's many restarts, so each boot is +# marked to keep it readable. +printf '=== start %s ===\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$LOGFILE" + +# With --config the server stays in the foreground (a bare `scramdb` would +# daemonize instead and own its own lifetime), so nohup plus a pidfile is what +# start and stop share. +# +# NO TUNING ENV. `results.json` declares "tuned": "no", so this script must +# start the shipped binary the way any user would. The allocator settings +# that used to live here (MIMALLOC_PURGE_DELAY / _ABANDONED_PAGE_PURGE) are +# now engine defaults applied in `main`, so every deployment gets them; and +# SCRAMDB_VACUUM_PARALLEL_WRITE=1 is GONE, not promoted - it re-arms a +# route whose lost-header-increment race is not root-caused yet, and a +# benchmark must never enable a path the engine itself keeps off. +SCRAMDB_LOG="${SCRAMDB_LOG:-warn}" \ + nohup "$BIN" --config "$CONFIG" >"$LOGFILE" 2>&1 & +pid=$! +echo "$pid" > "$PIDFILE" + +# Bounded wait for pgwire: ready the moment it accepts a connection, and a +# server that dies during startup is reported instead of waited on. +deadline=$(( $(date +%s) + READY_TIMEOUT )) +delay_ms=100 +while [ "$(date +%s)" -lt "$deadline" ]; do + if "$BENCH_DIR/check" >/dev/null 2>&1; then + exit 0 + fi + if ! kill -0 "$pid" 2>/dev/null; then + echo "start: the server exited during startup; last lines of $LOGFILE:" >&2 + tail -n 20 "$LOGFILE" >&2 + exit 1 + fi + printf -v delay '%d.%03d' $(( delay_ms / 1000 )) $(( delay_ms % 1000 )) + sleep "$delay" + if [ "$delay_ms" -lt 1000 ]; then + delay_ms=$(( delay_ms * 2 )) + fi +done + +echo "start: pgwire did not accept a connection within ${READY_TIMEOUT}s; last lines of $LOGFILE:" >&2 +tail -n 20 "$LOGFILE" >&2 +exit 1 diff --git a/scramdb/stop b/scramdb/stop new file mode 100755 index 0000000000..feaf55dac8 --- /dev/null +++ b/scramdb/stop @@ -0,0 +1,47 @@ +#!/bin/bash +set -euo pipefail + +BENCH_DIR="$(cd "$(dirname "$0")" && pwd)" +PIDFILE="$BENCH_DIR/scramdb.pid" +STOP_TIMEOUT="${SCRAMDB_STOP_TIMEOUT:-120}" + +[ -f "$PIDFILE" ] || exit 0 +pid=$(cat "$PIDFILE") + +if ! [[ "$pid" =~ ^[0-9]+$ ]]; then + echo "stop: $PIDFILE holds '${pid}', not a pid; removing it" >&2 + rm -f "$PIDFILE" + exit 0 +fi + +if ! kill -0 "$pid" 2>/dev/null; then + rm -f "$PIDFILE" + exit 0 +fi + +# SIGTERM is the graceful path: the server flushes and closes its segments. +kill "$pid" + +deadline=$(( $(date +%s) + STOP_TIMEOUT )) +while [ "$(date +%s)" -lt "$deadline" ]; do + if ! kill -0 "$pid" 2>/dev/null; then + rm -f "$PIDFILE" + exit 0 + fi + sleep 0.2 +done + +echo "stop: pid $pid ignored SIGTERM for ${STOP_TIMEOUT}s, sending SIGKILL" >&2 +kill -9 "$pid" 2>/dev/null || true + +deadline=$(( $(date +%s) + 10 )) +while [ "$(date +%s)" -lt "$deadline" ]; do + if ! kill -0 "$pid" 2>/dev/null; then + rm -f "$PIDFILE" + exit 0 + fi + sleep 0.2 +done + +echo "stop: pid $pid survived SIGKILL" >&2 +exit 1 diff --git a/scramdb/template.json b/scramdb/template.json new file mode 100644 index 0000000000..6da71701f2 --- /dev/null +++ b/scramdb/template.json @@ -0,0 +1,13 @@ +{ + "system": "ScramDB", + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "Rust", + "column-oriented", + "PostgreSQL compatible", + "UTAP", + "JIT" + ] +}