diff --git a/spiceai-cayenne/README.md b/spiceai-cayenne/README.md new file mode 100644 index 0000000000..302a9bb1b1 --- /dev/null +++ b/spiceai-cayenne/README.md @@ -0,0 +1,25 @@ +# Spice.ai OSS (Cayenne) + +[Spice.ai OSS](https://github.com/spiceai/spiceai) is a portable, single-binary +runtime built on Apache DataFusion that federates and accelerates SQL queries +across databases, data warehouses, and data lakes. This entry benchmarks its +native Cayenne acceleration engine: `./load` ingests `hits.parquet` into a +disk-backed Cayenne table (`acceleration: {engine: cayenne, mode: file}`), and +queries are served from that acceleration through the runtime's HTTP API. + +Notes: + +- `spicepod.yaml` takes the place of `create.sql`: it defines the dataset and + its acceleration. The dataset keeps the source parquet schema, where + `EventDate` is an integer (days since epoch); the queries referencing + `EventDate` wrap it as `to_timestamp("EventDate" * 86400)`, matching how + Spice's own benchmark suite runs ClickBench. Everything else in + `queries.sql` is identical to the `datafusion` entry's. +- The SQL results cache (enabled by default in Spice) is disabled in + `spicepod.yaml` per the benchmark caching rules. +- The acceleration persists across restarts and the runtime skips re-ingestion + when the on-disk data is current, so the true-cold-run restart cycle serves + disk-resident data like other durable systems. + +Run `./benchmark.sh` on a fresh Ubuntu VM (see `lib/benchmark-common.sh` for +the flow), or use the repo's `run-benchmark.sh` automation. diff --git a/spiceai-cayenne/benchmark.sh b/spiceai-cayenne/benchmark.sh new file mode 100755 index 0000000000..ddd7f28318 --- /dev/null +++ b/spiceai-cayenne/benchmark.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Thin shim — actual flow is in lib/benchmark-common.sh. +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +exec ../lib/benchmark-common.sh diff --git a/spiceai-cayenne/check b/spiceai-cayenne/check new file mode 100755 index 0000000000..5f5f7dec56 --- /dev/null +++ b/spiceai-cayenne/check @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +# The driver starts the system before the dataset is downloaded; until +# hits.parquet exists the dataset cannot load (the runtime stays up and +# retries), so gate on process liveness only. Afterwards gate on +# /v1/ready, which returns 200 only once the accelerated dataset is +# ready to serve queries. +if [ -e hits.parquet ]; then + curl -fsS http://localhost:8090/v1/ready >/dev/null +else + curl -fsS http://localhost:8090/health >/dev/null +fi diff --git a/spiceai-cayenne/data-size b/spiceai-cayenne/data-size new file mode 100755 index 0000000000..fb91faf722 --- /dev/null +++ b/spiceai-cayenne/data-size @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +# Cayenne file-mode acceleration data (segments + metastore), written by +# the runtime under ./.spice. +du -sb .spice | cut -f1 diff --git a/spiceai-cayenne/install b/spiceai-cayenne/install new file mode 100755 index 0000000000..f855f18912 --- /dev/null +++ b/spiceai-cayenne/install @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# install.spiceai.org ships the CLI only; `spice install` then downloads +# the runtime binary (spiced) into ~/.spice/bin. +if [ ! -x "$HOME/.spice/bin/spice" ]; then + curl https://install.spiceai.org | /bin/bash +fi +if [ ! -x "$HOME/.spice/bin/spiced" ]; then + "$HOME/.spice/bin/spice" install >/dev/null 2>&1 +fi + +# Symlink into /usr/local/bin so sibling scripts (check, load, query, +# start) find them unconditionally. +sudo ln -sf "$HOME/.spice/bin/spiced" /usr/local/bin/spiced +sudo ln -sf "$HOME/.spice/bin/spice" /usr/local/bin/spice + +spiced --version diff --git a/spiceai-cayenne/load b/spiceai-cayenne/load new file mode 100755 index 0000000000..83487c8cc4 --- /dev/null +++ b/spiceai-cayenne/load @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +# Idempotent: drop any prior acceleration state and restart the runtime so +# the Cayenne acceleration performs a full ingest of hits.parquet. +./stop >/dev/null 2>&1 || true +rm -rf .spice spiced.log +./start + +# /v1/ready returns 200 once the initial refresh has completed and the +# dataset is queryable from the acceleration. +for _ in $(seq 1 7200); do + if curl -fsS http://localhost:8090/v1/ready >/dev/null 2>&1; then + exit 0 + fi + sleep 1 +done + +echo "load: dataset did not become ready within 7200s" >&2 +tail -n 50 spiced.log >&2 +exit 1 diff --git a/spiceai-cayenne/make-json.sh b/spiceai-cayenne/make-json.sh new file mode 100755 index 0000000000..34c4b1dfa0 --- /dev/null +++ b/spiceai-cayenne/make-json.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Converts the raw `result.csv` from `benchmark.sh` plus the `Load time:` / +# `Data size:` lines in `log` into the final json format used by the +# benchmark dashboard. +# +# usage: ./make-json.sh # saves results//.json +set -e + +MACHINE=$1 +[ -n "$MACHINE" ] || { echo "usage: $0 " >&2; exit 1; } + +DATE=$(date -u +%Y-%m-%d) +YYYYMMDD=${DATE//-/} +mkdir -p "results/${YYYYMMDD}" + +python3 - "$MACHINE" "$DATE" > "results/${YYYYMMDD}/${MACHINE}.json" <<'PY' +import csv, json, re, sys + +machine, date = sys.argv[1], sys.argv[2] + +runs = {} +with open("result.csv") as f: + for query, _try, timing in csv.reader(f): + runs.setdefault(int(query), []).append( + None if timing == "null" else float(timing)) + +load_time = data_size = None +qps = err_ratio = None +with open("log") as f: + for line in f: + if m := re.match(r"Load time: ([\d.]+)", line): + load_time = float(m.group(1)) + elif m := re.match(r"Data size: (\d+)", line): + data_size = int(m.group(1)) + elif m := re.match(r"Concurrent QPS: ([\d.]+)", line): + qps = float(m.group(1)) + elif m := re.match(r"Concurrent error ratio: ([\d.]+)", line): + err_ratio = float(m.group(1)) + +template = json.load(open("template.json")) +result = { + "system": template["system"], + "date": date, + "machine": machine, + "cluster_size": 1, + "proprietary": template["proprietary"], + "hardware": template["hardware"], + "tuned": template["tuned"], + "tags": template["tags"], + "load_time": load_time, + "data_size": data_size, + "concurrent_qps": qps, + "concurrent_error_ratio": err_ratio, + "result": [runs[q] for q in sorted(runs)], +} +print(json.dumps(result, indent=4)) +PY + +echo "results/${YYYYMMDD}/${MACHINE}.json" diff --git a/spiceai-cayenne/queries.sql b/spiceai-cayenne/queries.sql new file mode 100644 index 0000000000..ab9fe7965e --- /dev/null +++ b/spiceai-cayenne/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0; +SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits; +SELECT AVG("UserID") FROM hits; +SELECT COUNT(DISTINCT "UserID") FROM hits; +SELECT COUNT(DISTINCT "SearchPhrase") FROM hits; +SELECT MIN(to_timestamp("EventDate" * 86400)), MAX(to_timestamp("EventDate" * 86400)) FROM hits; +SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC; +SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10; +SELECT "RegionID", SUM("AdvEngineID"), COUNT(*) AS c, AVG("ResolutionWidth"), COUNT(DISTINCT "UserID") FROM hits GROUP BY "RegionID" ORDER BY c DESC LIMIT 10; +SELECT "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhoneModel" ORDER BY u DESC LIMIT 10; +SELECT "MobilePhone", "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhone", "MobilePhoneModel" ORDER BY u DESC LIMIT 10; +SELECT "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "SearchPhrase", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY u DESC LIMIT 10; +SELECT "SearchEngineID", "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "UserID", COUNT(*) FROM hits GROUP BY "UserID" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" LIMIT 10; +SELECT "UserID", extract(minute FROM to_timestamp_seconds("EventTime")) AS m, "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", m, "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID" FROM hits WHERE "UserID" = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE "URL" LIKE '%google%'; +SELECT "SearchPhrase", MIN("URL"), COUNT(*) AS c FROM hits WHERE "URL" LIKE '%google%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "SearchPhrase", MIN("URL"), MIN("Title"), COUNT(*) AS c, COUNT(DISTINCT "UserID") FROM hits WHERE "Title" LIKE '%Google%' AND "URL" NOT LIKE '%.google.%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE "URL" LIKE '%google%' ORDER BY "EventTime" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "SearchPhrase" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime", "SearchPhrase" LIMIT 10; +SELECT "CounterID", AVG(length("URL")) AS l, COUNT(*) AS c FROM hits WHERE "URL" <> '' GROUP BY "CounterID" HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE("Referer", '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length("Referer")) AS l, COUNT(*) AS c, MIN("Referer") FROM hits WHERE "Referer" <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM("ResolutionWidth"), SUM("ResolutionWidth" + 1), SUM("ResolutionWidth" + 2), SUM("ResolutionWidth" + 3), SUM("ResolutionWidth" + 4), SUM("ResolutionWidth" + 5), SUM("ResolutionWidth" + 6), SUM("ResolutionWidth" + 7), SUM("ResolutionWidth" + 8), SUM("ResolutionWidth" + 9), SUM("ResolutionWidth" + 10), SUM("ResolutionWidth" + 11), SUM("ResolutionWidth" + 12), SUM("ResolutionWidth" + 13), SUM("ResolutionWidth" + 14), SUM("ResolutionWidth" + 15), SUM("ResolutionWidth" + 16), SUM("ResolutionWidth" + 17), SUM("ResolutionWidth" + 18), SUM("ResolutionWidth" + 19), SUM("ResolutionWidth" + 20), SUM("ResolutionWidth" + 21), SUM("ResolutionWidth" + 22), SUM("ResolutionWidth" + 23), SUM("ResolutionWidth" + 24), SUM("ResolutionWidth" + 25), SUM("ResolutionWidth" + 26), SUM("ResolutionWidth" + 27), SUM("ResolutionWidth" + 28), SUM("ResolutionWidth" + 29), SUM("ResolutionWidth" + 30), SUM("ResolutionWidth" + 31), SUM("ResolutionWidth" + 32), SUM("ResolutionWidth" + 33), SUM("ResolutionWidth" + 34), SUM("ResolutionWidth" + 35), SUM("ResolutionWidth" + 36), SUM("ResolutionWidth" + 37), SUM("ResolutionWidth" + 38), SUM("ResolutionWidth" + 39), SUM("ResolutionWidth" + 40), SUM("ResolutionWidth" + 41), SUM("ResolutionWidth" + 42), SUM("ResolutionWidth" + 43), SUM("ResolutionWidth" + 44), SUM("ResolutionWidth" + 45), SUM("ResolutionWidth" + 46), SUM("ResolutionWidth" + 47), SUM("ResolutionWidth" + 48), SUM("ResolutionWidth" + 49), SUM("ResolutionWidth" + 50), SUM("ResolutionWidth" + 51), SUM("ResolutionWidth" + 52), SUM("ResolutionWidth" + 53), SUM("ResolutionWidth" + 54), SUM("ResolutionWidth" + 55), SUM("ResolutionWidth" + 56), SUM("ResolutionWidth" + 57), SUM("ResolutionWidth" + 58), SUM("ResolutionWidth" + 59), SUM("ResolutionWidth" + 60), SUM("ResolutionWidth" + 61), SUM("ResolutionWidth" + 62), SUM("ResolutionWidth" + 63), SUM("ResolutionWidth" + 64), SUM("ResolutionWidth" + 65), SUM("ResolutionWidth" + 66), SUM("ResolutionWidth" + 67), SUM("ResolutionWidth" + 68), SUM("ResolutionWidth" + 69), SUM("ResolutionWidth" + 70), SUM("ResolutionWidth" + 71), SUM("ResolutionWidth" + 72), SUM("ResolutionWidth" + 73), SUM("ResolutionWidth" + 74), SUM("ResolutionWidth" + 75), SUM("ResolutionWidth" + 76), SUM("ResolutionWidth" + 77), SUM("ResolutionWidth" + 78), SUM("ResolutionWidth" + 79), SUM("ResolutionWidth" + 80), SUM("ResolutionWidth" + 81), SUM("ResolutionWidth" + 82), SUM("ResolutionWidth" + 83), SUM("ResolutionWidth" + 84), SUM("ResolutionWidth" + 85), SUM("ResolutionWidth" + 86), SUM("ResolutionWidth" + 87), SUM("ResolutionWidth" + 88), SUM("ResolutionWidth" + 89) FROM hits; +SELECT "SearchEngineID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "URL", COUNT(*) AS c FROM hits GROUP BY "URL" ORDER BY c DESC LIMIT 10; +SELECT 1, "URL", COUNT(*) AS c FROM hits GROUP BY 1, "URL" ORDER BY c DESC LIMIT 10; +SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c FROM hits GROUP BY "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3 ORDER BY c DESC LIMIT 10; +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "URL" <> '' GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10; +SELECT "Title", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY PageViews DESC LIMIT 10; +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT "URLHash", to_timestamp("EventDate" * 86400), COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", to_timestamp("EventDate" * 86400) ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', to_timestamp("EventTime")::timestamp) AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-14' AND to_timestamp("EventDate" * 86400) <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp("EventTime")::timestamp) ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000; diff --git a/spiceai-cayenne/query b/spiceai-cayenne/query new file mode 100755 index 0000000000..cc45644401 --- /dev/null +++ b/spiceai-cayenne/query @@ -0,0 +1,24 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it via the Spice HTTP API. +# Stdout: query result (JSON). +# Stderr: query runtime in fractional seconds on the last line. +# Exit non-zero on error. +set -e + +resp=$(mktemp /tmp/spiceai-query.XXXXXX) +trap 'rm -f "$resp"' EXIT + +metrics=$(curl -sS -o "$resp" -w '%{http_code} %{time_total}' \ + -H 'Content-Type: text/plain' --data-binary @- \ + 'http://localhost:8090/v1/sql') +code=${metrics% *} +time_total=${metrics#* } + +if [ "$code" != "200" ]; then + cat "$resp" >&2 + echo >&2 + exit 1 +fi + +cat "$resp" +printf '%s\n' "$time_total" >&2 diff --git a/spiceai-cayenne/results/20260708/c6a.4xlarge.json b/spiceai-cayenne/results/20260708/c6a.4xlarge.json new file mode 100644 index 0000000000..db12b61868 --- /dev/null +++ b/spiceai-cayenne/results/20260708/c6a.4xlarge.json @@ -0,0 +1,235 @@ +{ + "system": "Spice.ai OSS (Cayenne)", + "date": "2026-07-08", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "comment": "v2.1.0 (d70bd02e)", + "tags": [ + "Rust", + "column-oriented" + ], + "load_time": 439.864, + "data_size": 16653368704, + "concurrent_qps": 0.138, + "concurrent_error_ratio": 0.078, + "result": [ + [ + 0.541025, + 0.002376, + 0.001888 + ], + [ + 0.333733, + 0.042184, + 0.039156 + ], + [ + 0.215984, + 0.002611, + 0.002312 + ], + [ + 0.829599, + 0.079815, + 0.073717 + ], + [ + 1.400724, + 0.590428, + 0.575713 + ], + [ + 1.731595, + 0.667146, + 0.65541 + ], + [ + 0.330928, + 0.059293, + 0.05664 + ], + [ + 0.34638, + 0.043362, + 0.040531 + ], + [ + 1.577182, + 0.741808, + 0.743441 + ], + [ + 1.934725, + 0.799657, + 0.782161 + ], + [ + 0.938643, + 0.127029, + 0.126503 + ], + [ + 1.065055, + 0.145258, + 0.144827 + ], + [ + 1.818895, + 0.601749, + 0.584595 + ], + [ + 3.463769, + 0.921314, + 0.916361 + ], + [ + 1.992817, + 0.627397, + 0.6229 + ], + [ + 1.179182, + 0.681079, + 0.662493 + ], + [ + 3.79147, + 1.518668, + 1.485087 + ], + [ + 3.740524, + 1.503758, + 1.472 + ], + [ + 5.699322, + 2.760679, + 2.700854 + ], + [ + 0.766611, + 0.052886, + 0.047767 + ], + [ + 17.255802, + 0.727122, + 0.724388 + ], + [ + 19.671555, + 0.821539, + 0.824372 + ], + [ + 28.473381, + 1.076091, + 1.076941 + ], + [ + 32.958324, + 1.330373, + 1.231765 + ], + [ + 1.159485, + 0.070288, + 0.068416 + ], + [ + 1.507051, + 0.119574, + 0.117828 + ], + [ + 1.89583, + 0.129746, + 0.128066 + ], + [ + 17.324405, + 1.471089, + 1.455798 + ], + [ + 13.637127, + 4.11283, + 4.046217 + ], + [ + 0.359811, + 0.063103, + 0.064229 + ], + [ + 3.352665, + 0.555896, + 0.551688 + ], + [ + 6.3518, + 0.627421, + 0.623536 + ], + [ + 4.202002, + 2.355144, + 2.297643 + ], + [ + 18.569898, + 3.870132, + 3.805599 + ], + [ + 18.760597, + 3.784668, + 3.809337 + ], + [ + 0.942918, + 0.564804, + 0.561066 + ], + [ + 0.423084, + 0.072023, + 0.069313 + ], + [ + 0.368602, + 0.039651, + 0.03655 + ], + [ + 0.372777, + 0.035007, + 0.032929 + ], + [ + 0.549817, + 0.144022, + 0.125882 + ], + [ + 0.382629, + 0.02407, + 0.023169 + ], + [ + 0.369341, + 0.023066, + 0.022332 + ], + [ + 0.377473, + 0.02277, + 0.021804 + ] + ] +} \ No newline at end of file diff --git a/spiceai-cayenne/spicepod.yaml b/spiceai-cayenne/spicepod.yaml new file mode 100644 index 0000000000..58da032f30 --- /dev/null +++ b/spiceai-cayenne/spicepod.yaml @@ -0,0 +1,18 @@ +version: v2 +kind: Spicepod +name: spiceai-cayenne-clickbench + +runtime: + caching: + # The SQL results cache is enabled by default; the benchmark rules + # require query result caches to be disabled. + sql_results: + enabled: false + +datasets: + - from: file:hits.parquet + name: hits + acceleration: + enabled: true + engine: cayenne + mode: file diff --git a/spiceai-cayenne/start b/spiceai-cayenne/start new file mode 100755 index 0000000000..f5c3edca85 --- /dev/null +++ b/spiceai-cayenne/start @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +# spiced loads ./spicepod.yaml from the working directory and serves the +# HTTP API on 127.0.0.1:8090. Logs append to spiced.log for debugging. +nohup spiced >> spiced.log 2>&1 & diff --git a/spiceai-cayenne/stop b/spiceai-cayenne/stop new file mode 100755 index 0000000000..1845b3e41c --- /dev/null +++ b/spiceai-cayenne/stop @@ -0,0 +1,10 @@ +#!/bin/bash + +pkill spiced || exit 0 + +# Wait for graceful shutdown, then escalate. +for _ in $(seq 1 30); do + pgrep spiced >/dev/null || exit 0 + sleep 1 +done +pkill -9 spiced || true diff --git a/spiceai-cayenne/template.json b/spiceai-cayenne/template.json new file mode 100644 index 0000000000..493d218cb1 --- /dev/null +++ b/spiceai-cayenne/template.json @@ -0,0 +1,10 @@ +{ + "system": "Spice.ai OSS (Cayenne)", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "Rust", + "column-oriented" + ] +} \ No newline at end of file diff --git a/spiceai-parquet-partitioned/README.md b/spiceai-parquet-partitioned/README.md new file mode 100644 index 0000000000..c61704698f --- /dev/null +++ b/spiceai-parquet-partitioned/README.md @@ -0,0 +1,27 @@ +# Spice.ai OSS (Parquet, partitioned) + +[Spice.ai OSS](https://github.com/spiceai/spiceai) is a portable, single-binary +runtime built on Apache DataFusion that federates and accelerates SQL queries +across databases, data warehouses, and data lakes. This entry benchmarks the +runtime querying the 100-file partitioned `hits_{0..99}.parquet` dataset +directly through its file connector — no acceleration, no data loading; the +files are scanned in place as a single table. See `spiceai-parquet` for the +single-file variant and `spiceai-cayenne` for the same runtime with its +Cayenne acceleration engine. + +Notes: + +- Spice's file connector registers a dataset from either a single file or a + directory (globbing it for files matching `file_format`); a literal glob + path like `hits_*.parquet` is not expanded. `./load` therefore symlinks + the downloaded `hits_*.parquet` partitions into a `data/` subdirectory, + and `spicepod.yaml` registers `file:data` with `file_format: parquet`. +- The source parquet stores `EventDate` as an integer (days since epoch); + the queries referencing `EventDate` wrap it as + `to_timestamp("EventDate" * 86400)`. `queries.sql` is identical to the + `spiceai-parquet` and `spiceai-cayenne` entries'. +- The SQL results cache (enabled by default in Spice) is disabled in + `spicepod.yaml` per the benchmark caching rules. + +Run `./benchmark.sh` on a fresh Ubuntu VM (see `lib/benchmark-common.sh` for +the flow), or use the repo's `run-benchmark.sh` automation. diff --git a/spiceai-parquet-partitioned/benchmark.sh b/spiceai-parquet-partitioned/benchmark.sh new file mode 100755 index 0000000000..12a00b626c --- /dev/null +++ b/spiceai-parquet-partitioned/benchmark.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Thin shim — actual flow is in lib/benchmark-common.sh. +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" +exec ../lib/benchmark-common.sh diff --git a/spiceai-parquet-partitioned/check b/spiceai-parquet-partitioned/check new file mode 100755 index 0000000000..e30f942c54 --- /dev/null +++ b/spiceai-parquet-partitioned/check @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +# The driver starts the system before the dataset is downloaded; until the +# partitioned files exist the dataset cannot load (the runtime stays up and +# retries), so gate on process liveness only. Afterwards gate on /v1/ready, +# which returns 200 only once the accelerated dataset is ready to serve +# queries. +if compgen -G "hits_*.parquet" >/dev/null; then + curl -fsS http://localhost:8090/v1/ready >/dev/null +else + curl -fsS http://localhost:8090/health >/dev/null +fi diff --git a/spiceai-parquet-partitioned/data-size b/spiceai-parquet-partitioned/data-size new file mode 100755 index 0000000000..39f20d40ad --- /dev/null +++ b/spiceai-parquet-partitioned/data-size @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +wc -c hits_*.parquet | tail -n1 | awk '{print $1}' diff --git a/spiceai-parquet-partitioned/install b/spiceai-parquet-partitioned/install new file mode 100755 index 0000000000..f855f18912 --- /dev/null +++ b/spiceai-parquet-partitioned/install @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# install.spiceai.org ships the CLI only; `spice install` then downloads +# the runtime binary (spiced) into ~/.spice/bin. +if [ ! -x "$HOME/.spice/bin/spice" ]; then + curl https://install.spiceai.org | /bin/bash +fi +if [ ! -x "$HOME/.spice/bin/spiced" ]; then + "$HOME/.spice/bin/spice" install >/dev/null 2>&1 +fi + +# Symlink into /usr/local/bin so sibling scripts (check, load, query, +# start) find them unconditionally. +sudo ln -sf "$HOME/.spice/bin/spiced" /usr/local/bin/spiced +sudo ln -sf "$HOME/.spice/bin/spice" /usr/local/bin/spice + +spiced --version diff --git a/spiceai-parquet-partitioned/load b/spiceai-parquet-partitioned/load new file mode 100755 index 0000000000..b74f48a062 --- /dev/null +++ b/spiceai-parquet-partitioned/load @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +# No acceleration: queries scan hits_*.parquet directly, so there is +# nothing to ingest. Spice's file connector globs a directory for files +# matching file_format (a literal `hits_*.parquet` path is not expanded +# as a glob), so symlink the downloaded partitions into ./data rather +# than copying them. +rm -rf data +mkdir -p data +for f in hits_*.parquet; do + ln -sf "$PWD/$f" "data/$f" +done + +./stop >/dev/null 2>&1 || true +rm -f spiced.log +./start + +for _ in $(seq 1 300); do + if curl -fsS http://localhost:8090/v1/ready >/dev/null 2>&1; then + exit 0 + fi + sleep 1 +done + +echo "load: dataset did not become ready within 300s" >&2 +tail -n 50 spiced.log >&2 +exit 1 diff --git a/spiceai-parquet-partitioned/make-json.sh b/spiceai-parquet-partitioned/make-json.sh new file mode 100755 index 0000000000..34c4b1dfa0 --- /dev/null +++ b/spiceai-parquet-partitioned/make-json.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Converts the raw `result.csv` from `benchmark.sh` plus the `Load time:` / +# `Data size:` lines in `log` into the final json format used by the +# benchmark dashboard. +# +# usage: ./make-json.sh # saves results//.json +set -e + +MACHINE=$1 +[ -n "$MACHINE" ] || { echo "usage: $0 " >&2; exit 1; } + +DATE=$(date -u +%Y-%m-%d) +YYYYMMDD=${DATE//-/} +mkdir -p "results/${YYYYMMDD}" + +python3 - "$MACHINE" "$DATE" > "results/${YYYYMMDD}/${MACHINE}.json" <<'PY' +import csv, json, re, sys + +machine, date = sys.argv[1], sys.argv[2] + +runs = {} +with open("result.csv") as f: + for query, _try, timing in csv.reader(f): + runs.setdefault(int(query), []).append( + None if timing == "null" else float(timing)) + +load_time = data_size = None +qps = err_ratio = None +with open("log") as f: + for line in f: + if m := re.match(r"Load time: ([\d.]+)", line): + load_time = float(m.group(1)) + elif m := re.match(r"Data size: (\d+)", line): + data_size = int(m.group(1)) + elif m := re.match(r"Concurrent QPS: ([\d.]+)", line): + qps = float(m.group(1)) + elif m := re.match(r"Concurrent error ratio: ([\d.]+)", line): + err_ratio = float(m.group(1)) + +template = json.load(open("template.json")) +result = { + "system": template["system"], + "date": date, + "machine": machine, + "cluster_size": 1, + "proprietary": template["proprietary"], + "hardware": template["hardware"], + "tuned": template["tuned"], + "tags": template["tags"], + "load_time": load_time, + "data_size": data_size, + "concurrent_qps": qps, + "concurrent_error_ratio": err_ratio, + "result": [runs[q] for q in sorted(runs)], +} +print(json.dumps(result, indent=4)) +PY + +echo "results/${YYYYMMDD}/${MACHINE}.json" diff --git a/spiceai-parquet-partitioned/queries.sql b/spiceai-parquet-partitioned/queries.sql new file mode 100644 index 0000000000..ab9fe7965e --- /dev/null +++ b/spiceai-parquet-partitioned/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0; +SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits; +SELECT AVG("UserID") FROM hits; +SELECT COUNT(DISTINCT "UserID") FROM hits; +SELECT COUNT(DISTINCT "SearchPhrase") FROM hits; +SELECT MIN(to_timestamp("EventDate" * 86400)), MAX(to_timestamp("EventDate" * 86400)) FROM hits; +SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC; +SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10; +SELECT "RegionID", SUM("AdvEngineID"), COUNT(*) AS c, AVG("ResolutionWidth"), COUNT(DISTINCT "UserID") FROM hits GROUP BY "RegionID" ORDER BY c DESC LIMIT 10; +SELECT "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhoneModel" ORDER BY u DESC LIMIT 10; +SELECT "MobilePhone", "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhone", "MobilePhoneModel" ORDER BY u DESC LIMIT 10; +SELECT "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "SearchPhrase", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY u DESC LIMIT 10; +SELECT "SearchEngineID", "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "UserID", COUNT(*) FROM hits GROUP BY "UserID" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" LIMIT 10; +SELECT "UserID", extract(minute FROM to_timestamp_seconds("EventTime")) AS m, "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", m, "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID" FROM hits WHERE "UserID" = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE "URL" LIKE '%google%'; +SELECT "SearchPhrase", MIN("URL"), COUNT(*) AS c FROM hits WHERE "URL" LIKE '%google%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "SearchPhrase", MIN("URL"), MIN("Title"), COUNT(*) AS c, COUNT(DISTINCT "UserID") FROM hits WHERE "Title" LIKE '%Google%' AND "URL" NOT LIKE '%.google.%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE "URL" LIKE '%google%' ORDER BY "EventTime" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "SearchPhrase" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime", "SearchPhrase" LIMIT 10; +SELECT "CounterID", AVG(length("URL")) AS l, COUNT(*) AS c FROM hits WHERE "URL" <> '' GROUP BY "CounterID" HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE("Referer", '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length("Referer")) AS l, COUNT(*) AS c, MIN("Referer") FROM hits WHERE "Referer" <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM("ResolutionWidth"), SUM("ResolutionWidth" + 1), SUM("ResolutionWidth" + 2), SUM("ResolutionWidth" + 3), SUM("ResolutionWidth" + 4), SUM("ResolutionWidth" + 5), SUM("ResolutionWidth" + 6), SUM("ResolutionWidth" + 7), SUM("ResolutionWidth" + 8), SUM("ResolutionWidth" + 9), SUM("ResolutionWidth" + 10), SUM("ResolutionWidth" + 11), SUM("ResolutionWidth" + 12), SUM("ResolutionWidth" + 13), SUM("ResolutionWidth" + 14), SUM("ResolutionWidth" + 15), SUM("ResolutionWidth" + 16), SUM("ResolutionWidth" + 17), SUM("ResolutionWidth" + 18), SUM("ResolutionWidth" + 19), SUM("ResolutionWidth" + 20), SUM("ResolutionWidth" + 21), SUM("ResolutionWidth" + 22), SUM("ResolutionWidth" + 23), SUM("ResolutionWidth" + 24), SUM("ResolutionWidth" + 25), SUM("ResolutionWidth" + 26), SUM("ResolutionWidth" + 27), SUM("ResolutionWidth" + 28), SUM("ResolutionWidth" + 29), SUM("ResolutionWidth" + 30), SUM("ResolutionWidth" + 31), SUM("ResolutionWidth" + 32), SUM("ResolutionWidth" + 33), SUM("ResolutionWidth" + 34), SUM("ResolutionWidth" + 35), SUM("ResolutionWidth" + 36), SUM("ResolutionWidth" + 37), SUM("ResolutionWidth" + 38), SUM("ResolutionWidth" + 39), SUM("ResolutionWidth" + 40), SUM("ResolutionWidth" + 41), SUM("ResolutionWidth" + 42), SUM("ResolutionWidth" + 43), SUM("ResolutionWidth" + 44), SUM("ResolutionWidth" + 45), SUM("ResolutionWidth" + 46), SUM("ResolutionWidth" + 47), SUM("ResolutionWidth" + 48), SUM("ResolutionWidth" + 49), SUM("ResolutionWidth" + 50), SUM("ResolutionWidth" + 51), SUM("ResolutionWidth" + 52), SUM("ResolutionWidth" + 53), SUM("ResolutionWidth" + 54), SUM("ResolutionWidth" + 55), SUM("ResolutionWidth" + 56), SUM("ResolutionWidth" + 57), SUM("ResolutionWidth" + 58), SUM("ResolutionWidth" + 59), SUM("ResolutionWidth" + 60), SUM("ResolutionWidth" + 61), SUM("ResolutionWidth" + 62), SUM("ResolutionWidth" + 63), SUM("ResolutionWidth" + 64), SUM("ResolutionWidth" + 65), SUM("ResolutionWidth" + 66), SUM("ResolutionWidth" + 67), SUM("ResolutionWidth" + 68), SUM("ResolutionWidth" + 69), SUM("ResolutionWidth" + 70), SUM("ResolutionWidth" + 71), SUM("ResolutionWidth" + 72), SUM("ResolutionWidth" + 73), SUM("ResolutionWidth" + 74), SUM("ResolutionWidth" + 75), SUM("ResolutionWidth" + 76), SUM("ResolutionWidth" + 77), SUM("ResolutionWidth" + 78), SUM("ResolutionWidth" + 79), SUM("ResolutionWidth" + 80), SUM("ResolutionWidth" + 81), SUM("ResolutionWidth" + 82), SUM("ResolutionWidth" + 83), SUM("ResolutionWidth" + 84), SUM("ResolutionWidth" + 85), SUM("ResolutionWidth" + 86), SUM("ResolutionWidth" + 87), SUM("ResolutionWidth" + 88), SUM("ResolutionWidth" + 89) FROM hits; +SELECT "SearchEngineID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "URL", COUNT(*) AS c FROM hits GROUP BY "URL" ORDER BY c DESC LIMIT 10; +SELECT 1, "URL", COUNT(*) AS c FROM hits GROUP BY 1, "URL" ORDER BY c DESC LIMIT 10; +SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c FROM hits GROUP BY "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3 ORDER BY c DESC LIMIT 10; +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "URL" <> '' GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10; +SELECT "Title", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY PageViews DESC LIMIT 10; +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT "URLHash", to_timestamp("EventDate" * 86400), COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", to_timestamp("EventDate" * 86400) ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', to_timestamp("EventTime")::timestamp) AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-14' AND to_timestamp("EventDate" * 86400) <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp("EventTime")::timestamp) ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000; diff --git a/spiceai-parquet-partitioned/query b/spiceai-parquet-partitioned/query new file mode 100755 index 0000000000..cc45644401 --- /dev/null +++ b/spiceai-parquet-partitioned/query @@ -0,0 +1,24 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it via the Spice HTTP API. +# Stdout: query result (JSON). +# Stderr: query runtime in fractional seconds on the last line. +# Exit non-zero on error. +set -e + +resp=$(mktemp /tmp/spiceai-query.XXXXXX) +trap 'rm -f "$resp"' EXIT + +metrics=$(curl -sS -o "$resp" -w '%{http_code} %{time_total}' \ + -H 'Content-Type: text/plain' --data-binary @- \ + 'http://localhost:8090/v1/sql') +code=${metrics% *} +time_total=${metrics#* } + +if [ "$code" != "200" ]; then + cat "$resp" >&2 + echo >&2 + exit 1 +fi + +cat "$resp" +printf '%s\n' "$time_total" >&2 diff --git a/spiceai-parquet-partitioned/results/20260708/c6a.4xlarge.json b/spiceai-parquet-partitioned/results/20260708/c6a.4xlarge.json new file mode 100644 index 0000000000..4b46026aae --- /dev/null +++ b/spiceai-parquet-partitioned/results/20260708/c6a.4xlarge.json @@ -0,0 +1,236 @@ +{ + "system": "Spice.ai OSS (Parquet, partitioned)", + "date": "2026-07-08", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "comment": "v2.1.0 (d70bd02e)", + "tags": [ + "Rust", + "column-oriented", + "stateless" + ], + "load_time": 21.639, + "data_size": 14737666736, + "concurrent_qps": 0.305, + "concurrent_error_ratio": 0.08, + "result": [ + [ + 0.266677, + 0.025565, + 0.026589 + ], + [ + 0.331105, + 0.044808, + 0.039505 + ], + [ + 0.40023, + 0.080613, + 0.077481 + ], + [ + 0.928506, + 0.084913, + 0.078737 + ], + [ + 1.239955, + 0.601085, + 0.597949 + ], + [ + 1.557156, + 0.667725, + 0.648925 + ], + [ + 0.387307, + 0.085309, + 0.083323 + ], + [ + 0.392488, + 0.046502, + 0.044859 + ], + [ + 1.550604, + 0.735967, + 0.724215 + ], + [ + 2.160796, + 0.76898, + 0.795667 + ], + [ + 1.027491, + 0.225705, + 0.225406 + ], + [ + 1.132752, + 0.243773, + 0.243606 + ], + [ + 1.644884, + 0.775922, + 0.767459 + ], + [ + 3.148758, + 1.129459, + 1.124296 + ], + [ + 1.660663, + 0.799313, + 0.78404 + ], + [ + 1.339073, + 0.690864, + 0.683859 + ], + [ + 3.485837, + 1.450033, + 1.431677 + ], + [ + 3.460219, + 1.470826, + 1.424778 + ], + [ + 5.757591, + 2.815286, + 2.785833 + ], + [ + 0.446934, + 0.098861, + 0.094031 + ], + [ + 9.927081, + 1.46401, + 1.467484 + ], + [ + 11.568063, + 1.885035, + 1.871188 + ], + [ + 22.094685, + 3.3485, + 3.3525 + ], + [ + 3.34397, + 0.419457, + 0.406351 + ], + [ + 0.486623, + 0.127648, + 0.124667 + ], + [ + 1.072092, + 0.341232, + 0.335827 + ], + [ + 0.452702, + 0.142313, + 0.139198 + ], + [ + 10.062491, + 2.024342, + 2.027589 + ], + [ + 8.732587, + 4.271268, + 4.273548 + ], + [ + 0.391648, + 0.077321, + 0.073968 + ], + [ + 3.399821, + 0.69817, + 0.695774 + ], + [ + 6.893871, + 0.818703, + 0.810634 + ], + [ + 5.308011, + 2.401455, + 2.352789 + ], + [ + 11.050388, + 4.07009, + 4.036636 + ], + [ + 11.136782, + 4.118912, + 4.090831 + ], + [ + 1.021579, + 0.602331, + 0.597663 + ], + [ + 0.546881, + 0.159117, + 0.139296 + ], + [ + 0.445326, + 0.083123, + 0.079534 + ], + [ + 0.495741, + 0.091233, + 0.086066 + ], + [ + 0.690627, + 0.245774, + 0.212929 + ], + [ + 0.422863, + 0.054441, + 0.050424 + ], + [ + 0.388706, + 0.04898, + 0.048294 + ], + [ + 0.393554, + 0.045629, + 0.043093 + ] + ] +} \ No newline at end of file diff --git a/spiceai-parquet-partitioned/spicepod.yaml b/spiceai-parquet-partitioned/spicepod.yaml new file mode 100644 index 0000000000..df26d4c8c4 --- /dev/null +++ b/spiceai-parquet-partitioned/spicepod.yaml @@ -0,0 +1,19 @@ +version: v2 +kind: Spicepod +name: spiceai-parquet-partitioned-clickbench + +runtime: + caching: + # The SQL results cache is enabled by default; the benchmark rules + # require query result caches to be disabled. + sql_results: + enabled: false + +datasets: + # Directory of hits_0.parquet … hits_99.parquet, registered as one table + # via the file connector's listing-table support (glob-scans the dir for + # files matching file_format). + - from: file:data + name: hits + params: + file_format: parquet diff --git a/spiceai-parquet-partitioned/start b/spiceai-parquet-partitioned/start new file mode 100755 index 0000000000..f5c3edca85 --- /dev/null +++ b/spiceai-parquet-partitioned/start @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +# spiced loads ./spicepod.yaml from the working directory and serves the +# HTTP API on 127.0.0.1:8090. Logs append to spiced.log for debugging. +nohup spiced >> spiced.log 2>&1 & diff --git a/spiceai-parquet-partitioned/stop b/spiceai-parquet-partitioned/stop new file mode 100755 index 0000000000..1845b3e41c --- /dev/null +++ b/spiceai-parquet-partitioned/stop @@ -0,0 +1,10 @@ +#!/bin/bash + +pkill spiced || exit 0 + +# Wait for graceful shutdown, then escalate. +for _ in $(seq 1 30); do + pgrep spiced >/dev/null || exit 0 + sleep 1 +done +pkill -9 spiced || true diff --git a/spiceai-parquet-partitioned/template.json b/spiceai-parquet-partitioned/template.json new file mode 100644 index 0000000000..e2f0f645f9 --- /dev/null +++ b/spiceai-parquet-partitioned/template.json @@ -0,0 +1,11 @@ +{ + "system": "Spice.ai OSS (Parquet, partitioned)", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "Rust", + "column-oriented", + "stateless" + ] +} diff --git a/spiceai-parquet/README.md b/spiceai-parquet/README.md new file mode 100644 index 0000000000..e3c8c23c87 --- /dev/null +++ b/spiceai-parquet/README.md @@ -0,0 +1,22 @@ +# Spice.ai OSS (Parquet, single) + +[Spice.ai OSS](https://github.com/spiceai/spiceai) is a portable, single-binary +runtime built on Apache DataFusion that federates and accelerates SQL queries +across databases, data warehouses, and data lakes. This entry benchmarks the +runtime querying `hits.parquet` directly through its file connector — no +acceleration, no data loading; the single parquet file is scanned in place. +See `spiceai-parquet-partitioned` for the 100-file variant and +`spiceai-cayenne` for the same runtime with its Cayenne acceleration engine. + +Notes: + +- `spicepod.yaml` takes the place of `create.sql`: it registers the parquet + file as a dataset. The source parquet stores `EventDate` as an integer + (days since epoch); the queries referencing `EventDate` wrap it as + `to_timestamp("EventDate" * 86400)`. `queries.sql` is identical to the + `spiceai-cayenne` entry's. +- The SQL results cache (enabled by default in Spice) is disabled in + `spicepod.yaml` per the benchmark caching rules. + +Run `./benchmark.sh` on a fresh Ubuntu VM (see `lib/benchmark-common.sh` for +the flow), or use the repo's `run-benchmark.sh` automation. diff --git a/spiceai-parquet/benchmark.sh b/spiceai-parquet/benchmark.sh new file mode 100755 index 0000000000..ddd7f28318 --- /dev/null +++ b/spiceai-parquet/benchmark.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Thin shim — actual flow is in lib/benchmark-common.sh. +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +exec ../lib/benchmark-common.sh diff --git a/spiceai-parquet/check b/spiceai-parquet/check new file mode 100755 index 0000000000..5f5f7dec56 --- /dev/null +++ b/spiceai-parquet/check @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +# The driver starts the system before the dataset is downloaded; until +# hits.parquet exists the dataset cannot load (the runtime stays up and +# retries), so gate on process liveness only. Afterwards gate on +# /v1/ready, which returns 200 only once the accelerated dataset is +# ready to serve queries. +if [ -e hits.parquet ]; then + curl -fsS http://localhost:8090/v1/ready >/dev/null +else + curl -fsS http://localhost:8090/health >/dev/null +fi diff --git a/spiceai-parquet/data-size b/spiceai-parquet/data-size new file mode 100755 index 0000000000..708c0b72e7 --- /dev/null +++ b/spiceai-parquet/data-size @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +wc -c < hits.parquet diff --git a/spiceai-parquet/install b/spiceai-parquet/install new file mode 100755 index 0000000000..f855f18912 --- /dev/null +++ b/spiceai-parquet/install @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# install.spiceai.org ships the CLI only; `spice install` then downloads +# the runtime binary (spiced) into ~/.spice/bin. +if [ ! -x "$HOME/.spice/bin/spice" ]; then + curl https://install.spiceai.org | /bin/bash +fi +if [ ! -x "$HOME/.spice/bin/spiced" ]; then + "$HOME/.spice/bin/spice" install >/dev/null 2>&1 +fi + +# Symlink into /usr/local/bin so sibling scripts (check, load, query, +# start) find them unconditionally. +sudo ln -sf "$HOME/.spice/bin/spiced" /usr/local/bin/spiced +sudo ln -sf "$HOME/.spice/bin/spice" /usr/local/bin/spice + +spiced --version diff --git a/spiceai-parquet/load b/spiceai-parquet/load new file mode 100755 index 0000000000..799be068d4 --- /dev/null +++ b/spiceai-parquet/load @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +# No acceleration: queries scan hits.parquet directly, so there is nothing +# to ingest. Restart the runtime so it (re)registers the dataset now that +# the file exists; /v1/ready returns 200 once the dataset is queryable. +./stop >/dev/null 2>&1 || true +rm -f spiced.log +./start + +for _ in $(seq 1 300); do + if curl -fsS http://localhost:8090/v1/ready >/dev/null 2>&1; then + exit 0 + fi + sleep 1 +done + +echo "load: dataset did not become ready within 300s" >&2 +tail -n 50 spiced.log >&2 +exit 1 diff --git a/spiceai-parquet/make-json.sh b/spiceai-parquet/make-json.sh new file mode 100755 index 0000000000..34c4b1dfa0 --- /dev/null +++ b/spiceai-parquet/make-json.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Converts the raw `result.csv` from `benchmark.sh` plus the `Load time:` / +# `Data size:` lines in `log` into the final json format used by the +# benchmark dashboard. +# +# usage: ./make-json.sh # saves results//.json +set -e + +MACHINE=$1 +[ -n "$MACHINE" ] || { echo "usage: $0 " >&2; exit 1; } + +DATE=$(date -u +%Y-%m-%d) +YYYYMMDD=${DATE//-/} +mkdir -p "results/${YYYYMMDD}" + +python3 - "$MACHINE" "$DATE" > "results/${YYYYMMDD}/${MACHINE}.json" <<'PY' +import csv, json, re, sys + +machine, date = sys.argv[1], sys.argv[2] + +runs = {} +with open("result.csv") as f: + for query, _try, timing in csv.reader(f): + runs.setdefault(int(query), []).append( + None if timing == "null" else float(timing)) + +load_time = data_size = None +qps = err_ratio = None +with open("log") as f: + for line in f: + if m := re.match(r"Load time: ([\d.]+)", line): + load_time = float(m.group(1)) + elif m := re.match(r"Data size: (\d+)", line): + data_size = int(m.group(1)) + elif m := re.match(r"Concurrent QPS: ([\d.]+)", line): + qps = float(m.group(1)) + elif m := re.match(r"Concurrent error ratio: ([\d.]+)", line): + err_ratio = float(m.group(1)) + +template = json.load(open("template.json")) +result = { + "system": template["system"], + "date": date, + "machine": machine, + "cluster_size": 1, + "proprietary": template["proprietary"], + "hardware": template["hardware"], + "tuned": template["tuned"], + "tags": template["tags"], + "load_time": load_time, + "data_size": data_size, + "concurrent_qps": qps, + "concurrent_error_ratio": err_ratio, + "result": [runs[q] for q in sorted(runs)], +} +print(json.dumps(result, indent=4)) +PY + +echo "results/${YYYYMMDD}/${MACHINE}.json" diff --git a/spiceai-parquet/queries.sql b/spiceai-parquet/queries.sql new file mode 100644 index 0000000000..ab9fe7965e --- /dev/null +++ b/spiceai-parquet/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0; +SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits; +SELECT AVG("UserID") FROM hits; +SELECT COUNT(DISTINCT "UserID") FROM hits; +SELECT COUNT(DISTINCT "SearchPhrase") FROM hits; +SELECT MIN(to_timestamp("EventDate" * 86400)), MAX(to_timestamp("EventDate" * 86400)) FROM hits; +SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC; +SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10; +SELECT "RegionID", SUM("AdvEngineID"), COUNT(*) AS c, AVG("ResolutionWidth"), COUNT(DISTINCT "UserID") FROM hits GROUP BY "RegionID" ORDER BY c DESC LIMIT 10; +SELECT "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhoneModel" ORDER BY u DESC LIMIT 10; +SELECT "MobilePhone", "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhone", "MobilePhoneModel" ORDER BY u DESC LIMIT 10; +SELECT "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "SearchPhrase", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY u DESC LIMIT 10; +SELECT "SearchEngineID", "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "UserID", COUNT(*) FROM hits GROUP BY "UserID" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" LIMIT 10; +SELECT "UserID", extract(minute FROM to_timestamp_seconds("EventTime")) AS m, "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", m, "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID" FROM hits WHERE "UserID" = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE "URL" LIKE '%google%'; +SELECT "SearchPhrase", MIN("URL"), COUNT(*) AS c FROM hits WHERE "URL" LIKE '%google%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "SearchPhrase", MIN("URL"), MIN("Title"), COUNT(*) AS c, COUNT(DISTINCT "UserID") FROM hits WHERE "Title" LIKE '%Google%' AND "URL" NOT LIKE '%.google.%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE "URL" LIKE '%google%' ORDER BY "EventTime" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "SearchPhrase" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime", "SearchPhrase" LIMIT 10; +SELECT "CounterID", AVG(length("URL")) AS l, COUNT(*) AS c FROM hits WHERE "URL" <> '' GROUP BY "CounterID" HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE("Referer", '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length("Referer")) AS l, COUNT(*) AS c, MIN("Referer") FROM hits WHERE "Referer" <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM("ResolutionWidth"), SUM("ResolutionWidth" + 1), SUM("ResolutionWidth" + 2), SUM("ResolutionWidth" + 3), SUM("ResolutionWidth" + 4), SUM("ResolutionWidth" + 5), SUM("ResolutionWidth" + 6), SUM("ResolutionWidth" + 7), SUM("ResolutionWidth" + 8), SUM("ResolutionWidth" + 9), SUM("ResolutionWidth" + 10), SUM("ResolutionWidth" + 11), SUM("ResolutionWidth" + 12), SUM("ResolutionWidth" + 13), SUM("ResolutionWidth" + 14), SUM("ResolutionWidth" + 15), SUM("ResolutionWidth" + 16), SUM("ResolutionWidth" + 17), SUM("ResolutionWidth" + 18), SUM("ResolutionWidth" + 19), SUM("ResolutionWidth" + 20), SUM("ResolutionWidth" + 21), SUM("ResolutionWidth" + 22), SUM("ResolutionWidth" + 23), SUM("ResolutionWidth" + 24), SUM("ResolutionWidth" + 25), SUM("ResolutionWidth" + 26), SUM("ResolutionWidth" + 27), SUM("ResolutionWidth" + 28), SUM("ResolutionWidth" + 29), SUM("ResolutionWidth" + 30), SUM("ResolutionWidth" + 31), SUM("ResolutionWidth" + 32), SUM("ResolutionWidth" + 33), SUM("ResolutionWidth" + 34), SUM("ResolutionWidth" + 35), SUM("ResolutionWidth" + 36), SUM("ResolutionWidth" + 37), SUM("ResolutionWidth" + 38), SUM("ResolutionWidth" + 39), SUM("ResolutionWidth" + 40), SUM("ResolutionWidth" + 41), SUM("ResolutionWidth" + 42), SUM("ResolutionWidth" + 43), SUM("ResolutionWidth" + 44), SUM("ResolutionWidth" + 45), SUM("ResolutionWidth" + 46), SUM("ResolutionWidth" + 47), SUM("ResolutionWidth" + 48), SUM("ResolutionWidth" + 49), SUM("ResolutionWidth" + 50), SUM("ResolutionWidth" + 51), SUM("ResolutionWidth" + 52), SUM("ResolutionWidth" + 53), SUM("ResolutionWidth" + 54), SUM("ResolutionWidth" + 55), SUM("ResolutionWidth" + 56), SUM("ResolutionWidth" + 57), SUM("ResolutionWidth" + 58), SUM("ResolutionWidth" + 59), SUM("ResolutionWidth" + 60), SUM("ResolutionWidth" + 61), SUM("ResolutionWidth" + 62), SUM("ResolutionWidth" + 63), SUM("ResolutionWidth" + 64), SUM("ResolutionWidth" + 65), SUM("ResolutionWidth" + 66), SUM("ResolutionWidth" + 67), SUM("ResolutionWidth" + 68), SUM("ResolutionWidth" + 69), SUM("ResolutionWidth" + 70), SUM("ResolutionWidth" + 71), SUM("ResolutionWidth" + 72), SUM("ResolutionWidth" + 73), SUM("ResolutionWidth" + 74), SUM("ResolutionWidth" + 75), SUM("ResolutionWidth" + 76), SUM("ResolutionWidth" + 77), SUM("ResolutionWidth" + 78), SUM("ResolutionWidth" + 79), SUM("ResolutionWidth" + 80), SUM("ResolutionWidth" + 81), SUM("ResolutionWidth" + 82), SUM("ResolutionWidth" + 83), SUM("ResolutionWidth" + 84), SUM("ResolutionWidth" + 85), SUM("ResolutionWidth" + 86), SUM("ResolutionWidth" + 87), SUM("ResolutionWidth" + 88), SUM("ResolutionWidth" + 89) FROM hits; +SELECT "SearchEngineID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "URL", COUNT(*) AS c FROM hits GROUP BY "URL" ORDER BY c DESC LIMIT 10; +SELECT 1, "URL", COUNT(*) AS c FROM hits GROUP BY 1, "URL" ORDER BY c DESC LIMIT 10; +SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c FROM hits GROUP BY "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3 ORDER BY c DESC LIMIT 10; +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "URL" <> '' GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10; +SELECT "Title", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY PageViews DESC LIMIT 10; +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT "URLHash", to_timestamp("EventDate" * 86400), COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", to_timestamp("EventDate" * 86400) ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-01' AND to_timestamp("EventDate" * 86400) <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', to_timestamp("EventTime")::timestamp) AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate" * 86400) >= '2013-07-14' AND to_timestamp("EventDate" * 86400) <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp("EventTime")::timestamp) ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000; diff --git a/spiceai-parquet/query b/spiceai-parquet/query new file mode 100755 index 0000000000..cc45644401 --- /dev/null +++ b/spiceai-parquet/query @@ -0,0 +1,24 @@ +#!/bin/bash +# Reads a SQL query from stdin, runs it via the Spice HTTP API. +# Stdout: query result (JSON). +# Stderr: query runtime in fractional seconds on the last line. +# Exit non-zero on error. +set -e + +resp=$(mktemp /tmp/spiceai-query.XXXXXX) +trap 'rm -f "$resp"' EXIT + +metrics=$(curl -sS -o "$resp" -w '%{http_code} %{time_total}' \ + -H 'Content-Type: text/plain' --data-binary @- \ + 'http://localhost:8090/v1/sql') +code=${metrics% *} +time_total=${metrics#* } + +if [ "$code" != "200" ]; then + cat "$resp" >&2 + echo >&2 + exit 1 +fi + +cat "$resp" +printf '%s\n' "$time_total" >&2 diff --git a/spiceai-parquet/results/20260708/c6a.4xlarge.json b/spiceai-parquet/results/20260708/c6a.4xlarge.json new file mode 100644 index 0000000000..25d46708b4 --- /dev/null +++ b/spiceai-parquet/results/20260708/c6a.4xlarge.json @@ -0,0 +1,236 @@ +{ + "system": "Spice.ai OSS (Parquet, single)", + "date": "2026-07-08", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "comment": "v2.1.0 (d70bd02e)", + "tags": [ + "Rust", + "column-oriented", + "stateless" + ], + "load_time": 11.932, + "data_size": 14779976446, + "concurrent_qps": 0.362, + "concurrent_error_ratio": 0.088, + "result": [ + [ + 0.211009, + 0.003885, + 0.003525 + ], + [ + 0.282437, + 0.027292, + 0.029358 + ], + [ + 0.331889, + 0.062931, + 0.068003 + ], + [ + 0.536968, + 0.068569, + 0.066755 + ], + [ + 0.973189, + 0.643578, + 0.620826 + ], + [ + 1.074261, + 0.68371, + 0.690844 + ], + [ + 0.346284, + 0.072276, + 0.068733 + ], + [ + 0.338211, + 0.036346, + 0.035573 + ], + [ + 1.14754, + 0.761359, + 0.737891 + ], + [ + 1.55788, + 0.858835, + 0.845141 + ], + [ + 0.678391, + 0.232269, + 0.22862 + ], + [ + 0.70658, + 0.249905, + 0.244121 + ], + [ + 1.203536, + 0.778886, + 0.765841 + ], + [ + 2.673992, + 1.140921, + 1.140717 + ], + [ + 1.222168, + 0.825785, + 0.806256 + ], + [ + 1.1031, + 0.702697, + 0.680414 + ], + [ + 2.738435, + 1.434943, + 1.439456 + ], + [ + 2.66888, + 1.441392, + 1.449803 + ], + [ + 5.079107, + 2.680419, + 2.641895 + ], + [ + 0.314576, + 0.091857, + 0.089189 + ], + [ + 9.745234, + 1.484237, + 1.484487 + ], + [ + 11.426155, + 1.876375, + 1.897992 + ], + [ + 22.031562, + 3.064249, + 3.083247 + ], + [ + 17.448496, + 2.100023, + 2.026402 + ], + [ + 2.7559, + 0.430171, + 0.424262 + ], + [ + 0.908535, + 0.355628, + 0.366803 + ], + [ + 2.825757, + 0.519144, + 0.518764 + ], + [ + 9.853503, + 2.417102, + 2.380355 + ], + [ + 9.060389, + 4.483492, + 4.577573 + ], + [ + 0.301784, + 0.058251, + 0.054621 + ], + [ + 3.051457, + 0.704216, + 0.681654 + ], + [ + 6.379854, + 0.823642, + 0.800725 + ], + [ + 4.779308, + 2.467153, + 2.331666 + ], + [ + 10.4145, + 4.061152, + 4.003772 + ], + [ + 10.459085, + 4.06808, + 3.959535 + ], + [ + 0.965567, + 0.627754, + 0.61588 + ], + [ + 0.639065, + 0.182472, + 0.174015 + ], + [ + 0.444201, + 0.088123, + 0.087452 + ], + [ + 0.551757, + 0.100718, + 0.091127 + ], + [ + 0.90978, + 0.29912, + 0.279999 + ], + [ + 0.420331, + 0.050594, + 0.047482 + ], + [ + 0.398249, + 0.044642, + 0.043891 + ], + [ + 0.38516, + 0.040607, + 0.03887 + ] + ] +} \ No newline at end of file diff --git a/spiceai-parquet/spicepod.yaml b/spiceai-parquet/spicepod.yaml new file mode 100644 index 0000000000..e6839f3ac5 --- /dev/null +++ b/spiceai-parquet/spicepod.yaml @@ -0,0 +1,14 @@ +version: v2 +kind: Spicepod +name: spiceai-parquet-clickbench + +runtime: + caching: + # The SQL results cache is enabled by default; the benchmark rules + # require query result caches to be disabled. + sql_results: + enabled: false + +datasets: + - from: file:hits.parquet + name: hits diff --git a/spiceai-parquet/start b/spiceai-parquet/start new file mode 100755 index 0000000000..f5c3edca85 --- /dev/null +++ b/spiceai-parquet/start @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +# spiced loads ./spicepod.yaml from the working directory and serves the +# HTTP API on 127.0.0.1:8090. Logs append to spiced.log for debugging. +nohup spiced >> spiced.log 2>&1 & diff --git a/spiceai-parquet/stop b/spiceai-parquet/stop new file mode 100755 index 0000000000..1845b3e41c --- /dev/null +++ b/spiceai-parquet/stop @@ -0,0 +1,10 @@ +#!/bin/bash + +pkill spiced || exit 0 + +# Wait for graceful shutdown, then escalate. +for _ in $(seq 1 30); do + pgrep spiced >/dev/null || exit 0 + sleep 1 +done +pkill -9 spiced || true diff --git a/spiceai-parquet/template.json b/spiceai-parquet/template.json new file mode 100644 index 0000000000..2a7ff52af8 --- /dev/null +++ b/spiceai-parquet/template.json @@ -0,0 +1,11 @@ +{ + "system": "Spice.ai OSS (Parquet, single)", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "Rust", + "column-oriented", + "stateless" + ] +}