diff --git a/README.md b/README.md index 0bfba7941..b5221d5f3 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,8 @@ To introduce a new result for an existing system with a different usage scenario `index.html` can be re-generated using `./generate-results.sh`. The CI (GitHub Actions) does this automatically, this step is optional. +The shared driver (`lib/benchmark-common.sh`) runs a supplementary concurrent-QPS test (`BENCH_CONCURRENT_CONNECTIONS` workers for `BENCH_CONCURRENT_DURATION` seconds) after the main sweep. Single-process engines must set `BENCH_CONCURRENT_DURATION=0` in their `benchmark.sh` to skip it: each query forks a fresh full-machine process with no shared scheduler, so concurrent connections only oversubscribe RAM (and can OOM the run) instead of measuring throughput. Rule of thumb — skip when `./start` launches no shared server (the embedded CLIs and Spark variants); keep it for daemons and the in-process server wrappers (pandas/polars/`*-dataframe`), which share one process. See issue #946. + All tests were originally run on AWS c6a.4xlarge EC2 VMs with 500 GB gp2 disks. With better automation, more EC2 machines were added later: c6a.2xlarge, c6a.metal, c8g.4xlarge, c6a.xlarge, c7a.metal-48xl, c6a.large, c8g.metal-48xl, and t3a.small. These represent older and modern machines, and small / medium / large systems (CPU and main memory). diff --git a/bqn/benchmark.sh b/bqn/benchmark.sh index 617422ddc..5427511a0 100755 --- a/bqn/benchmark.sh +++ b/bqn/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/bqn/results/20260529/c6a.4xlarge.json b/bqn/results/20260529/c6a.4xlarge.json index 30d42d51d..ed57253aa 100644 --- a/bqn/results/20260529/c6a.4xlarge.json +++ b/bqn/results/20260529/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 526, "data_size": 118169498884, - "concurrent_qps": 0.042, - "concurrent_error_ratio": 0.554, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.002, 0.001, 0.001], [3.551, 1.308, 1.306], diff --git a/bqn/results/20260529/c6a.metal.json b/bqn/results/20260529/c6a.metal.json index 42e3e4acb..fdabbd82c 100644 --- a/bqn/results/20260529/c6a.metal.json +++ b/bqn/results/20260529/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 472, "data_size": 118169498884, - "concurrent_qps": 0.223, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.001, 0.001, 0.001], [4.181, 1.272, 1.22], diff --git a/bqn/results/20260529/c7a.metal-48xl.json b/bqn/results/20260529/c7a.metal-48xl.json index 921b88b44..6541120d2 100644 --- a/bqn/results/20260529/c7a.metal-48xl.json +++ b/bqn/results/20260529/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 473, "data_size": 118169498884, - "concurrent_qps": 0.235, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.001, 0.001, 0.001], [4.122, 1.161, 1.094], diff --git a/bqn/results/20260529/c8g.4xlarge.json b/bqn/results/20260529/c8g.4xlarge.json index a3709b950..9a15aa6ca 100644 --- a/bqn/results/20260529/c8g.4xlarge.json +++ b/bqn/results/20260529/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 518, "data_size": 118169498884, - "concurrent_qps": 0.042, - "concurrent_error_ratio": 0.597, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.001, 0, 0], [3.349, 0.676, 0.693], diff --git a/bqn/results/20260529/c8g.metal-48xl.json b/bqn/results/20260529/c8g.metal-48xl.json index aaf7f4d1c..9450d3709 100644 --- a/bqn/results/20260529/c8g.metal-48xl.json +++ b/bqn/results/20260529/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C","embedded","array language"], "load_time": 472, "data_size": 118169498884, - "concurrent_qps": 0.237, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.001, 0, 0], [3.657, 0.702, 0.699], diff --git a/chdb-parquet-partitioned/benchmark.sh b/chdb-parquet-partitioned/benchmark.sh index e236dbbd2..5972a9845 100755 --- a/chdb-parquet-partitioned/benchmark.sh +++ b/chdb-parquet-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json b/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json index fb8c49965..ad6ab77aa 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 9, "data_size": 14737666736, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.061, 0.027, 0.026], [0.151, 0.058, 0.058], diff --git a/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json b/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json index 5f6ff9c9a..ef3f0be45 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.108, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.049, 0.026, 0.026], [0.112, 0.045, 0.045], diff --git a/chdb-parquet-partitioned/results/20260511/c6a.large.json b/chdb-parquet-partitioned/results/20260511/c6a.large.json index 1f54d9cd9..67072e2cd 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.large.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.045, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.17, 0.044, 0.041], [0.352, 0.141, 0.144], diff --git a/chdb-parquet-partitioned/results/20260511/c6a.metal.json b/chdb-parquet-partitioned/results/20260511/c6a.metal.json index 9ebc69ea0..af5abfdec 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.metal.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 66, "data_size": 14737666736, - "concurrent_qps": 3.643, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.045, 0.012, 0.018], [0.114, 0.054, 0.052], diff --git a/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json b/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json index 009709246..5473e5b5b 100644 --- a/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json +++ b/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.101, 0.026, 0.027], [0.217, 0.083, 0.082], diff --git a/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json b/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json index 0f34b6189..020657e93 100644 --- a/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json +++ b/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 62, "data_size": 14737666736, - "concurrent_qps": 4.03, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.066, 0.015, 0.015], [0.134, 0.088, 0.089], diff --git a/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json b/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json index 3b3d43e24..73c4e8d84 100644 --- a/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json +++ b/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.072, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.033, 0.017, 0.018], [0.082, 0.034, 0.036], diff --git a/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json b/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json index 5b090867c..09dfde1fb 100644 --- a/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json +++ b/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 66, "data_size": 14737666736, - "concurrent_qps": 4.042, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.037, 0.011, 0.012], [0.076, 0.033, 0.029], diff --git a/chdb-parquet-partitioned/results/20260511/t3a.small.json b/chdb-parquet-partitioned/results/20260511/t3a.small.json index 777b8bd5f..49ac7f229 100644 --- a/chdb-parquet-partitioned/results/20260511/t3a.small.json +++ b/chdb-parquet-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.023, - "concurrent_error_ratio": 0.176, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.224, 0.069, 0.068], [0.515, 0.286, 0.268], diff --git a/chdb/benchmark.sh b/chdb/benchmark.sh index 2e5741b39..bf0286804 100755 --- a/chdb/benchmark.sh +++ b/chdb/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/chdb/results/20260511/c6a.2xlarge.json b/chdb/results/20260511/c6a.2xlarge.json index e81217f1c..144774696 100644 --- a/chdb/results/20260511/c6a.2xlarge.json +++ b/chdb/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 573, "data_size": 15259766974, - "concurrent_qps": 0.307, - "concurrent_error_ratio": 0.959, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.014, 0.003, 0.003], [0.064, 0.014, 0.015], diff --git a/chdb/results/20260511/c6a.4xlarge.json b/chdb/results/20260511/c6a.4xlarge.json index db5f829e6..5b87eb65d 100644 --- a/chdb/results/20260511/c6a.4xlarge.json +++ b/chdb/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 532, "data_size": 15261827371, - "concurrent_qps": 0.58, - "concurrent_error_ratio": 0.94, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.015, 0.003, 0.003], [0.054, 0.012, 0.011], diff --git a/chdb/results/20260511/c6a.large.json b/chdb/results/20260511/c6a.large.json index 77d96d613..edde07843 100644 --- a/chdb/results/20260511/c6a.large.json +++ b/chdb/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 774, "data_size": 15257484538, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.015, 0.005, 0.005], [0.132, 0.069, 0.052], diff --git a/chdb/results/20260511/c6a.metal.json b/chdb/results/20260511/c6a.metal.json index 3084da182..f6f6b7a80 100644 --- a/chdb/results/20260511/c6a.metal.json +++ b/chdb/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 163, "data_size": 15266676267, - "concurrent_qps": 1.358, - "concurrent_error_ratio": 0.942, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.019, 0.007, 0.007], [0.096, 0.06, 0.053], diff --git a/chdb/results/20260511/c6a.xlarge.json b/chdb/results/20260511/c6a.xlarge.json index dbe122e20..ecc7ca054 100644 --- a/chdb/results/20260511/c6a.xlarge.json +++ b/chdb/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 602, "data_size": 15261958294, - "concurrent_qps": 0.13, - "concurrent_error_ratio": 0.97, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.014, 0.002, 0.002], [0.072, 0.023, 0.022], diff --git a/chdb/results/20260511/c7a.metal-48xl.json b/chdb/results/20260511/c7a.metal-48xl.json index a496a3a79..013fc8990 100644 --- a/chdb/results/20260511/c7a.metal-48xl.json +++ b/chdb/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 156, "data_size": 15266229827, - "concurrent_qps": 1.185, - "concurrent_error_ratio": 0.96, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.02, 0.004, 0.004], [0.132, 0.058, 0.075], diff --git a/chdb/results/20260511/c8g.4xlarge.json b/chdb/results/20260511/c8g.4xlarge.json index 1096197e1..1b4d7669a 100644 --- a/chdb/results/20260511/c8g.4xlarge.json +++ b/chdb/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 602, "data_size": 15260683017, - "concurrent_qps": 1.028, - "concurrent_error_ratio": 0.942, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.01, 0.002, 0.002], [0.046, 0.014, 0.01], diff --git a/chdb/results/20260511/c8g.metal-48xl.json b/chdb/results/20260511/c8g.metal-48xl.json index 9012a0a21..fc81ec78e 100644 --- a/chdb/results/20260511/c8g.metal-48xl.json +++ b/chdb/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 149, "data_size": 15264739392, - "concurrent_qps": 1.5150000000000001, - "concurrent_error_ratio": 0.954, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.012, 0.007, 0.007], [0.064, 0.027, 0.027], diff --git a/chdb/results/20260511/t3a.small.json b/chdb/results/20260511/t3a.small.json index a0051cd52..b00f25076 100644 --- a/chdb/results/20260511/t3a.small.json +++ b/chdb/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"], "load_time": 1516, "data_size": 15257125380, - "concurrent_qps": 0.007, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.025, 0.009, 0.008], [0.239, 0.12, 0.118], diff --git a/clickhouse-datalake-partitioned/benchmark.sh b/clickhouse-datalake-partitioned/benchmark.sh index 08d52c39f..9c514c480 100755 --- a/clickhouse-datalake-partitioned/benchmark.sh +++ b/clickhouse-datalake-partitioned/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json index dba6ed420..bf6b2ec07 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.13, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.422, 0.378, 0.376], [1.431, 1.414, 1.311], diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json index 6e5211d1a..91f938b96 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.378, - "concurrent_error_ratio": 0.995, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.453, 0.228, 0.224], [2.036, 0.904, 0.781], diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.large.json b/clickhouse-datalake-partitioned/results/20260511/c6a.large.json index 4eb2f06b6..f3f7e0ad5 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.large.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.008, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [1.478, 1.47, 1.411], [4.133, 3.975, 4.106], diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json b/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json index 5dfd90b4f..4f14353cb 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.008, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.181, 0.141, 0.164], [0.382, 0.3, 0.308], diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json index 59388834d..ff4521330 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.042, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.72, 0.742, 0.753], [2.651, 2.642, 2.479], diff --git a/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json index 991e7a068..d6dfbbb78 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.102, - "concurrent_error_ratio": 0.991, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.199, 0.165, 0.181], [0.337, 0.296, 0.297], diff --git a/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json index 814817e72..9e15c1bce 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.74, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.239, 0.222, 0.223], [0.726, 0.75, 0.721], diff --git a/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json index 4993286c8..067d6dcfc 100644 --- a/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.143, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.183, 0.145, 0.146], [0.307, 0.312, 0.296], diff --git a/clickhouse-datalake-partitioned/results/20260511/t3a.small.json b/clickhouse-datalake-partitioned/results/20260511/t3a.small.json index 4cf59e4f3..4fef9ba5b 100644 --- a/clickhouse-datalake-partitioned/results/20260511/t3a.small.json +++ b/clickhouse-datalake-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.012, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.045, 3.076, 3.281], [7.062, 7.301, 7.262], diff --git a/clickhouse-datalake-partitioned/results/20260516/c6a.2xlarge.json b/clickhouse-datalake-partitioned/results/20260516/c6a.2xlarge.json index 354d0418c..f5204b1c6 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c6a.2xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260516/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.135, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.482, 0.385, 0.394], [1.442, 1.362, 1.311], diff --git a/clickhouse-datalake-partitioned/results/20260516/c6a.4xlarge.json b/clickhouse-datalake-partitioned/results/20260516/c6a.4xlarge.json index 03e3a99d2..e726cb9e2 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c6a.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260516/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.392, - "concurrent_error_ratio": 0.995, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.643, 0.299, 0.235], [2.039, 1.078, 0.76], diff --git a/clickhouse-datalake-partitioned/results/20260516/c6a.metal.json b/clickhouse-datalake-partitioned/results/20260516/c6a.metal.json index 7e9c1ebac..463704de7 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c6a.metal.json +++ b/clickhouse-datalake-partitioned/results/20260516/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.958, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.238, 0.132, 0.119], [0.379, 0.317, 0.28], diff --git a/clickhouse-datalake-partitioned/results/20260516/c6a.xlarge.json b/clickhouse-datalake-partitioned/results/20260516/c6a.xlarge.json index 122b0fb8a..101ead2f3 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c6a.xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260516/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.032, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.788, 0.709, 0.716], [2.651, 2.555, 2.516], diff --git a/clickhouse-datalake-partitioned/results/20260516/c7a.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260516/c7a.metal-48xl.json index 6b72b3cfe..65983b5dd 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c7a.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/20260516/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.078, - "concurrent_error_ratio": 0.991, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.244, 0.131, 0.14], [0.373, 0.267, 0.271], diff --git a/clickhouse-datalake-partitioned/results/20260516/c8g.4xlarge.json b/clickhouse-datalake-partitioned/results/20260516/c8g.4xlarge.json index 7185b4c33..d7649a5ce 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c8g.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/20260516/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.487, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.279, 0.219, 0.229], [0.792, 0.715, 0.742], diff --git a/clickhouse-datalake-partitioned/results/20260516/c8g.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260516/c8g.metal-48xl.json index 71ab6a57c..27432caad 100644 --- a/clickhouse-datalake-partitioned/results/20260516/c8g.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/20260516/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 1.272, - "concurrent_error_ratio": 0.991, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.174, 0.153, 0.157], [0.315, 0.282, 0.28], diff --git a/clickhouse-datalake-partitioned/results/20260517/c6a.large.json b/clickhouse-datalake-partitioned/results/20260517/c6a.large.json index 8bf816417..6a8dd4bb2 100644 --- a/clickhouse-datalake-partitioned/results/20260517/c6a.large.json +++ b/clickhouse-datalake-partitioned/results/20260517/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [1.496, 1.432, 1.405], [4.177, 3.961, 3.944], diff --git a/clickhouse-datalake-partitioned/results/20260517/t3a.small.json b/clickhouse-datalake-partitioned/results/20260517/t3a.small.json index 59a08bf37..70fcd44ea 100644 --- a/clickhouse-datalake-partitioned/results/20260517/t3a.small.json +++ b/clickhouse-datalake-partitioned/results/20260517/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.007, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.155, 2.982, 2.899], [7.073, 6.892, 6.605], diff --git a/clickhouse-datalake/benchmark.sh b/clickhouse-datalake/benchmark.sh index 08d52c39f..9c514c480 100755 --- a/clickhouse-datalake/benchmark.sh +++ b/clickhouse-datalake/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-datalake/results/20260511/c6a.2xlarge.json b/clickhouse-datalake/results/20260511/c6a.2xlarge.json index ef68c7a32..746f03b33 100644 --- a/clickhouse-datalake/results/20260511/c6a.2xlarge.json +++ b/clickhouse-datalake/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.197, - "concurrent_error_ratio": 0.996, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.311, 0.21, 0.211], [0.987, 0.864, 0.926], diff --git a/clickhouse-datalake/results/20260511/c6a.4xlarge.json b/clickhouse-datalake/results/20260511/c6a.4xlarge.json index d259d68f0..6eab8180f 100644 --- a/clickhouse-datalake/results/20260511/c6a.4xlarge.json +++ b/clickhouse-datalake/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.37, - "concurrent_error_ratio": 0.996, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.285, 0.205, 0.206], [0.746, 0.536, 0.52], diff --git a/clickhouse-datalake/results/20260511/c6a.large.json b/clickhouse-datalake/results/20260511/c6a.large.json index 1d42988e4..9c78be1c5 100644 --- a/clickhouse-datalake/results/20260511/c6a.large.json +++ b/clickhouse-datalake/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.333, 0.231, 0.223], [2.554, 2.373, 2.309], diff --git a/clickhouse-datalake/results/20260511/c6a.metal.json b/clickhouse-datalake/results/20260511/c6a.metal.json index b163df750..d6e7be9d3 100644 --- a/clickhouse-datalake/results/20260511/c6a.metal.json +++ b/clickhouse-datalake/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.88, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.315, 0.201, 0.204], [0.389, 0.251, 0.269], diff --git a/clickhouse-datalake/results/20260511/c6a.xlarge.json b/clickhouse-datalake/results/20260511/c6a.xlarge.json index 064d156dc..f8d32220a 100644 --- a/clickhouse-datalake/results/20260511/c6a.xlarge.json +++ b/clickhouse-datalake/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.038, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.291, 0.206, 0.201], [1.599, 1.464, 1.505], diff --git a/clickhouse-datalake/results/20260511/c7a.metal-48xl.json b/clickhouse-datalake/results/20260511/c7a.metal-48xl.json index b39e031b1..ed1640fe6 100644 --- a/clickhouse-datalake/results/20260511/c7a.metal-48xl.json +++ b/clickhouse-datalake/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.798, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.338, 0.2, 0.195], [0.411, 0.289, 0.274], diff --git a/clickhouse-datalake/results/20260511/c8g.4xlarge.json b/clickhouse-datalake/results/20260511/c8g.4xlarge.json index 28c0213bf..8bbc318fe 100644 --- a/clickhouse-datalake/results/20260511/c8g.4xlarge.json +++ b/clickhouse-datalake/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.477, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.215, 0.169, 0.148], [0.593, 0.527, 0.568], diff --git a/clickhouse-datalake/results/20260511/c8g.metal-48xl.json b/clickhouse-datalake/results/20260511/c8g.metal-48xl.json index 9269c5e59..54dfd2203 100644 --- a/clickhouse-datalake/results/20260511/c8g.metal-48xl.json +++ b/clickhouse-datalake/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.855, - "concurrent_error_ratio": 0.995, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.217, 0.174, 0.161], [0.367, 0.259, 0.265], diff --git a/clickhouse-datalake/results/20260511/t3a.small.json b/clickhouse-datalake/results/20260511/t3a.small.json index 8262e358a..6225bee0d 100644 --- a/clickhouse-datalake/results/20260511/t3a.small.json +++ b/clickhouse-datalake/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.007, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.449, 0.344, 0.36], [2.773, 2.534, 2.698], diff --git a/clickhouse-datalake/results/20260516/c6a.2xlarge.json b/clickhouse-datalake/results/20260516/c6a.2xlarge.json index f34cac434..7d4f3d972 100644 --- a/clickhouse-datalake/results/20260516/c6a.2xlarge.json +++ b/clickhouse-datalake/results/20260516/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.115, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.325, 0.213, 0.217], [1.161, 0.908, 0.94], diff --git a/clickhouse-datalake/results/20260516/c6a.4xlarge.json b/clickhouse-datalake/results/20260516/c6a.4xlarge.json index f661269e1..507b03d99 100644 --- a/clickhouse-datalake/results/20260516/c6a.4xlarge.json +++ b/clickhouse-datalake/results/20260516/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.287, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.36, 0.208, 0.213], [0.998, 0.581, 0.601], diff --git a/clickhouse-datalake/results/20260516/c6a.metal.json b/clickhouse-datalake/results/20260516/c6a.metal.json index bcd4fe5ab..f8ce94751 100644 --- a/clickhouse-datalake/results/20260516/c6a.metal.json +++ b/clickhouse-datalake/results/20260516/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.942, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.37, 0.222, 0.219], [0.481, 0.376, 0.288], diff --git a/clickhouse-datalake/results/20260516/c6a.xlarge.json b/clickhouse-datalake/results/20260516/c6a.xlarge.json index 25d1e7efc..ac0a137c7 100644 --- a/clickhouse-datalake/results/20260516/c6a.xlarge.json +++ b/clickhouse-datalake/results/20260516/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.032, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.341, 0.217, 0.209], [1.641, 1.451, 1.486], diff --git a/clickhouse-datalake/results/20260516/c7a.metal-48xl.json b/clickhouse-datalake/results/20260516/c7a.metal-48xl.json index 8071f1338..933e407dc 100644 --- a/clickhouse-datalake/results/20260516/c7a.metal-48xl.json +++ b/clickhouse-datalake/results/20260516/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.91, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.455, 0.207, 0.222], [0.639, 0.347, 0.271], diff --git a/clickhouse-datalake/results/20260516/c8g.4xlarge.json b/clickhouse-datalake/results/20260516/c8g.4xlarge.json index ccfa13276..c6b1e6a92 100644 --- a/clickhouse-datalake/results/20260516/c8g.4xlarge.json +++ b/clickhouse-datalake/results/20260516/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.453, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.264, 0.167, 0.168], [0.682, 0.556, 0.542], diff --git a/clickhouse-datalake/results/20260516/c8g.metal-48xl.json b/clickhouse-datalake/results/20260516/c8g.metal-48xl.json index aadb1f4aa..8f3065448 100644 --- a/clickhouse-datalake/results/20260516/c8g.metal-48xl.json +++ b/clickhouse-datalake/results/20260516/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.955, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.255, 0.167, 0.174], [0.427, 0.28, 0.289], diff --git a/clickhouse-datalake/results/20260517/c6a.large.json b/clickhouse-datalake/results/20260517/c6a.large.json index 4c8d29bf1..97d76dd22 100644 --- a/clickhouse-datalake/results/20260517/c6a.large.json +++ b/clickhouse-datalake/results/20260517/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.005, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.408, 0.228, 0.224], [2.574, 2.408, 2.455], diff --git a/clickhouse-datalake/results/20260517/t3a.small.json b/clickhouse-datalake/results/20260517/t3a.small.json index 8cf759f7a..584455b51 100644 --- a/clickhouse-datalake/results/20260517/t3a.small.json +++ b/clickhouse-datalake/results/20260517/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.01, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.541, 0.375, 0.379], [2.806, 2.65, 2.682], diff --git a/clickhouse-parquet-partitioned/benchmark.sh b/clickhouse-parquet-partitioned/benchmark.sh index e236dbbd2..5972a9845 100755 --- a/clickhouse-parquet-partitioned/benchmark.sh +++ b/clickhouse-parquet-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.2xlarge.json b/clickhouse-parquet-partitioned/results/20260511/c6a.2xlarge.json index 4df9a6175..6b83acc52 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.2xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 9, "data_size": 14737666736, - "concurrent_qps": 0.048, - "concurrent_error_ratio": 0.237, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.061, 0.015, 0.015], [0.136, 0.045, 0.043], diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.4xlarge.json b/clickhouse-parquet-partitioned/results/20260511/c6a.4xlarge.json index dc5f2f8e0..5d9373661 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.4xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.088, - "concurrent_error_ratio": 0.086, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.065, 0.011, 0.011], [0.117, 0.03, 0.029], diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.large.json b/clickhouse-parquet-partitioned/results/20260511/c6a.large.json index b342c90ef..383903947 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.large.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.087, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.191, 0.063, 0.065], [0.484, 0.308, 0.297], diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.metal.json b/clickhouse-parquet-partitioned/results/20260511/c6a.metal.json index 7af2ff74d..d2fa3379e 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.metal.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.88, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.05, 0.012, 0.012], [0.096, 0.055, 0.041], diff --git a/clickhouse-parquet-partitioned/results/20260511/c6a.xlarge.json b/clickhouse-parquet-partitioned/results/20260511/c6a.xlarge.json index 8fa6a7649..64c42f28c 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c6a.xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.047, - "concurrent_error_ratio": 0.067, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.119, 0.024, 0.024], [0.229, 0.072, 0.073], diff --git a/clickhouse-parquet-partitioned/results/20260511/c7a.metal-48xl.json b/clickhouse-parquet-partitioned/results/20260511/c7a.metal-48xl.json index 7c6ff3522..c30bc2b64 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c7a.metal-48xl.json +++ b/clickhouse-parquet-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.855, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.088, 0.015, 0.014], [0.153, 0.069, 0.066], diff --git a/clickhouse-parquet-partitioned/results/20260511/c8g.4xlarge.json b/clickhouse-parquet-partitioned/results/20260511/c8g.4xlarge.json index e1e065c19..b19e5be95 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c8g.4xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.137, - "concurrent_error_ratio": 0.079, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.041, 0.006, 0.006], [0.078, 0.023, 0.023], diff --git a/clickhouse-parquet-partitioned/results/20260511/c8g.metal-48xl.json b/clickhouse-parquet-partitioned/results/20260511/c8g.metal-48xl.json index 018aebd01..f4d951b9c 100644 --- a/clickhouse-parquet-partitioned/results/20260511/c8g.metal-48xl.json +++ b/clickhouse-parquet-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.732, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.047, 0.026, 0.033], [0.078, 0.065, 0.041], diff --git a/clickhouse-parquet-partitioned/results/20260511/t3a.small.json b/clickhouse-parquet-partitioned/results/20260511/t3a.small.json index c2375896f..1a8a5fadd 100644 --- a/clickhouse-parquet-partitioned/results/20260511/t3a.small.json +++ b/clickhouse-parquet-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0.111, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.427, 0.15, 0.15], [1.2, 0.765, 0.703], diff --git a/clickhouse-parquet-partitioned/results/20260516/c6a.2xlarge.json b/clickhouse-parquet-partitioned/results/20260516/c6a.2xlarge.json index 9a5dbd9ac..796e3de8a 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c6a.2xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260516/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.047, - "concurrent_error_ratio": 0.067, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.108, 0.015, 0.016], [0.2, 0.045, 0.045], diff --git a/clickhouse-parquet-partitioned/results/20260516/c6a.4xlarge.json b/clickhouse-parquet-partitioned/results/20260516/c6a.4xlarge.json index fdedfebcf..100a94d92 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c6a.4xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260516/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 22, "data_size": 14737666736, - "concurrent_qps": 0.1, - "concurrent_error_ratio": 0.062, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.09, 0.012, 0.012], [0.169, 0.032, 0.03], diff --git a/clickhouse-parquet-partitioned/results/20260516/c6a.metal.json b/clickhouse-parquet-partitioned/results/20260516/c6a.metal.json index bd45e3058..9ecdda9e0 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c6a.metal.json +++ b/clickhouse-parquet-partitioned/results/20260516/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.842, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.109, 0.011, 0.012], [0.159, 0.064, 0.059], diff --git a/clickhouse-parquet-partitioned/results/20260516/c7a.metal-48xl.json b/clickhouse-parquet-partitioned/results/20260516/c7a.metal-48xl.json index aab2fc8e6..447a245e6 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c7a.metal-48xl.json +++ b/clickhouse-parquet-partitioned/results/20260516/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.822, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.148, 0.015, 0.014], [0.229, 0.083, 0.064], diff --git a/clickhouse-parquet-partitioned/results/20260516/c8g.4xlarge.json b/clickhouse-parquet-partitioned/results/20260516/c8g.4xlarge.json index 935828cbe..d0a9367c4 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c8g.4xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260516/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.188, - "concurrent_error_ratio": 0.066, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.076, 0.006, 0.007], [0.144, 0.024, 0.024], diff --git a/clickhouse-parquet-partitioned/results/20260516/c8g.metal-48xl.json b/clickhouse-parquet-partitioned/results/20260516/c8g.metal-48xl.json index 73aee9b90..7bba344cf 100644 --- a/clickhouse-parquet-partitioned/results/20260516/c8g.metal-48xl.json +++ b/clickhouse-parquet-partitioned/results/20260516/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 1.9449999999999998, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.083, 0.025, 0.022], [0.118, 0.046, 0.043], diff --git a/clickhouse-parquet-partitioned/results/20260517/c6a.large.json b/clickhouse-parquet-partitioned/results/20260517/c6a.large.json index f93ae6876..49416b858 100644 --- a/clickhouse-parquet-partitioned/results/20260517/c6a.large.json +++ b/clickhouse-parquet-partitioned/results/20260517/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.16, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.219, 0.062, 0.063], [0.568, 0.313, 0.3], diff --git a/clickhouse-parquet-partitioned/results/20260517/c6a.xlarge.json b/clickhouse-parquet-partitioned/results/20260517/c6a.xlarge.json index bb43cf976..b453f6af2 100644 --- a/clickhouse-parquet-partitioned/results/20260517/c6a.xlarge.json +++ b/clickhouse-parquet-partitioned/results/20260517/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.057, - "concurrent_error_ratio": 0.056, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.167, 0.024, 0.024], [0.286, 0.088, 0.077], diff --git a/clickhouse-parquet-partitioned/results/20260517/t3a.small.json b/clickhouse-parquet-partitioned/results/20260517/t3a.small.json index 108afc5fe..135060bee 100644 --- a/clickhouse-parquet-partitioned/results/20260517/t3a.small.json +++ b/clickhouse-parquet-partitioned/results/20260517/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.028, - "concurrent_error_ratio": 0.227, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.429, 0.132, 0.155], [1.219, 0.633, 0.66], diff --git a/clickhouse-parquet/benchmark.sh b/clickhouse-parquet/benchmark.sh index 617422ddc..5427511a0 100755 --- a/clickhouse-parquet/benchmark.sh +++ b/clickhouse-parquet/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-parquet/results/20260511/c6a.2xlarge.json b/clickhouse-parquet/results/20260511/c6a.2xlarge.json index 17eca5c1c..f63bf5554 100644 --- a/clickhouse-parquet/results/20260511/c6a.2xlarge.json +++ b/clickhouse-parquet/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.08, - "concurrent_error_ratio": 0.059, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.081, 0.037, 0.037], [0.159, 0.078, 0.08], diff --git a/clickhouse-parquet/results/20260511/c6a.4xlarge.json b/clickhouse-parquet/results/20260511/c6a.4xlarge.json index ede89e703..0c76a42bf 100644 --- a/clickhouse-parquet/results/20260511/c6a.4xlarge.json +++ b/clickhouse-parquet/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.103, - "concurrent_error_ratio": 0.061, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.084, 0.037, 0.037], [0.143, 0.067, 0.065], diff --git a/clickhouse-parquet/results/20260511/c6a.large.json b/clickhouse-parquet/results/20260511/c6a.large.json index 2ad80a56f..cc7d15b91 100644 --- a/clickhouse-parquet/results/20260511/c6a.large.json +++ b/clickhouse-parquet/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.027, - "concurrent_error_ratio": 0.111, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.088, 0.048, 0.042], [0.381, 0.281, 0.279], diff --git a/clickhouse-parquet/results/20260511/c6a.metal.json b/clickhouse-parquet/results/20260511/c6a.metal.json index 666e24c21..92fed31a8 100644 --- a/clickhouse-parquet/results/20260511/c6a.metal.json +++ b/clickhouse-parquet/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 2.013, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.079, 0.038, 0.039], [0.139, 0.076, 0.08], diff --git a/clickhouse-parquet/results/20260511/c6a.xlarge.json b/clickhouse-parquet/results/20260511/c6a.xlarge.json index 7ce2c1213..447e0d35e 100644 --- a/clickhouse-parquet/results/20260511/c6a.xlarge.json +++ b/clickhouse-parquet/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 0.069, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.084, 0.038, 0.038], [0.2, 0.104, 0.102], diff --git a/clickhouse-parquet/results/20260511/c7a.metal-48xl.json b/clickhouse-parquet/results/20260511/c7a.metal-48xl.json index 69c11d965..f7e68eaaf 100644 --- a/clickhouse-parquet/results/20260511/c7a.metal-48xl.json +++ b/clickhouse-parquet/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 2.04, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.078, 0.039, 0.038], [0.143, 0.087, 0.082], diff --git a/clickhouse-parquet/results/20260511/c8g.4xlarge.json b/clickhouse-parquet/results/20260511/c8g.4xlarge.json index 532bfee46..32178d8ab 100644 --- a/clickhouse-parquet/results/20260511/c8g.4xlarge.json +++ b/clickhouse-parquet/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.082, - "concurrent_error_ratio": 0.058, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.066, 0.029, 0.029], [0.113, 0.054, 0.054], diff --git a/clickhouse-parquet/results/20260511/c8g.metal-48xl.json b/clickhouse-parquet/results/20260511/c8g.metal-48xl.json index 84f1244bd..1cdf9d57d 100644 --- a/clickhouse-parquet/results/20260511/c8g.metal-48xl.json +++ b/clickhouse-parquet/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 2.078, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.055, 0.034, 0.035], [0.104, 0.061, 0.062], diff --git a/clickhouse-parquet/results/20260511/t3a.small.json b/clickhouse-parquet/results/20260511/t3a.small.json index e27e135a3..5d345e082 100644 --- a/clickhouse-parquet/results/20260511/t3a.small.json +++ b/clickhouse-parquet/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0.118, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.136, 0.105, 0.087], [0.609, 0.53, 0.494], diff --git a/clickhouse-parquet/results/20260516/c6a.2xlarge.json b/clickhouse-parquet/results/20260516/c6a.2xlarge.json index 9bacce653..9592f8d58 100644 --- a/clickhouse-parquet/results/20260516/c6a.2xlarge.json +++ b/clickhouse-parquet/results/20260516/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.06, - "concurrent_error_ratio": 0.077, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.119, 0.036, 0.036], [0.245, 0.079, 0.078], diff --git a/clickhouse-parquet/results/20260516/c6a.metal.json b/clickhouse-parquet/results/20260516/c6a.metal.json index c75d926d3..5e3a0d5af 100644 --- a/clickhouse-parquet/results/20260516/c6a.metal.json +++ b/clickhouse-parquet/results/20260516/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 1.995, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.143, 0.039, 0.038], [0.204, 0.08, 0.079], diff --git a/clickhouse-parquet/results/20260516/c6a.xlarge.json b/clickhouse-parquet/results/20260516/c6a.xlarge.json index f2112444e..b04425cba 100644 --- a/clickhouse-parquet/results/20260516/c6a.xlarge.json +++ b/clickhouse-parquet/results/20260516/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.043, - "concurrent_error_ratio": 0.103, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.119, 0.036, 0.036], [0.267, 0.11, 0.108], diff --git a/clickhouse-parquet/results/20260516/c7a.metal-48xl.json b/clickhouse-parquet/results/20260516/c7a.metal-48xl.json index 2213760f7..808a78ecd 100644 --- a/clickhouse-parquet/results/20260516/c7a.metal-48xl.json +++ b/clickhouse-parquet/results/20260516/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 2.052, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.137, 0.04, 0.038], [0.215, 0.083, 0.084], diff --git a/clickhouse-parquet/results/20260516/c8g.4xlarge.json b/clickhouse-parquet/results/20260516/c8g.4xlarge.json index 258298c20..5744759c5 100644 --- a/clickhouse-parquet/results/20260516/c8g.4xlarge.json +++ b/clickhouse-parquet/results/20260516/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.12, - "concurrent_error_ratio": 0.065, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.101, 0.031, 0.031], [0.159, 0.056, 0.058], diff --git a/clickhouse-parquet/results/20260516/c8g.metal-48xl.json b/clickhouse-parquet/results/20260516/c8g.metal-48xl.json index dfe3114be..ec7ff2a26 100644 --- a/clickhouse-parquet/results/20260516/c8g.metal-48xl.json +++ b/clickhouse-parquet/results/20260516/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 2.135, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.105, 0.034, 0.034], [0.154, 0.061, 0.06], diff --git a/clickhouse-parquet/results/20260517/c6a.4xlarge.json b/clickhouse-parquet/results/20260517/c6a.4xlarge.json index 953ae5921..e7199f3bc 100644 --- a/clickhouse-parquet/results/20260517/c6a.4xlarge.json +++ b/clickhouse-parquet/results/20260517/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 8, "data_size": 14779976446, - "concurrent_qps": 0.22, - "concurrent_error_ratio": 0.05, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.137, 0.037, 0.037], [0.205, 0.065, 0.067], diff --git a/clickhouse-parquet/results/20260517/c6a.large.json b/clickhouse-parquet/results/20260517/c6a.large.json index c8dd52b5b..422f7f015 100644 --- a/clickhouse-parquet/results/20260517/c6a.large.json +++ b/clickhouse-parquet/results/20260517/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.033, - "concurrent_error_ratio": 0.091, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.148, 0.041, 0.041], [0.444, 0.282, 0.289], diff --git a/clickhouse-parquet/results/20260517/t3a.small.json b/clickhouse-parquet/results/20260517/t3a.small.json index 138078ab1..87d115fe7 100644 --- a/clickhouse-parquet/results/20260517/t3a.small.json +++ b/clickhouse-parquet/results/20260517/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.03, - "concurrent_error_ratio": 0.053, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.205, 0.076, 0.086], [0.694, 0.479, 0.307], diff --git a/datafusion-partitioned/results/20260511/c6a.2xlarge.json b/datafusion-partitioned/results/20260511/c6a.2xlarge.json index 5ffea5fe2..5a9e1d1a4 100644 --- a/datafusion-partitioned/results/20260511/c6a.2xlarge.json +++ b/datafusion-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 8, "data_size": 14737666736, - "concurrent_qps": 0.062, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.024, 0.002, 0.002], [0.114, 0.036, 0.038], diff --git a/datafusion-partitioned/results/20260511/c6a.4xlarge.json b/datafusion-partitioned/results/20260511/c6a.4xlarge.json index 366d9d09e..daa19b16d 100644 --- a/datafusion-partitioned/results/20260511/c6a.4xlarge.json +++ b/datafusion-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 17, "data_size": 14737666736, - "concurrent_qps": 0.097, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.02, 0.002, 0.002], [0.079, 0.021, 0.022], diff --git a/datafusion-partitioned/results/20260511/c6a.metal.json b/datafusion-partitioned/results/20260511/c6a.metal.json index bf0f0bac7..b1e13f47f 100644 --- a/datafusion-partitioned/results/20260511/c6a.metal.json +++ b/datafusion-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 51, "data_size": 14737666736, - "concurrent_qps": 4.915, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.037, 0.003, 0.002], [0.105, 0.036, 0.036], diff --git a/datafusion-partitioned/results/20260511/c6a.xlarge.json b/datafusion-partitioned/results/20260511/c6a.xlarge.json index 6bfb4e428..7e0558b5f 100644 --- a/datafusion-partitioned/results/20260511/c6a.xlarge.json +++ b/datafusion-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 3, "data_size": 14737666736, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.022, 0.002, 0.002], [0.139, 0.063, 0.062], diff --git a/datafusion-partitioned/results/20260511/c7a.metal-48xl.json b/datafusion-partitioned/results/20260511/c7a.metal-48xl.json index 649e626a6..ec4dec3f1 100644 --- a/datafusion-partitioned/results/20260511/c7a.metal-48xl.json +++ b/datafusion-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 54, "data_size": 14737666736, - "concurrent_qps": 7.017, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.018, 0.003, 0.002], [0.07, 0.039, 0.042], diff --git a/datafusion-partitioned/results/20260511/c8g.4xlarge.json b/datafusion-partitioned/results/20260511/c8g.4xlarge.json index 5b79c7315..22db69ba2 100644 --- a/datafusion-partitioned/results/20260511/c8g.4xlarge.json +++ b/datafusion-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 18, "data_size": 14737666736, - "concurrent_qps": 0.095, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.015, 0.002, 0.002], [0.075, 0.014, 0.012], diff --git a/datafusion-partitioned/results/20260511/c8g.metal-48xl.json b/datafusion-partitioned/results/20260511/c8g.metal-48xl.json index e9f2981bd..7502a879f 100644 --- a/datafusion-partitioned/results/20260511/c8g.metal-48xl.json +++ b/datafusion-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 45, "data_size": 14737666736, - "concurrent_qps": 6.498, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.013, 0.002, 0.002], [0.058, 0.018, 0.019], diff --git a/datafusion-vortex-partitioned/benchmark.sh b/datafusion-vortex-partitioned/benchmark.sh index 51153ec49..6bbd1cebb 100755 --- a/datafusion-vortex-partitioned/benchmark.sh +++ b/datafusion-vortex-partitioned/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/datafusion-vortex-partitioned/results/20260511/c6a.2xlarge.json b/datafusion-vortex-partitioned/results/20260511/c6a.2xlarge.json index 9899ec8c2..d79c49726 100644 --- a/datafusion-vortex-partitioned/results/20260511/c6a.2xlarge.json +++ b/datafusion-vortex-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 299, "data_size": 15812419580, - "concurrent_qps": 0.022, - "concurrent_error_ratio": 0.519, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.223, 0.026, 0.025], [0.273, 0.049, 0.048], diff --git a/datafusion-vortex-partitioned/results/20260511/c6a.4xlarge.json b/datafusion-vortex-partitioned/results/20260511/c6a.4xlarge.json index fd0f93c81..e72b8ad28 100644 --- a/datafusion-vortex-partitioned/results/20260511/c6a.4xlarge.json +++ b/datafusion-vortex-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 157, "data_size": 15812419580, - "concurrent_qps": 0.022, - "concurrent_error_ratio": 0.567, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.208, 0.026, 0.026], [0.205, 0.039, 0.04], diff --git a/datafusion-vortex-partitioned/results/20260511/c6a.metal.json b/datafusion-vortex-partitioned/results/20260511/c6a.metal.json index 8ec6088aa..85e127a69 100644 --- a/datafusion-vortex-partitioned/results/20260511/c6a.metal.json +++ b/datafusion-vortex-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 129, "data_size": 15812419580, - "concurrent_qps": 0.952, - "concurrent_error_ratio": 0.789, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.186, 0.018, 0.019], [0.165, 0.027, 0.029], diff --git a/datafusion-vortex-partitioned/results/20260511/c6a.xlarge.json b/datafusion-vortex-partitioned/results/20260511/c6a.xlarge.json index cf9d4c08c..7f6bc3951 100644 --- a/datafusion-vortex-partitioned/results/20260511/c6a.xlarge.json +++ b/datafusion-vortex-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 969, "data_size": 15812419580, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 0.5, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.216, 0.028, 0.027], [0.251, 0.069, 0.069], diff --git a/datafusion-vortex-partitioned/results/20260511/c7a.metal-48xl.json b/datafusion-vortex-partitioned/results/20260511/c7a.metal-48xl.json index c534a95f3..e63a8b7e9 100644 --- a/datafusion-vortex-partitioned/results/20260511/c7a.metal-48xl.json +++ b/datafusion-vortex-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 130, "data_size": 15812419580, - "concurrent_qps": 1.345, - "concurrent_error_ratio": 0.79, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.216, 0.017, 0.016], [0.206, 0.026, 0.029], diff --git a/datafusion-vortex-partitioned/results/20260511/c8g.4xlarge.json b/datafusion-vortex-partitioned/results/20260511/c8g.4xlarge.json index 3d07cea9d..82deff4eb 100644 --- a/datafusion-vortex-partitioned/results/20260511/c8g.4xlarge.json +++ b/datafusion-vortex-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 239, "data_size": 15812419580, - "concurrent_qps": 0.028, - "concurrent_error_ratio": 0.646, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.217, 0.009, 0.01], [0.268, 0.018, 0.019], diff --git a/datafusion-vortex/benchmark.sh b/datafusion-vortex/benchmark.sh index dce465f2f..6dd84d0d3 100755 --- a/datafusion-vortex/benchmark.sh +++ b/datafusion-vortex/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/datafusion/results/20260511/c6a.2xlarge.json b/datafusion/results/20260511/c6a.2xlarge.json index c260fc1fa..d009f1845 100644 --- a/datafusion/results/20260511/c6a.2xlarge.json +++ b/datafusion/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.042, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.026, 0.001, 0.001], [0.121, 0.039, 0.039], diff --git a/datafusion/results/20260511/c6a.4xlarge.json b/datafusion/results/20260511/c6a.4xlarge.json index 8d65274f6..20f9c2abb 100644 --- a/datafusion/results/20260511/c6a.4xlarge.json +++ b/datafusion/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.11, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.027, 0.001, 0.001], [0.114, 0.035, 0.036], diff --git a/datafusion/results/20260511/c6a.metal.json b/datafusion/results/20260511/c6a.metal.json index 42ef6f7a5..002ccf020 100644 --- a/datafusion/results/20260511/c6a.metal.json +++ b/datafusion/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 4.017, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.023, 0.001, 0.002], [0.136, 0.076, 0.096], diff --git a/datafusion/results/20260511/c6a.xlarge.json b/datafusion/results/20260511/c6a.xlarge.json index 4245154e8..b40baff23 100644 --- a/datafusion/results/20260511/c6a.xlarge.json +++ b/datafusion/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.026, 0.001, 0.001], [0.165, 0.059, 0.059], diff --git a/datafusion/results/20260511/c7a.metal-48xl.json b/datafusion/results/20260511/c7a.metal-48xl.json index 27c3c0a09..62845a038 100644 --- a/datafusion/results/20260511/c7a.metal-48xl.json +++ b/datafusion/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 5.888, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.024, 0.001, 0.001], [0.147, 0.101, 0.103], diff --git a/datafusion/results/20260511/c8g.4xlarge.json b/datafusion/results/20260511/c8g.4xlarge.json index f959b3d85..d00a01b61 100644 --- a/datafusion/results/20260511/c8g.4xlarge.json +++ b/datafusion/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.092, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.013, 0.001, 0.001], [0.068, 0.018, 0.017], diff --git a/datafusion/results/20260511/c8g.metal-48xl.json b/datafusion/results/20260511/c8g.metal-48xl.json index 8b28d3f03..5d6cf28b2 100644 --- a/datafusion/results/20260511/c8g.metal-48xl.json +++ b/datafusion/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","embedded","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 5.388, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.013, 0.001, 0.001], [0.101, 0.067, 0.079], diff --git a/drill/benchmark.sh b/drill/benchmark.sh index 617422ddc..5427511a0 100755 --- a/drill/benchmark.sh +++ b/drill/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-datalake-partitioned/benchmark.sh b/duckdb-datalake-partitioned/benchmark.sh index 08d52c39f..9c514c480 100755 --- a/duckdb-datalake-partitioned/benchmark.sh +++ b/duckdb-datalake-partitioned/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.2xlarge.json b/duckdb-datalake-partitioned/results/20260511/c6a.2xlarge.json index 96c23dca1..bdd428bc1 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.2xlarge.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.18, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.896, 0.752, 0.808], [2.036, 2.01, 2.055], diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.4xlarge.json b/duckdb-datalake-partitioned/results/20260511/c6a.4xlarge.json index c815c397b..cf6ca89ca 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.4xlarge.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.378, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.659, 0.518, 0.436], [1.207, 1.098, 1.039], diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.large.json b/duckdb-datalake-partitioned/results/20260511/c6a.large.json index d853d0625..8c9aff115 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.large.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14737666736, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [2.504, 2.324, 2.666], [7.67, 7.597, 7.455], diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.metal.json b/duckdb-datalake-partitioned/results/20260511/c6a.metal.json index 3d069c3a1..fc9a581b6 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.metal.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.628, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.619, 0.558, 0.582], [5.483, 0.676, 0.621], diff --git a/duckdb-datalake-partitioned/results/20260511/c6a.xlarge.json b/duckdb-datalake-partitioned/results/20260511/c6a.xlarge.json index 202fea0d7..e0491519d 100644 --- a/duckdb-datalake-partitioned/results/20260511/c6a.xlarge.json +++ b/duckdb-datalake-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.067, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [1.486, 1.281, 1.474], [4.146, 3.676, 3.939], diff --git a/duckdb-datalake-partitioned/results/20260511/c7a.metal-48xl.json b/duckdb-datalake-partitioned/results/20260511/c7a.metal-48xl.json index c9ec0c284..d9c282d95 100644 --- a/duckdb-datalake-partitioned/results/20260511/c7a.metal-48xl.json +++ b/duckdb-datalake-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.76, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.586, 0.547, 0.603], [0.661, 5.358, 5.35], diff --git a/duckdb-datalake-partitioned/results/20260511/c8g.4xlarge.json b/duckdb-datalake-partitioned/results/20260511/c8g.4xlarge.json index cafb0e56a..926d4f403 100644 --- a/duckdb-datalake-partitioned/results/20260511/c8g.4xlarge.json +++ b/duckdb-datalake-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14737666736, - "concurrent_qps": 0.482, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.488, 0.484, 0.392], [1.101, 0.922, 0.907], diff --git a/duckdb-datalake-partitioned/results/20260511/c8g.metal-48xl.json b/duckdb-datalake-partitioned/results/20260511/c8g.metal-48xl.json index 1afa74526..cc49d8fac 100644 --- a/duckdb-datalake-partitioned/results/20260511/c8g.metal-48xl.json +++ b/duckdb-datalake-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14737666736, - "concurrent_qps": 0.842, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.419, 0.402, 0.459], [0.476, 0.439, 0.445], diff --git a/duckdb-datalake-partitioned/results/20260511/t3a.small.json b/duckdb-datalake-partitioned/results/20260511/t3a.small.json index 196ba09b0..c7d934c3a 100644 --- a/duckdb-datalake-partitioned/results/20260511/t3a.small.json +++ b/duckdb-datalake-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.012, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.187, 3.345, 3.31], [9.425, 9.408, 9.526], diff --git a/duckdb-datalake/benchmark.sh b/duckdb-datalake/benchmark.sh index 08d52c39f..9c514c480 100755 --- a/duckdb-datalake/benchmark.sh +++ b/duckdb-datalake/benchmark.sh @@ -4,4 +4,9 @@ export BENCH_DOWNLOAD_SCRIPT="" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-datalake/results/20260511/c6a.2xlarge.json b/duckdb-datalake/results/20260511/c6a.2xlarge.json index b0b497544..b77423868 100644 --- a/duckdb-datalake/results/20260511/c6a.2xlarge.json +++ b/duckdb-datalake/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.245, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.182, 0.162, 0.151], [0.885, 0.822, 0.822], diff --git a/duckdb-datalake/results/20260511/c6a.4xlarge.json b/duckdb-datalake/results/20260511/c6a.4xlarge.json index 0f84b3f72..585355711 100644 --- a/duckdb-datalake/results/20260511/c6a.4xlarge.json +++ b/duckdb-datalake/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.475, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.184, 0.142, 0.15], [0.567, 0.51, 0.506], diff --git a/duckdb-datalake/results/20260511/c6a.large.json b/duckdb-datalake/results/20260511/c6a.large.json index d13b7affe..5aa3bb784 100644 --- a/duckdb-datalake/results/20260511/c6a.large.json +++ b/duckdb-datalake/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.043, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.239, 0.216, 0.191], [2.835, 2.884, 2.863], diff --git a/duckdb-datalake/results/20260511/c6a.metal.json b/duckdb-datalake/results/20260511/c6a.metal.json index 05553b898..2a7a16bca 100644 --- a/duckdb-datalake/results/20260511/c6a.metal.json +++ b/duckdb-datalake/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.645, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.178, 0.146, 0.124], [5.334, 0.492, 5.279], diff --git a/duckdb-datalake/results/20260511/c6a.xlarge.json b/duckdb-datalake/results/20260511/c6a.xlarge.json index 45866ee4a..e85f403ca 100644 --- a/duckdb-datalake/results/20260511/c6a.xlarge.json +++ b/duckdb-datalake/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.212, 0.169, 0.168], [1.537, 1.508, 1.512], diff --git a/duckdb-datalake/results/20260511/c7a.metal-48xl.json b/duckdb-datalake/results/20260511/c7a.metal-48xl.json index 084e1b936..2a8392001 100644 --- a/duckdb-datalake/results/20260511/c7a.metal-48xl.json +++ b/duckdb-datalake/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.838, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.165, 0.158, 0.128], [0.53, 0.448, 0.446], diff --git a/duckdb-datalake/results/20260511/c8g.4xlarge.json b/duckdb-datalake/results/20260511/c8g.4xlarge.json index b8ef59ca2..6cb34a397 100644 --- a/duckdb-datalake/results/20260511/c8g.4xlarge.json +++ b/duckdb-datalake/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.66, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.189, 0.133, 0.124], [0.528, 0.495, 0.487], diff --git a/duckdb-datalake/results/20260511/c8g.metal-48xl.json b/duckdb-datalake/results/20260511/c8g.metal-48xl.json index 79740ca16..0ff3f954b 100644 --- a/duckdb-datalake/results/20260511/c8g.metal-48xl.json +++ b/duckdb-datalake/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.547, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.15, 0.111, 0.147], [5.271, 0.355, 0.342], diff --git a/duckdb-datalake/results/20260511/t3a.small.json b/duckdb-datalake/results/20260511/t3a.small.json index 2864184c7..351ed7db3 100644 --- a/duckdb-datalake/results/20260511/t3a.small.json +++ b/duckdb-datalake/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.382, 0.338, 0.351], [3.213, 3.143, 3.205], diff --git a/duckdb-parquet-partitioned/benchmark.sh b/duckdb-parquet-partitioned/benchmark.sh index e236dbbd2..5972a9845 100755 --- a/duckdb-parquet-partitioned/benchmark.sh +++ b/duckdb-parquet-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.2xlarge.json b/duckdb-parquet-partitioned/results/20260511/c6a.2xlarge.json index d6d47f33f..38b4645cb 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.2xlarge.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.383, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.073, 0.038, 0.036], [0.118, 0.048, 0.047], diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.4xlarge.json b/duckdb-parquet-partitioned/results/20260511/c6a.4xlarge.json index 4593218e0..dbce9f806 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.4xlarge.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.917, - "concurrent_error_ratio": 0.996, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.065, 0.034, 0.037], [0.076, 0.027, 0.028], diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.large.json b/duckdb-parquet-partitioned/results/20260511/c6a.large.json index ec3d8050c..70482ccec 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.large.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.241, 0.104, 0.107], [0.427, 0.174, 0.176], diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.metal.json b/duckdb-parquet-partitioned/results/20260511/c6a.metal.json index 7262b301a..0c474c13c 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.metal.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.865, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.378, 0.363, 0.331], [0.076, 0.055, 0.066], diff --git a/duckdb-parquet-partitioned/results/20260511/c6a.xlarge.json b/duckdb-parquet-partitioned/results/20260511/c6a.xlarge.json index a67735701..203ebe03c 100644 --- a/duckdb-parquet-partitioned/results/20260511/c6a.xlarge.json +++ b/duckdb-parquet-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 5, "data_size": 14737666736, - "concurrent_qps": 0.097, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.13, 0.06, 0.061], [0.212, 0.092, 0.09], diff --git a/duckdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json b/duckdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json index 669164377..9e848738a 100644 --- a/duckdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json +++ b/duckdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 2.078, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.308, 0.329, 0.285], [0.083, 0.059, 0.053], diff --git a/duckdb-parquet-partitioned/results/20260511/c8g.4xlarge.json b/duckdb-parquet-partitioned/results/20260511/c8g.4xlarge.json index 0daec4986..310bb94bd 100644 --- a/duckdb-parquet-partitioned/results/20260511/c8g.4xlarge.json +++ b/duckdb-parquet-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 1.597, - "concurrent_error_ratio": 0.996, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.044, 0.02, 0.022], [0.06, 0.016, 0.017], diff --git a/duckdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json b/duckdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json index 0b92bcd99..a2467e1fc 100644 --- a/duckdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json +++ b/duckdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 2.442, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.189, 0.18, 0.223], [0.067, 0.057, 0.066], diff --git a/duckdb-parquet-partitioned/results/20260511/t3a.small.json b/duckdb-parquet-partitioned/results/20260511/t3a.small.json index 59f3900f7..2e4033d41 100644 --- a/duckdb-parquet-partitioned/results/20260511/t3a.small.json +++ b/duckdb-parquet-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14737666736, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.309, 0.185, 0.195], [0.522, 0.28, 0.285], diff --git a/duckdb-parquet/benchmark.sh b/duckdb-parquet/benchmark.sh index 617422ddc..5427511a0 100755 --- a/duckdb-parquet/benchmark.sh +++ b/duckdb-parquet/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-parquet/results/20260511/c6a.2xlarge.json b/duckdb-parquet/results/20260511/c6a.2xlarge.json index fadc9dec9..dd86a1a5a 100644 --- a/duckdb-parquet/results/20260511/c6a.2xlarge.json +++ b/duckdb-parquet/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.387, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.076, 0.051, 0.051], [0.146, 0.083, 0.081], diff --git a/duckdb-parquet/results/20260511/c6a.4xlarge.json b/duckdb-parquet/results/20260511/c6a.4xlarge.json index 160929868..bcbf648c8 100644 --- a/duckdb-parquet/results/20260511/c6a.4xlarge.json +++ b/duckdb-parquet/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.88, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.079, 0.053, 0.052], [0.12, 0.069, 0.07], diff --git a/duckdb-parquet/results/20260511/c6a.large.json b/duckdb-parquet/results/20260511/c6a.large.json index 1b87e3f10..234fab3f2 100644 --- a/duckdb-parquet/results/20260511/c6a.large.json +++ b/duckdb-parquet/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.079, 0.053, 0.052], [0.29, 0.168, 0.165], diff --git a/duckdb-parquet/results/20260511/c6a.metal.json b/duckdb-parquet/results/20260511/c6a.metal.json index 8a275137e..8ed389f4f 100644 --- a/duckdb-parquet/results/20260511/c6a.metal.json +++ b/duckdb-parquet/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 1.815, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.072, 0.05, 0.051], [0.122, 0.079, 0.077], diff --git a/duckdb-parquet/results/20260511/c6a.xlarge.json b/duckdb-parquet/results/20260511/c6a.xlarge.json index 453fe02b5..48802cb44 100644 --- a/duckdb-parquet/results/20260511/c6a.xlarge.json +++ b/duckdb-parquet/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.073, 0.055, 0.055], [0.193, 0.114, 0.113], diff --git a/duckdb-parquet/results/20260511/c7a.metal-48xl.json b/duckdb-parquet/results/20260511/c7a.metal-48xl.json index 7e23cbc8b..cb6d5b01f 100644 --- a/duckdb-parquet/results/20260511/c7a.metal-48xl.json +++ b/duckdb-parquet/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 2.057, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.072, 0.044, 0.046], [0.112, 0.07, 0.073], diff --git a/duckdb-parquet/results/20260511/c8g.4xlarge.json b/duckdb-parquet/results/20260511/c8g.4xlarge.json index 832c26886..89719e70b 100644 --- a/duckdb-parquet/results/20260511/c8g.4xlarge.json +++ b/duckdb-parquet/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 1.463, - "concurrent_error_ratio": 0.997, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.055, 0.039, 0.039], [0.097, 0.049, 0.049], diff --git a/duckdb-parquet/results/20260511/c8g.metal-48xl.json b/duckdb-parquet/results/20260511/c8g.metal-48xl.json index c10a8750d..4d4c2e9df 100644 --- a/duckdb-parquet/results/20260511/c8g.metal-48xl.json +++ b/duckdb-parquet/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 2.61, - "concurrent_error_ratio": 0.994, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.051, 0.043, 0.04], [0.094, 0.07, 0.076], diff --git a/duckdb-parquet/results/20260511/t3a.small.json b/duckdb-parquet/results/20260511/t3a.small.json index 6b284d92b..bd98cdb3f 100644 --- a/duckdb-parquet/results/20260511/t3a.small.json +++ b/duckdb-parquet/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.145, 0.114, 0.114], [0.473, 0.301, 0.304], diff --git a/duckdb-vortex-partitioned/benchmark.sh b/duckdb-vortex-partitioned/benchmark.sh index e236dbbd2..5972a9845 100755 --- a/duckdb-vortex-partitioned/benchmark.sh +++ b/duckdb-vortex-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-vortex/benchmark.sh b/duckdb-vortex/benchmark.sh index 617422ddc..5427511a0 100755 --- a/duckdb-vortex/benchmark.sh +++ b/duckdb-vortex/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-vortex/results/20260511/c6a.2xlarge.json b/duckdb-vortex/results/20260511/c6a.2xlarge.json index c83e07ead..b816f92e5 100644 --- a/duckdb-vortex/results/20260511/c6a.2xlarge.json +++ b/duckdb-vortex/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 197, "data_size": 15731820628, - "concurrent_qps": 0.198, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.042, 0.027, 0.027], [0.062, 0.04, 0.04], diff --git a/duckdb-vortex/results/20260511/c6a.4xlarge.json b/duckdb-vortex/results/20260511/c6a.4xlarge.json index 5dd611074..ddf7d6c98 100644 --- a/duckdb-vortex/results/20260511/c6a.4xlarge.json +++ b/duckdb-vortex/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 141, "data_size": 15731820628, - "concurrent_qps": 0.735, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.043, 0.027, 0.027], [0.054, 0.032, 0.032], diff --git a/duckdb-vortex/results/20260511/c6a.metal.json b/duckdb-vortex/results/20260511/c6a.metal.json index 1d8ee97a3..0c83d3947 100644 --- a/duckdb-vortex/results/20260511/c6a.metal.json +++ b/duckdb-vortex/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 222, "data_size": 15731820628, - "concurrent_qps": 0.665, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.048, 0.031, 0.032], [0.082, 0.059, 0.061], diff --git a/duckdb-vortex/results/20260511/c7a.metal-48xl.json b/duckdb-vortex/results/20260511/c7a.metal-48xl.json index 078b2c45d..0fb50ab47 100644 --- a/duckdb-vortex/results/20260511/c7a.metal-48xl.json +++ b/duckdb-vortex/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 292, "data_size": 15731820628, - "concurrent_qps": 0.585, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.057, 0.029, 0.03], [0.095, 0.057, 0.057], diff --git a/duckdb-vortex/results/20260511/c8g.4xlarge.json b/duckdb-vortex/results/20260511/c8g.4xlarge.json index af970a7b0..cd3292391 100644 --- a/duckdb-vortex/results/20260511/c8g.4xlarge.json +++ b/duckdb-vortex/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 118, "data_size": 15731820628, - "concurrent_qps": 0.977, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.042, 0.022, 0.022], [0.05, 0.021, 0.021], diff --git a/duckdb-vortex/results/20260511/c8g.metal-48xl.json b/duckdb-vortex/results/20260511/c8g.metal-48xl.json index a464de31d..c237ffc1a 100644 --- a/duckdb-vortex/results/20260511/c8g.metal-48xl.json +++ b/duckdb-vortex/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","column-oriented","DuckDB derivative","embedded","stateless"], "load_time": 148, "data_size": 15731820628, - "concurrent_qps": 0.913, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.037, 0.024, 0.025], [0.06, 0.046, 0.04], diff --git a/duckdb/benchmark.sh b/duckdb/benchmark.sh index 617422ddc..5427511a0 100755 --- a/duckdb/benchmark.sh +++ b/duckdb/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb/results/20260511/c6a.2xlarge.json b/duckdb/results/20260511/c6a.2xlarge.json index 18f563e1a..bf9b68850 100644 --- a/duckdb/results/20260511/c6a.2xlarge.json +++ b/duckdb/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 208, "data_size": 20436758528, - "concurrent_qps": 0.337, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.027, 0.018, 0.018], [0.202, 0.052, 0.051], diff --git a/duckdb/results/20260511/c6a.4xlarge.json b/duckdb/results/20260511/c6a.4xlarge.json index e0c5d0e34..8f7dbe2d3 100644 --- a/duckdb/results/20260511/c6a.4xlarge.json +++ b/duckdb/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 126, "data_size": 20457467904, - "concurrent_qps": 0.705, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.03, 0.018, 0.018], [0.132, 0.041, 0.041], diff --git a/duckdb/results/20260511/c6a.large.json b/duckdb/results/20260511/c6a.large.json index 7c90fdd4c..308ddeae2 100644 --- a/duckdb/results/20260511/c6a.large.json +++ b/duckdb/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 508, "data_size": 20422340608, - "concurrent_qps": 0.03, - "concurrent_error_ratio": 1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.026, 0.018, 0.018], [0.559, 0.092, 0.092], diff --git a/duckdb/results/20260511/c6a.metal.json b/duckdb/results/20260511/c6a.metal.json index 91d377e83..429ed173d 100644 --- a/duckdb/results/20260511/c6a.metal.json +++ b/duckdb/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 102, "data_size": 20635725824, - "concurrent_qps": 1.258, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.024, 0.017, 0.018], [0.082, 0.045, 0.043], diff --git a/duckdb/results/20260511/c6a.xlarge.json b/duckdb/results/20260511/c6a.xlarge.json index 518105065..e4cbb8da1 100644 --- a/duckdb/results/20260511/c6a.xlarge.json +++ b/duckdb/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 279, "data_size": 20429680640, - "concurrent_qps": 0.138, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.029, 0.018, 0.019], [0.324, 0.067, 0.068], diff --git a/duckdb/results/20260511/c7a.metal-48xl.json b/duckdb/results/20260511/c7a.metal-48xl.json index 6f24176b5..f69b4c5a6 100644 --- a/duckdb/results/20260511/c7a.metal-48xl.json +++ b/duckdb/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 98, "data_size": 20636512256, - "concurrent_qps": 1.553, - "concurrent_error_ratio": 0.992, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.025, 0.017, 0.017], [0.098, 0.041, 0.04], diff --git a/duckdb/results/20260511/c8g.4xlarge.json b/duckdb/results/20260511/c8g.4xlarge.json index 9159fc9f6..e389a496a 100644 --- a/duckdb/results/20260511/c8g.4xlarge.json +++ b/duckdb/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 120, "data_size": 20462448640, - "concurrent_qps": 1.067, - "concurrent_error_ratio": 0.998, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.021, 0.013, 0.013], [0.083, 0.025, 0.026], diff --git a/duckdb/results/20260511/c8g.metal-48xl.json b/duckdb/results/20260511/c8g.metal-48xl.json index d5bc8ca9c..ca2540849 100644 --- a/duckdb/results/20260511/c8g.metal-48xl.json +++ b/duckdb/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","DuckDB derivative","embedded"], "load_time": 96, "data_size": 20639133696, - "concurrent_qps": 2.1, - "concurrent_error_ratio": 0.993, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.017, 0.014, 0.015], [0.059, 0.032, 0.034], diff --git a/gendb/benchmark.sh b/gendb/benchmark.sh index 31b5bc717..68314250b 100755 --- a/gendb/benchmark.sh +++ b/gendb/benchmark.sh @@ -12,4 +12,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no export BENCH_DURABLE=yes +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/gendb/results/20260518/c6a.2xlarge.json b/gendb/results/20260518/c6a.2xlarge.json index 291ae3180..1ead20d20 100644 --- a/gendb/results/20260518/c6a.2xlarge.json +++ b/gendb/results/20260518/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 180, "data_size": 37351308970, - "concurrent_qps": 0.047, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.738, 0.015, 0.015], diff --git a/gendb/results/20260518/c6a.4xlarge.json b/gendb/results/20260518/c6a.4xlarge.json index 7221abb72..14c674dee 100644 --- a/gendb/results/20260518/c6a.4xlarge.json +++ b/gendb/results/20260518/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 166, "data_size": 37351308970, - "concurrent_qps": 0.093, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.638, 0.016, 0.015], diff --git a/gendb/results/20260518/c6a.metal.json b/gendb/results/20260518/c6a.metal.json index 818e9aa7f..2a251de1d 100644 --- a/gendb/results/20260518/c6a.metal.json +++ b/gendb/results/20260518/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 177, "data_size": 37351308970, - "concurrent_qps": 2.002, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.663, 0.027, 0.031], diff --git a/gendb/results/20260518/c7a.metal-48xl.json b/gendb/results/20260518/c7a.metal-48xl.json index 4f4940b57..766c6c40d 100644 --- a/gendb/results/20260518/c7a.metal-48xl.json +++ b/gendb/results/20260518/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 181, "data_size": 37351308970, - "concurrent_qps": 2.4, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.791, 0.022, 0.022], diff --git a/gendb/results/20260518/c8g.4xlarge.json b/gendb/results/20260518/c8g.4xlarge.json index 5e6ed8526..80ad9f233 100644 --- a/gendb/results/20260518/c8g.4xlarge.json +++ b/gendb/results/20260518/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 162, "data_size": 37351308970, - "concurrent_qps": 0.175, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.887, 0.006, 0.005], diff --git a/gendb/results/20260518/c8g.metal-48xl.json b/gendb/results/20260518/c8g.metal-48xl.json index 336684bbc..706301c35 100644 --- a/gendb/results/20260518/c8g.metal-48xl.json +++ b/gendb/results/20260518/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 173, "data_size": 37351308970, - "concurrent_qps": 3.41, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0, 0, 0], [0.613, 0.008, 0.008], diff --git a/glaredb-partitioned/benchmark.sh b/glaredb-partitioned/benchmark.sh index e236dbbd2..5972a9845 100755 --- a/glaredb-partitioned/benchmark.sh +++ b/glaredb-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/glaredb-partitioned/results/20260511/c6a.2xlarge.json b/glaredb-partitioned/results/20260511/c6a.2xlarge.json index 51c7287e2..5e9109a67 100644 --- a/glaredb-partitioned/results/20260511/c6a.2xlarge.json +++ b/glaredb-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 9, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.06, 0.046, 0.042], [0.208, 0.184, 0.183], diff --git a/glaredb-partitioned/results/20260511/c6a.4xlarge.json b/glaredb-partitioned/results/20260511/c6a.4xlarge.json index dc1dba0f0..cd8cb04fb 100644 --- a/glaredb-partitioned/results/20260511/c6a.4xlarge.json +++ b/glaredb-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.052, - "concurrent_error_ratio": 0.031, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.03, 0.02, 0.028], [0.116, 0.098, 0.12], diff --git a/glaredb-partitioned/results/20260511/c6a.large.json b/glaredb-partitioned/results/20260511/c6a.large.json index abc75a015..0be07f7d0 100644 --- a/glaredb-partitioned/results/20260511/c6a.large.json +++ b/glaredb-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.027, - "concurrent_error_ratio": 0.059, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.235, 0.13, 0.129], [0.817, 0.711, 0.712], diff --git a/glaredb-partitioned/results/20260511/c6a.metal.json b/glaredb-partitioned/results/20260511/c6a.metal.json index 97a82f5ad..1052e25c9 100644 --- a/glaredb-partitioned/results/20260511/c6a.metal.json +++ b/glaredb-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.705, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.047, 0.062, 0.057], [0.081, 0.077, 0.077], diff --git a/glaredb-partitioned/results/20260511/c7a.metal-48xl.json b/glaredb-partitioned/results/20260511/c7a.metal-48xl.json index faeba470e..f479d7dc1 100644 --- a/glaredb-partitioned/results/20260511/c7a.metal-48xl.json +++ b/glaredb-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 40, "data_size": 14737666736, - "concurrent_qps": 1.987, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.065, 0.06, 0.068], [0.082, 0.072, 0.081], diff --git a/glaredb-partitioned/results/20260511/c8g.4xlarge.json b/glaredb-partitioned/results/20260511/c8g.4xlarge.json index 3ca7be616..3bc2a5297 100644 --- a/glaredb-partitioned/results/20260511/c8g.4xlarge.json +++ b/glaredb-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.07, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.045, 0.038, 0.036], [0.109, 0.058, 0.075], diff --git a/glaredb-partitioned/results/20260511/c8g.metal-48xl.json b/glaredb-partitioned/results/20260511/c8g.metal-48xl.json index 24e7db067..15bf3d3da 100644 --- a/glaredb-partitioned/results/20260511/c8g.metal-48xl.json +++ b/glaredb-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 1.373, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.071, 0.068, 0.065], [0.067, 0.075, 0.073], diff --git a/glaredb/benchmark.sh b/glaredb/benchmark.sh index 617422ddc..5427511a0 100755 --- a/glaredb/benchmark.sh +++ b/glaredb/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/glaredb/results/20260511/c6a.2xlarge.json b/glaredb/results/20260511/c6a.2xlarge.json index 279c0c8e3..66821a921 100644 --- a/glaredb/results/20260511/c6a.2xlarge.json +++ b/glaredb/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.037, - "concurrent_error_ratio": 0.043, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.062, 0.057, 0.053], [0.208, 0.186, 0.203], diff --git a/glaredb/results/20260511/c6a.4xlarge.json b/glaredb/results/20260511/c6a.4xlarge.json index 3e3a1e2bd..d97574d6a 100644 --- a/glaredb/results/20260511/c6a.4xlarge.json +++ b/glaredb/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.052, - "concurrent_error_ratio": 0.061, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.045, 0.036, 0.037], [0.119, 0.126, 0.121], diff --git a/glaredb/results/20260511/c6a.large.json b/glaredb/results/20260511/c6a.large.json index 5ebc43ae9..4488056e8 100644 --- a/glaredb/results/20260511/c6a.large.json +++ b/glaredb/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.028, - "concurrent_error_ratio": 0.056, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.121, 0.114, 0.115], [0.698, 0.643, 0.645], diff --git a/glaredb/results/20260511/c6a.metal.json b/glaredb/results/20260511/c6a.metal.json index 2a82a2b52..13b0eca24 100644 --- a/glaredb/results/20260511/c6a.metal.json +++ b/glaredb/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 1.6680000000000001, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.052, 0.066, 0.046], [0.077, 0.081, 0.071], diff --git a/glaredb/results/20260511/c7a.metal-48xl.json b/glaredb/results/20260511/c7a.metal-48xl.json index c0cba3770..aab081cc5 100644 --- a/glaredb/results/20260511/c7a.metal-48xl.json +++ b/glaredb/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 1.817, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.045, 0.058, 0.057], [0.063, 0.063, 0.073], diff --git a/glaredb/results/20260511/c8g.4xlarge.json b/glaredb/results/20260511/c8g.4xlarge.json index 7c7daa008..9b889cfae 100644 --- a/glaredb/results/20260511/c8g.4xlarge.json +++ b/glaredb/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.063, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.053, 0.05, 0.057], [0.087, 0.072, 0.071], diff --git a/glaredb/results/20260511/c8g.metal-48xl.json b/glaredb/results/20260511/c8g.metal-48xl.json index 13f94099c..f5acc6c22 100644 --- a/glaredb/results/20260511/c8g.metal-48xl.json +++ b/glaredb/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Rust","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 1.383, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.054, 0.053, 0.057], [0.059, 0.056, 0.058], diff --git a/hyper-parquet/benchmark.sh b/hyper-parquet/benchmark.sh index e236dbbd2..5972a9845 100755 --- a/hyper-parquet/benchmark.sh +++ b/hyper-parquet/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/hyper-parquet/results/20260511/c6a.2xlarge.json b/hyper-parquet/results/20260511/c6a.2xlarge.json index 0065cb469..41074c43f 100644 --- a/hyper-parquet/results/20260511/c6a.2xlarge.json +++ b/hyper-parquet/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0.226, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.867, 0.397, 0.386], [0.916, 0.413, 0.415], diff --git a/hyper-parquet/results/20260511/c6a.4xlarge.json b/hyper-parquet/results/20260511/c6a.4xlarge.json index 5cd1500ac..24b68a6ba 100644 --- a/hyper-parquet/results/20260511/c6a.4xlarge.json +++ b/hyper-parquet/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.095, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.668, 0.321, 0.309], [0.696, 0.305, 0.345], diff --git a/hyper-parquet/results/20260511/c6a.metal.json b/hyper-parquet/results/20260511/c6a.metal.json index 33d48d616..eb2842709 100644 --- a/hyper-parquet/results/20260511/c6a.metal.json +++ b/hyper-parquet/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 5.122, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.599, 0.261, 0.287], [0.649, 0.294, 0.297], diff --git a/hyper-parquet/results/20260511/c6a.xlarge.json b/hyper-parquet/results/20260511/c6a.xlarge.json index 9bd89c84a..033fbc6ed 100644 --- a/hyper-parquet/results/20260511/c6a.xlarge.json +++ b/hyper-parquet/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 0.389, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [1.204, 0.55, 0.604], [1.339, 0.659, 0.622], diff --git a/hyper-parquet/results/20260511/c7a.metal-48xl.json b/hyper-parquet/results/20260511/c7a.metal-48xl.json index 91db48270..db84ffb0c 100644 --- a/hyper-parquet/results/20260511/c7a.metal-48xl.json +++ b/hyper-parquet/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented","stateless"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 6.622, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.628, 0.252, 0.247], [0.644, 0.252, 0.253], diff --git a/hyper/benchmark.sh b/hyper/benchmark.sh index 2e5741b39..bf0286804 100755 --- a/hyper/benchmark.sh +++ b/hyper/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/hyper/results/20260511/c6a.2xlarge.json b/hyper/results/20260511/c6a.2xlarge.json index 509e57d0d..371e0c65f 100644 --- a/hyper/results/20260511/c6a.2xlarge.json +++ b/hyper/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 638, "data_size": 18959040512, - "concurrent_qps": 0.305, - "concurrent_error_ratio": 0.991, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.064, 0.041, 0.038], [0.173, 0.041, 0.04], diff --git a/hyper/results/20260511/c6a.4xlarge.json b/hyper/results/20260511/c6a.4xlarge.json index e9ae99397..3f188d0c2 100644 --- a/hyper/results/20260511/c6a.4xlarge.json +++ b/hyper/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 655, "data_size": 18959040512, - "concurrent_qps": 0.793, - "concurrent_error_ratio": 0.981, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.06, 0.022, 0.022], [0.109, 0.03, 0.023], diff --git a/hyper/results/20260511/c6a.large.json b/hyper/results/20260511/c6a.large.json index bb33d932a..128df8ac9 100644 --- a/hyper/results/20260511/c6a.large.json +++ b/hyper/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 750, "data_size": 18959040512, - "concurrent_qps": 0.01, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.094, 0.056, 0.054], [0.541, 0.105, 0.104], diff --git a/hyper/results/20260511/c6a.metal.json b/hyper/results/20260511/c6a.metal.json index a5b58206f..4104b4489 100644 --- a/hyper/results/20260511/c6a.metal.json +++ b/hyper/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 428, "data_size": 18959040512, - "concurrent_qps": 1.74, - "concurrent_error_ratio": 0.963, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.044, 0.014, 0.014], [0.075, 0.029, 0.028], diff --git a/hyper/results/20260511/c6a.xlarge.json b/hyper/results/20260511/c6a.xlarge.json index 5c35cbccf..a68c5ccc5 100644 --- a/hyper/results/20260511/c6a.xlarge.json +++ b/hyper/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 623, "data_size": 18959040512, - "concurrent_qps": 0.017, - "concurrent_error_ratio": 0.999, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.073, 0.035, 0.035], [0.285, 0.058, 0.059], diff --git a/hyper/results/20260511/c7a.metal-48xl.json b/hyper/results/20260511/c7a.metal-48xl.json index 84321e569..219066b6c 100644 --- a/hyper/results/20260511/c7a.metal-48xl.json +++ b/hyper/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["C++","column-oriented"], "load_time": 356, "data_size": 18959040512, - "concurrent_qps": 2.01, - "concurrent_error_ratio": 0.963, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.04, 0.011, 0.012], [0.083, 0.026, 0.025], diff --git a/kdb/benchmark.sh b/kdb/benchmark.sh index 617422ddc..5427511a0 100755 --- a/kdb/benchmark.sh +++ b/kdb/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/lib/benchmark-common.sh b/lib/benchmark-common.sh index 5e796ba6f..49f47be4e 100755 --- a/lib/benchmark-common.sh +++ b/lib/benchmark-common.sh @@ -55,7 +55,16 @@ # that runs after the cold/warm sweep. Default 10. # BENCH_CONCURRENT_DURATION # Wall-clock window for the QPS test, in seconds. -# Default 600. +# Default 600. Set to 0 to skip the test (emits +# null). Single-process engines should set 0: each +# query forks a fresh full-machine process with no +# shared scheduler, so N concurrent connections only +# oversubscribe RAM instead of measuring throughput. +# The rule: skip when ./start launches no shared +# server (the embedded CLIs and spark variants), keep +# for daemons and the in-process server wrappers +# (pandas/polars/*-dataframe), which do share one +# process. See issue #946. # BENCH_CONCURRENT_SEED Seed shared across systems so that connection # N hits the same query order on every engine # (the per-connection permutation is derived @@ -324,6 +333,15 @@ bench_concurrent_qps() { local duration="${BENCH_CONCURRENT_DURATION}" local seed="${BENCH_CONCURRENT_SEED}" + # Duration 0 means the test is disabled (single-process engines set this; + # see the BENCH_CONCURRENT_DURATION docs). Emit null and skip the restart, + # cache flush, and worker setup entirely. + if [ "$duration" -le 0 ]; then + echo "Concurrent QPS: null" + echo "Concurrent error ratio: null" + return 0 + fi + # Read the same query file that bench_run_query consumed. local queries=() q while IFS= read -r q; do diff --git a/octosql/benchmark.sh b/octosql/benchmark.sh index 617422ddc..5427511a0 100755 --- a/octosql/benchmark.sh +++ b/octosql/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/opteryx/benchmark.sh b/opteryx/benchmark.sh index e236dbbd2..5972a9845 100755 --- a/opteryx/benchmark.sh +++ b/opteryx/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/opteryx/results/20260511/c6a.2xlarge.json b/opteryx/results/20260511/c6a.2xlarge.json index 398482794..1c9951b15 100644 --- a/opteryx/results/20260511/c6a.2xlarge.json +++ b/opteryx/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0.087, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.628, 0.497, 0.499], [1.18, 0.723, 0.728], diff --git a/opteryx/results/20260511/c6a.4xlarge.json b/opteryx/results/20260511/c6a.4xlarge.json index 58b68bb46..b3a00faa0 100644 --- a/opteryx/results/20260511/c6a.4xlarge.json +++ b/opteryx/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.04, - "concurrent_error_ratio": 0.04, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.631, 0.496, 0.497], [1.181, 0.723, 0.714], diff --git a/opteryx/results/20260511/c6a.large.json b/opteryx/results/20260511/c6a.large.json index 370e668a2..10681a466 100644 --- a/opteryx/results/20260511/c6a.large.json +++ b/opteryx/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.015, - "concurrent_error_ratio": 0.308, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.659, 0.494, 0.494], [1.289, 0.866, 0.871], diff --git a/opteryx/results/20260511/c6a.metal.json b/opteryx/results/20260511/c6a.metal.json index a5146ec2f..4ab59bf09 100644 --- a/opteryx/results/20260511/c6a.metal.json +++ b/opteryx/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 64, "data_size": 14737666736, - "concurrent_qps": 0.875, - "concurrent_error_ratio": 0.047, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.58, 0.46, 0.473], [1.192, 0.767, 0.74], diff --git a/opteryx/results/20260511/c6a.xlarge.json b/opteryx/results/20260511/c6a.xlarge.json index 79d64349f..f644feec2 100644 --- a/opteryx/results/20260511/c6a.xlarge.json +++ b/opteryx/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.023, - "concurrent_error_ratio": 0.125, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.705, 0.511, 0.502], [1.321, 0.768, 0.764], diff --git a/opteryx/results/20260511/c7a.metal-48xl.json b/opteryx/results/20260511/c7a.metal-48xl.json index a7e76e339..ec645afdc 100644 --- a/opteryx/results/20260511/c7a.metal-48xl.json +++ b/opteryx/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["stateless","column-oriented","embedded"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 0.993, - "concurrent_error_ratio": 0.046, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.531, 0.435, 0.434], [1.135, 0.678, 0.702], diff --git a/sail-partitioned/benchmark.sh b/sail-partitioned/benchmark.sh index e236dbbd2..5972a9845 100755 --- a/sail-partitioned/benchmark.sh +++ b/sail-partitioned/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/sail-partitioned/results/20260511/c6a.2xlarge.json b/sail-partitioned/results/20260511/c6a.2xlarge.json index 741f73159..bb227ae2c 100644 --- a/sail-partitioned/results/20260511/c6a.2xlarge.json +++ b/sail-partitioned/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 10, "data_size": 14737666736, - "concurrent_qps": 0.045, - "concurrent_error_ratio": 0.036, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.257, 0.082, 0.083], [0.338, 0.119, 0.118], diff --git a/sail-partitioned/results/20260511/c6a.4xlarge.json b/sail-partitioned/results/20260511/c6a.4xlarge.json index 1b077b8d0..35da4a0c0 100644 --- a/sail-partitioned/results/20260511/c6a.4xlarge.json +++ b/sail-partitioned/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.223, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.258, 0.086, 0.08], [0.325, 0.102, 0.102], diff --git a/sail-partitioned/results/20260511/c6a.large.json b/sail-partitioned/results/20260511/c6a.large.json index aec3b7dfb..fb12c8085 100644 --- a/sail-partitioned/results/20260511/c6a.large.json +++ b/sail-partitioned/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 2, "data_size": 14737666736, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0.167, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.262, 0.089, 0.087], [0.477, 0.213, 0.211], diff --git a/sail-partitioned/results/20260511/c6a.metal.json b/sail-partitioned/results/20260511/c6a.metal.json index 3cdd77b5c..705ff53c6 100644 --- a/sail-partitioned/results/20260511/c6a.metal.json +++ b/sail-partitioned/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 3.038, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.248, 0.086, 0.082], [0.297, 0.12, 0.124], diff --git a/sail-partitioned/results/20260511/c6a.xlarge.json b/sail-partitioned/results/20260511/c6a.xlarge.json index 17028d466..7de462da6 100644 --- a/sail-partitioned/results/20260511/c6a.xlarge.json +++ b/sail-partitioned/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 4, "data_size": 14737666736, - "concurrent_qps": 0.035, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.34, 0.084, 0.084], [0.473, 0.152, 0.156], diff --git a/sail-partitioned/results/20260511/c7a.metal-48xl.json b/sail-partitioned/results/20260511/c7a.metal-48xl.json index be9a605f4..0def127fe 100644 --- a/sail-partitioned/results/20260511/c7a.metal-48xl.json +++ b/sail-partitioned/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 3.735, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.23, 0.075, 0.075], [0.301, 0.11, 0.107], diff --git a/sail-partitioned/results/20260511/c8g.4xlarge.json b/sail-partitioned/results/20260511/c8g.4xlarge.json index d829f15d1..300ea1de6 100644 --- a/sail-partitioned/results/20260511/c8g.4xlarge.json +++ b/sail-partitioned/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 21, "data_size": 14737666736, - "concurrent_qps": 0.317, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.215, 0.072, 0.076], [0.275, 0.09, 0.093], diff --git a/sail-partitioned/results/20260511/c8g.metal-48xl.json b/sail-partitioned/results/20260511/c8g.metal-48xl.json index c3613c2d7..ca427c54f 100644 --- a/sail-partitioned/results/20260511/c8g.metal-48xl.json +++ b/sail-partitioned/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 65, "data_size": 14737666736, - "concurrent_qps": 4.01, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.207, 0.073, 0.073], [0.242, 0.096, 0.092], diff --git a/sail-partitioned/results/20260511/t3a.small.json b/sail-partitioned/results/20260511/t3a.small.json index 679068385..245cbbf5a 100644 --- a/sail-partitioned/results/20260511/t3a.small.json +++ b/sail-partitioned/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 0, "data_size": 14737666736, - "concurrent_qps": 0.015, - "concurrent_error_ratio": 0.471, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.365, 0.174, 0.181], [0.681, 0.398, 0.413], diff --git a/sail/benchmark.sh b/sail/benchmark.sh index 617422ddc..5427511a0 100755 --- a/sail/benchmark.sh +++ b/sail/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/sail/results/20260511/c6a.2xlarge.json b/sail/results/20260511/c6a.2xlarge.json index 09d0cae9f..b94548d40 100644 --- a/sail/results/20260511/c6a.2xlarge.json +++ b/sail/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.047, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.208, 0.014, 0.014], [0.306, 0.058, 0.058], diff --git a/sail/results/20260511/c6a.4xlarge.json b/sail/results/20260511/c6a.4xlarge.json index 532915b87..1fce537cf 100644 --- a/sail/results/20260511/c6a.4xlarge.json +++ b/sail/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.205, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.213, 0.014, 0.014], [0.296, 0.05, 0.051], diff --git a/sail/results/20260511/c6a.large.json b/sail/results/20260511/c6a.large.json index 6f068ebf4..344fb6a32 100644 --- a/sail/results/20260511/c6a.large.json +++ b/sail/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.023, - "concurrent_error_ratio": 0.222, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.226, 0.015, 0.019], [0.443, 0.13, 0.128], diff --git a/sail/results/20260511/c6a.metal.json b/sail/results/20260511/c6a.metal.json index 0a1fc1ecf..1a5fabc9d 100644 --- a/sail/results/20260511/c6a.metal.json +++ b/sail/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 6, "data_size": 14779976446, - "concurrent_qps": 2.575, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.196, 0.015, 0.017], [0.713, 0.48, 0.475], diff --git a/sail/results/20260511/c6a.xlarge.json b/sail/results/20260511/c6a.xlarge.json index 4194495ad..65e9a32ed 100644 --- a/sail/results/20260511/c6a.xlarge.json +++ b/sail/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.025, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.21, 0.017, 0.017], [0.354, 0.084, 0.083], diff --git a/sail/results/20260511/c7a.metal-48xl.json b/sail/results/20260511/c7a.metal-48xl.json index c7b5e606d..af6c02dd5 100644 --- a/sail/results/20260511/c7a.metal-48xl.json +++ b/sail/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 3.217, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.214, 0.017, 0.015], [0.643, 0.43, 0.428], diff --git a/sail/results/20260511/c8g.4xlarge.json b/sail/results/20260511/c8g.4xlarge.json index 9e9a80222..22802dbcd 100644 --- a/sail/results/20260511/c8g.4xlarge.json +++ b/sail/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 2, "data_size": 14779976446, - "concurrent_qps": 0.275, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.146, 0.013, 0.013], [0.212, 0.041, 0.039], diff --git a/sail/results/20260511/c8g.metal-48xl.json b/sail/results/20260511/c8g.metal-48xl.json index 4a1ff2a77..9167309e3 100644 --- a/sail/results/20260511/c8g.metal-48xl.json +++ b/sail/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 7, "data_size": 14779976446, - "concurrent_qps": 3.477, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.089, 0.011, 0.011], [0.276, 0.221, 0.274], diff --git a/sail/results/20260511/t3a.small.json b/sail/results/20260511/t3a.small.json index 7b6ccd336..2a0579bc4 100644 --- a/sail/results/20260511/t3a.small.json +++ b/sail/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["column-oriented","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.018, - "concurrent_error_ratio": 0.353, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.249, 0.029, 0.037], [0.679, 0.237, 0.228], diff --git a/spark-auron/benchmark.sh b/spark-auron/benchmark.sh index 617422ddc..5427511a0 100755 --- a/spark-auron/benchmark.sh +++ b/spark-auron/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-auron/results/20260511/c6a.2xlarge.json b/spark-auron/results/20260511/c6a.2xlarge.json index 0f73988e6..be77886eb 100644 --- a/spark-auron/results/20260511/c6a.2xlarge.json +++ b/spark-auron/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.107, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [58.052, 47.978, 47.209], [3.761, 3.519, 3.612], diff --git a/spark-auron/results/20260511/c6a.4xlarge.json b/spark-auron/results/20260511/c6a.4xlarge.json index ca6152fa3..a86f24c0c 100644 --- a/spark-auron/results/20260511/c6a.4xlarge.json +++ b/spark-auron/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.177, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [60.775, 12.96, 13.058], [3.135, 2.89, 2.951], diff --git a/spark-auron/results/20260511/c6a.large.json b/spark-auron/results/20260511/c6a.large.json index 41b3d7f65..008fef705 100644 --- a/spark-auron/results/20260511/c6a.large.json +++ b/spark-auron/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.027, - "concurrent_error_ratio": 0.673, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [85.079, 84.822, 86.685], [10.121, 9.197, 9.338], diff --git a/spark-auron/results/20260511/c6a.metal.json b/spark-auron/results/20260511/c6a.metal.json index 6caa46b24..d61665cb0 100644 --- a/spark-auron/results/20260511/c6a.metal.json +++ b/spark-auron/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.103, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [75.476, 40.827, 40.45], [39.232, 39.05, 39.061], diff --git a/spark-auron/results/20260511/c6a.xlarge.json b/spark-auron/results/20260511/c6a.xlarge.json index f2a0316ba..fd6874b14 100644 --- a/spark-auron/results/20260511/c6a.xlarge.json +++ b/spark-auron/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.043, - "concurrent_error_ratio": 0.297, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [65.718, 51.592, 51.302], [5.491, 5.624, 5.38], diff --git a/spark-auron/results/20260511/c7a.metal-48xl.json b/spark-auron/results/20260511/c7a.metal-48xl.json index c277f9172..97eb943d0 100644 --- a/spark-auron/results/20260511/c7a.metal-48xl.json +++ b/spark-auron/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.13, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [73.191, 42.478, 42.628], [40.636, 39.718, 40.684], diff --git a/spark-comet/benchmark.sh b/spark-comet/benchmark.sh index 617422ddc..5427511a0 100755 --- a/spark-comet/benchmark.sh +++ b/spark-comet/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-comet/results/20260511/c6a.2xlarge.json b/spark-comet/results/20260511/c6a.2xlarge.json index 6425de687..a3cbedded 100644 --- a/spark-comet/results/20260511/c6a.2xlarge.json +++ b/spark-comet/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 0.2, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [6.946, 6.925, 6.644], [7.363, 6.723, 7.095], diff --git a/spark-comet/results/20260511/c6a.4xlarge.json b/spark-comet/results/20260511/c6a.4xlarge.json index 27ed3f204..d07cc8b19 100644 --- a/spark-comet/results/20260511/c6a.4xlarge.json +++ b/spark-comet/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.008, - "concurrent_error_ratio": 0.5, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.677, 4.421, 4.342], [4.763, 4.434, 4.607], diff --git a/spark-comet/results/20260511/c6a.large.json b/spark-comet/results/20260511/c6a.large.json index 55a48ef48..7cf9ed2c7 100644 --- a/spark-comet/results/20260511/c6a.large.json +++ b/spark-comet/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.01, - "concurrent_error_ratio": 0.4, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [17.778, 17.718, 17.454], [18.684, 18.37, 18.418], diff --git a/spark-comet/results/20260511/c6a.metal.json b/spark-comet/results/20260511/c6a.metal.json index f15016a75..5871048f0 100644 --- a/spark-comet/results/20260511/c6a.metal.json +++ b/spark-comet/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.3, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.209, 5.429, 5.368], [5.248, 5.338, 5.354], diff --git a/spark-comet/results/20260511/c6a.xlarge.json b/spark-comet/results/20260511/c6a.xlarge.json index ffa076ce8..387afb485 100644 --- a/spark-comet/results/20260511/c6a.xlarge.json +++ b/spark-comet/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 0.2, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [10.699, 9.82, 9.716], [10.407, 10.268, 10.193], diff --git a/spark-comet/results/20260511/c7a.metal-48xl.json b/spark-comet/results/20260511/c7a.metal-48xl.json index 1845d11ce..0a755c63d 100644 --- a/spark-comet/results/20260511/c7a.metal-48xl.json +++ b/spark-comet/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 0.385, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.288, 4.643, 4.884], [5.335, 5.637, 5.122], diff --git a/spark-comet/results/20260511/c8g.4xlarge.json b/spark-comet/results/20260511/c8g.4xlarge.json index 771b2459e..ec8dfa8d0 100644 --- a/spark-comet/results/20260511/c8g.4xlarge.json +++ b/spark-comet/results/20260511/c8g.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.005, - "concurrent_error_ratio": 0.7, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.331, 3.082, 3.007], [3.455, 3.137, 3.095], diff --git a/spark-comet/results/20260511/c8g.metal-48xl.json b/spark-comet/results/20260511/c8g.metal-48xl.json index 4eec42226..641e810cb 100644 --- a/spark-comet/results/20260511/c8g.metal-48xl.json +++ b/spark-comet/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","Rust","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.342, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.542, 4.4, 5.007], [5.498, 5.086, 4.329], diff --git a/spark-gluten/benchmark.sh b/spark-gluten/benchmark.sh index 617422ddc..5427511a0 100755 --- a/spark-gluten/benchmark.sh +++ b/spark-gluten/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-gluten/results/20260511/c6a.2xlarge.json b/spark-gluten/results/20260511/c6a.2xlarge.json index 8775fdf0d..bec004135 100644 --- a/spark-gluten/results/20260511/c6a.2xlarge.json +++ b/spark-gluten/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.11, - "concurrent_error_ratio": 0.029, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.095, 3.583, 3.521], [4.278, 3.792, 3.757], diff --git a/spark-gluten/results/20260511/c6a.4xlarge.json b/spark-gluten/results/20260511/c6a.4xlarge.json index 242305549..83d2781a9 100644 --- a/spark-gluten/results/20260511/c6a.4xlarge.json +++ b/spark-gluten/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 11, "data_size": 14779976446, - "concurrent_qps": 0.183, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.441, 2.905, 2.934], [3.577, 3.132, 3.003], diff --git a/spark-gluten/results/20260511/c6a.large.json b/spark-gluten/results/20260511/c6a.large.json index afea681fa..36ab22ca2 100644 --- a/spark-gluten/results/20260511/c6a.large.json +++ b/spark-gluten/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.017, - "concurrent_error_ratio": 0.231, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [10.073, 9.513, 9.553], [10.725, 10.379, 10.318], diff --git a/spark-gluten/results/20260511/c6a.metal.json b/spark-gluten/results/20260511/c6a.metal.json index ac5dc6e2c..e9ca52372 100644 --- a/spark-gluten/results/20260511/c6a.metal.json +++ b/spark-gluten/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.512, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.086, 4.489, 4.714], [5.324, 4.703, 4.831], diff --git a/spark-gluten/results/20260511/c6a.xlarge.json b/spark-gluten/results/20260511/c6a.xlarge.json index d82b51601..fcfb8d38b 100644 --- a/spark-gluten/results/20260511/c6a.xlarge.json +++ b/spark-gluten/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.032, - "concurrent_error_ratio": 0.345, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [6.157, 5.598, 5.608], [6.366, 5.885, 5.763], diff --git a/spark-gluten/results/20260511/c7a.metal-48xl.json b/spark-gluten/results/20260511/c7a.metal-48xl.json index 2580b3c8e..edfb90f64 100644 --- a/spark-gluten/results/20260511/c7a.metal-48xl.json +++ b/spark-gluten/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 0.605, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.551, 4.001, 3.975], [4.62, 4.068, 4.141], diff --git a/spark-gluten/results/20260511/t3a.small.json b/spark-gluten/results/20260511/t3a.small.json index 94cd0ccbd..c8e7299b9 100644 --- a/spark-gluten/results/20260511/t3a.small.json +++ b/spark-gluten/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.015, - "concurrent_error_ratio": 0.1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [17.782, 17.036, 17.335], [18.613, 18.098, 17.83], diff --git a/spark-velox/benchmark.sh b/spark-velox/benchmark.sh index 617422ddc..5427511a0 100755 --- a/spark-velox/benchmark.sh +++ b/spark-velox/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-velox/results/20260511/c6a.2xlarge.json b/spark-velox/results/20260511/c6a.2xlarge.json index 18d92e93d..841319da5 100644 --- a/spark-velox/results/20260511/c6a.2xlarge.json +++ b/spark-velox/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.108, - "concurrent_error_ratio": 0.03, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.092, 3.645, 3.659], [4.371, 3.824, 3.749], diff --git a/spark-velox/results/20260511/c6a.4xlarge.json b/spark-velox/results/20260511/c6a.4xlarge.json index ccb3f1a4f..cd067734c 100644 --- a/spark-velox/results/20260511/c6a.4xlarge.json +++ b/spark-velox/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 10, "data_size": 14779976446, - "concurrent_qps": 0.18, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.441, 2.95, 2.937], [3.573, 3.001, 2.978], diff --git a/spark-velox/results/20260511/c6a.large.json b/spark-velox/results/20260511/c6a.large.json index 7eb6a5efd..5fe86e958 100644 --- a/spark-velox/results/20260511/c6a.large.json +++ b/spark-velox/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.015, - "concurrent_error_ratio": 0.1, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [9.776, 9.458, 9.537], [10.542, 10.205, 10.015], diff --git a/spark-velox/results/20260511/c6a.metal.json b/spark-velox/results/20260511/c6a.metal.json index 3bed8fc13..f67822c00 100644 --- a/spark-velox/results/20260511/c6a.metal.json +++ b/spark-velox/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 7, "data_size": 14779976446, - "concurrent_qps": 0.513, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.104, 4.608, 4.638], [5.057, 4.653, 4.661], diff --git a/spark-velox/results/20260511/c6a.xlarge.json b/spark-velox/results/20260511/c6a.xlarge.json index 7c5704699..6e5c20da2 100644 --- a/spark-velox/results/20260511/c6a.xlarge.json +++ b/spark-velox/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.032, - "concurrent_error_ratio": 0.472, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [6.052, 5.51, 5.528], [6.261, 5.842, 5.874], diff --git a/spark-velox/results/20260511/c7a.metal-48xl.json b/spark-velox/results/20260511/c7a.metal-48xl.json index 4a9812160..36aaf20a5 100644 --- a/spark-velox/results/20260511/c7a.metal-48xl.json +++ b/spark-velox/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 5, "data_size": 14779976446, - "concurrent_qps": 0.607, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [4.522, 4.082, 4.224], [4.674, 4.142, 4.258], diff --git a/spark-velox/results/20260511/t3a.small.json b/spark-velox/results/20260511/t3a.small.json index 421d3d779..230645127 100644 --- a/spark-velox/results/20260511/t3a.small.json +++ b/spark-velox/results/20260511/t3a.small.json @@ -9,8 +9,8 @@ "tags": ["Java","C++","column-oriented","Spark derivative","stateless"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.017, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [19.718, 20.133, 19.386], [21.277, 20.722, 20.455], diff --git a/spark/benchmark.sh b/spark/benchmark.sh index 5c71433ad..3d8a0f486 100755 --- a/spark/benchmark.sh +++ b/spark/benchmark.sh @@ -5,4 +5,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" # (and would re-download nothing). Skip restart. export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark/results/20260511/c6a.2xlarge.json b/spark/results/20260511/c6a.2xlarge.json index 80b6e5168..5770597af 100644 --- a/spark/results/20260511/c6a.2xlarge.json +++ b/spark/results/20260511/c6a.2xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 4, "data_size": 14779976446, - "concurrent_qps": 0.01, - "concurrent_error_ratio": 0.4, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.912, 3.65, 3.675], [4.21, 3.852, 3.985], diff --git a/spark/results/20260511/c6a.4xlarge.json b/spark/results/20260511/c6a.4xlarge.json index 5b75c4f4e..e1a4c0141 100644 --- a/spark/results/20260511/c6a.4xlarge.json +++ b/spark/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.003, - "concurrent_error_ratio": 0.8, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.275, 3.111, 2.948], [3.512, 3.224, 3.245], diff --git a/spark/results/20260511/c6a.large.json b/spark/results/20260511/c6a.large.json index d62d02252..a60b71175 100644 --- a/spark/results/20260511/c6a.large.json +++ b/spark/results/20260511/c6a.large.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.007, - "concurrent_error_ratio": 0.6, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [10.198, 10.136, 9.943], [11.14, 10.324, 10.688], diff --git a/spark/results/20260511/c6a.metal.json b/spark/results/20260511/c6a.metal.json index 236c5e678..e0f7707eb 100644 --- a/spark/results/20260511/c6a.metal.json +++ b/spark/results/20260511/c6a.metal.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 0, "data_size": 14779976446, - "concurrent_qps": 0.203, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [3.511, 3.602, 3.089], [5.46, 17.568, 5.541], diff --git a/spark/results/20260511/c6a.xlarge.json b/spark/results/20260511/c6a.xlarge.json index c68527c91..808cccf70 100644 --- a/spark/results/20260511/c6a.xlarge.json +++ b/spark/results/20260511/c6a.xlarge.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 1, "data_size": 14779976446, - "concurrent_qps": 0.013, - "concurrent_error_ratio": 0.2, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [5.962, 5.576, 5.768], [6.188, 6.054, 6.545], diff --git a/spark/results/20260511/c7a.metal-48xl.json b/spark/results/20260511/c7a.metal-48xl.json index 286fe6156..1f85d63be 100644 --- a/spark/results/20260511/c7a.metal-48xl.json +++ b/spark/results/20260511/c7a.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 0.225, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [2.893, 2.672, 2.863], [5.848, 10.434, 3.3], diff --git a/spark/results/20260511/c8g.4xlarge.json b/spark/results/20260511/c8g.4xlarge.json index a9282ae91..5951a1fed 100644 --- a/spark/results/20260511/c8g.4xlarge.json +++ b/spark/results/20260511/c8g.4xlarge.json @@ -10,7 +10,7 @@ "load_time": 1, "data_size": 14779976446, "concurrent_qps": null, - "concurrent_error_ratio": 1, + "concurrent_error_ratio": null, "result": [ [2.517, 2.265, 2.269], [2.867, 2.517, 2.554], diff --git a/spark/results/20260511/c8g.metal-48xl.json b/spark/results/20260511/c8g.metal-48xl.json index 67d828871..a70450e99 100644 --- a/spark/results/20260511/c8g.metal-48xl.json +++ b/spark/results/20260511/c8g.metal-48xl.json @@ -9,8 +9,8 @@ "tags": ["Java","column-oriented","stateless","Spark derivative"], "load_time": 3, "data_size": 14779976446, - "concurrent_qps": 0.078, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [2.984, 3.11, 2.639], [5.773, 4.555, 4.157], diff --git a/sqlite/benchmark.sh b/sqlite/benchmark.sh index 2e5741b39..bf0286804 100755 --- a/sqlite/benchmark.sh +++ b/sqlite/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/turso/benchmark.sh b/turso/benchmark.sh index 2e5741b39..bf0286804 100755 --- a/turso/benchmark.sh +++ b/turso/benchmark.sh @@ -3,4 +3,9 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/zighouse/benchmark.sh b/zighouse/benchmark.sh index c722454d5..8f6103ef3 100755 --- a/zighouse/benchmark.sh +++ b/zighouse/benchmark.sh @@ -3,4 +3,9 @@ set -e export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes +# Single-process engine: each query forks a fresh full-machine process with no +# shared scheduler across connections, so the concurrent-QPS test only +# oversubscribes RAM rather than measuring throughput. Skip it by default; +# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/zighouse/results/20260511/c6a.4xlarge.json b/zighouse/results/20260511/c6a.4xlarge.json index cffe5a5d3..018479e3d 100644 --- a/zighouse/results/20260511/c6a.4xlarge.json +++ b/zighouse/results/20260511/c6a.4xlarge.json @@ -9,8 +9,8 @@ "tags": ["Zig","column-oriented","embedded","parquet","tuned"], "load_time": 86, "data_size": 10907660070, - "concurrent_qps": 3.71, - "concurrent_error_ratio": 0, + "concurrent_qps": null, + "concurrent_error_ratio": null, "result": [ [0.003, 0, 0], [0.626, 0.017, 0.017],