ClickHouse · george-larionov · Jun 15, 2026 · Jun 15, 2026 · Jun 15, 2026 · Jun 16, 2026
diff --git a/README.md b/README.md
@@ -89,6 +89,8 @@ To introduce a new result for an existing system with a different usage scenario
 `index.html` can be re-generated using `./generate-results.sh`.
 The CI (GitHub Actions) does this automatically, this step is optional.
 
+The shared driver (`lib/benchmark-common.sh`) runs a supplementary concurrent-QPS test (`BENCH_CONCURRENT_CONNECTIONS` workers for `BENCH_CONCURRENT_DURATION` seconds) after the main sweep. Single-process engines must set `BENCH_CONCURRENT_DURATION=0` in their `benchmark.sh` to skip it: each query forks a fresh full-machine process with no shared scheduler, so concurrent connections only oversubscribe RAM (and can OOM the run) instead of measuring throughput. Rule of thumb — skip when `./start` launches no shared server (the embedded CLIs and Spark variants); keep it for daemons and the in-process server wrappers (pandas/polars/`*-dataframe`), which share one process. See issue #946.
+
 All tests were originally run on AWS c6a.4xlarge EC2 VMs with 500 GB gp2 disks.
 With better automation, more EC2 machines were added later: c6a.2xlarge, c6a.metal, c8g.4xlarge, c6a.xlarge, c7a.metal-48xl, c6a.large, c8g.metal-48xl, and t3a.small.
 These represent older and modern machines, and small / medium / large systems (CPU and main memory).

diff --git a/bqn/benchmark.sh b/bqn/benchmark.sh
@@ -3,4 +3,9 @@
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_DURABLE=yes
 export BENCH_RESTARTABLE=no
+# Single-process engine: each query forks a fresh full-machine process with no
+# shared scheduler across connections, so the concurrent-QPS test only
+# oversubscribes RAM rather than measuring throughput. Skip it by default;
+# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946.
+export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}"
 exec ../lib/benchmark-common.sh
diff --git a/bqn/results/20260529/c6a.4xlarge.json b/bqn/results/20260529/c6a.4xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C","embedded","array language"],
     "load_time": 526,
     "data_size": 118169498884,
-    "concurrent_qps": 0.042,
-    "concurrent_error_ratio": 0.554,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.002, 0.001, 0.001],
         [3.551, 1.308, 1.306],

diff --git a/bqn/results/20260529/c6a.metal.json b/bqn/results/20260529/c6a.metal.json
@@ -9,8 +9,8 @@
     "tags": ["C","embedded","array language"],
     "load_time": 472,
     "data_size": 118169498884,
-    "concurrent_qps": 0.223,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.001, 0.001, 0.001],
         [4.181, 1.272, 1.22],

diff --git a/bqn/results/20260529/c7a.metal-48xl.json b/bqn/results/20260529/c7a.metal-48xl.json
@@ -9,8 +9,8 @@
     "tags": ["C","embedded","array language"],
     "load_time": 473,
     "data_size": 118169498884,
-    "concurrent_qps": 0.235,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.001, 0.001, 0.001],
         [4.122, 1.161, 1.094],

diff --git a/bqn/results/20260529/c8g.4xlarge.json b/bqn/results/20260529/c8g.4xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C","embedded","array language"],
     "load_time": 518,
     "data_size": 118169498884,
-    "concurrent_qps": 0.042,
-    "concurrent_error_ratio": 0.597,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.001, 0, 0],
         [3.349, 0.676, 0.693],

diff --git a/bqn/results/20260529/c8g.metal-48xl.json b/bqn/results/20260529/c8g.metal-48xl.json
@@ -9,8 +9,8 @@
     "tags": ["C","embedded","array language"],
     "load_time": 472,
     "data_size": 118169498884,
-    "concurrent_qps": 0.237,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.001, 0, 0],
         [3.657, 0.702, 0.699],

diff --git a/chdb-parquet-partitioned/benchmark.sh b/chdb-parquet-partitioned/benchmark.sh
@@ -3,4 +3,9 @@
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
 export BENCH_RESTARTABLE=no
+# Single-process engine: each query forks a fresh full-machine process with no
+# shared scheduler across connections, so the concurrent-QPS test only
+# oversubscribes RAM rather than measuring throughput. Skip it by default;
+# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946.
+export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}"
 exec ../lib/benchmark-common.sh
diff --git a/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json b/chdb-parquet-partitioned/results/20260511/c6a.2xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 9,
     "data_size": 14737666736,
-    "concurrent_qps": 0.045,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.061, 0.027, 0.026],
         [0.151, 0.058, 0.058],

diff --git a/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json b/chdb-parquet-partitioned/results/20260511/c6a.4xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 21,
     "data_size": 14737666736,
-    "concurrent_qps": 0.108,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.049, 0.026, 0.026],
         [0.112, 0.045, 0.045],

diff --git a/chdb-parquet-partitioned/results/20260511/c6a.large.json b/chdb-parquet-partitioned/results/20260511/c6a.large.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 2,
     "data_size": 14737666736,
-    "concurrent_qps": 0.035,
-    "concurrent_error_ratio": 0.045,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.17, 0.044, 0.041],
         [0.352, 0.141, 0.144],

diff --git a/chdb-parquet-partitioned/results/20260511/c6a.metal.json b/chdb-parquet-partitioned/results/20260511/c6a.metal.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 66,
     "data_size": 14737666736,
-    "concurrent_qps": 3.643,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.045, 0.012, 0.018],
         [0.114, 0.054, 0.052],

diff --git a/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json b/chdb-parquet-partitioned/results/20260511/c6a.xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 4,
     "data_size": 14737666736,
-    "concurrent_qps": 0.04,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.101, 0.026, 0.027],
         [0.217, 0.083, 0.082],

diff --git a/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json b/chdb-parquet-partitioned/results/20260511/c7a.metal-48xl.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 62,
     "data_size": 14737666736,
-    "concurrent_qps": 4.03,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.066, 0.015, 0.015],
         [0.134, 0.088, 0.089],

diff --git a/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json b/chdb-parquet-partitioned/results/20260511/c8g.4xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 21,
     "data_size": 14737666736,
-    "concurrent_qps": 0.072,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.033, 0.017, 0.018],
         [0.082, 0.034, 0.036],

diff --git a/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json b/chdb-parquet-partitioned/results/20260511/c8g.metal-48xl.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 66,
     "data_size": 14737666736,
-    "concurrent_qps": 4.042,
-    "concurrent_error_ratio": 0,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.037, 0.011, 0.012],
         [0.076, 0.033, 0.029],

diff --git a/chdb-parquet-partitioned/results/20260511/t3a.small.json b/chdb-parquet-partitioned/results/20260511/t3a.small.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 0,
     "data_size": 14737666736,
-    "concurrent_qps": 0.023,
-    "concurrent_error_ratio": 0.176,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.224, 0.069, 0.068],
         [0.515, 0.286, 0.268],

diff --git a/chdb/benchmark.sh b/chdb/benchmark.sh
@@ -3,4 +3,9 @@
 export BENCH_DOWNLOAD_SCRIPT="download-hits-csv"
 export BENCH_DURABLE=yes
 export BENCH_RESTARTABLE=no
+# Single-process engine: each query forks a fresh full-machine process with no
+# shared scheduler across connections, so the concurrent-QPS test only
+# oversubscribes RAM rather than measuring throughput. Skip it by default;
+# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946.
+export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}"
 exec ../lib/benchmark-common.sh
diff --git a/chdb/results/20260511/c6a.2xlarge.json b/chdb/results/20260511/c6a.2xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 573,
     "data_size": 15259766974,
-    "concurrent_qps": 0.307,
-    "concurrent_error_ratio": 0.959,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.014, 0.003, 0.003],
         [0.064, 0.014, 0.015],

diff --git a/chdb/results/20260511/c6a.4xlarge.json b/chdb/results/20260511/c6a.4xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 532,
     "data_size": 15261827371,
-    "concurrent_qps": 0.58,
-    "concurrent_error_ratio": 0.94,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.015, 0.003, 0.003],
         [0.054, 0.012, 0.011],

diff --git a/chdb/results/20260511/c6a.large.json b/chdb/results/20260511/c6a.large.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 774,
     "data_size": 15257484538,
-    "concurrent_qps": 0.025,
-    "concurrent_error_ratio": 0.993,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.015, 0.005, 0.005],
         [0.132, 0.069, 0.052],

diff --git a/chdb/results/20260511/c6a.metal.json b/chdb/results/20260511/c6a.metal.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 163,
     "data_size": 15266676267,
-    "concurrent_qps": 1.358,
-    "concurrent_error_ratio": 0.942,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.019, 0.007, 0.007],
         [0.096, 0.06, 0.053],

diff --git a/chdb/results/20260511/c6a.xlarge.json b/chdb/results/20260511/c6a.xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 602,
     "data_size": 15261958294,
-    "concurrent_qps": 0.13,
-    "concurrent_error_ratio": 0.97,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.014, 0.002, 0.002],
         [0.072, 0.023, 0.022],

diff --git a/chdb/results/20260511/c7a.metal-48xl.json b/chdb/results/20260511/c7a.metal-48xl.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 156,
     "data_size": 15266229827,
-    "concurrent_qps": 1.185,
-    "concurrent_error_ratio": 0.96,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.02, 0.004, 0.004],
         [0.132, 0.058, 0.075],

diff --git a/chdb/results/20260511/c8g.4xlarge.json b/chdb/results/20260511/c8g.4xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 602,
     "data_size": 15260683017,
-    "concurrent_qps": 1.028,
-    "concurrent_error_ratio": 0.942,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.01, 0.002, 0.002],
         [0.046, 0.014, 0.01],

diff --git a/chdb/results/20260511/c8g.metal-48xl.json b/chdb/results/20260511/c8g.metal-48xl.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 149,
     "data_size": 15264739392,
-    "concurrent_qps": 1.5150000000000001,
-    "concurrent_error_ratio": 0.954,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.012, 0.007, 0.007],
         [0.064, 0.027, 0.027],

diff --git a/chdb/results/20260511/t3a.small.json b/chdb/results/20260511/t3a.small.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","ClickHouse derivative","embedded","stateless"],
     "load_time": 1516,
     "data_size": 15257125380,
-    "concurrent_qps": 0.007,
-    "concurrent_error_ratio": 0.997,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.025, 0.009, 0.008],
         [0.239, 0.12, 0.118],

diff --git a/clickhouse-datalake-partitioned/benchmark.sh b/clickhouse-datalake-partitioned/benchmark.sh
@@ -4,4 +4,9 @@
 export BENCH_DOWNLOAD_SCRIPT=""
 export BENCH_DURABLE=yes
 export BENCH_RESTARTABLE=no
+# Single-process engine: each query forks a fresh full-machine process with no
+# shared scheduler across connections, so the concurrent-QPS test only
+# oversubscribes RAM rather than measuring throughput. Skip it by default;
+# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946.
+export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}"
 exec ../lib/benchmark-common.sh
diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c6a.2xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
     "load_time": 2,
     "data_size": 14737666736,
-    "concurrent_qps": 0.13,
-    "concurrent_error_ratio": 0.998,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.422, 0.378, 0.376],
         [1.431, 1.414, 1.311],

diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c6a.4xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
     "load_time": 2,
     "data_size": 14737666736,
-    "concurrent_qps": 0.378,
-    "concurrent_error_ratio": 0.995,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.453, 0.228, 0.224],
         [2.036, 0.904, 0.781],

diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.large.json b/clickhouse-datalake-partitioned/results/20260511/c6a.large.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
     "load_time": 0,
     "data_size": 14737666736,
-    "concurrent_qps": 0.008,
-    "concurrent_error_ratio": 1,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [1.478, 1.47, 1.411],
         [4.133, 3.975, 4.106],

diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json b/clickhouse-datalake-partitioned/results/20260511/c6a.metal.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
     "load_time": 2,
     "data_size": 14737666736,
-    "concurrent_qps": 1.008,
-    "concurrent_error_ratio": 0.992,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.181, 0.141, 0.164],
         [0.382, 0.3, 0.308],

diff --git a/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c6a.xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
     "load_time": 0,
     "data_size": 14737666736,
-    "concurrent_qps": 0.042,
-    "concurrent_error_ratio": 0.999,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.72, 0.742, 0.753],
         [2.651, 2.642, 2.479],

diff --git a/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260511/c7a.metal-48xl.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
     "load_time": 2,
     "data_size": 14737666736,
-    "concurrent_qps": 1.102,
-    "concurrent_error_ratio": 0.991,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.199, 0.165, 0.181],
         [0.337, 0.296, 0.297],

diff --git a/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json b/clickhouse-datalake-partitioned/results/20260511/c8g.4xlarge.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
     "load_time": 2,
     "data_size": 14737666736,
-    "concurrent_qps": 0.74,
-    "concurrent_error_ratio": 0.994,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.239, 0.222, 0.223],
         [0.726, 0.75, 0.721],

diff --git a/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json b/clickhouse-datalake-partitioned/results/20260511/c8g.metal-48xl.json
@@ -9,8 +9,8 @@
     "tags": ["C++","column-oriented","embedded","stateless","ClickHouse derivative"],
     "load_time": 2,
     "data_size": 14737666736,
-    "concurrent_qps": 1.143,
-    "concurrent_error_ratio": 0.992,
+    "concurrent_qps": null,
+    "concurrent_error_ratio": null,
     "result": [
         [0.183, 0.145, 0.146],
         [0.307, 0.312, 0.296],