From 5d9604cee3afe8802c3edd994a9945d716fcb35c Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 9 Jun 2026 18:59:24 +0530 Subject: [PATCH 01/16] runtime: enforce backend selection --- crates/runtime/Cargo.toml | 4 +-- crates/runtime/src/lib.rs | 76 ++++++++++++++++++++++++--------------- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 4cd0af60869..d17a0224797 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -10,7 +10,7 @@ rust-version.workspace = true workspace = true [dependencies] -tokio = { workspace = true, optional = true } +tokio.workspace = true async-task = { version = "4.4", default-features = false, optional = true } spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } libc = { version = "0.2", optional = true } @@ -20,5 +20,5 @@ futures.workspace = true [features] default = ["tokio"] -tokio = ["dep:tokio"] +tokio = [] simulation = ["dep:async-task", "dep:spin", "dep:libc"] diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index eaed2f35f46..c1795df8ab8 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -1,3 +1,11 @@ +#[cfg(all(feature = "tokio", feature = "simulation"))] +compile_error!( + "spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`, not both" +); + +#[cfg(not(any(feature = "tokio", feature = "simulation")))] +compile_error!("spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`"); + #[cfg(feature = "simulation")] extern crate alloc; @@ -15,8 +23,11 @@ pub mod sim; #[cfg(feature = "simulation")] pub mod sim_std; -#[cfg(feature = "tokio")] pub type TokioHandle = tokio::runtime::Handle; +pub type TokioRuntime = tokio::runtime::Runtime; +pub type TokioRuntimeBuilder = tokio::runtime::Builder; + +pub use tokio::sync; #[derive(Clone)] pub enum Handle { @@ -74,6 +85,15 @@ enum JoinErrorInner { Simulation(sim::JoinError), } +#[cfg(feature = "tokio")] +impl From for AbortHandle { + fn from(handle: tokio::task::AbortHandle) -> Self { + Self { + inner: AbortHandleInner::Tokio(handle), + } + } +} + impl AbortHandle { pub fn abort(&self) { match &self.inner { @@ -81,8 +101,6 @@ impl AbortHandle { AbortHandleInner::Tokio(handle) => handle.abort(), #[cfg(feature = "simulation")] AbortHandleInner::Simulation(handle) => handle.abort(), - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - _ => unreachable!("runtime abort handle has no enabled backend"), } } } @@ -100,16 +118,10 @@ impl JoinErrorInner { impl fmt::Display for JoinError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - let _ = f; - #[cfg(any(feature = "tokio", feature = "simulation"))] - return self.inner.fmt(f); - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - unreachable!("runtime join error has no enabled backend") + self.inner.fmt(f) } } -#[cfg(any(feature = "tokio", feature = "simulation"))] impl std::error::Error for JoinError {} impl JoinHandleInner { @@ -160,8 +172,6 @@ impl Future for JoinHandle { type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - let _ = cx; match self.inner.poll_result(cx) { Poll::Ready(Ok(output)) => { self.inner = JoinHandleInner::Detached(PhantomData); @@ -197,17 +207,30 @@ impl fmt::Display for RuntimeTimeout { } } -#[cfg(any(feature = "tokio", feature = "simulation"))] impl std::error::Error for RuntimeTimeout {} -#[cfg(feature = "tokio")] impl Handle { pub fn tokio(handle: TokioHandle) -> Self { - Self::Tokio(handle) + #[cfg(feature = "tokio")] + { + Self::Tokio(handle) + } + #[cfg(not(feature = "tokio"))] + { + let _ = handle; + panic!("spacetimedb-runtime tokio handle requested without the `tokio` backend enabled") + } } pub fn tokio_current() -> Self { - Self::tokio(TokioHandle::current()) + #[cfg(feature = "tokio")] + { + Self::tokio(TokioHandle::current()) + } + #[cfg(not(feature = "tokio"))] + { + panic!("spacetimedb-runtime current tokio handle requested without the `tokio` backend enabled") + } } } @@ -220,8 +243,6 @@ impl Handle { impl Handle { pub fn spawn(&self, future: impl Future + Send + 'static) -> JoinHandle { - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - let _ = future; match self { #[cfg(feature = "tokio")] Self::Tokio(handle) => JoinHandle { @@ -231,8 +252,6 @@ impl Handle { Self::Simulation(handle) => JoinHandle { inner: JoinHandleInner::Simulation(handle.spawn_on(sim::NodeId::MAIN, future)), }, - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - _ => unreachable!("runtime dispatch has no enabled backend"), } } @@ -241,8 +260,6 @@ impl Handle { F: FnOnce() -> R + Send + 'static, R: Send + 'static, { - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - let _ = &f; match self { #[cfg(feature = "tokio")] Self::Tokio(_) => tokio::task::spawn_blocking(f) @@ -261,8 +278,6 @@ impl Handle { .spawn_on(sim::NodeId::MAIN, async move { f() }) .await .expect("simulation spawn_blocking task should not be cancelled"), - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - _ => unreachable!("runtime dispatch has no enabled backend"), } } @@ -271,8 +286,6 @@ impl Handle { timeout_after: Duration, future: impl Future, ) -> Result { - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - let _ = (timeout_after, future); match self { #[cfg(feature = "tokio")] Self::Tokio(_) => tokio::time::timeout(timeout_after, future) @@ -280,8 +293,15 @@ impl Handle { .map_err(|_| RuntimeTimeout), #[cfg(feature = "simulation")] Self::Simulation(handle) => handle.timeout(timeout_after, future).await.map_err(|_| RuntimeTimeout), - #[cfg(not(any(feature = "tokio", feature = "simulation")))] - _ => unreachable!("runtime dispatch has no enabled backend"), + } + } + + pub async fn sleep(&self, duration: Duration) { + match self { + #[cfg(feature = "tokio")] + Self::Tokio(_) => tokio::time::sleep(duration).await, + #[cfg(feature = "simulation")] + Self::Simulation(handle) => handle.sleep(duration).await, } } } From 0fd6c69f81da195e12d9550d1e9e9610115bfee8 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 9 Jun 2026 19:21:45 +0530 Subject: [PATCH 02/16] comment --- crates/runtime/src/lib.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index c1795df8ab8..8093ca61ca4 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -1,6 +1,6 @@ #[cfg(all(feature = "tokio", feature = "simulation"))] compile_error!( - "spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`, not both" + "spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`, not both" ); #[cfg(not(any(feature = "tokio", feature = "simulation")))] @@ -27,6 +27,14 @@ pub type TokioHandle = tokio::runtime::Handle; pub type TokioRuntime = tokio::runtime::Runtime; pub type TokioRuntimeBuilder = tokio::runtime::Builder; +// We intentionally re-export `tokio::sync` even when the simulation backend is +// selected. Async and non-blocking synchronization operations are +// executor-agnostic, so driving them from the deterministic simulation runtime +// remains deterministic. +// +// Callers must avoid APIs that block or park OS threads on their own, such as +// `blocking_send`, because those semantics are outside the simulation runtime's +// deterministic scheduler. pub use tokio::sync; #[derive(Clone)] From 60ea93f346f80a1084f0eae5fcf1233d66094944 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 9 Jun 2026 19:00:19 +0530 Subject: [PATCH 03/16] move engine into its own crate --- .github/workflows/ci.yml | 3 + Cargo.lock | 43 +++ Cargo.toml | 4 +- crates/core/Cargo.toml | 5 +- crates/core/src/database_logger.rs | 6 + crates/core/src/error.rs | 267 ++---------------- crates/core/src/host/host_controller.rs | 19 +- crates/core/src/host/module_host.rs | 21 +- crates/core/src/lib.rs | 3 +- crates/core/src/sql/execute.rs | 6 +- crates/core/src/sql/mod.rs | 1 - crates/core/src/subscription/mod.rs | 75 +---- crates/core/src/worker_metrics/mod.rs | 108 ------- crates/datastore/Cargo.toml | 8 +- crates/durability/Cargo.toml | 5 +- crates/engine/Cargo.toml | 59 ++++ crates/engine/LICENSE | 1 + crates/{core => engine}/src/db/durability.rs | 0 crates/{core => engine}/src/db/mod.rs | 33 ++- crates/{core => engine}/src/db/persistence.rs | 30 +- .../{core => engine}/src/db/relational_db.rs | 101 +++---- crates/{core => engine}/src/db/snapshot.rs | 24 +- crates/{core => engine}/src/db/update.rs | 30 +- crates/engine/src/error.rs | 262 +++++++++++++++++ crates/engine/src/lib.rs | 10 + crates/engine/src/metrics.rs | 181 ++++++++++++ .../src/sql/parser.rs => engine/src/rls.rs} | 0 crates/engine/src/sql/ast.rs | 77 +++++ crates/engine/src/sql/mod.rs | 1 + crates/engine/src/util.rs | 21 ++ crates/{core => engine}/testdata/README.md | 0 .../clog/00000000000000000000.stdb.log | Bin .../clog/00000000000000000000.stdb.ofs | Bin .../testdata/v1.2/replicas/22000001/db.lock | 0 .../22000001/module_logs/2025-08-18.log | 0 .../00000000000000000000.snapshot_bsatn | Bin ...4ae0065d053adb2efbe1b5b7af457331d330e481e8 | Bin ...0d8175307d6f205756ed163f4237c6cba2936798dc | Bin ...de72d6277eb285fc2d6b0a5ac6f92498e08a9e5ecc | Bin ...e12161246a1d5917c61ada5d81b8dcce12fd5780b3 | Bin ...d471f5203209169321083ef99de254ad24af0f6d5a | Bin ...6be31fdef38f83c2fd3bcc05f4934e53bdbfa21f10 | Bin ...bc4de886ce0281f11037c3424494e58fee92411241 | Bin crates/snapshot/Cargo.toml | 7 +- crates/standalone/src/lib.rs | 2 + 45 files changed, 849 insertions(+), 564 deletions(-) create mode 100644 crates/engine/Cargo.toml create mode 120000 crates/engine/LICENSE rename crates/{core => engine}/src/db/durability.rs (100%) rename crates/{core => engine}/src/db/mod.rs (86%) rename crates/{core => engine}/src/db/persistence.rs (92%) rename crates/{core => engine}/src/db/relational_db.rs (98%) rename crates/{core => engine}/src/db/snapshot.rs (95%) rename crates/{core => engine}/src/db/update.rs (97%) create mode 100644 crates/engine/src/error.rs create mode 100644 crates/engine/src/lib.rs create mode 100644 crates/engine/src/metrics.rs rename crates/{core/src/sql/parser.rs => engine/src/rls.rs} (100%) create mode 100644 crates/engine/src/sql/ast.rs create mode 100644 crates/engine/src/sql/mod.rs create mode 100644 crates/engine/src/util.rs rename crates/{core => engine}/testdata/README.md (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.log (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.ofs (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/db.lock (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/module_logs/2025-08-18.log (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/00000000000000000000.snapshot_bsatn (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/19/30ce81246a4cdc25e9024ae0065d053adb2efbe1b5b7af457331d330e481e8 (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/41/bb11b6d2cdc488192ee70d8175307d6f205756ed163f4237c6cba2936798dc (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/45/4d2e2c62ff5d46c5b3e6de72d6277eb285fc2d6b0a5ac6f92498e08a9e5ecc (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/62/22df0e5ca93d3fb22762e12161246a1d5917c61ada5d81b8dcce12fd5780b3 (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/79/4dced5633eca2ffee784d471f5203209169321083ef99de254ad24af0f6d5a (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/95/74dd6d2857fa771a1cd16be31fdef38f83c2fd3bcc05f4934e53bdbfa21f10 (100%) rename crates/{core => engine}/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/9a/b95f5aaed7541289faa8bc4de886ce0281f11037c3424494e58fee92411241 (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ab0e18e9ad..e91936edcdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -261,6 +261,9 @@ jobs: wasm-bindgen --version + - name: Check engine simulation build + run: cargo check -p spacetimedb-engine --no-default-features --features simulation + # Source emsdk environment to make emcc (Emscripten compiler) available in PATH. - name: Run tests run: | diff --git a/Cargo.lock b/Cargo.lock index ef5953bbd2e..f0bd9f88e83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8081,6 +8081,7 @@ dependencies = [ "spacetimedb-data-structures", "spacetimedb-datastore", "spacetimedb-durability", + "spacetimedb-engine", "spacetimedb-execution", "spacetimedb-expr", "spacetimedb-fs-utils", @@ -8199,6 +8200,48 @@ dependencies = [ "tracing", ] +[[package]] +name = "spacetimedb-engine" +version = "2.4.1" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "enum-map", + "env_logger 0.10.2", + "fs_extra", + "futures", + "hex", + "itertools 0.12.1", + "log", + "once_cell", + "parking_lot 0.12.5", + "pretty_assertions", + "prometheus", + "serde", + "sled", + "spacetimedb-commitlog", + "spacetimedb-data-structures", + "spacetimedb-datastore", + "spacetimedb-durability", + "spacetimedb-expr", + "spacetimedb-fs-utils", + "spacetimedb-lib", + "spacetimedb-metrics", + "spacetimedb-paths", + "spacetimedb-primitives", + "spacetimedb-runtime", + "spacetimedb-sats", + "spacetimedb-schema", + "spacetimedb-snapshot", + "spacetimedb-table", + "sqlparser", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tracing", +] + [[package]] name = "spacetimedb-execution" version = "2.4.1" diff --git a/Cargo.toml b/Cargo.toml index ebc8df65d10..63851fb5ae0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "crates/data-structures", "crates/datastore", "crates/durability", + "crates/engine", "crates/execution", "crates/expr", "crates/guard", @@ -132,6 +133,7 @@ spacetimedb-core = { path = "crates/core", version = "=2.4.1" } spacetimedb-data-structures = { path = "crates/data-structures", version = "=2.4.1" } spacetimedb-datastore = { path = "crates/datastore", version = "=2.4.1" } spacetimedb-durability = { path = "crates/durability", version = "=2.4.1" } +spacetimedb-engine = { path = "crates/engine", version = "=2.4.1" } spacetimedb-execution = { path = "crates/execution", version = "=2.4.1" } spacetimedb-expr = { path = "crates/expr", version = "=2.4.1" } spacetimedb-guard = { path = "crates/guard", version = "=2.4.1" } @@ -143,6 +145,7 @@ spacetimedb-pg = { path = "crates/pg", version = "=2.4.1" } spacetimedb-physical-plan = { path = "crates/physical-plan", version = "=2.4.1" } spacetimedb-primitives = { path = "crates/primitives", version = "=2.4.1" } spacetimedb-query = { path = "crates/query", version = "=2.4.1" } +spacetimedb-runtime = { path = "crates/runtime", version = "=2.4.1" } spacetimedb-sats = { path = "crates/sats", version = "=2.4.1" } spacetimedb-schema = { path = "crates/schema", version = "=2.4.1" } spacetimedb-standalone = { path = "crates/standalone", version = "=2.4.1" } @@ -152,7 +155,6 @@ spacetimedb-fs-utils = { path = "crates/fs-utils", version = "=2.4.1" } spacetimedb-snapshot = { path = "crates/snapshot", version = "=2.4.1" } spacetimedb-subscription = { path = "crates/subscription", version = "=2.4.1" } spacetimedb-query-builder = { path = "crates/query-builder", version = "=2.4.1" } -spacetimedb-runtime = { path = "crates/runtime", version = "=2.4.1" } # Prevent `ahash` from pulling in `getrandom` by disabling default features. # Modules use `getrandom02` and we need to prevent an incompatible version diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 6e7075536c2..83f44967d6e 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -22,13 +22,14 @@ spacetimedb-client-api-messages.workspace = true spacetimedb-commitlog.workspace = true spacetimedb-datastore.workspace = true spacetimedb-durability.workspace = true +spacetimedb-engine.workspace = true spacetimedb-memory-usage.workspace = true spacetimedb-metrics.workspace = true spacetimedb-primitives.workspace = true spacetimedb-paths.workspace = true spacetimedb-physical-plan.workspace = true spacetimedb-query.workspace = true -spacetimedb-runtime = { workspace = true, features = ["tokio"] } +spacetimedb-runtime.workspace = true spacetimedb-sats = { workspace = true, features = ["serde"] } spacetimedb-schema.workspace = true spacetimedb-table.workspace = true @@ -144,7 +145,7 @@ allow_loopback_http_for_tests = [] # Enable timing for wasm ABI calls spacetimedb-wasm-instance-env-times = [] # Enable test helpers and utils -test = ["spacetimedb-commitlog/test", "spacetimedb-datastore/test"] +test = ["spacetimedb-commitlog/test", "spacetimedb-datastore/test", "spacetimedb-engine/test"] # Perfmaps for profiling modules perfmap = [] # Enables core pinning. diff --git a/crates/core/src/database_logger.rs b/crates/core/src/database_logger.rs index 0e202229dea..3b5db5d7e70 100644 --- a/crates/core/src/database_logger.rs +++ b/crates/core/src/database_logger.rs @@ -674,6 +674,12 @@ impl SystemLogger { } } +impl spacetimedb_engine::db::update::UpdateLogger for SystemLogger { + fn info(&self, msg: &str) { + self.info(msg); + } +} + #[cfg(test)] mod tests { use std::{ops::Range, sync::Arc}; diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 4ba103979ec..efee537dbbe 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -1,240 +1,33 @@ -use std::io; -use std::num::ParseIntError; -use std::path::PathBuf; -use std::sync::{MutexGuard, PoisonError}; +pub use spacetimedb_engine::error::*; -use hex::FromHexError; -use spacetimedb_commitlog::repo::TxOffset; -use spacetimedb_durability::DurabilityExited; -use spacetimedb_expr::errors::TypingError; -use spacetimedb_fs_utils::lockfile::advisory::LockError; -use spacetimedb_lib::Identity; -use spacetimedb_schema::error::ValidationErrors; -use spacetimedb_schema::table_name::TableName; -use spacetimedb_snapshot::SnapshotError; -use spacetimedb_table::table::ReadViaBsatnError; -use thiserror::Error; - -use crate::client::ClientActorId; use crate::host::module_host::ViewCallError; use crate::host::scheduler::ScheduleError; use crate::host::AbiCall; use spacetimedb_lib::buffer::DecodeError; -use spacetimedb_primitives::*; -use spacetimedb_sats::hash::Hash; -use spacetimedb_sats::product_value::InvalidFieldError; -use spacetimedb_schema::def::error::{LibError, RelationError, SchemaErrors}; -use spacetimedb_schema::relation::FieldName; - -pub use spacetimedb_datastore::error::{DatastoreError, IndexError, SequenceError, TableError}; - -#[derive(Error, Debug)] -pub enum ClientError { - #[error("Client not found: {0}")] - NotFound(ClientActorId), -} - -#[derive(Error, Debug, PartialEq, Eq)] -pub enum SubscriptionError { - #[error("Index not found: {0:?}")] - NotFound(IndexId), - #[error("Empty string")] - Empty, - #[error("Unsupported query on subscription: {0:?}")] - Unsupported(String), - #[error("Subscribing to queries in one call is not supported")] - Multiple, -} - -#[derive(Error, Debug)] -pub enum PlanError { - #[error("Unsupported feature: `{feature}`")] - Unsupported { feature: String }, - #[error("Unknown table: `{table}`")] - UnknownTable { table: Box }, - #[error("Qualified Table `{expect}` not found")] - TableNotFoundQualified { expect: String }, - #[error("Unknown field: `{field}` not found in the table(s): `{tables:?}`")] - UnknownField { field: String, tables: Vec }, - #[error("Unknown field name: `{field}` not found in the table(s): `{tables:?}`")] - UnknownFieldName { field: FieldName, tables: Vec }, - #[error("Field(s): `{fields:?}` not found in the table(s): `{tables:?}`")] - UnknownFields { - fields: Vec, - tables: Vec, - }, - #[error("Ambiguous field: `{field}`. Also found in {found:?}")] - AmbiguousField { field: String, found: Vec }, - #[error("Plan error: `{0}`")] - Unstructured(String), - #[error("Internal DBError: `{0}`")] - DatabaseInternal(Box), - #[error("Relation Error: `{0}`")] - Relation(#[from] RelationError), -} - -#[derive(Error, Debug)] -pub enum DatabaseError { - #[error("Replica not found: {0}")] - NotFound(u64), - #[error("Database is already opened. Path: `{0}`. Error: {1}")] - DatabasedOpened(PathBuf, anyhow::Error), -} - -impl From for DatabaseError { - fn from(LockError { path, source, .. }: LockError) -> Self { - Self::DatabasedOpened(path, source.into()) - } -} - -#[derive(Error, Debug)] -pub enum DBError { - #[error("LibError: {0}")] - Lib(#[from] LibError), - #[error("BufferError: {0}")] - Buffer(#[from] DecodeError), - #[error("DatastoreError: {0}")] - Datastore(#[from] DatastoreError), - #[error("SequenceError: {0}")] - Sequence2(#[from] SequenceError), - #[error("SchemaError: {0}")] - Schema(SchemaErrors), - #[error("IOError: {0}.")] - IoError(#[from] std::io::Error), - #[error("ParseIntError: {0}.")] - ParseInt(#[from] ParseIntError), - #[error("Hex representation of hash decoded to incorrect number of bytes: {0}.")] - DecodeHexHash(usize), - #[error("DecodeHexError: {0}.")] - DecodeHex(#[from] FromHexError), - #[error("DatabaseError: {0}.")] - Database(#[from] DatabaseError), - #[error("SledError: {0}.")] - SledDbError(#[from] sled::Error), - #[error("Mutex was poisoned acquiring lock on MessageLog: {0}")] - MessageLogPoisoned(String), - #[error("SubscriptionError: {0}")] - Subscription(#[from] SubscriptionError), - #[error("ClientError: {0}")] - Client(#[from] ClientError), - #[error("SqlParserError: {error}, executing: `{sql}`")] - SqlParser { - sql: String, - error: sqlparser::parser::ParserError, - }, - #[error("SqlError: {error}, executing: `{sql}`")] - Plan { sql: String, error: PlanError }, - #[error("Error replaying the commit log: {0}")] - LogReplay(#[from] LogReplayError), - #[error(transparent)] - // Box the inner [`SnapshotError`] to keep Clippy quiet about large `Err` variants. - Snapshot(#[from] Box), - #[error("Error reading a value from a table through BSATN: {0}")] - ReadViaBsatnError(#[from] ReadViaBsatnError), - #[error("Module validation errors: {0}")] - ModuleValidationErrors(#[from] ValidationErrors), - #[error(transparent)] - Other(#[from] anyhow::Error), - #[error(transparent)] - TypeError(#[from] TypingError), - #[error("{error}, executing: `{sql}`")] - WithSql { - #[source] - error: Box, - sql: Box, - }, - #[error(transparent)] - RestoreSnapshot(#[from] RestoreSnapshotError), - #[error(transparent)] - DurabilityGone(#[from] DurabilityExited), - #[error(transparent)] - View(#[from] ViewCallError), -} - -impl From for DBError { - fn from(value: InvalidFieldError) -> Self { - LibError::from(value).into() - } -} - -impl From for DBError { - fn from(err: spacetimedb_table::read_column::TypeError) -> Self { - DatastoreError::Table(TableError::from(err)).into() - } -} - -impl From for PlanError { - fn from(err: DBError) -> Self { - PlanError::DatabaseInternal(Box::new(err)) - } -} - -impl<'a, T: ?Sized + 'a> From>> for DBError { - fn from(err: PoisonError>) -> Self { - DBError::MessageLogPoisoned(err.to_string()) - } -} - -impl From for DBError { - fn from(e: spacetimedb_durability::local::OpenError) -> Self { - use spacetimedb_durability::local::OpenError::*; +use spacetimedb_schema::def::error::RelationError; +use spacetimedb_schema::table_name::TableName; +use thiserror::Error; - match e { - Lock(e) => Self::from(DatabaseError::from(e)), - Commitlog(e) => Self::Other(e.into()), +impl From for DBError { + fn from(err: ViewCallError) -> Self { + match err { + ViewCallError::Args(err) => spacetimedb_engine::error::ViewError::Args(err.to_string()).into(), + ViewCallError::NoSuchModule(err) => { + spacetimedb_engine::error::ViewError::NoSuchModule(err.to_string()).into() + } + ViewCallError::NoSuchView => spacetimedb_engine::error::ViewError::NoSuchView.into(), + ViewCallError::TableDoesNotExist(view_id) => { + spacetimedb_engine::error::ViewError::TableDoesNotExist(view_id).into() + } + ViewCallError::MissingClientConnection => { + spacetimedb_engine::error::ViewError::MissingClientConnection.into() + } + ViewCallError::DatastoreError(err) => spacetimedb_engine::error::ViewError::DatastoreError(err).into(), + ViewCallError::InternalError(err) => spacetimedb_engine::error::ViewError::InternalError(err).into(), } } } -#[derive(Debug, Error)] -pub enum LogReplayError { - #[error( - "Out-of-order commit detected: {} in segment {} after offset {}", - .commit_offset, - .segment_offset, - .last_commit_offset - )] - OutOfOrderCommit { - commit_offset: u64, - segment_offset: usize, - last_commit_offset: u64, - }, - #[error( - "Error reading segment {}/{} at commit {}: {}", - .segment_offset, - .total_segments, - .commit_offset, - .source - )] - TrailingSegments { - segment_offset: usize, - total_segments: usize, - commit_offset: u64, - #[source] - source: io::Error, - }, - #[error("Could not reset log to offset {}: {}", .offset, .source)] - Reset { - offset: u64, - #[source] - source: io::Error, - }, - #[error("Missing object {} referenced from commit {}", .hash, .commit_offset)] - MissingObject { hash: Hash, commit_offset: u64 }, - #[error( - "Unexpected I/O error reading commit {} from segment {}: {}", - .commit_offset, - .segment_offset, - .source - )] - Io { - segment_offset: usize, - commit_offset: u64, - #[source] - source: io::Error, - }, -} - #[derive(Error, Debug)] pub enum NodesError { #[error("Failed to decode row: {0}")] @@ -294,23 +87,3 @@ impl From for NodesError { } } } - -#[derive(Debug, Error)] -pub enum RestoreSnapshotError { - #[error("Snapshot has incorrect database_identity: expected {expected} but found {actual}")] - IdentityMismatch { expected: Identity, actual: Identity }, - #[error("Failed to restore datastore from snapshot")] - Datastore(#[source] Box), - #[error("Failed to read snapshot")] - Snapshot(#[from] Box), - #[error("Failed to bootstrap datastore without snapshot")] - Bootstrap(#[source] Box), - #[error("No connected snapshot found, commitlog starts at {min_commitlog_offset}")] - NoConnectedSnapshot { min_commitlog_offset: TxOffset }, - #[error("Failed to invalidate snapshots at or newer than {offset}")] - Invalidate { - offset: TxOffset, - #[source] - source: Box, - }, -} diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index f2092412639..b28c5ea01cd 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -38,6 +38,7 @@ use spacetimedb_datastore::traits::Program; use spacetimedb_durability::{self as durability}; use spacetimedb_lib::{AlgebraicValue, Identity, Timestamp}; use spacetimedb_paths::server::{ModuleLogsDir, ServerDataDir}; +use spacetimedb_runtime::AbortHandle; use spacetimedb_sats::hash::Hash; use spacetimedb_schema::auto_migrate::{ponder_migrate, AutoMigrateError, MigrationPolicy, PrettyPrintStyle}; use spacetimedb_schema::def::{ModuleDef, RawModuleDefVersion}; @@ -47,7 +48,6 @@ use std::ops::Deref; use std::sync::Arc; use std::time::Duration; use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock as AsyncRwLock}; -use tokio::task::AbortHandle; use tokio::time::error::Elapsed; use tokio::time::{interval_at, timeout, Instant}; @@ -889,7 +889,8 @@ impl Host { .. } = host_controller; let replica_dir = data_dir.replica(replica_id); - let (tx_metrics_queue, tx_metrics_recorder_task) = spawn_tx_metrics_recorder(); + let runtime = spacetimedb_runtime::Handle::tokio_current(); + let (tx_metrics_queue, tx_metrics_recorder_task) = spawn_tx_metrics_recorder(&runtime); let (db, connected_clients) = match config.storage { db::Storage::Memory => RelationalDB::open( @@ -902,8 +903,13 @@ impl Host { )?, db::Storage::Disk => { // Replay from the local state. - let history = relational_db::local_history(&replica_dir).await?; - let persistence = persistence.persistence(&database, replica_id).await?; + let history = relational_db::local_history(&replica_dir, &runtime).await?; + let persistence_db = db::persistence::Database { + id: database.id, + database_identity: database.database_identity, + owner_identity: database.owner_identity, + }; + let persistence = persistence.persistence(&persistence_db, replica_id).await?; // Loading a database from persistent storage involves heavy // blocking I/O. `asyncify` to avoid blocking the async worker. let (db, clients) = asyncify({ @@ -1084,8 +1090,9 @@ impl Host { module_host.clear_all_clients().await?; scheduler_starter.start(&module_host)?; - let disk_metrics_recorder_task = tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle(); - let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone()); + let disk_metrics_recorder_task: spacetimedb_runtime::AbortHandle = + tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle().into(); + let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone(), &runtime); let module = watch::Sender::new(module_host); diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 9ce704426f7..d10239bd7f0 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -20,7 +20,6 @@ use crate::messages::control_db::{Database, HostType}; use crate::replica_context::ReplicaContext; use crate::sql::ast::SchemaViewer; use crate::sql::execute::SqlResult; -use crate::sql::parser::RowLevelExpr; use crate::subscription::module_subscription_actor::ModuleSubscriptions; use crate::subscription::module_subscription_manager::BroadcastError; pub use crate::subscription::module_subscription_manager::TransactionOffset; @@ -51,6 +50,7 @@ use spacetimedb_datastore::execution_context::{Workload, WorkloadType}; use spacetimedb_datastore::locking_tx_datastore::{MutTxId, ViewCallInfo}; use spacetimedb_datastore::traits::{IsolationLevel, Program, TxData}; pub use spacetimedb_durability::{DurabilityExited, DurableOffset}; +use spacetimedb_engine::rls::RowLevelExpr; use spacetimedb_execution::pipelined::{PipelinedProject, ViewProject}; use spacetimedb_execution::RelValue; use spacetimedb_expr::expr::CollectViews; @@ -64,10 +64,10 @@ use spacetimedb_query::compile_subscription; use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductValue}; use spacetimedb_schema::auto_migrate::{AutoMigrateError, MigrationPolicy}; -use spacetimedb_schema::def::{ModuleDef, ProcedureDef, ReducerDef, TableDef, ViewDef}; +use spacetimedb_schema::def::{ModuleDef, ProcedureDef, ReducerDef, ViewDef}; use spacetimedb_schema::identifier::Identifier; use spacetimedb_schema::reducer_name::ReducerName; -use spacetimedb_schema::schema::{Schema, TableSchema}; + use spacetimedb_schema::table_name::TableName; use std::collections::VecDeque; use std::fmt; @@ -549,19 +549,6 @@ impl GenericModuleInstance for super::v8::JsProcedureInstance { } } -/// Creates the table for `table_def` in `stdb`. -pub fn create_table_from_def( - stdb: &RelationalDB, - tx: &mut MutTxId, - module_def: &ModuleDef, - table_def: &TableDef, -) -> anyhow::Result<()> { - let schema = TableSchema::from_module_def(module_def, table_def, (), TableId::SENTINEL); - stdb.create_table(tx, schema) - .with_context(|| format!("failed to create table {}", &table_def.name))?; - Ok(()) -} - /// Creates the table for `view_def` in `stdb`. pub fn create_table_from_view_def( stdb: &RelationalDB, @@ -614,7 +601,7 @@ fn init_database_inner( table_defs.sort_by_key(|x| &x.name); for def in table_defs { logger.info(&format!("Creating table `{}`", &def.name)); - create_table_from_def(stdb, tx, module_def, def)?; + spacetimedb_engine::db::update::create_table_from_def(stdb, tx, module_def, def)?; } // Create all in-memory views defined by the module. diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 26b35230b1f..04140b7036c 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -4,7 +4,8 @@ pub mod energy; pub mod sql; pub mod auth; -pub mod db; +pub use spacetimedb_engine::db; +pub use spacetimedb_engine::metrics; pub mod messages; pub use spacetimedb_lib::Identity; pub mod error; diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 6ff2cd05566..255f57ff8b7 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -7,8 +7,8 @@ use crate::energy::FunctionBudget; use crate::error::DBError; use crate::estimation::{check_row_limit, estimate_rows_scanned}; use crate::host::module_host::{ - DatabaseUpdate, EventStatus, ModuleEvent, ModuleFunctionCall, RefInstance, ViewCallError, ViewCallResult, - ViewOutcome, WasmInstance, + DatabaseUpdate, EventStatus, ModuleEvent, ModuleFunctionCall, RefInstance, ViewCallResult, ViewOutcome, + WasmInstance, }; use crate::host::{ArgsTuple, ModuleHost}; use crate::subscription::module_subscription_actor::{commit_and_broadcast_event, ModuleSubscriptions}; @@ -163,7 +163,7 @@ fn run_inner( if let ViewOutcome::Failed(err) = result.outcome { let (_, metrics, reducer) = db.rollback_mut_tx(result.tx); db.report_mut_tx_metrics(reducer, metrics, None); - return Err(DBError::View(ViewCallError::InternalError(err))); + return Err(DBError::View(spacetimedb_engine::error::ViewError::InternalError(err))); } let tx = result.tx; diff --git a/crates/core/src/sql/mod.rs b/crates/core/src/sql/mod.rs index 741105f0e75..78c7f6bbfe7 100644 --- a/crates/core/src/sql/mod.rs +++ b/crates/core/src/sql/mod.rs @@ -1,3 +1,2 @@ pub mod ast; pub mod execute; -pub mod parser; diff --git a/crates/core/src/subscription/mod.rs b/crates/core/src/subscription/mod.rs index 40d6d712504..3a06571db57 100644 --- a/crates/core/src/subscription/mod.rs +++ b/crates/core/src/subscription/mod.rs @@ -1,19 +1,16 @@ +use crate::error::DBError; use crate::subscription::websocket_building::{BuildableWebsocketFormat, RowListBuilder as _, RowListBuilderSource}; -use crate::{error::DBError, worker_metrics::WORKER_METRICS}; use anyhow::Result; use metrics::QueryMetrics; use module_subscription_manager::Plan; -use prometheus::IntCounter; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use spacetimedb_client_api_messages::websocket::common::ByteListLen as _; use spacetimedb_client_api_messages::websocket::v1::{self as ws_v1}; -use spacetimedb_datastore::{ - db_metrics::DB_METRICS, execution_context::WorkloadType, locking_tx_datastore::datastore::MetricsRecorder, -}; +pub use spacetimedb_engine::metrics::ExecutionCounters; use spacetimedb_execution::pipelined::ViewProject; use spacetimedb_execution::{pipelined::PipelinedProject, Datastore, DeltaStore}; use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_lib::{metrics::ExecutionMetrics, Identity}; +use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_primitives::TableId; use spacetimedb_sats::bsatn::ToBsatn; use spacetimedb_sats::Serialize; @@ -32,72 +29,6 @@ pub mod subscription; pub mod tx; pub mod websocket_building; -#[derive(Debug)] -pub struct ExecutionCounters { - rdb_num_index_seeks: IntCounter, - rdb_num_rows_scanned: IntCounter, - rdb_num_bytes_scanned: IntCounter, - rdb_num_bytes_written: IntCounter, - bytes_sent_to_clients: IntCounter, - delta_queries_matched: IntCounter, - delta_queries_evaluated: IntCounter, - duplicate_rows_evaluated: IntCounter, - duplicate_rows_sent: IntCounter, -} - -impl ExecutionCounters { - pub fn new(workload: &WorkloadType, db: &Identity) -> Self { - Self { - rdb_num_index_seeks: DB_METRICS.rdb_num_index_seeks.with_label_values(workload, db), - rdb_num_rows_scanned: DB_METRICS.rdb_num_rows_scanned.with_label_values(workload, db), - rdb_num_bytes_scanned: DB_METRICS.rdb_num_bytes_scanned.with_label_values(workload, db), - rdb_num_bytes_written: DB_METRICS.rdb_num_bytes_written.with_label_values(workload, db), - bytes_sent_to_clients: WORKER_METRICS.bytes_sent_to_clients.with_label_values(workload, db), - delta_queries_matched: DB_METRICS.delta_queries_matched.with_label_values(db), - delta_queries_evaluated: DB_METRICS.delta_queries_evaluated.with_label_values(db), - duplicate_rows_evaluated: DB_METRICS.duplicate_rows_evaluated.with_label_values(db), - duplicate_rows_sent: DB_METRICS.duplicate_rows_sent.with_label_values(db), - } - } - - /// Update the global system metrics with transaction-level execution metrics. - pub(crate) fn record(&self, metrics: &ExecutionMetrics) { - if metrics.index_seeks > 0 { - self.rdb_num_index_seeks.inc_by(metrics.index_seeks as u64); - } - if metrics.rows_scanned > 0 { - self.rdb_num_rows_scanned.inc_by(metrics.rows_scanned as u64); - } - if metrics.bytes_scanned > 0 { - self.rdb_num_bytes_scanned.inc_by(metrics.bytes_scanned as u64); - } - if metrics.bytes_written > 0 { - self.rdb_num_bytes_written.inc_by(metrics.bytes_written as u64); - } - if metrics.bytes_sent_to_clients > 0 { - self.bytes_sent_to_clients.inc_by(metrics.bytes_sent_to_clients as u64); - } - if metrics.delta_queries_matched > 0 { - self.delta_queries_matched.inc_by(metrics.delta_queries_matched); - } - if metrics.delta_queries_evaluated > 0 { - self.delta_queries_evaluated.inc_by(metrics.delta_queries_evaluated); - } - if metrics.duplicate_rows_evaluated > 0 { - self.duplicate_rows_evaluated.inc_by(metrics.duplicate_rows_evaluated); - } - if metrics.duplicate_rows_sent > 0 { - self.duplicate_rows_sent.inc_by(metrics.duplicate_rows_sent); - } - } -} - -impl MetricsRecorder for ExecutionCounters { - fn record(&self, metrics: &ExecutionMetrics) { - self.record(metrics); - } -} - /// Execute a subscription query over a view. /// /// Specifically this utility is for queries that return rows from a view. diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index 9246022ced2..20b7d437a57 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -352,11 +352,6 @@ metrics_group!( #[labels(db: Identity, reducer: str)] pub reducer_plus_query_duration: HistogramVec, - #[name = spacetime_num_bytes_sent_to_clients_total] - #[help = "The cumulative number of bytes sent to clients"] - #[labels(txn_type: WorkloadType, db: Identity)] - pub bytes_sent_to_clients: IntCounterVec, - #[name = spacetime_subscription_send_queue_length] #[help = "The number of `ComputedQueries` waiting in the queue to be aggregated and broadcast by the `send_worker`"] #[labels(database_identity: Identity)] @@ -372,34 +367,6 @@ metrics_group!( #[labels(db: Identity)] pub total_outgoing_queue_length: IntGaugeVec, - #[name = spacetime_replay_total_time_seconds] - #[help = "Total time spent replaying a database upon restart, including snapshot read, snapshot restore and commitlog replay"] - #[labels(db: Identity)] - // We expect a small number of observations per label - // (exactly one, for non-replicated databases, and one per leader change for replicated databases) - // so we'll just store a `Gauge` with the most recent observation for each database. - pub replay_total_time_seconds: GaugeVec, - - #[name = spacetime_replay_snapshot_read_time_seconds] - #[help = "Time spent reading a snapshot from disk before restoring the snapshot upon restart"] - #[labels(db: Identity)] - pub replay_snapshot_read_time_seconds: GaugeVec, - - #[name = spacetime_replay_snapshot_restore_time_seconds] - #[help = "Time spent restoring a database from a snapshot after reading the snapshot and before commitlog replay upon restart"] - #[labels(db: Identity)] - pub replay_snapshot_restore_time_seconds: GaugeVec, - - #[name = spacetime_replay_commitlog_time_seconds] - #[help = "Time spent replaying the commitlog after restoring from a snapshot upon restart"] - #[labels(db: Identity)] - pub replay_commitlog_time_seconds: GaugeVec, - - #[name = spacetime_replay_commitlog_num_commits] - #[help = "Number of commits replayed after restoring from a snapshot upon restart"] - #[labels(db: Identity)] - pub replay_commitlog_num_commits: IntGaugeVec, - #[name = spacetime_module_create_instance_time_seconds] #[help = "Time taken to construct a WASM instance or V8 isolate to run module code"] #[labels(db: Identity, module_type: HostType)] @@ -411,75 +378,6 @@ metrics_group!( #[buckets(0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100)] pub module_create_instance_time_seconds: HistogramVec, - #[name = spacetime_snapshot_creation_time_total_sec] - #[help = "The time (in seconds) it took to take and store a database snapshot, including scheduling overhead"] - #[labels(db: Identity)] - // Snapshot creation should take in the order of milliseconds, - // but log data suggests that there are outliers. - // So let's track a wide range of buckets to get a better picture. - // - // We also track the timing without `asyncify` scheduling overhead - // (`snapshot_creation_time_inner`), and the snapshot compression - // timing with / without scheduling overhead (`snapshot_compression_time_total` - // and `snapshot_compression_time_inner`, respectively). - // - // Compression may have contributed to observed outliers, but is no - // longer included in the snapshot creation timing. - #[buckets(0.0005, 0.001, 0.005, 0.01, 0.1, 1.0, 5.0, 10.0)] - pub snapshot_creation_time_total: HistogramVec, - - #[name = spacetime_snapshot_creation_time_inner_sec] - #[help = "The time (in seconds) it took to take and store a database snapshot, excluding scheduling overhead"] - #[labels(db: Identity)] - #[buckets(0.0005, 0.001, 0.005, 0.01, 0.1, 1.0, 5.0, 10.0)] - pub snapshot_creation_time_inner: HistogramVec, - - #[name = spacetime_snapshot_creation_time_fsync_sec] - #[help = "The time (in seconds) it took to fsync a database snapshot, excluding scheduling overhead"] - #[labels(db: Identity)] - #[buckets(0.0005, 0.001, 0.005, 0.01, 0.1, 1.0, 5.0, 10.0)] - pub snapshot_creation_time_fsync: HistogramVec, - - #[name = spacetime_snapshot_compression_time_total_sec] - #[help = "The time (in seconds) it took to do a compression pass on the snapshot repository, including scheduling overhead"] - #[labels(db: Identity)] - // Not sure what range to expect, but certainly slower than snapshot - // creation. - #[buckets(0.001, 0.01, 0.1, 1.0, 5.0, 10.0)] - pub snapshot_compression_time_total: HistogramVec, - - #[name = spacetime_snapshot_compression_time_inner_sec] - #[help = "The time (in seconds) it took to do a compression pass on the snapshot repository, excluding scheduling overhead"] - #[labels(db: Identity)] - #[buckets(0.001, 0.01, 0.1, 1.0, 5.0, 10.0)] - pub snapshot_compression_time_inner: HistogramVec, - - #[name = spacetime_snapshot_compression_time_per_snapshot_sec] - #[help = "The time (in seconds) it took to compress a single snapshot"] - #[labels(db: Identity)] - #[buckets(0.001, 0.01, 0.1, 1.0, 5.0, 10.0)] - pub snapshot_compression_time_single: HistogramVec, - - #[name = spacetime_snapshot_compression_skipped] - #[help = "The number of snapshots skipped in a single compression pass because they were already compressed"] - #[labels(db: Identity)] - pub snapshot_compression_skipped: IntGaugeVec, - - #[name = spacetime_snapshot_compression_compressed] - #[help = "The number of snapshots compressed in a single compression pass"] - #[labels(db: Identity)] - pub snapshot_compression_compressed: IntGaugeVec, - - #[name = spacetime_snapshot_compression_objects_compressed] - #[help = "The number of snapshot objects compressed in a single compression pass"] - #[labels(db: Identity)] - pub snapshot_compression_objects_compressed: IntGaugeVec, - - #[name = spacetime_snapshot_compression_objects_hardlinked] - #[help = "The number of snapshot objects hardlinked in a single compression pass"] - #[labels(db: Identity)] - pub snapshot_compression_objects_hardlinked: IntGaugeVec, - #[name = spacetime_subscription_rows_examined] #[help = "Distribution of rows examined per subscription query"] #[labels(db: Identity, scan_type: str, table: str, unindexed_columns: str)] @@ -496,12 +394,6 @@ metrics_group!( #[help = "Total number of subscription queries by scan strategy"] #[labels(db: Identity, scan_type: str, table: str, unindexed_columns: str)] pub subscription_queries_total: IntCounterVec, - - #[name = spacetime_durability_blocking_send_duration_sec] - #[help = "Latency of blocking sends in request_durability (seconds); _count gives the number of times the channel was full"] - #[labels(database_identity: Identity)] - #[buckets(0.001, 0.01, 0.1, 1.0, 10.0)] - pub durability_blocking_send_duration: HistogramVec, } ); diff --git a/crates/datastore/Cargo.toml b/crates/datastore/Cargo.toml index 86cb2aca0b6..ad31b5cd931 100644 --- a/crates/datastore/Cargo.toml +++ b/crates/datastore/Cargo.toml @@ -10,14 +10,14 @@ rust-version.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-lib = { workspace = true, features = ["serde", "metrics_impls"] } spacetimedb-commitlog.workspace = true -spacetimedb-durability.workspace = true +spacetimedb-durability = { path = "../durability", default-features = false } spacetimedb-metrics.workspace = true spacetimedb-primitives.workspace = true spacetimedb-paths.workspace = true spacetimedb-sats = { workspace = true, features = ["serde"] } spacetimedb-schema.workspace = true spacetimedb-table.workspace = true -spacetimedb-snapshot.workspace = true +spacetimedb-snapshot = { path = "../snapshot", default-features = false } spacetimedb-execution.workspace = true anyhow = { workspace = true, features = ["backtrace"] } @@ -39,7 +39,9 @@ thin-vec.workspace = true [features] # Print a warning when doing an unindexed `iter_by_col_range` on a large table. unindexed_iter_by_col_range_warn = [] -default = ["unindexed_iter_by_col_range_warn"] +default = ["unindexed_iter_by_col_range_warn", "tokio"] +tokio = ["spacetimedb-durability/tokio", "spacetimedb-snapshot/tokio"] +simulation = ["spacetimedb-durability/simulation", "spacetimedb-snapshot/simulation"] # Enable test helpers and utils test = ["spacetimedb-commitlog/test", "spacetimedb-schema/test"] diff --git a/crates/durability/Cargo.toml b/crates/durability/Cargo.toml index 4eaa3870001..198dd4461fc 100644 --- a/crates/durability/Cargo.toml +++ b/crates/durability/Cargo.toml @@ -8,6 +8,9 @@ license-file = "LICENSE" description = "Traits and single-node implementation of durability for SpacetimeDB." [features] +default = ["tokio"] +tokio = ["spacetimedb-runtime/tokio"] +simulation = ["spacetimedb-runtime/simulation"] test = [] fallocate = ["spacetimedb-commitlog/fallocate"] @@ -21,7 +24,7 @@ scopeguard.workspace = true spacetimedb-commitlog.workspace = true spacetimedb-fs-utils.workspace = true spacetimedb-paths.workspace = true -spacetimedb-runtime = { workspace = true, features = ["tokio"] } +spacetimedb-runtime = { path = "../runtime", default-features = false } spacetimedb-sats.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml new file mode 100644 index 00000000000..7eafb2914a6 --- /dev/null +++ b/crates/engine/Cargo.toml @@ -0,0 +1,59 @@ +[package] +name = "spacetimedb-engine" +version.workspace = true +edition.workspace = true +license-file = "LICENSE" +description = "Database engine and local persistence runtime for SpacetimeDB" +rust-version.workspace = true + +[lints] +workspace = true + +[features] +default = ["tokio"] +tokio = ["spacetimedb-runtime/tokio", "spacetimedb-datastore/tokio", "spacetimedb-durability/tokio", "spacetimedb-snapshot/tokio"] +simulation = ["spacetimedb-runtime/simulation", "spacetimedb-datastore/simulation", "spacetimedb-durability/simulation", "spacetimedb-snapshot/simulation"] +test = ["spacetimedb-commitlog/test", "spacetimedb-datastore/test"] + +[dependencies] +anyhow = { workspace = true, features = ["backtrace"] } +async-trait.workspace = true +enum-map.workspace = true +futures.workspace = true +hex.workspace = true +itertools.workspace = true +log.workspace = true +once_cell.workspace = true +parking_lot.workspace = true +prometheus.workspace = true +serde.workspace = true +sled.workspace = true +spacetimedb-commitlog.workspace = true +spacetimedb-data-structures.workspace = true +spacetimedb-datastore = { path = "../datastore", default-features = false } +spacetimedb-durability = { path = "../durability", default-features = false } +spacetimedb-expr.workspace = true +spacetimedb-fs-utils.workspace = true +spacetimedb-lib = { workspace = true, features = ["serde", "metrics_impls"] } +spacetimedb-metrics.workspace = true +spacetimedb-paths.workspace = true +spacetimedb-primitives.workspace = true +spacetimedb-runtime = { path = "../runtime", default-features = false } +spacetimedb-sats = { workspace = true, features = ["serde"] } +spacetimedb-schema.workspace = true +spacetimedb-snapshot = { path = "../snapshot", default-features = false } +spacetimedb-table.workspace = true +sqlparser.workspace = true +tempfile.workspace = true +thiserror.workspace = true +tracing.workspace = true + +[dev-dependencies] +bytes.workspace = true +env_logger.workspace = true +fs_extra.workspace = true +pretty_assertions.workspace = true +spacetimedb-commitlog = { workspace = true, features = ["test"] } +spacetimedb-datastore = { path = "../datastore", default-features = false, features = ["test"] } +spacetimedb-schema = { workspace = true, features = ["test"] } +tokio.workspace = true diff --git a/crates/engine/LICENSE b/crates/engine/LICENSE new file mode 120000 index 00000000000..8540cf8a991 --- /dev/null +++ b/crates/engine/LICENSE @@ -0,0 +1 @@ +../../licenses/BSL.txt \ No newline at end of file diff --git a/crates/core/src/db/durability.rs b/crates/engine/src/db/durability.rs similarity index 100% rename from crates/core/src/db/durability.rs rename to crates/engine/src/db/durability.rs diff --git a/crates/core/src/db/mod.rs b/crates/engine/src/db/mod.rs similarity index 86% rename from crates/core/src/db/mod.rs rename to crates/engine/src/db/mod.rs index 45777fbd612..363123c0dba 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/engine/src/db/mod.rs @@ -2,10 +2,8 @@ use std::sync::Arc; use enum_map::EnumMap; use spacetimedb_schema::reducer_name::ReducerName; -use tokio::sync::mpsc; -use tokio::time::MissedTickBehavior; -use crate::subscription::ExecutionCounters; +use crate::metrics::ExecutionCounters; use spacetimedb_datastore::execution_context::WorkloadType; use spacetimedb_datastore::{locking_tx_datastore::datastore::TxMetrics, traits::TxData}; @@ -56,7 +54,7 @@ pub struct MetricsMessage { /// The handle used to send work to the tx metrics recorder. #[derive(Clone)] pub struct MetricsRecorderQueue { - tx: mpsc::UnboundedSender, + tx: spacetimedb_runtime::sync::mpsc::UnboundedSender, } impl MetricsRecorderQueue { @@ -127,19 +125,20 @@ const TX_METRICS_RECORDING_INTERVAL: std::time::Duration = std::time::Duration:: /// Spawns a task for recording transaction metrics. /// Returns the handle for pushing metrics to the recorder. -pub fn spawn_tx_metrics_recorder() -> (MetricsRecorderQueue, tokio::task::AbortHandle) { - let (tx, mut rx) = mpsc::unbounded_channel(); - let abort_handle = tokio::spawn(async move { - let mut interval = tokio::time::interval(TX_METRICS_RECORDING_INTERVAL); - interval.set_missed_tick_behavior(MissedTickBehavior::Skip); - - loop { - interval.tick().await; - while let Ok(metrics) = rx.try_recv() { - record_metrics(metrics); +pub fn spawn_tx_metrics_recorder( + handle: &spacetimedb_runtime::Handle, +) -> (MetricsRecorderQueue, spacetimedb_runtime::AbortHandle) { + let handle_clone = handle.clone(); + let (tx, mut rx) = spacetimedb_runtime::sync::mpsc::unbounded_channel(); + let abort_handle = handle + .spawn(async move { + loop { + handle_clone.sleep(TX_METRICS_RECORDING_INTERVAL).await; + while let Ok(metrics) = rx.try_recv() { + record_metrics(metrics); + } } - } - }) - .abort_handle(); + }) + .abort_handle(); (MetricsRecorderQueue { tx }, abort_handle) } diff --git a/crates/core/src/db/persistence.rs b/crates/engine/src/db/persistence.rs similarity index 92% rename from crates/core/src/db/persistence.rs rename to crates/engine/src/db/persistence.rs index 0edef7ab4b8..de152b0c6f7 100644 --- a/crates/core/src/db/persistence.rs +++ b/crates/engine/src/db/persistence.rs @@ -10,7 +10,8 @@ use spacetimedb_durability::{DurabilityExited, TxOffset}; use spacetimedb_paths::server::ServerDataDir; use spacetimedb_snapshot::DynSnapshotRepo; -use crate::{messages::control_db::Database, util::asyncify}; +use crate::util::asyncify; +use spacetimedb_lib::Identity; use spacetimedb_runtime::Handle; use super::{ @@ -84,6 +85,13 @@ pub type Durability = dyn spacetimedb_durability::Durability; /// configured or the database is in follower state. pub type DiskSizeFn = Arc io::Result + Send + Sync>; +#[derive(Clone, Copy, Debug)] +pub struct Database { + pub id: u64, + pub database_identity: Identity, + pub owner_identity: Identity, +} + /// Persistence services for a database. pub struct Persistence { /// The [Durability] to use, for persisting transactions. @@ -109,9 +117,9 @@ impl Persistence { durability: impl spacetimedb_durability::Durability + 'static, disk_size: impl Fn() -> io::Result + Send + Sync + 'static, snapshots: Option, - runtime: tokio::runtime::Handle, + runtime: Handle, ) -> Self { - Self::new_with_runtime(durability, disk_size, snapshots, Handle::tokio(runtime)) + Self::new_with_runtime(durability, disk_size, snapshots, runtime) } pub fn new_with_runtime( @@ -175,7 +183,7 @@ impl Persistence { /// A persistence provider is a "factory" of sorts that can produce [Persistence] /// services for a given replica. /// -/// The [crate::host::HostController] uses this to obtain [Persistence]s from +/// The host controller uses this to obtain [Persistence]s from /// an external source, and construct [relational_db::RelationalDB]s with it. /// /// This is an `async_trait` to allow it to be used as a trait object. @@ -215,15 +223,16 @@ impl LocalPersistenceProvider { #[async_trait] impl PersistenceProvider for LocalPersistenceProvider { async fn persistence(&self, database: &Database, replica_id: u64) -> anyhow::Result { + let database_identity = database.database_identity; let replica_dir = self.data_dir.replica(replica_id); let snapshot_dir = replica_dir.snapshots(); let runtime = Handle::tokio_current(); - let database_identity = database.database_identity; - let snapshot_worker = - asyncify(move || relational_db::open_snapshot_repo(snapshot_dir, database_identity, replica_id)) - .await - .map(|repo| SnapshotWorker::new(repo, snapshot::Compression::Enabled, runtime.clone()))?; + let snapshot_worker = asyncify(&runtime, move || { + relational_db::open_snapshot_repo(snapshot_dir, database_identity, replica_id) + }) + .await + .map(|repo| SnapshotWorker::new(repo, snapshot::Compression::Enabled, runtime.clone()))?; let (durability, disk_size) = relational_db::local_durability_with_options( replica_dir, runtime.clone(), @@ -232,11 +241,12 @@ impl PersistenceProvider for LocalPersistenceProvider { ) .await?; - tokio::spawn(relational_db::snapshot_watching_commitlog_compressor( + runtime.spawn(relational_db::snapshot_watching_commitlog_compressor( snapshot_worker.subscribe(), None, None, durability.clone(), + runtime.clone(), )); Ok(Persistence { diff --git a/crates/core/src/db/relational_db.rs b/crates/engine/src/db/relational_db.rs similarity index 98% rename from crates/core/src/db/relational_db.rs rename to crates/engine/src/db/relational_db.rs index 3bbcc0ba20f..0882e1e8243 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/engine/src/db/relational_db.rs @@ -1,9 +1,9 @@ use crate::db::durability::{request_durability, spawn_close as spawn_durability_close}; use crate::db::MetricsRecorderQueue; use crate::error::{DBError, RestoreSnapshotError}; -use crate::subscription::ExecutionCounters; +use crate::metrics::ExecutionCounters; +use crate::metrics::ENGINE_METRICS; use crate::util::asyncify; -use crate::worker_metrics::WORKER_METRICS; use anyhow::{anyhow, Context}; use enum_map::EnumMap; use spacetimedb_commitlog::repo::OnNewSegmentFn; @@ -43,6 +43,7 @@ use spacetimedb_lib::ConnectionId; use spacetimedb_lib::Identity; use spacetimedb_paths::server::{ReplicaDir, SnapshotsPath}; use spacetimedb_primitives::*; +use spacetimedb_runtime::sync::watch; use spacetimedb_runtime::Handle; use spacetimedb_sats::memory_usage::MemoryUsage; use spacetimedb_sats::raw_identifier::RawIdentifier; @@ -62,7 +63,6 @@ use std::borrow::Cow; use std::io; use std::ops::RangeBounds; use std::sync::Arc; -use tokio::sync::watch; pub use super::persistence::{DiskSizeFn, Durability, Persistence}; pub use super::snapshot::SnapshotWorker; @@ -303,7 +303,7 @@ impl RelationalDB { apply_history(&inner, database_identity, history)?; let elapsed_time = start_time.elapsed(); - WORKER_METRICS + ENGINE_METRICS .replay_total_time_seconds .with_label_values(&database_identity) .set(elapsed_time.as_secs_f64()); @@ -495,7 +495,7 @@ impl RelationalDB { let elapsed_time = start.elapsed(); - WORKER_METRICS + ENGINE_METRICS .replay_snapshot_read_time_seconds .with_label_values(database_identity) .set(elapsed_time.as_secs_f64()); @@ -519,7 +519,7 @@ impl RelationalDB { .inspect(|_| { let elapsed_time = start.elapsed(); - WORKER_METRICS.replay_snapshot_restore_time_seconds.with_label_values(database_identity).set(elapsed_time.as_secs_f64()); + ENGINE_METRICS.replay_snapshot_restore_time_seconds.with_label_values(database_identity).set(elapsed_time.as_secs_f64()); log::info!( "[{database_identity}] DATABASE: restored from snapshot of tx_offset {snapshot_offset} in {elapsed_time:?}", @@ -1050,7 +1050,7 @@ impl RelationalDB { /// Reports the `TxMetrics`s passed. /// /// Should only be called after the tx lock has been fully released. - pub(crate) fn report_tx_metrics( + pub fn report_tx_metrics( &self, reducer: Option, tx_data: Option>, @@ -1079,7 +1079,10 @@ const VIEWS_EXPIRATION: std::time::Duration = std::time::Duration::from_secs(10 const VIEW_CLEANUP_BUDGET: std::time::Duration = std::time::Duration::from_millis(10); /// Spawn a background task that periodically cleans up expired views -pub fn spawn_view_cleanup_loop(db: Arc) -> tokio::task::AbortHandle { +pub fn spawn_view_cleanup_loop( + db: Arc, + handle: &spacetimedb_runtime::Handle, +) -> spacetimedb_runtime::AbortHandle { fn run_view_cleanup(db: &RelationalDB) { match db.with_auto_commit(Workload::Internal, |tx| { tx.clear_expired_views(VIEWS_EXPIRATION, VIEW_CLEANUP_BUDGET) @@ -1106,23 +1109,19 @@ pub fn spawn_view_cleanup_loop(db: Arc) -> tokio::task::AbortHandl } } - tokio::spawn(async move { - loop { - // Offload actual cleanup to blocking thread pool, as `VIEW_CLEANUP_BUDGET` is defined - // in milliseconds, which may be too long for async tasks. - let db = db.clone(); - let db_identity = db.database_identity(); - tokio::task::spawn_blocking(move || run_view_cleanup(&db)) - .await - .inspect_err(|e| { - log::error!("[{}] DATABASE: failed to run view cleanup task: {}", db_identity, e); - }) - .ok(); + let handle_clone = handle.clone(); + handle + .spawn(async move { + loop { + // Offload actual cleanup to blocking thread pool, as `VIEW_CLEANUP_BUDGET` is defined + // in milliseconds, which may be too long for async tasks. + let db = db.clone(); + handle_clone.spawn_blocking(move || run_view_cleanup(&db)).await; - tokio::time::sleep(VIEWS_EXPIRATION).await; - } - }) - .abort_handle() + handle_clone.sleep(VIEWS_EXPIRATION).await; + } + }) + .abort_handle() } impl RelationalDB { pub fn create_table(&self, tx: &mut MutTx, schema: TableSchema) -> Result { @@ -1528,7 +1527,7 @@ impl RelationalDB { } /// Read the value of [ST_VARNAME_ROW_LIMIT] from `st_var` - pub(crate) fn row_limit(&self, tx: &Tx) -> Result, DBError> { + pub fn row_limit(&self, tx: &Tx) -> Result, DBError> { let data = self.read_var(tx, StVarName::RowLimit); if let Some(StVarValue::U64(limit)) = data? { @@ -1669,10 +1668,10 @@ fn apply_history( history: impl durability::History, ) -> Result<(), DBError> { let counters = ApplyHistoryCounters { - replay_commitlog_time_seconds: WORKER_METRICS + replay_commitlog_time_seconds: ENGINE_METRICS .replay_commitlog_time_seconds .with_label_values(&database_identity), - replay_commitlog_num_commits: WORKER_METRICS + replay_commitlog_num_commits: ENGINE_METRICS .replay_commitlog_num_commits .with_label_values(&database_identity), }; @@ -1715,10 +1714,11 @@ pub async fn local_durability_with_options( snapshot_worker.request_snapshot_ignore_closed(); }) as Arc }); - let local = asyncify(move || { + let durability_runtime = runtime.clone(); + let local = asyncify(&runtime, move || { durability::Local::open( replica_dir.clone(), - runtime, + durability_runtime, opts, // Give the durability a handle to request a new snapshot run, // which it will send down whenever we rotate commitlog segments. @@ -1738,9 +1738,12 @@ pub async fn local_durability_with_options( /// Open a [History] for replay from the local durable state. /// /// Currently, this is simply a read-only copy of the commitlog. -pub async fn local_history(replica_dir: &ReplicaDir) -> io::Result + use<>> { +pub async fn local_history( + replica_dir: &ReplicaDir, + runtime: &Handle, +) -> io::Result + use<>> { let commitlog_dir = replica_dir.commit_log(); - asyncify(move || Commitlog::open(commitlog_dir, <_>::default(), None)).await + asyncify(runtime, move || Commitlog::open(commitlog_dir, <_>::default(), None)).await } /// Watches snapshot creation events and compresses all commitlog segments older @@ -1749,9 +1752,10 @@ pub async fn local_history(replica_dir: &ReplicaDir) -> io::Result, - mut clog_tx: Option>, - mut snap_tx: Option>, + mut clog_tx: Option>, + mut snap_tx: Option>, durability: LocalDurability, + runtime: Handle, ) { let mut prev_snapshot_offset = *snapshot_rx.borrow_and_update(); while snapshot_rx.changed().await.is_ok() { @@ -1764,7 +1768,7 @@ pub async fn snapshot_watching_commitlog_compressor( tracing::warn!("failed to send offset {snapshot_offset} after snapshot creation: {err}"); } - let res: io::Result<_> = asyncify(move || { + let res: io::Result<_> = asyncify(&runtime, move || { let segment_offsets = durability.existing_segment_offsets()?; let start_idx = segment_offsets .binary_search(&prev_snapshot_offset) @@ -1828,17 +1832,18 @@ fn default_row_count_fn(db: Identity) -> RowCountFn { pub mod tests_utils { use crate::db::snapshot; use crate::db::snapshot::SnapshotWorker; - use crate::messages::control_db::HostType; use super::*; use core::ops::Deref; use durability::{Durability, EmptyHistory}; use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_datastore::locking_tx_datastore::TxId; + use spacetimedb_datastore::system_tables::ModuleKind; use spacetimedb_fs_utils::compression::CompressType; use spacetimedb_lib::{bsatn::to_vec, ser::Serialize}; use spacetimedb_paths::server::ReplicaDir; use spacetimedb_paths::FromPathUnchecked; + use spacetimedb_runtime::{TokioHandle, TokioRuntime, TokioRuntimeBuilder}; use tempfile::TempDir; pub enum TestDBDir { @@ -1867,7 +1872,7 @@ pub mod tests_utils { pub type TestDBParts = ( Arc, Option>>, - Option, + Option, Option, ); @@ -1912,7 +1917,7 @@ pub mod tests_utils { struct DurableState { durability: Arc>, - rt: tokio::runtime::Runtime, + rt: TokioRuntime, replica_dir: TestDBDir, } @@ -1938,7 +1943,7 @@ pub mod tests_utils { /// database. pub fn durable() -> Result { let dir = TempReplicaDir::new()?; - let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + let rt = TokioRuntimeBuilder::new_multi_thread().enable_all().build()?; // Enter the runtime so that `Self::durable_internal` can spawn a `SnapshotWorker`. let _rt = rt.enter(); let (db, durability) = Self::durable_internal(&dir, rt.handle().clone(), true)?; @@ -1957,7 +1962,7 @@ pub mod tests_utils { pub fn durable_without_snapshot_repo() -> Result { let dir = TempReplicaDir::new()?; - let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + let rt = TokioRuntimeBuilder::new_multi_thread().enable_all().build()?; // Enter the runtime so that `Self::durable_internal` can spawn a `SnapshotWorker`. let _rt = rt.enter(); let (db, durability) = Self::durable_internal(&dir, rt.handle().clone(), false)?; @@ -1976,7 +1981,7 @@ pub mod tests_utils { pub fn open_existing_durable( root: &ReplicaDir, - rt: tokio::runtime::Handle, + rt: TokioHandle, replica_id: u64, db_identity: Identity, owner_identity: Identity, @@ -1991,9 +1996,8 @@ pub mod tests_utils { .transpose()?; let runtime = Handle::tokio(rt.clone()); - let (local, disk_size_fn) = + let (local, disk_size_fn): (LocalDurability, DiskSizeFn) = rt.block_on(local_durability(root.clone(), runtime.clone(), snapshots.as_ref()))?; - let history = local.as_history(); let persistence = Persistence { durability: local.clone(), @@ -2005,7 +2009,7 @@ pub mod tests_utils { let (db, _) = RelationalDB::open( db_identity, owner_identity, - history, + local.as_history(), Some(persistence), None, PagePool::new_for_test(), @@ -2083,7 +2087,7 @@ pub mod tests_utils { /// Handle to the tokio runtime, available if [`Self::durable`] was used /// to create the [`TestDB`]. - pub fn runtime(&self) -> Option<&tokio::runtime::Handle> { + pub fn runtime(&self) -> Option<&TokioHandle> { self.durable.as_ref().map(|ds| ds.rt.handle()) } @@ -2107,7 +2111,7 @@ pub mod tests_utils { fn durable_internal( root: &ReplicaDir, - rt: tokio::runtime::Handle, + rt: TokioHandle, want_snapshot_repo: bool, ) -> Result<(RelationalDB, Arc>), DBError> { let snapshots = want_snapshot_repo @@ -2118,16 +2122,15 @@ pub mod tests_utils { }) .transpose()?; let runtime = Handle::tokio(rt.clone()); - let (local, disk_size_fn) = + let (local, disk_size_fn): (LocalDurability, DiskSizeFn) = rt.block_on(local_durability(root.clone(), runtime.clone(), snapshots.as_ref()))?; - let history = local.as_history(); let persistence = Persistence { durability: local.clone(), disk_size: disk_size_fn, snapshots, runtime, }; - let db = Self::open_db(history, Some(persistence), None, 0)?; + let db = Self::open_db(local.as_history(), Some(persistence), None, 0)?; Ok((db, local)) } @@ -2149,7 +2152,7 @@ pub mod tests_utils { assert_eq!(connected_clients.len(), expected_num_clients); let db = db.with_row_count(Self::row_count_fn()); db.with_auto_commit(Workload::Internal, |tx| { - db.set_initialized(tx, Program::empty(HostType::Wasm.into())) + db.set_initialized(tx, Program::empty(ModuleKind::WASM)) })?; Ok(db) } diff --git a/crates/core/src/db/snapshot.rs b/crates/engine/src/db/snapshot.rs similarity index 95% rename from crates/core/src/db/snapshot.rs rename to crates/engine/src/db/snapshot.rs index 7f2474e0d74..11b08727b7d 100644 --- a/crates/core/src/db/snapshot.rs +++ b/crates/engine/src/db/snapshot.rs @@ -14,10 +14,10 @@ use prometheus::{Histogram, IntGauge}; use spacetimedb_datastore::locking_tx_datastore::{committed_state::CommittedState, datastore::Locking}; use spacetimedb_durability::TxOffset; use spacetimedb_lib::Identity; +use spacetimedb_runtime::sync::watch; use spacetimedb_snapshot::{CompressionStats, DynSnapshotRepo}; -use tokio::sync::watch; -use crate::worker_metrics::WORKER_METRICS; +use crate::metrics::ENGINE_METRICS; use spacetimedb_runtime::Handle; pub type SnapshotDatabaseState = Arc>; @@ -158,9 +158,9 @@ struct SnapshotMetrics { impl SnapshotMetrics { fn new(db: Identity) -> Self { Self { - snapshot_timing_total: WORKER_METRICS.snapshot_creation_time_total.with_label_values(&db), - snapshot_timing_inner: WORKER_METRICS.snapshot_creation_time_inner.with_label_values(&db), - snapshot_timing_fsync: WORKER_METRICS.snapshot_creation_time_fsync.with_label_values(&db), + snapshot_timing_total: ENGINE_METRICS.snapshot_creation_time_total.with_label_values(&db), + snapshot_timing_inner: ENGINE_METRICS.snapshot_creation_time_inner.with_label_values(&db), + snapshot_timing_fsync: ENGINE_METRICS.snapshot_creation_time_fsync.with_label_values(&db), } } } @@ -280,15 +280,15 @@ struct CompressionMetrics { impl CompressionMetrics { fn new(db: Identity) -> Self { Self { - timing_total: WORKER_METRICS.snapshot_compression_time_total.with_label_values(&db), - timing_inner: WORKER_METRICS.snapshot_compression_time_inner.with_label_values(&db), - timing_single: WORKER_METRICS.snapshot_compression_time_single.with_label_values(&db), - skipped: WORKER_METRICS.snapshot_compression_skipped.with_label_values(&db), - compressed: WORKER_METRICS.snapshot_compression_compressed.with_label_values(&db), - objects_compressed: WORKER_METRICS + timing_total: ENGINE_METRICS.snapshot_compression_time_total.with_label_values(&db), + timing_inner: ENGINE_METRICS.snapshot_compression_time_inner.with_label_values(&db), + timing_single: ENGINE_METRICS.snapshot_compression_time_single.with_label_values(&db), + skipped: ENGINE_METRICS.snapshot_compression_skipped.with_label_values(&db), + compressed: ENGINE_METRICS.snapshot_compression_compressed.with_label_values(&db), + objects_compressed: ENGINE_METRICS .snapshot_compression_objects_compressed .with_label_values(&db), - objects_hardlinked: WORKER_METRICS + objects_hardlinked: ENGINE_METRICS .snapshot_compression_objects_hardlinked .with_label_values(&db), } diff --git a/crates/core/src/db/update.rs b/crates/engine/src/db/update.rs similarity index 97% rename from crates/core/src/db/update.rs rename to crates/engine/src/db/update.rs index f9ca4c110d9..6752ad78d10 100644 --- a/crates/core/src/db/update.rs +++ b/crates/engine/src/db/update.rs @@ -1,13 +1,13 @@ use super::relational_db::RelationalDB; -use crate::database_logger::SystemLogger; -use crate::sql::parser::RowLevelExpr; +use crate::rls::RowLevelExpr; +use anyhow::Context; use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_lib::db::auth::StTableType; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::AlgebraicValue; use spacetimedb_primitives::{ColSet, TableId}; use spacetimedb_schema::auto_migrate::{AutoMigratePlan, ManualMigratePlan, MigratePlan}; -use spacetimedb_schema::def::{TableDef, ViewDef}; +use spacetimedb_schema::def::{ModuleDef, TableDef, ViewDef}; use spacetimedb_schema::schema::{column_schemas_from_defs, IndexSchema, Schema, SequenceSchema, TableSchema}; /// The logger used for by [`update_database`] and friends. @@ -15,12 +15,6 @@ pub trait UpdateLogger { fn info(&self, msg: &str); } -impl UpdateLogger for SystemLogger { - fn info(&self, msg: &str) { - self.info(msg); - } -} - /// The result of a database update. /// Indicates whether clients should be disconnected when the update is complete. #[must_use] @@ -332,13 +326,23 @@ fn auto_migrate_database( Ok(res) } +/// Creates the table for `table_def` in `stdb`. +pub fn create_table_from_def( + stdb: &RelationalDB, + tx: &mut MutTxId, + module_def: &ModuleDef, + table_def: &TableDef, +) -> anyhow::Result<()> { + let schema = TableSchema::from_module_def(module_def, table_def, (), TableId::SENTINEL); + stdb.create_table(tx, schema) + .with_context(|| format!("failed to create table {}", &table_def.name))?; + Ok(()) +} + #[cfg(test)] mod test { use super::*; - use crate::{ - db::relational_db::tests_utils::{begin_mut_tx, insert, TestDB}, - host::module_host::create_table_from_def, - }; + use crate::db::relational_db::tests_utils::{begin_mut_tx, insert, TestDB}; use spacetimedb_datastore::locking_tx_datastore::PendingSchemaChange; use spacetimedb_lib::db::raw_def::v9::{btree, RawIndexAlgorithm, RawModuleDefV9Builder, TableAccess}; use spacetimedb_sats::{product, AlgebraicType, AlgebraicType::U64}; diff --git a/crates/engine/src/error.rs b/crates/engine/src/error.rs new file mode 100644 index 00000000000..7c1a3e11129 --- /dev/null +++ b/crates/engine/src/error.rs @@ -0,0 +1,262 @@ +use std::io; +use std::num::ParseIntError; +use std::path::PathBuf; +use std::sync::{MutexGuard, PoisonError}; + +use hex::FromHexError; +use spacetimedb_commitlog::repo::TxOffset; +use spacetimedb_durability::DurabilityExited; +use spacetimedb_expr::errors::TypingError; +use spacetimedb_fs_utils::lockfile::advisory::LockError; +use spacetimedb_lib::Identity; +use spacetimedb_schema::error::ValidationErrors; +use spacetimedb_schema::table_name::TableName; +use spacetimedb_snapshot::SnapshotError; +use spacetimedb_table::table::ReadViaBsatnError; +use thiserror::Error; + +use spacetimedb_lib::buffer::DecodeError; +use spacetimedb_primitives::*; +use spacetimedb_sats::hash::Hash; +use spacetimedb_sats::product_value::InvalidFieldError; +use spacetimedb_schema::def::error::{LibError, RelationError, SchemaErrors}; +use spacetimedb_schema::relation::FieldName; + +pub use spacetimedb_datastore::error::{DatastoreError, IndexError, SequenceError, TableError}; + +#[derive(Error, Debug, PartialEq, Eq)] +pub enum SubscriptionError { + #[error("Index not found: {0:?}")] + NotFound(IndexId), + #[error("Empty string")] + Empty, + #[error("Unsupported query on subscription: {0:?}")] + Unsupported(String), + #[error("Subscribing to queries in one call is not supported")] + Multiple, +} + +#[derive(Error, Debug)] +pub enum PlanError { + #[error("Unsupported feature: `{feature}`")] + Unsupported { feature: String }, + #[error("Unknown table: `{table}`")] + UnknownTable { table: Box }, + #[error("Qualified Table `{expect}` not found")] + TableNotFoundQualified { expect: String }, + #[error("Unknown field: `{field}` not found in the table(s): `{tables:?}`")] + UnknownField { field: String, tables: Vec }, + #[error("Unknown field name: `{field}` not found in the table(s): `{tables:?}`")] + UnknownFieldName { field: FieldName, tables: Vec }, + #[error("Field(s): `{fields:?}` not found in the table(s): `{tables:?}`")] + UnknownFields { + fields: Vec, + tables: Vec, + }, + #[error("Ambiguous field: `{field}`. Also found in {found:?}")] + AmbiguousField { field: String, found: Vec }, + #[error("Plan error: `{0}`")] + Unstructured(String), + #[error("Internal DBError: `{0}`")] + DatabaseInternal(Box), + #[error("Relation Error: `{0}`")] + Relation(#[from] RelationError), +} + +#[derive(Error, Debug)] +pub enum DatabaseError { + #[error("Replica not found: {0}")] + NotFound(u64), + #[error("Database is already opened. Path: `{0}`. Error: {1}")] + DatabasedOpened(PathBuf, anyhow::Error), +} + +impl From for DatabaseError { + fn from(LockError { path, source, .. }: LockError) -> Self { + Self::DatabasedOpened(path, source.into()) + } +} + +#[derive(Error, Debug)] +pub enum ViewError { + #[error("{0}")] + Args(String), + #[error("{0}")] + NoSuchModule(String), + #[error("no such view")] + NoSuchView, + #[error("Table does not exist for view `{0}`")] + TableDoesNotExist(ViewId), + #[error("missing client connection for view call trigged by subscription")] + MissingClientConnection, + #[error("DB error during view call: {0}")] + DatastoreError(#[from] DatastoreError), + #[error("The module instance encountered a fatal error: {0}")] + InternalError(String), +} + +#[derive(Error, Debug)] +pub enum DBError { + #[error("LibError: {0}")] + Lib(#[from] LibError), + #[error("BufferError: {0}")] + Buffer(#[from] DecodeError), + #[error("DatastoreError: {0}")] + Datastore(#[from] DatastoreError), + #[error("SequenceError: {0}")] + Sequence2(#[from] SequenceError), + #[error("SchemaError: {0}")] + Schema(SchemaErrors), + #[error("IOError: {0}.")] + IoError(#[from] std::io::Error), + #[error("ParseIntError: {0}.")] + ParseInt(#[from] ParseIntError), + #[error("Hex representation of hash decoded to incorrect number of bytes: {0}.")] + DecodeHexHash(usize), + #[error("DecodeHexError: {0}.")] + DecodeHex(#[from] FromHexError), + #[error("DatabaseError: {0}.")] + Database(#[from] DatabaseError), + #[error("SledError: {0}.")] + SledDbError(#[from] sled::Error), + #[error("Mutex was poisoned acquiring lock on MessageLog: {0}")] + MessageLogPoisoned(String), + #[error("SubscriptionError: {0}")] + Subscription(#[from] SubscriptionError), + #[error("SqlParserError: {error}, executing: `{sql}`")] + SqlParser { + sql: String, + error: sqlparser::parser::ParserError, + }, + #[error("SqlError: {error}, executing: `{sql}`")] + Plan { sql: String, error: PlanError }, + #[error("Error replaying the commit log: {0}")] + LogReplay(#[from] LogReplayError), + #[error(transparent)] + // Box the inner [`SnapshotError`] to keep Clippy quiet about large `Err` variants. + Snapshot(#[from] Box), + #[error("Error reading a value from a table through BSATN: {0}")] + ReadViaBsatnError(#[from] ReadViaBsatnError), + #[error("Module validation errors: {0}")] + ModuleValidationErrors(#[from] ValidationErrors), + #[error(transparent)] + Other(#[from] anyhow::Error), + #[error(transparent)] + TypeError(#[from] TypingError), + #[error("{error}, executing: `{sql}`")] + WithSql { + #[source] + error: Box, + sql: Box, + }, + #[error(transparent)] + RestoreSnapshot(#[from] RestoreSnapshotError), + #[error(transparent)] + DurabilityGone(#[from] DurabilityExited), + #[error(transparent)] + View(#[from] ViewError), +} + +impl From for DBError { + fn from(value: InvalidFieldError) -> Self { + LibError::from(value).into() + } +} + +impl From for DBError { + fn from(err: spacetimedb_table::read_column::TypeError) -> Self { + DatastoreError::Table(TableError::from(err)).into() + } +} + +impl From for PlanError { + fn from(err: DBError) -> Self { + PlanError::DatabaseInternal(Box::new(err)) + } +} + +impl<'a, T: ?Sized + 'a> From>> for DBError { + fn from(err: PoisonError>) -> Self { + DBError::MessageLogPoisoned(err.to_string()) + } +} + +impl From for DBError { + fn from(e: spacetimedb_durability::local::OpenError) -> Self { + use spacetimedb_durability::local::OpenError::*; + + match e { + Lock(e) => Self::from(DatabaseError::from(e)), + Commitlog(e) => Self::Other(e.into()), + } + } +} + +#[derive(Debug, Error)] +pub enum LogReplayError { + #[error( + "Out-of-order commit detected: {} in segment {} after offset {}", + .commit_offset, + .segment_offset, + .last_commit_offset + )] + OutOfOrderCommit { + commit_offset: u64, + segment_offset: usize, + last_commit_offset: u64, + }, + #[error( + "Error reading segment {}/{} at commit {}: {}", + .segment_offset, + .total_segments, + .commit_offset, + .source + )] + TrailingSegments { + segment_offset: usize, + total_segments: usize, + commit_offset: u64, + #[source] + source: io::Error, + }, + #[error("Could not reset log to offset {}: {}", .offset, .source)] + Reset { + offset: u64, + #[source] + source: io::Error, + }, + #[error("Missing object {} referenced from commit {}", .hash, .commit_offset)] + MissingObject { hash: Hash, commit_offset: u64 }, + #[error( + "Unexpected I/O error reading commit {} from segment {}: {}", + .commit_offset, + .segment_offset, + .source + )] + Io { + segment_offset: usize, + commit_offset: u64, + #[source] + source: io::Error, + }, +} + +#[derive(Debug, Error)] +pub enum RestoreSnapshotError { + #[error("Snapshot has incorrect database_identity: expected {expected} but found {actual}")] + IdentityMismatch { expected: Identity, actual: Identity }, + #[error("Failed to restore datastore from snapshot")] + Datastore(#[source] Box), + #[error("Failed to read snapshot")] + Snapshot(#[from] Box), + #[error("Failed to bootstrap datastore without snapshot")] + Bootstrap(#[source] Box), + #[error("No connected snapshot found, commitlog starts at {min_commitlog_offset}")] + NoConnectedSnapshot { min_commitlog_offset: TxOffset }, + #[error("Failed to invalidate snapshots at or newer than {offset}")] + Invalidate { + offset: TxOffset, + #[source] + source: Box, + }, +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs new file mode 100644 index 00000000000..3c9cb364d80 --- /dev/null +++ b/crates/engine/src/lib.rs @@ -0,0 +1,10 @@ +pub mod db; +pub mod error; +pub mod metrics; +pub mod rls; +mod sql; +pub mod util; + +pub use spacetimedb_lib::identity; +pub use spacetimedb_lib::Identity; +pub use spacetimedb_sats::hash; diff --git a/crates/engine/src/metrics.rs b/crates/engine/src/metrics.rs new file mode 100644 index 00000000000..f5ee0011b23 --- /dev/null +++ b/crates/engine/src/metrics.rs @@ -0,0 +1,181 @@ +use once_cell::sync::Lazy; +use prometheus::{GaugeVec, HistogramVec, IntCounter, IntCounterVec, IntGaugeVec}; +use spacetimedb_datastore::{ + db_metrics::DB_METRICS, execution_context::WorkloadType, locking_tx_datastore::datastore::MetricsRecorder, +}; +use spacetimedb_lib::{metrics::ExecutionMetrics, Identity}; +use spacetimedb_metrics::metrics_group; + +metrics_group!( + pub struct EngineMetrics { + #[name = spacetime_num_bytes_sent_to_clients_total] + #[help = "The cumulative number of bytes sent to clients"] + #[labels(txn_type: WorkloadType, db: Identity)] + pub bytes_sent_to_clients: IntCounterVec, + + #[name = spacetime_replay_total_time_seconds] + #[help = "Total time spent replaying a database upon restart, including snapshot read, snapshot restore and commitlog replay"] + #[labels(db: Identity)] + pub replay_total_time_seconds: GaugeVec, + + #[name = spacetime_replay_snapshot_read_time_seconds] + #[help = "Time spent reading a snapshot from disk before restoring the snapshot upon restart"] + #[labels(db: Identity)] + pub replay_snapshot_read_time_seconds: GaugeVec, + + #[name = spacetime_replay_snapshot_restore_time_seconds] + #[help = "Time spent restoring a database from a snapshot after reading the snapshot and before commitlog replay upon restart"] + #[labels(db: Identity)] + pub replay_snapshot_restore_time_seconds: GaugeVec, + + #[name = spacetime_replay_commitlog_time_seconds] + #[help = "Time spent replaying the commitlog after restoring from a snapshot upon restart"] + #[labels(db: Identity)] + pub replay_commitlog_time_seconds: GaugeVec, + + #[name = spacetime_replay_commitlog_num_commits] + #[help = "Number of commits replayed after restoring from a snapshot upon restart"] + #[labels(db: Identity)] + pub replay_commitlog_num_commits: IntGaugeVec, + + // Snapshot creation should take in the order of milliseconds, + // but log data suggests that there are outliers. + // So let's track a wide range of buckets to get a better picture. + // + // We also track the timing without `asyncify` scheduling overhead + // (`snapshot_creation_time_inner`), and the snapshot compression + // timing with / without scheduling overhead (`snapshot_compression_time_total` + // and `snapshot_compression_time_inner`, respectively). + // + // Compression may have contributed to observed outliers, but is no + // longer included in the snapshot creation timing. + #[name = spacetime_snapshot_creation_time_total_sec] + #[help = "The time (in seconds) it took to take and store a database snapshot, including scheduling overhead"] + #[labels(db: Identity)] + #[buckets(0.0005, 0.001, 0.005, 0.01, 0.1, 1.0, 5.0, 10.0)] + pub snapshot_creation_time_total: HistogramVec, + + #[name = spacetime_snapshot_creation_time_inner_sec] + #[help = "The time (in seconds) it took to take and store a database snapshot, excluding scheduling overhead"] + #[labels(db: Identity)] + #[buckets(0.0005, 0.001, 0.005, 0.01, 0.1, 1.0, 5.0, 10.0)] + pub snapshot_creation_time_inner: HistogramVec, + + #[name = spacetime_snapshot_creation_time_fsync_sec] + #[help = "The time (in seconds) it took to fsync a database snapshot, excluding scheduling overhead"] + #[labels(db: Identity)] + #[buckets(0.0005, 0.001, 0.005, 0.01, 0.1, 1.0, 5.0, 10.0)] + pub snapshot_creation_time_fsync: HistogramVec, + + #[name = spacetime_snapshot_compression_time_total_sec] + #[help = "The time (in seconds) it took to do a compression pass on the snapshot repository, including scheduling overhead"] + #[labels(db: Identity)] + #[buckets(0.001, 0.01, 0.1, 1.0, 5.0, 10.0)] + pub snapshot_compression_time_total: HistogramVec, + + #[name = spacetime_snapshot_compression_time_inner_sec] + #[help = "The time (in seconds) it took to do a compression pass on the snapshot repository, excluding scheduling overhead"] + #[labels(db: Identity)] + #[buckets(0.001, 0.01, 0.1, 1.0, 5.0, 10.0)] + pub snapshot_compression_time_inner: HistogramVec, + + #[name = spacetime_snapshot_compression_time_per_snapshot_sec] + #[help = "The time (in seconds) it took to compress a single snapshot"] + #[labels(db: Identity)] + #[buckets(0.001, 0.01, 0.1, 1.0, 5.0, 10.0)] + pub snapshot_compression_time_single: HistogramVec, + + #[name = spacetime_snapshot_compression_skipped] + #[help = "The number of snapshots skipped in a single compression pass because they were already compressed"] + #[labels(db: Identity)] + pub snapshot_compression_skipped: IntGaugeVec, + + #[name = spacetime_snapshot_compression_compressed] + #[help = "The number of snapshots compressed in a single compression pass"] + #[labels(db: Identity)] + pub snapshot_compression_compressed: IntGaugeVec, + + #[name = spacetime_snapshot_compression_objects_compressed] + #[help = "The number of snapshot objects compressed in a single compression pass"] + #[labels(db: Identity)] + pub snapshot_compression_objects_compressed: IntGaugeVec, + + #[name = spacetime_snapshot_compression_objects_hardlinked] + #[help = "The number of snapshot objects hardlinked in a single compression pass"] + #[labels(db: Identity)] + pub snapshot_compression_objects_hardlinked: IntGaugeVec, + + #[name = spacetime_durability_blocking_send_duration_sec] + #[help = "Latency of blocking sends in request_durability (seconds); _count gives the number of times the channel was full"] + #[labels(database_identity: Identity)] + #[buckets(0.001, 0.01, 0.1, 1.0, 10.0)] + pub durability_blocking_send_duration: HistogramVec, + } +); + +pub static ENGINE_METRICS: Lazy = Lazy::new(EngineMetrics::new); + +#[derive(Debug)] +pub struct ExecutionCounters { + rdb_num_index_seeks: IntCounter, + rdb_num_rows_scanned: IntCounter, + rdb_num_bytes_scanned: IntCounter, + rdb_num_bytes_written: IntCounter, + bytes_sent_to_clients: IntCounter, + delta_queries_matched: IntCounter, + delta_queries_evaluated: IntCounter, + duplicate_rows_evaluated: IntCounter, + duplicate_rows_sent: IntCounter, +} + +impl ExecutionCounters { + pub fn new(workload: &WorkloadType, db: &Identity) -> Self { + Self { + rdb_num_index_seeks: DB_METRICS.rdb_num_index_seeks.with_label_values(workload, db), + rdb_num_rows_scanned: DB_METRICS.rdb_num_rows_scanned.with_label_values(workload, db), + rdb_num_bytes_scanned: DB_METRICS.rdb_num_bytes_scanned.with_label_values(workload, db), + rdb_num_bytes_written: DB_METRICS.rdb_num_bytes_written.with_label_values(workload, db), + bytes_sent_to_clients: ENGINE_METRICS.bytes_sent_to_clients.with_label_values(workload, db), + delta_queries_matched: DB_METRICS.delta_queries_matched.with_label_values(db), + delta_queries_evaluated: DB_METRICS.delta_queries_evaluated.with_label_values(db), + duplicate_rows_evaluated: DB_METRICS.duplicate_rows_evaluated.with_label_values(db), + duplicate_rows_sent: DB_METRICS.duplicate_rows_sent.with_label_values(db), + } + } + + pub fn record(&self, metrics: &ExecutionMetrics) { + if metrics.index_seeks > 0 { + self.rdb_num_index_seeks.inc_by(metrics.index_seeks as u64); + } + if metrics.rows_scanned > 0 { + self.rdb_num_rows_scanned.inc_by(metrics.rows_scanned as u64); + } + if metrics.bytes_scanned > 0 { + self.rdb_num_bytes_scanned.inc_by(metrics.bytes_scanned as u64); + } + if metrics.bytes_written > 0 { + self.rdb_num_bytes_written.inc_by(metrics.bytes_written as u64); + } + if metrics.bytes_sent_to_clients > 0 { + self.bytes_sent_to_clients.inc_by(metrics.bytes_sent_to_clients as u64); + } + if metrics.delta_queries_matched > 0 { + self.delta_queries_matched.inc_by(metrics.delta_queries_matched); + } + if metrics.delta_queries_evaluated > 0 { + self.delta_queries_evaluated.inc_by(metrics.delta_queries_evaluated); + } + if metrics.duplicate_rows_evaluated > 0 { + self.duplicate_rows_evaluated.inc_by(metrics.duplicate_rows_evaluated); + } + if metrics.duplicate_rows_sent > 0 { + self.duplicate_rows_sent.inc_by(metrics.duplicate_rows_sent); + } + } +} + +impl MetricsRecorder for ExecutionCounters { + fn record(&self, metrics: &ExecutionMetrics) { + self.record(metrics); + } +} diff --git a/crates/core/src/sql/parser.rs b/crates/engine/src/rls.rs similarity index 100% rename from crates/core/src/sql/parser.rs rename to crates/engine/src/rls.rs diff --git a/crates/engine/src/sql/ast.rs b/crates/engine/src/sql/ast.rs new file mode 100644 index 00000000000..892430ba1ea --- /dev/null +++ b/crates/engine/src/sql/ast.rs @@ -0,0 +1,77 @@ +use anyhow::Context; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{StRowLevelSecurityFields, ST_ROW_LEVEL_SECURITY_ID}; +use spacetimedb_expr::check::SchemaView; +use spacetimedb_lib::identity::AuthCtx; +use spacetimedb_primitives::TableId; +use spacetimedb_sats::AlgebraicValue; +use spacetimedb_schema::schema::TableOrViewSchema; +use std::ops::Deref; +use std::sync::Arc; + +pub struct SchemaViewer<'a, T> { + tx: &'a T, + auth: &'a AuthCtx, +} + +impl Deref for SchemaViewer<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.tx + } +} + +impl SchemaView for SchemaViewer<'_, T> { + fn table_id(&self, name: &str) -> Option { + self.tx + .table_id_from_name_or_alias(name) + .ok() + .flatten() + .and_then(|table_id| self.schema_for_table(table_id)) + .filter(|schema| self.auth.has_read_access(schema.table_access)) + .map(|schema| schema.table_id) + } + + fn schema_for_table(&self, table_id: TableId) -> Option> { + self.tx + .get_schema(table_id) + .filter(|schema| self.auth.has_read_access(schema.table_access)) + .map(Arc::clone) + .map(TableOrViewSchema::from) + .map(Arc::new) + } + + fn rls_rules_for_table(&self, table_id: TableId) -> anyhow::Result>> { + self.tx + .iter_by_col_eq( + ST_ROW_LEVEL_SECURITY_ID, + StRowLevelSecurityFields::TableId, + &AlgebraicValue::from(table_id), + )? + .map(|row| { + row.read_col::(StRowLevelSecurityFields::Sql) + .with_context(|| { + format!( + "Failed to read value from the `{}` column of `{}` for table_id `{}`", + "sql", "st_row_level_security", table_id + ) + }) + .and_then(|sql| { + sql.into_string().map_err(|_| { + anyhow::anyhow!(format!( + "Failed to read value from the `{}` column of `{}` for table_id `{}`", + "sql", "st_row_level_security", table_id + )) + }) + }) + }) + .collect::>() + } +} + +impl<'a, T> SchemaViewer<'a, T> { + pub fn new(tx: &'a T, auth: &'a AuthCtx) -> Self { + Self { tx, auth } + } +} diff --git a/crates/engine/src/sql/mod.rs b/crates/engine/src/sql/mod.rs new file mode 100644 index 00000000000..851c0bc27ff --- /dev/null +++ b/crates/engine/src/sql/mod.rs @@ -0,0 +1 @@ +pub mod ast; diff --git a/crates/engine/src/util.rs b/crates/engine/src/util.rs new file mode 100644 index 00000000000..4d07d7d3c06 --- /dev/null +++ b/crates/engine/src/util.rs @@ -0,0 +1,21 @@ +use spacetimedb_runtime::Handle; +use tracing::Span; + +/// Ergonomic wrapper for `runtime.spawn_blocking(f).await`. +/// +/// If `f` panics, it will be bubbled up to the calling task. +pub async fn asyncify(runtime: &Handle, f: F) -> R +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + // Ensure that `f` executes in the current span context. + // If there is no current span, or it is disabled, `span` is disabled. + let span = Span::current(); + runtime + .spawn_blocking(move || { + let _enter = span.enter(); + f() + }) + .await +} diff --git a/crates/core/testdata/README.md b/crates/engine/testdata/README.md similarity index 100% rename from crates/core/testdata/README.md rename to crates/engine/testdata/README.md diff --git a/crates/core/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.log b/crates/engine/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.log similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.log rename to crates/engine/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.log diff --git a/crates/core/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.ofs b/crates/engine/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.ofs similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.ofs rename to crates/engine/testdata/v1.2/replicas/22000001/clog/00000000000000000000.stdb.ofs diff --git a/crates/core/testdata/v1.2/replicas/22000001/db.lock b/crates/engine/testdata/v1.2/replicas/22000001/db.lock similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/db.lock rename to crates/engine/testdata/v1.2/replicas/22000001/db.lock diff --git a/crates/core/testdata/v1.2/replicas/22000001/module_logs/2025-08-18.log b/crates/engine/testdata/v1.2/replicas/22000001/module_logs/2025-08-18.log similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/module_logs/2025-08-18.log rename to crates/engine/testdata/v1.2/replicas/22000001/module_logs/2025-08-18.log diff --git a/crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/00000000000000000000.snapshot_bsatn b/crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/00000000000000000000.snapshot_bsatn similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/00000000000000000000.snapshot_bsatn rename to crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/00000000000000000000.snapshot_bsatn diff --git a/crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/19/30ce81246a4cdc25e9024ae0065d053adb2efbe1b5b7af457331d330e481e8 b/crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/19/30ce81246a4cdc25e9024ae0065d053adb2efbe1b5b7af457331d330e481e8 similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/19/30ce81246a4cdc25e9024ae0065d053adb2efbe1b5b7af457331d330e481e8 rename to crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/19/30ce81246a4cdc25e9024ae0065d053adb2efbe1b5b7af457331d330e481e8 diff --git a/crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/41/bb11b6d2cdc488192ee70d8175307d6f205756ed163f4237c6cba2936798dc b/crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/41/bb11b6d2cdc488192ee70d8175307d6f205756ed163f4237c6cba2936798dc similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/41/bb11b6d2cdc488192ee70d8175307d6f205756ed163f4237c6cba2936798dc rename to crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/41/bb11b6d2cdc488192ee70d8175307d6f205756ed163f4237c6cba2936798dc diff --git a/crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/45/4d2e2c62ff5d46c5b3e6de72d6277eb285fc2d6b0a5ac6f92498e08a9e5ecc b/crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/45/4d2e2c62ff5d46c5b3e6de72d6277eb285fc2d6b0a5ac6f92498e08a9e5ecc similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/45/4d2e2c62ff5d46c5b3e6de72d6277eb285fc2d6b0a5ac6f92498e08a9e5ecc rename to crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/45/4d2e2c62ff5d46c5b3e6de72d6277eb285fc2d6b0a5ac6f92498e08a9e5ecc diff --git a/crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/62/22df0e5ca93d3fb22762e12161246a1d5917c61ada5d81b8dcce12fd5780b3 b/crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/62/22df0e5ca93d3fb22762e12161246a1d5917c61ada5d81b8dcce12fd5780b3 similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/62/22df0e5ca93d3fb22762e12161246a1d5917c61ada5d81b8dcce12fd5780b3 rename to crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/62/22df0e5ca93d3fb22762e12161246a1d5917c61ada5d81b8dcce12fd5780b3 diff --git a/crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/79/4dced5633eca2ffee784d471f5203209169321083ef99de254ad24af0f6d5a b/crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/79/4dced5633eca2ffee784d471f5203209169321083ef99de254ad24af0f6d5a similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/79/4dced5633eca2ffee784d471f5203209169321083ef99de254ad24af0f6d5a rename to crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/79/4dced5633eca2ffee784d471f5203209169321083ef99de254ad24af0f6d5a diff --git a/crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/95/74dd6d2857fa771a1cd16be31fdef38f83c2fd3bcc05f4934e53bdbfa21f10 b/crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/95/74dd6d2857fa771a1cd16be31fdef38f83c2fd3bcc05f4934e53bdbfa21f10 similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/95/74dd6d2857fa771a1cd16be31fdef38f83c2fd3bcc05f4934e53bdbfa21f10 rename to crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/95/74dd6d2857fa771a1cd16be31fdef38f83c2fd3bcc05f4934e53bdbfa21f10 diff --git a/crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/9a/b95f5aaed7541289faa8bc4de886ce0281f11037c3424494e58fee92411241 b/crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/9a/b95f5aaed7541289faa8bc4de886ce0281f11037c3424494e58fee92411241 similarity index 100% rename from crates/core/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/9a/b95f5aaed7541289faa8bc4de886ce0281f11037c3424494e58fee92411241 rename to crates/engine/testdata/v1.2/replicas/22000001/snapshots/00000000000000000000.snapshot_dir/objects/9a/b95f5aaed7541289faa8bc4de886ce0281f11037c3424494e58fee92411241 diff --git a/crates/snapshot/Cargo.toml b/crates/snapshot/Cargo.toml index aa51c4e3bd8..ebf4d577670 100644 --- a/crates/snapshot/Cargo.toml +++ b/crates/snapshot/Cargo.toml @@ -6,10 +6,15 @@ rust-version.workspace = true license-file = "LICENSE" description = "Low-level interfaces for capturing and restoring snapshots of database states" +[features] +default = ["tokio"] +tokio = ["spacetimedb-durability/tokio"] +simulation = ["spacetimedb-durability/simulation"] + [dependencies] spacetimedb-table.workspace = true spacetimedb-data-structures.workspace = true -spacetimedb-durability.workspace = true +spacetimedb-durability = { path = "../durability", default-features = false } spacetimedb-lib.workspace = true spacetimedb-sats = { workspace = true, features = ["blake3"] } spacetimedb-primitives.workspace = true diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index 189bafa11bd..90597c29611 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -17,6 +17,7 @@ use spacetimedb::energy::{EnergyBalance, EnergyQuanta, NullEnergyMonitor}; use spacetimedb::host::{DiskStorage, HostController, HostRuntimeConfig, MigratePlanResult, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; use spacetimedb::messages::control_db::{Database, Node, Replica}; +use spacetimedb::metrics::ENGINE_METRICS; use spacetimedb::subscription::row_list_builder_pool::BsatnRowListBuilderPool; use spacetimedb::util::jobs::JobCores; use spacetimedb::worker_metrics::WORKER_METRICS; @@ -95,6 +96,7 @@ impl StandaloneEnv { let metrics_registry = prometheus::Registry::new(); metrics_registry.register(Box::new(&*WORKER_METRICS)).unwrap(); + metrics_registry.register(Box::new(&*ENGINE_METRICS)).unwrap(); metrics_registry.register(Box::new(&*DB_METRICS)).unwrap(); metrics_registry.register(Box::new(&*DATA_SIZE_METRICS)).unwrap(); From e731109f153bb3d27492557fae47b8431eb0cf24 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 9 Jun 2026 19:50:46 +0530 Subject: [PATCH 04/16] flatten db --- crates/core/src/database_logger.rs | 2 +- crates/core/src/db/mod.rs | 43 ++++++ crates/core/src/host/module_host.rs | 2 +- crates/core/src/lib.rs | 2 +- crates/engine/src/{sql => }/ast.rs | 0 crates/engine/src/db/mod.rs | 144 -------------------- crates/engine/src/{db => }/durability.rs | 2 +- crates/engine/src/lib.rs | 8 +- crates/engine/src/{db => }/persistence.rs | 0 crates/engine/src/{db => }/relational_db.rs | 113 ++++++++++++++- crates/engine/src/rls.rs | 2 +- crates/engine/src/{db => }/snapshot.rs | 0 crates/engine/src/sql/mod.rs | 1 - crates/engine/src/{db => }/update.rs | 2 +- 14 files changed, 163 insertions(+), 158 deletions(-) create mode 100644 crates/core/src/db/mod.rs rename crates/engine/src/{sql => }/ast.rs (100%) delete mode 100644 crates/engine/src/db/mod.rs rename crates/engine/src/{db => }/durability.rs (98%) rename crates/engine/src/{db => }/persistence.rs (100%) rename crates/engine/src/{db => }/relational_db.rs (97%) rename crates/engine/src/{db => }/snapshot.rs (100%) delete mode 100644 crates/engine/src/sql/mod.rs rename crates/engine/src/{db => }/update.rs (99%) diff --git a/crates/core/src/database_logger.rs b/crates/core/src/database_logger.rs index 3b5db5d7e70..9a2891955f8 100644 --- a/crates/core/src/database_logger.rs +++ b/crates/core/src/database_logger.rs @@ -674,7 +674,7 @@ impl SystemLogger { } } -impl spacetimedb_engine::db::update::UpdateLogger for SystemLogger { +impl spacetimedb_engine::update::UpdateLogger for SystemLogger { fn info(&self, msg: &str) { self.info(msg); } diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs new file mode 100644 index 00000000000..98d89edb1ea --- /dev/null +++ b/crates/core/src/db/mod.rs @@ -0,0 +1,43 @@ +pub mod persistence { + pub use spacetimedb_engine::persistence::*; +} + +pub mod relational_db { + pub use spacetimedb_engine::relational_db::*; +} + +pub mod snapshot { + pub use spacetimedb_engine::snapshot::*; +} + +pub mod update { + pub use spacetimedb_engine::update::*; +} + +/// Whether SpacetimeDB is run in memory, or persists objects and +/// a message log to disk. +#[derive(Clone, Copy)] +pub enum Storage { + /// The object store is in memory, and no message log is kept. + Memory, + + /// The object store is persisted to disk, and a message log is kept. + Disk, +} + +/// Internal database config parameters +#[derive(Clone, Copy)] +pub struct Config { + /// Specifies the object storage model. + pub storage: Storage, + /// Specifies the page pool max size in bytes. + pub page_pool_max_size: Option, +} + +pub type MetricsRecorderQueue = spacetimedb_engine::relational_db::MetricsRecorderQueue; + +pub fn spawn_tx_metrics_recorder( + handle: &spacetimedb_runtime::Handle, +) -> (MetricsRecorderQueue, spacetimedb_runtime::AbortHandle) { + spacetimedb_engine::relational_db::spawn_tx_metrics_recorder(handle) +} diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index d10239bd7f0..2a98e7fc4e7 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -601,7 +601,7 @@ fn init_database_inner( table_defs.sort_by_key(|x| &x.name); for def in table_defs { logger.info(&format!("Creating table `{}`", &def.name)); - spacetimedb_engine::db::update::create_table_from_def(stdb, tx, module_def, def)?; + spacetimedb_engine::update::create_table_from_def(stdb, tx, module_def, def)?; } // Create all in-memory views defined by the module. diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 04140b7036c..d630faab09a 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -4,7 +4,7 @@ pub mod energy; pub mod sql; pub mod auth; -pub use spacetimedb_engine::db; +pub mod db; pub use spacetimedb_engine::metrics; pub mod messages; pub use spacetimedb_lib::Identity; diff --git a/crates/engine/src/sql/ast.rs b/crates/engine/src/ast.rs similarity index 100% rename from crates/engine/src/sql/ast.rs rename to crates/engine/src/ast.rs diff --git a/crates/engine/src/db/mod.rs b/crates/engine/src/db/mod.rs deleted file mode 100644 index 363123c0dba..00000000000 --- a/crates/engine/src/db/mod.rs +++ /dev/null @@ -1,144 +0,0 @@ -use std::sync::Arc; - -use enum_map::EnumMap; -use spacetimedb_schema::reducer_name::ReducerName; - -use crate::metrics::ExecutionCounters; -use spacetimedb_datastore::execution_context::WorkloadType; -use spacetimedb_datastore::{locking_tx_datastore::datastore::TxMetrics, traits::TxData}; - -mod durability; -pub mod persistence; -pub mod relational_db; -pub mod snapshot; -pub mod update; - -/// Whether SpacetimeDB is run in memory, or persists objects and -/// a message log to disk. -#[derive(Clone, Copy)] -pub enum Storage { - /// The object store is in memory, and no message log is kept. - Memory, - - /// The object store is persisted to disk, and a message log is kept. - Disk, -} - -/// Internal database config parameters -#[derive(Clone, Copy)] -pub struct Config { - /// Specifies the object storage model. - pub storage: Storage, - /// Specifies the page pool max size in bytes. - pub page_pool_max_size: Option, -} - -/// A message that is processed by the [`spawn_metrics_recorder`] actor. -/// We use a separate task to record metrics to avoid blocking transactions. -pub struct MetricsMessage { - /// The reducer the produced these metrics. - reducer: Option, - /// Metrics from a mutable transaction. - metrics_for_writer: Option, - /// Metrics from a read-only transaction. - /// A message may have metrics for both types of transactions, - /// because metrics for a reducer and its subscription updates are recorded together. - metrics_for_reader: Option, - /// The row updates for an immutable transaction. - /// Needed for insert and delete counters. - tx_data: Option>, - /// Cached metrics counters for each workload type. - counters: Arc>, -} - -/// The handle used to send work to the tx metrics recorder. -#[derive(Clone)] -pub struct MetricsRecorderQueue { - tx: spacetimedb_runtime::sync::mpsc::UnboundedSender, -} - -impl MetricsRecorderQueue { - pub fn send_metrics( - &self, - reducer: Option, - metrics_for_writer: Option, - metrics_for_reader: Option, - tx_data: Option>, - counters: Arc>, - ) { - if let Err(err) = self.tx.send(MetricsMessage { - reducer, - metrics_for_writer, - metrics_for_reader, - tx_data, - counters, - }) { - log::warn!("failed to send metrics: {err}"); - } - } -} - -fn record_metrics( - MetricsMessage { - reducer, - metrics_for_writer, - metrics_for_reader, - tx_data, - counters, - }: MetricsMessage, -) { - if let Some(tx_metrics) = metrics_for_writer { - tx_metrics.report( - // If row updates are present, - // they will always belong to the writer transaction. - tx_data.as_deref(), - reducer.as_ref(), - |wl| &counters[wl], - ); - } - if let Some(tx_metrics) = metrics_for_reader { - tx_metrics.report( - // If row updates are present, - // they will never belong to the reader transaction. - // Passing row updates here will most likely panic. - None, - reducer.as_ref(), - |wl| &counters[wl], - ); - } -} - -/// The metrics recorder is a side channel that the main database thread forwards metrics to. -/// While we want to avoid unnecessary compute on the critical path, communicating with other -/// threads is not free, and for this case in particular waking a parked task is not free. -/// -/// Previously, each tx would send its metrics to the recorder task. As soon as the recorder -/// task `recv`d a message, it would update the counters and gauges, and immediately wait for -/// the next tx's message. This meant that the tx would need to be more expensive than the -/// recording of its metrics in order for the recorder task not to be parked on `recv` when -/// the tx would `send` its metrics. This would obviously never be the case, and so each `send` -/// would incur the overhead of waking the task. -/// -/// To mitigate this, we now record metrics, for potentially many transactions, periodically -/// every 5ms. -const TX_METRICS_RECORDING_INTERVAL: std::time::Duration = std::time::Duration::from_millis(5); - -/// Spawns a task for recording transaction metrics. -/// Returns the handle for pushing metrics to the recorder. -pub fn spawn_tx_metrics_recorder( - handle: &spacetimedb_runtime::Handle, -) -> (MetricsRecorderQueue, spacetimedb_runtime::AbortHandle) { - let handle_clone = handle.clone(); - let (tx, mut rx) = spacetimedb_runtime::sync::mpsc::unbounded_channel(); - let abort_handle = handle - .spawn(async move { - loop { - handle_clone.sleep(TX_METRICS_RECORDING_INTERVAL).await; - while let Ok(metrics) = rx.try_recv() { - record_metrics(metrics); - } - } - }) - .abort_handle(); - (MetricsRecorderQueue { tx }, abort_handle) -} diff --git a/crates/engine/src/db/durability.rs b/crates/engine/src/durability.rs similarity index 98% rename from crates/engine/src/db/durability.rs rename to crates/engine/src/durability.rs index f749f72850a..9c938a6401b 100644 --- a/crates/engine/src/db/durability.rs +++ b/crates/engine/src/durability.rs @@ -10,7 +10,7 @@ use spacetimedb_durability::Transaction; use spacetimedb_lib::Identity; use spacetimedb_sats::ProductValue; -use crate::db::persistence::Durability; +use crate::persistence::Durability; use spacetimedb_runtime::Handle; pub(super) fn request_durability( diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 3c9cb364d80..d2e8860bb2a 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1,8 +1,12 @@ -pub mod db; +mod ast; +pub(crate) mod durability; pub mod error; pub mod metrics; +pub mod persistence; +pub mod relational_db; pub mod rls; -mod sql; +pub mod snapshot; +pub mod update; pub mod util; pub use spacetimedb_lib::identity; diff --git a/crates/engine/src/db/persistence.rs b/crates/engine/src/persistence.rs similarity index 100% rename from crates/engine/src/db/persistence.rs rename to crates/engine/src/persistence.rs diff --git a/crates/engine/src/db/relational_db.rs b/crates/engine/src/relational_db.rs similarity index 97% rename from crates/engine/src/db/relational_db.rs rename to crates/engine/src/relational_db.rs index 0882e1e8243..00e001c2e45 100644 --- a/crates/engine/src/db/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -1,8 +1,111 @@ -use crate::db::durability::{request_durability, spawn_close as spawn_durability_close}; -use crate::db::MetricsRecorderQueue; +use crate::durability::{request_durability, spawn_close as spawn_durability_close}; use crate::error::{DBError, RestoreSnapshotError}; use crate::metrics::ExecutionCounters; use crate::metrics::ENGINE_METRICS; + +/// Whether SpacetimeDB is run in memory, or persists objects and +/// a message log to disk. +#[derive(Clone, Copy)] +pub enum Storage { + /// The object store is in memory, and no message log is kept. + Memory, + + /// The object store is persisted to disk, and a message log is kept. + Disk, +} + +/// Internal database config parameters +#[derive(Clone, Copy)] +pub struct Config { + /// Specifies the object storage model. + pub storage: Storage, + /// Specifies the page pool max size in bytes. + pub page_pool_max_size: Option, +} + +/// A message that is processed by the [`spawn_metrics_recorder`] actor. +/// We use a separate task to record metrics to avoid blocking transactions. +pub struct MetricsMessage { + /// The reducer the produced these metrics. + reducer: Option, + /// Metrics from a mutable transaction. + metrics_for_writer: Option, + /// Metrics from a read-only transaction. + /// A message may have metrics for both types of transactions, + /// because metrics for a reducer and its subscription updates are recorded together. + metrics_for_reader: Option, + /// The row updates for an immutable transaction. + /// Needed for insert and delete counters. + tx_data: Option>, + /// Cached metrics counters for each workload type. + counters: Arc>, +} + +/// The handle used to send work to the tx metrics recorder. +#[derive(Clone)] +pub struct MetricsRecorderQueue { + tx: spacetimedb_runtime::sync::mpsc::UnboundedSender, +} + +impl MetricsRecorderQueue { + pub fn send_metrics( + &self, + reducer: Option, + metrics_for_writer: Option, + metrics_for_reader: Option, + tx_data: Option>, + counters: Arc>, + ) { + if let Err(err) = self.tx.send(MetricsMessage { + reducer, + metrics_for_writer, + metrics_for_reader, + tx_data, + counters, + }) { + log::warn!("failed to send metrics: {err}"); + } + } +} + +fn record_metrics( + MetricsMessage { + reducer, + metrics_for_writer, + metrics_for_reader, + tx_data, + counters, + }: MetricsMessage, +) { + if let Some(tx_metrics) = metrics_for_writer { + tx_metrics.report(tx_data.as_deref(), reducer.as_ref(), |wl| &counters[wl]); + } + if let Some(tx_metrics) = metrics_for_reader { + tx_metrics.report(None, reducer.as_ref(), |wl| &counters[wl]); + } +} + +const TX_METRICS_RECORDING_INTERVAL: std::time::Duration = std::time::Duration::from_millis(5); + +/// Spawns a task for recording transaction metrics. +/// Returns the handle for pushing metrics to the recorder. +pub fn spawn_tx_metrics_recorder( + handle: &spacetimedb_runtime::Handle, +) -> (MetricsRecorderQueue, spacetimedb_runtime::AbortHandle) { + let handle_clone = handle.clone(); + let (tx, mut rx) = spacetimedb_runtime::sync::mpsc::unbounded_channel(); + let abort_handle = handle + .spawn(async move { + loop { + handle_clone.sleep(TX_METRICS_RECORDING_INTERVAL).await; + while let Ok(metrics) = rx.try_recv() { + record_metrics(metrics); + } + } + }) + .abort_handle(); + (MetricsRecorderQueue { tx }, abort_handle) +} use crate::util::asyncify; use anyhow::{anyhow, Context}; use enum_map::EnumMap; @@ -1830,8 +1933,8 @@ fn default_row_count_fn(db: Identity) -> RowCountFn { #[cfg(any(test, feature = "test"))] pub mod tests_utils { - use crate::db::snapshot; - use crate::db::snapshot::SnapshotWorker; + use crate::snapshot; + use crate::snapshot::SnapshotWorker; use super::*; use core::ops::Deref; @@ -2331,7 +2434,7 @@ mod tests { use super::tests_utils::begin_mut_tx; use super::*; - use crate::db::relational_db::tests_utils::{begin_tx, create_view_for_test, insert, make_snapshot, TestDB}; + use crate::relational_db::tests_utils::{begin_tx, create_view_for_test, insert, make_snapshot, TestDB}; use anyhow::bail; use bytes::Bytes; use commitlog::payload::txdata; diff --git a/crates/engine/src/rls.rs b/crates/engine/src/rls.rs index 66d216a5590..5fc606352ec 100644 --- a/crates/engine/src/rls.rs +++ b/crates/engine/src/rls.rs @@ -1,4 +1,4 @@ -use crate::sql::ast::SchemaViewer; +use crate::ast::SchemaViewer; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_expr::check::parse_and_type_sub; diff --git a/crates/engine/src/db/snapshot.rs b/crates/engine/src/snapshot.rs similarity index 100% rename from crates/engine/src/db/snapshot.rs rename to crates/engine/src/snapshot.rs diff --git a/crates/engine/src/sql/mod.rs b/crates/engine/src/sql/mod.rs deleted file mode 100644 index 851c0bc27ff..00000000000 --- a/crates/engine/src/sql/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod ast; diff --git a/crates/engine/src/db/update.rs b/crates/engine/src/update.rs similarity index 99% rename from crates/engine/src/db/update.rs rename to crates/engine/src/update.rs index 6752ad78d10..bfd6d7648bd 100644 --- a/crates/engine/src/db/update.rs +++ b/crates/engine/src/update.rs @@ -342,7 +342,7 @@ pub fn create_table_from_def( #[cfg(test)] mod test { use super::*; - use crate::db::relational_db::tests_utils::{begin_mut_tx, insert, TestDB}; + use crate::relational_db::tests_utils::{begin_mut_tx, insert, TestDB}; use spacetimedb_datastore::locking_tx_datastore::PendingSchemaChange; use spacetimedb_lib::db::raw_def::v9::{btree, RawIndexAlgorithm, RawModuleDefV9Builder, TableAccess}; use spacetimedb_sats::{product, AlgebraicType, AlgebraicType::U64}; From 04b7ea8ba7bc5b84b74ef8a48a3c2815dceeaf2b Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 10 Jun 2026 18:49:45 +0530 Subject: [PATCH 05/16] fix db error --- crates/core/src/db/mod.rs | 4 +- crates/engine/src/lib.rs | 93 +++++++++++++++++++++++++ crates/engine/src/relational_db.rs | 106 +---------------------------- crates/runtime/src/lib.rs | 10 +-- 4 files changed, 102 insertions(+), 111 deletions(-) diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 98d89edb1ea..992076a7917 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -34,10 +34,10 @@ pub struct Config { pub page_pool_max_size: Option, } -pub type MetricsRecorderQueue = spacetimedb_engine::relational_db::MetricsRecorderQueue; +pub type MetricsRecorderQueue = spacetimedb_engine::MetricsRecorderQueue; pub fn spawn_tx_metrics_recorder( handle: &spacetimedb_runtime::Handle, ) -> (MetricsRecorderQueue, spacetimedb_runtime::AbortHandle) { - spacetimedb_engine::relational_db::spawn_tx_metrics_recorder(handle) + spacetimedb_engine::spawn_tx_metrics_recorder(handle) } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index d2e8860bb2a..42466557835 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -9,6 +9,99 @@ pub mod snapshot; pub mod update; pub mod util; +use std::sync::Arc; + +use enum_map::EnumMap; +use spacetimedb_datastore::execution_context::WorkloadType; +use spacetimedb_datastore::locking_tx_datastore::datastore::TxMetrics; +use spacetimedb_datastore::traits::TxData; pub use spacetimedb_lib::identity; pub use spacetimedb_lib::Identity; pub use spacetimedb_sats::hash; +use spacetimedb_schema::reducer_name::ReducerName; + +use crate::metrics::ExecutionCounters; + +/// A message that is processed by the [`spawn_metrics_recorder`] actor. +/// We use a separate task to record metrics to avoid blocking transactions. +pub struct MetricsMessage { + /// The reducer the produced these metrics. + reducer: Option, + /// Metrics from a mutable transaction. + metrics_for_writer: Option, + /// Metrics from a read-only transaction. + /// A message may have metrics for both types of transactions, + /// because metrics for a reducer and its subscription updates are recorded together. + metrics_for_reader: Option, + /// The row updates for an immutable transaction. + /// Needed for insert and delete counters. + tx_data: Option>, + /// Cached metrics counters for each workload type. + counters: Arc>, +} + +/// The handle used to send work to the tx metrics recorder. +#[derive(Clone)] +pub struct MetricsRecorderQueue { + tx: spacetimedb_runtime::sync::mpsc::UnboundedSender, +} + +impl MetricsRecorderQueue { + pub fn send_metrics( + &self, + reducer: Option, + metrics_for_writer: Option, + metrics_for_reader: Option, + tx_data: Option>, + counters: Arc>, + ) { + if let Err(err) = self.tx.send(MetricsMessage { + reducer, + metrics_for_writer, + metrics_for_reader, + tx_data, + counters, + }) { + log::warn!("failed to send metrics: {err}"); + } + } +} + +fn record_metrics( + MetricsMessage { + reducer, + metrics_for_writer, + metrics_for_reader, + tx_data, + counters, + }: MetricsMessage, +) { + if let Some(tx_metrics) = metrics_for_writer { + tx_metrics.report(tx_data.as_deref(), reducer.as_ref(), |wl| &counters[wl]); + } + if let Some(tx_metrics) = metrics_for_reader { + tx_metrics.report(None, reducer.as_ref(), |wl| &counters[wl]); + } +} + +const TX_METRICS_RECORDING_INTERVAL: std::time::Duration = std::time::Duration::from_millis(5); + +/// Spawns a task for recording transaction metrics. +/// Returns the handle for pushing metrics to the recorder. +pub fn spawn_tx_metrics_recorder( + handle: &spacetimedb_runtime::Handle, +) -> (MetricsRecorderQueue, spacetimedb_runtime::AbortHandle) { + let handle_clone = handle.clone(); + let (tx, mut rx) = spacetimedb_runtime::sync::mpsc::unbounded_channel(); + let abort_handle = handle + .spawn(async move { + loop { + handle_clone.sleep(TX_METRICS_RECORDING_INTERVAL).await; + while let Ok(metrics) = rx.try_recv() { + record_metrics(metrics); + } + } + }) + .abort_handle(); + (MetricsRecorderQueue { tx }, abort_handle) +} diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 00e001c2e45..a896c270765 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -2,111 +2,8 @@ use crate::durability::{request_durability, spawn_close as spawn_durability_clos use crate::error::{DBError, RestoreSnapshotError}; use crate::metrics::ExecutionCounters; use crate::metrics::ENGINE_METRICS; - -/// Whether SpacetimeDB is run in memory, or persists objects and -/// a message log to disk. -#[derive(Clone, Copy)] -pub enum Storage { - /// The object store is in memory, and no message log is kept. - Memory, - - /// The object store is persisted to disk, and a message log is kept. - Disk, -} - -/// Internal database config parameters -#[derive(Clone, Copy)] -pub struct Config { - /// Specifies the object storage model. - pub storage: Storage, - /// Specifies the page pool max size in bytes. - pub page_pool_max_size: Option, -} - -/// A message that is processed by the [`spawn_metrics_recorder`] actor. -/// We use a separate task to record metrics to avoid blocking transactions. -pub struct MetricsMessage { - /// The reducer the produced these metrics. - reducer: Option, - /// Metrics from a mutable transaction. - metrics_for_writer: Option, - /// Metrics from a read-only transaction. - /// A message may have metrics for both types of transactions, - /// because metrics for a reducer and its subscription updates are recorded together. - metrics_for_reader: Option, - /// The row updates for an immutable transaction. - /// Needed for insert and delete counters. - tx_data: Option>, - /// Cached metrics counters for each workload type. - counters: Arc>, -} - -/// The handle used to send work to the tx metrics recorder. -#[derive(Clone)] -pub struct MetricsRecorderQueue { - tx: spacetimedb_runtime::sync::mpsc::UnboundedSender, -} - -impl MetricsRecorderQueue { - pub fn send_metrics( - &self, - reducer: Option, - metrics_for_writer: Option, - metrics_for_reader: Option, - tx_data: Option>, - counters: Arc>, - ) { - if let Err(err) = self.tx.send(MetricsMessage { - reducer, - metrics_for_writer, - metrics_for_reader, - tx_data, - counters, - }) { - log::warn!("failed to send metrics: {err}"); - } - } -} - -fn record_metrics( - MetricsMessage { - reducer, - metrics_for_writer, - metrics_for_reader, - tx_data, - counters, - }: MetricsMessage, -) { - if let Some(tx_metrics) = metrics_for_writer { - tx_metrics.report(tx_data.as_deref(), reducer.as_ref(), |wl| &counters[wl]); - } - if let Some(tx_metrics) = metrics_for_reader { - tx_metrics.report(None, reducer.as_ref(), |wl| &counters[wl]); - } -} - -const TX_METRICS_RECORDING_INTERVAL: std::time::Duration = std::time::Duration::from_millis(5); - -/// Spawns a task for recording transaction metrics. -/// Returns the handle for pushing metrics to the recorder. -pub fn spawn_tx_metrics_recorder( - handle: &spacetimedb_runtime::Handle, -) -> (MetricsRecorderQueue, spacetimedb_runtime::AbortHandle) { - let handle_clone = handle.clone(); - let (tx, mut rx) = spacetimedb_runtime::sync::mpsc::unbounded_channel(); - let abort_handle = handle - .spawn(async move { - loop { - handle_clone.sleep(TX_METRICS_RECORDING_INTERVAL).await; - while let Ok(metrics) = rx.try_recv() { - record_metrics(metrics); - } - } - }) - .abort_handle(); - (MetricsRecorderQueue { tx }, abort_handle) -} use crate::util::asyncify; +use crate::MetricsRecorderQueue; use anyhow::{anyhow, Context}; use enum_map::EnumMap; use spacetimedb_commitlog::repo::OnNewSegmentFn; @@ -1935,6 +1832,7 @@ fn default_row_count_fn(db: Identity) -> RowCountFn { pub mod tests_utils { use crate::snapshot; use crate::snapshot::SnapshotWorker; + use crate::MetricsRecorderQueue; use super::*; use core::ops::Deref; diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 8093ca61ca4..11a0e4990fb 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -1,8 +1,8 @@ -#[cfg(all(feature = "tokio", feature = "simulation"))] -compile_error!( - "spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`, not both" -); - +//#[cfg(all(feature = "tokio", feature = "simulation"))] +//compile_error!( +// "spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`, not both" +//); +// #[cfg(not(any(feature = "tokio", feature = "simulation")))] compile_error!("spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`"); From 52078b57895662eeb497f19669dcdc9302b039d6 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 10 Jun 2026 19:11:58 +0530 Subject: [PATCH 06/16] move ast --- crates/core/src/estimation.rs | 2 +- crates/core/src/host/module_host.rs | 3 +- .../src/host/wasm_common/module_host_actor.rs | 2 +- crates/core/src/sql/ast.rs | 77 ------------------- crates/core/src/sql/execute.rs | 4 +- crates/core/src/sql/mod.rs | 1 - .../module_subscription_manager.rs | 2 +- crates/core/src/subscription/query.rs | 2 +- crates/core/src/subscription/subscription.rs | 2 +- crates/engine/src/lib.rs | 2 +- 10 files changed, 9 insertions(+), 88 deletions(-) delete mode 100644 crates/core/src/sql/ast.rs diff --git a/crates/core/src/estimation.rs b/crates/core/src/estimation.rs index 70dda1b3e2a..47f196d1c4d 100644 --- a/crates/core/src/estimation.rs +++ b/crates/core/src/estimation.rs @@ -132,7 +132,7 @@ mod tests { use crate::db::relational_db::tests_utils::{begin_tx, insert, with_auto_commit}; use crate::db::relational_db::{tests_utils::TestDB, RelationalDB}; use crate::error::DBError; - use crate::sql::ast::SchemaViewer; + use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_lib::{identity::AuthCtx, AlgebraicType}; use spacetimedb_query::compile_subscription; use spacetimedb_sats::product; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 2a98e7fc4e7..579068f77d4 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -18,7 +18,6 @@ use crate::host::{InvalidFunctionArguments, InvalidViewArguments}; use crate::identity::Identity; use crate::messages::control_db::{Database, HostType}; use crate::replica_context::ReplicaContext; -use crate::sql::ast::SchemaViewer; use crate::sql::execute::SqlResult; use crate::subscription::module_subscription_actor::ModuleSubscriptions; use crate::subscription::module_subscription_manager::BroadcastError; @@ -50,6 +49,7 @@ use spacetimedb_datastore::execution_context::{Workload, WorkloadType}; use spacetimedb_datastore::locking_tx_datastore::{MutTxId, ViewCallInfo}; use spacetimedb_datastore::traits::{IsolationLevel, Program, TxData}; pub use spacetimedb_durability::{DurabilityExited, DurableOffset}; +use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_engine::rls::RowLevelExpr; use spacetimedb_execution::pipelined::{PipelinedProject, ViewProject}; use spacetimedb_execution::RelValue; @@ -67,7 +67,6 @@ use spacetimedb_schema::auto_migrate::{AutoMigrateError, MigrationPolicy}; use spacetimedb_schema::def::{ModuleDef, ProcedureDef, ReducerDef, ViewDef}; use spacetimedb_schema::identifier::Identifier; use spacetimedb_schema::reducer_name::ReducerName; - use spacetimedb_schema::table_name::TableName; use std::collections::VecDeque; use std::fmt; diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 4489fb9f8bd..cdb185f8e4e 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -22,7 +22,6 @@ use crate::identity::Identity; use crate::messages::control_db::HostType; use crate::module_host_context::ModuleCreationContext; use crate::replica_context::ReplicaContext; -use crate::sql::ast::SchemaViewer; use crate::sql::execute::run_with_instance; use crate::subscription::module_subscription_actor::{commit_and_broadcast_event, CommitAndBroadcastEventSuccess}; use crate::subscription::module_subscription_manager::TransactionOffset; @@ -39,6 +38,7 @@ use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo}; use spacetimedb_datastore::traits::{IsolationLevel, Program}; +use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_execution::pipelined::PipelinedProject; use spacetimedb_lib::buffer::DecodeError; use spacetimedb_lib::db::raw_def::v9::{Lifecycle, ViewResultHeader}; diff --git a/crates/core/src/sql/ast.rs b/crates/core/src/sql/ast.rs deleted file mode 100644 index 892430ba1ea..00000000000 --- a/crates/core/src/sql/ast.rs +++ /dev/null @@ -1,77 +0,0 @@ -use anyhow::Context; -use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; -use spacetimedb_datastore::system_tables::{StRowLevelSecurityFields, ST_ROW_LEVEL_SECURITY_ID}; -use spacetimedb_expr::check::SchemaView; -use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_primitives::TableId; -use spacetimedb_sats::AlgebraicValue; -use spacetimedb_schema::schema::TableOrViewSchema; -use std::ops::Deref; -use std::sync::Arc; - -pub struct SchemaViewer<'a, T> { - tx: &'a T, - auth: &'a AuthCtx, -} - -impl Deref for SchemaViewer<'_, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.tx - } -} - -impl SchemaView for SchemaViewer<'_, T> { - fn table_id(&self, name: &str) -> Option { - self.tx - .table_id_from_name_or_alias(name) - .ok() - .flatten() - .and_then(|table_id| self.schema_for_table(table_id)) - .filter(|schema| self.auth.has_read_access(schema.table_access)) - .map(|schema| schema.table_id) - } - - fn schema_for_table(&self, table_id: TableId) -> Option> { - self.tx - .get_schema(table_id) - .filter(|schema| self.auth.has_read_access(schema.table_access)) - .map(Arc::clone) - .map(TableOrViewSchema::from) - .map(Arc::new) - } - - fn rls_rules_for_table(&self, table_id: TableId) -> anyhow::Result>> { - self.tx - .iter_by_col_eq( - ST_ROW_LEVEL_SECURITY_ID, - StRowLevelSecurityFields::TableId, - &AlgebraicValue::from(table_id), - )? - .map(|row| { - row.read_col::(StRowLevelSecurityFields::Sql) - .with_context(|| { - format!( - "Failed to read value from the `{}` column of `{}` for table_id `{}`", - "sql", "st_row_level_security", table_id - ) - }) - .and_then(|sql| { - sql.into_string().map_err(|_| { - anyhow::anyhow!(format!( - "Failed to read value from the `{}` column of `{}` for table_id `{}`", - "sql", "st_row_level_security", table_id - )) - }) - }) - }) - .collect::>() - } -} - -impl<'a, T> SchemaViewer<'a, T> { - pub fn new(tx: &'a T, auth: &'a AuthCtx) -> Self { - Self { tx, auth } - } -} diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 255f57ff8b7..d5b471d3696 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1,8 +1,6 @@ use std::sync::Arc; use std::time::Duration; -use super::ast::SchemaViewer; -use crate::db::relational_db::RelationalDB; use crate::energy::FunctionBudget; use crate::error::DBError; use crate::estimation::{check_row_limit, estimate_rows_scanned}; @@ -17,6 +15,8 @@ use crate::subscription::tx::DeltaTx; use anyhow::anyhow; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_engine::ast::SchemaViewer; +use spacetimedb_engine::relational_db::RelationalDB; use spacetimedb_expr::statement::Statement; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::metrics::ExecutionMetrics; diff --git a/crates/core/src/sql/mod.rs b/crates/core/src/sql/mod.rs index 78c7f6bbfe7..2e8bdddf980 100644 --- a/crates/core/src/sql/mod.rs +++ b/crates/core/src/sql/mod.rs @@ -1,2 +1 @@ -pub mod ast; pub mod execute; diff --git a/crates/core/src/subscription/module_subscription_manager.rs b/crates/core/src/subscription/module_subscription_manager.rs index 0700846fc74..ccbee836c25 100644 --- a/crates/core/src/subscription/module_subscription_manager.rs +++ b/crates/core/src/subscription/module_subscription_manager.rs @@ -2204,7 +2204,6 @@ mod tests { use super::{Plan, SubscriptionManager}; use crate::db::relational_db::tests_utils::with_read_only; use crate::host::module_host::DatabaseTableUpdate; - use crate::sql::ast::SchemaViewer; use crate::subscription::module_subscription_manager::ClientQueryId; use crate::subscription::row_list_builder_pool::BsatnRowListBuilderPool; use crate::subscription::tx::DeltaTx; @@ -2219,6 +2218,7 @@ mod tests { subscription::execution_unit::QueryHash, }; use spacetimedb_datastore::execution_context::Workload; + use spacetimedb_engine::ast::SchemaViewer; fn create_table(db: &RelationalDB, name: &str) -> ResultTest { Ok(db.create_table_for_test(name, &[("a", AlgebraicType::U8)], &[])?) diff --git a/crates/core/src/subscription/query.rs b/crates/core/src/subscription/query.rs index bea30f96b7f..e09f0b87674 100644 --- a/crates/core/src/subscription/query.rs +++ b/crates/core/src/subscription/query.rs @@ -2,10 +2,10 @@ use super::execution_unit::QueryHash; use super::module_subscription_manager::Plan; use crate::db::relational_db::Tx; use crate::error::{DBError, SubscriptionError}; -use crate::sql::ast::SchemaViewer; use once_cell::sync::Lazy; use regex::Regex; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_execution::Datastore; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_subscription::SubscriptionPlan; diff --git a/crates/core/src/subscription/subscription.rs b/crates/core/src/subscription/subscription.rs index a9f3bd12f62..fd7c5fc3fcb 100644 --- a/crates/core/src/subscription/subscription.rs +++ b/crates/core/src/subscription/subscription.rs @@ -2,8 +2,8 @@ use super::execution_unit::QueryHash; use super::module_subscription_manager::Plan; use crate::db::relational_db::RelationalDB; use crate::error::DBError; -use crate::sql::ast::SchemaViewer; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_lib::db::auth::StTableType; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_schema::schema::TableSchema; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 42466557835..0a0ebb96a09 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1,4 +1,4 @@ -mod ast; +pub mod ast; pub(crate) mod durability; pub mod error; pub mod metrics; From 198967d58c63058fc67cc9e878ef187d5c509f3a Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 10 Jun 2026 20:39:22 +0530 Subject: [PATCH 07/16] clippy --- crates/bench/benches/subscription.rs | 2 +- crates/core/src/db/mod.rs | 2 ++ crates/core/src/estimation.rs | 2 +- crates/core/src/host/module_host.rs | 2 +- crates/core/src/host/wasm_common/module_host_actor.rs | 2 +- crates/core/src/sql/execute.rs | 2 +- crates/core/src/subscription/module_subscription_manager.rs | 2 +- crates/core/src/subscription/query.rs | 2 +- crates/core/src/subscription/subscription.rs | 2 +- 9 files changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/bench/benches/subscription.rs b/crates/bench/benches/subscription.rs index ebd8e83e35c..359ad86746b 100644 --- a/crates/bench/benches/subscription.rs +++ b/crates/bench/benches/subscription.rs @@ -1,9 +1,9 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use spacetimedb::client::consume_each_list::ConsumeEachBuffer; use spacetimedb::db::relational_db::RelationalDB; +use spacetimedb::db::SchemaViewer; use spacetimedb::error::DBError; use spacetimedb::identity::AuthCtx; -use spacetimedb::sql::ast::SchemaViewer; use spacetimedb::subscription::row_list_builder_pool::BsatnRowListBuilderPool; use spacetimedb::subscription::tx::DeltaTx; use spacetimedb::subscription::{collect_table_update, TableUpdateType}; diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 992076a7917..8ba71616db0 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -36,6 +36,8 @@ pub struct Config { pub type MetricsRecorderQueue = spacetimedb_engine::MetricsRecorderQueue; +pub type SchemaViewer<'a, T> = spacetimedb_engine::ast::SchemaViewer<'a, T>; + pub fn spawn_tx_metrics_recorder( handle: &spacetimedb_runtime::Handle, ) -> (MetricsRecorderQueue, spacetimedb_runtime::AbortHandle) { diff --git a/crates/core/src/estimation.rs b/crates/core/src/estimation.rs index 47f196d1c4d..ff704c92cd5 100644 --- a/crates/core/src/estimation.rs +++ b/crates/core/src/estimation.rs @@ -131,8 +131,8 @@ mod tests { use super::{estimate_rows_scanned, row_estimate}; use crate::db::relational_db::tests_utils::{begin_tx, insert, with_auto_commit}; use crate::db::relational_db::{tests_utils::TestDB, RelationalDB}; + use crate::db::SchemaViewer; use crate::error::DBError; - use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_lib::{identity::AuthCtx, AlgebraicType}; use spacetimedb_query::compile_subscription; use spacetimedb_sats::product; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 579068f77d4..194eafb0334 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -6,6 +6,7 @@ use crate::client::messages::{OneOffQueryResponseMessage, ProcedureResultMessage use crate::client::{ClientActorId, ClientConnectionSender, WsVersion}; use crate::database_logger::{DatabaseLogger, LogLevel, Record}; use crate::db::relational_db::{RelationalDB, Tx}; +use crate::db::SchemaViewer; use crate::error::DBError; use crate::estimation::{check_row_limit, estimate_rows_scanned}; use crate::hash::Hash; @@ -49,7 +50,6 @@ use spacetimedb_datastore::execution_context::{Workload, WorkloadType}; use spacetimedb_datastore::locking_tx_datastore::{MutTxId, ViewCallInfo}; use spacetimedb_datastore::traits::{IsolationLevel, Program, TxData}; pub use spacetimedb_durability::{DurabilityExited, DurableOffset}; -use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_engine::rls::RowLevelExpr; use spacetimedb_execution::pipelined::{PipelinedProject, ViewProject}; use spacetimedb_execution::RelValue; diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index cdb185f8e4e..a3e9d618f09 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -2,6 +2,7 @@ use super::instrumentation::CallTimes; use super::*; use crate::client::ClientActorId; use crate::database_logger; +use crate::db::SchemaViewer; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; use crate::host::host_controller::CallProcedureReturn; @@ -38,7 +39,6 @@ use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo}; use spacetimedb_datastore::traits::{IsolationLevel, Program}; -use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_execution::pipelined::PipelinedProject; use spacetimedb_lib::buffer::DecodeError; use spacetimedb_lib::db::raw_def::v9::{Lifecycle, ViewResultHeader}; diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index d5b471d3696..1ad8ee7b770 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use std::time::Duration; +use crate::db::SchemaViewer; use crate::energy::FunctionBudget; use crate::error::DBError; use crate::estimation::{check_row_limit, estimate_rows_scanned}; @@ -15,7 +16,6 @@ use crate::subscription::tx::DeltaTx; use anyhow::anyhow; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::traits::IsolationLevel; -use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_engine::relational_db::RelationalDB; use spacetimedb_expr::statement::Statement; use spacetimedb_lib::identity::AuthCtx; diff --git a/crates/core/src/subscription/module_subscription_manager.rs b/crates/core/src/subscription/module_subscription_manager.rs index ccbee836c25..d3e95fb3044 100644 --- a/crates/core/src/subscription/module_subscription_manager.rs +++ b/crates/core/src/subscription/module_subscription_manager.rs @@ -2203,6 +2203,7 @@ mod tests { use super::{Plan, SubscriptionManager}; use crate::db::relational_db::tests_utils::with_read_only; + use crate::db::SchemaViewer; use crate::host::module_host::DatabaseTableUpdate; use crate::subscription::module_subscription_manager::ClientQueryId; use crate::subscription::row_list_builder_pool::BsatnRowListBuilderPool; @@ -2218,7 +2219,6 @@ mod tests { subscription::execution_unit::QueryHash, }; use spacetimedb_datastore::execution_context::Workload; - use spacetimedb_engine::ast::SchemaViewer; fn create_table(db: &RelationalDB, name: &str) -> ResultTest { Ok(db.create_table_for_test(name, &[("a", AlgebraicType::U8)], &[])?) diff --git a/crates/core/src/subscription/query.rs b/crates/core/src/subscription/query.rs index e09f0b87674..e2ef3676c10 100644 --- a/crates/core/src/subscription/query.rs +++ b/crates/core/src/subscription/query.rs @@ -1,11 +1,11 @@ use super::execution_unit::QueryHash; use super::module_subscription_manager::Plan; use crate::db::relational_db::Tx; +use crate::db::SchemaViewer; use crate::error::{DBError, SubscriptionError}; use once_cell::sync::Lazy; use regex::Regex; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; -use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_execution::Datastore; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_subscription::SubscriptionPlan; diff --git a/crates/core/src/subscription/subscription.rs b/crates/core/src/subscription/subscription.rs index fd7c5fc3fcb..82021e37e30 100644 --- a/crates/core/src/subscription/subscription.rs +++ b/crates/core/src/subscription/subscription.rs @@ -1,9 +1,9 @@ use super::execution_unit::QueryHash; use super::module_subscription_manager::Plan; use crate::db::relational_db::RelationalDB; +use crate::db::SchemaViewer; use crate::error::DBError; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; -use spacetimedb_engine::ast::SchemaViewer; use spacetimedb_lib::db::auth::StTableType; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_schema::schema::TableSchema; From 6abd749826492aac91db047c4e9dbe995c3238c6 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 10 Jun 2026 20:52:09 +0530 Subject: [PATCH 08/16] sql module --- crates/core/src/db/mod.rs | 12 +++++++++++- crates/core/src/host/module_host.rs | 2 +- crates/engine/src/lib.rs | 3 +-- crates/engine/src/{ => sql}/ast.rs | 0 crates/engine/src/sql/mod.rs | 2 ++ crates/engine/src/{ => sql}/rls.rs | 2 +- crates/engine/src/update.rs | 2 +- 7 files changed, 17 insertions(+), 6 deletions(-) rename crates/engine/src/{ => sql}/ast.rs (100%) create mode 100644 crates/engine/src/sql/mod.rs rename crates/engine/src/{ => sql}/rls.rs (97%) diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 8ba71616db0..81a250922a6 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -6,6 +6,16 @@ pub mod relational_db { pub use spacetimedb_engine::relational_db::*; } +pub mod sql { + pub mod ast { + pub use spacetimedb_engine::sql::ast::*; + } + + pub mod rls { + pub use spacetimedb_engine::sql::rls::*; + } +} + pub mod snapshot { pub use spacetimedb_engine::snapshot::*; } @@ -36,7 +46,7 @@ pub struct Config { pub type MetricsRecorderQueue = spacetimedb_engine::MetricsRecorderQueue; -pub type SchemaViewer<'a, T> = spacetimedb_engine::ast::SchemaViewer<'a, T>; +pub type SchemaViewer<'a, T> = spacetimedb_engine::sql::ast::SchemaViewer<'a, T>; pub fn spawn_tx_metrics_recorder( handle: &spacetimedb_runtime::Handle, diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 194eafb0334..0ab921d3ec3 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -50,7 +50,7 @@ use spacetimedb_datastore::execution_context::{Workload, WorkloadType}; use spacetimedb_datastore::locking_tx_datastore::{MutTxId, ViewCallInfo}; use spacetimedb_datastore::traits::{IsolationLevel, Program, TxData}; pub use spacetimedb_durability::{DurabilityExited, DurableOffset}; -use spacetimedb_engine::rls::RowLevelExpr; +use spacetimedb_engine::sql::rls::RowLevelExpr; use spacetimedb_execution::pipelined::{PipelinedProject, ViewProject}; use spacetimedb_execution::RelValue; use spacetimedb_expr::expr::CollectViews; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 0a0ebb96a09..bfcc2886d92 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1,10 +1,9 @@ -pub mod ast; pub(crate) mod durability; pub mod error; pub mod metrics; pub mod persistence; pub mod relational_db; -pub mod rls; +pub mod sql; pub mod snapshot; pub mod update; pub mod util; diff --git a/crates/engine/src/ast.rs b/crates/engine/src/sql/ast.rs similarity index 100% rename from crates/engine/src/ast.rs rename to crates/engine/src/sql/ast.rs diff --git a/crates/engine/src/sql/mod.rs b/crates/engine/src/sql/mod.rs new file mode 100644 index 00000000000..7ac171e4ed9 --- /dev/null +++ b/crates/engine/src/sql/mod.rs @@ -0,0 +1,2 @@ +pub mod ast; +pub mod rls; \ No newline at end of file diff --git a/crates/engine/src/rls.rs b/crates/engine/src/sql/rls.rs similarity index 97% rename from crates/engine/src/rls.rs rename to crates/engine/src/sql/rls.rs index 5fc606352ec..df063a6a63e 100644 --- a/crates/engine/src/rls.rs +++ b/crates/engine/src/sql/rls.rs @@ -1,4 +1,4 @@ -use crate::ast::SchemaViewer; +use super::ast::SchemaViewer; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_expr::check::parse_and_type_sub; diff --git a/crates/engine/src/update.rs b/crates/engine/src/update.rs index bfd6d7648bd..bc2464590bc 100644 --- a/crates/engine/src/update.rs +++ b/crates/engine/src/update.rs @@ -1,5 +1,5 @@ use super::relational_db::RelationalDB; -use crate::rls::RowLevelExpr; +use crate::sql::rls::RowLevelExpr; use anyhow::Context; use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_lib::db::auth::StTableType; From 7148de25f208200b59e24eaf12aca075bc3b81db Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 10 Jun 2026 21:32:20 +0530 Subject: [PATCH 09/16] formatting --- crates/bench/benches/subscription.rs | 2 +- crates/core/src/db/mod.rs | 2 -- crates/core/src/estimation.rs | 2 +- crates/core/src/host/module_host.rs | 2 +- crates/core/src/host/wasm_common/module_host_actor.rs | 2 +- crates/core/src/sql/execute.rs | 2 +- crates/core/src/subscription/module_subscription_manager.rs | 2 +- crates/core/src/subscription/query.rs | 2 +- crates/core/src/subscription/subscription.rs | 2 +- crates/engine/src/lib.rs | 2 +- crates/engine/src/sql/mod.rs | 2 +- 11 files changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/bench/benches/subscription.rs b/crates/bench/benches/subscription.rs index 359ad86746b..e54be943b4e 100644 --- a/crates/bench/benches/subscription.rs +++ b/crates/bench/benches/subscription.rs @@ -1,7 +1,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use spacetimedb::client::consume_each_list::ConsumeEachBuffer; use spacetimedb::db::relational_db::RelationalDB; -use spacetimedb::db::SchemaViewer; +use spacetimedb::db::sql::ast::SchemaViewer; use spacetimedb::error::DBError; use spacetimedb::identity::AuthCtx; use spacetimedb::subscription::row_list_builder_pool::BsatnRowListBuilderPool; diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 81a250922a6..6b1d2f6700b 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -46,8 +46,6 @@ pub struct Config { pub type MetricsRecorderQueue = spacetimedb_engine::MetricsRecorderQueue; -pub type SchemaViewer<'a, T> = spacetimedb_engine::sql::ast::SchemaViewer<'a, T>; - pub fn spawn_tx_metrics_recorder( handle: &spacetimedb_runtime::Handle, ) -> (MetricsRecorderQueue, spacetimedb_runtime::AbortHandle) { diff --git a/crates/core/src/estimation.rs b/crates/core/src/estimation.rs index ff704c92cd5..92aa9370d92 100644 --- a/crates/core/src/estimation.rs +++ b/crates/core/src/estimation.rs @@ -131,7 +131,7 @@ mod tests { use super::{estimate_rows_scanned, row_estimate}; use crate::db::relational_db::tests_utils::{begin_tx, insert, with_auto_commit}; use crate::db::relational_db::{tests_utils::TestDB, RelationalDB}; - use crate::db::SchemaViewer; + use crate::db::sql::ast::SchemaViewer; use crate::error::DBError; use spacetimedb_lib::{identity::AuthCtx, AlgebraicType}; use spacetimedb_query::compile_subscription; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 0ab921d3ec3..e9cda26ea01 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -6,7 +6,7 @@ use crate::client::messages::{OneOffQueryResponseMessage, ProcedureResultMessage use crate::client::{ClientActorId, ClientConnectionSender, WsVersion}; use crate::database_logger::{DatabaseLogger, LogLevel, Record}; use crate::db::relational_db::{RelationalDB, Tx}; -use crate::db::SchemaViewer; +use crate::db::sql::ast::SchemaViewer; use crate::error::DBError; use crate::estimation::{check_row_limit, estimate_rows_scanned}; use crate::hash::Hash; diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index a3e9d618f09..a76d814ce7b 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -2,7 +2,7 @@ use super::instrumentation::CallTimes; use super::*; use crate::client::ClientActorId; use crate::database_logger; -use crate::db::SchemaViewer; +use crate::db::sql::ast::SchemaViewer; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; use crate::host::host_controller::CallProcedureReturn; diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 1ad8ee7b770..7fe2fe229f8 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::time::Duration; -use crate::db::SchemaViewer; +use crate::db::sql::ast::SchemaViewer; use crate::energy::FunctionBudget; use crate::error::DBError; use crate::estimation::{check_row_limit, estimate_rows_scanned}; diff --git a/crates/core/src/subscription/module_subscription_manager.rs b/crates/core/src/subscription/module_subscription_manager.rs index d3e95fb3044..404a8230040 100644 --- a/crates/core/src/subscription/module_subscription_manager.rs +++ b/crates/core/src/subscription/module_subscription_manager.rs @@ -2203,7 +2203,7 @@ mod tests { use super::{Plan, SubscriptionManager}; use crate::db::relational_db::tests_utils::with_read_only; - use crate::db::SchemaViewer; + use crate::db::sql::ast::SchemaViewer; use crate::host::module_host::DatabaseTableUpdate; use crate::subscription::module_subscription_manager::ClientQueryId; use crate::subscription::row_list_builder_pool::BsatnRowListBuilderPool; diff --git a/crates/core/src/subscription/query.rs b/crates/core/src/subscription/query.rs index e2ef3676c10..1a82ac6c5a7 100644 --- a/crates/core/src/subscription/query.rs +++ b/crates/core/src/subscription/query.rs @@ -1,7 +1,7 @@ use super::execution_unit::QueryHash; use super::module_subscription_manager::Plan; use crate::db::relational_db::Tx; -use crate::db::SchemaViewer; +use crate::db::sql::ast::SchemaViewer; use crate::error::{DBError, SubscriptionError}; use once_cell::sync::Lazy; use regex::Regex; diff --git a/crates/core/src/subscription/subscription.rs b/crates/core/src/subscription/subscription.rs index 82021e37e30..54b25acbfd2 100644 --- a/crates/core/src/subscription/subscription.rs +++ b/crates/core/src/subscription/subscription.rs @@ -1,7 +1,7 @@ use super::execution_unit::QueryHash; use super::module_subscription_manager::Plan; use crate::db::relational_db::RelationalDB; -use crate::db::SchemaViewer; +use crate::db::sql::ast::SchemaViewer; use crate::error::DBError; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_lib::db::auth::StTableType; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index bfcc2886d92..204f54f7187 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -3,8 +3,8 @@ pub mod error; pub mod metrics; pub mod persistence; pub mod relational_db; -pub mod sql; pub mod snapshot; +pub mod sql; pub mod update; pub mod util; diff --git a/crates/engine/src/sql/mod.rs b/crates/engine/src/sql/mod.rs index 7ac171e4ed9..70aae2831d7 100644 --- a/crates/engine/src/sql/mod.rs +++ b/crates/engine/src/sql/mod.rs @@ -1,2 +1,2 @@ pub mod ast; -pub mod rls; \ No newline at end of file +pub mod rls; From 545b1ffc17d63b482cc4a3340e8e4802df1c1422 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 10 Jun 2026 21:44:51 +0530 Subject: [PATCH 10/16] uncomment --- crates/runtime/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 11a0e4990fb..8093ca61ca4 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -1,8 +1,8 @@ -//#[cfg(all(feature = "tokio", feature = "simulation"))] -//compile_error!( -// "spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`, not both" -//); -// +#[cfg(all(feature = "tokio", feature = "simulation"))] +compile_error!( + "spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`, not both" +); + #[cfg(not(any(feature = "tokio", feature = "simulation")))] compile_error!("spacetimedb-runtime requires exactly one runtime backend: enable either `tokio` or `simulation`"); From 6aedc521e6c83b31a8e0668a30a1cbf5169c3d6c Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Thu, 11 Jun 2026 18:00:51 +0530 Subject: [PATCH 11/16] add back commentary --- crates/engine/src/lib.rs | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 204f54f7187..09229362bd7 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -76,13 +76,39 @@ fn record_metrics( }: MetricsMessage, ) { if let Some(tx_metrics) = metrics_for_writer { - tx_metrics.report(tx_data.as_deref(), reducer.as_ref(), |wl| &counters[wl]); + tx_metrics.report( + // If row updates are present, + // they will always belong to the writer transaction. + tx_data.as_deref(), + reducer.as_ref(), + |wl| &counters[wl], + ); } if let Some(tx_metrics) = metrics_for_reader { - tx_metrics.report(None, reducer.as_ref(), |wl| &counters[wl]); + tx_metrics.report( + // If row updates are present, + // they will never belong to the reader transaction. + // Passing row updates here will most likely panic. + None, + reducer.as_ref(), + |wl| &counters[wl], + ); } } +/// The metrics recorder is a side channel that the main database thread forwards metrics to. +/// While we want to avoid unnecessary compute on the critical path, communicating with other +/// threads is not free, and for this case in particular waking a parked task is not free. +/// +/// Previously, each tx would send its metrics to the recorder task. As soon as the recorder +/// task `recv`d a message, it would update the counters and gauges, and immediately wait for +/// the next tx's message. This meant that the tx would need to be more expensive than the +/// recording of its metrics in order for the recorder task not to be parked on `recv` when +/// the tx would `send` its metrics. This would obviously never be the case, and so each `send` +/// would incur the overhead of waking the task. +/// +/// To mitigate this, we now record metrics, for potentially many transactions, periodically +/// every 5ms. const TX_METRICS_RECORDING_INTERVAL: std::time::Duration = std::time::Duration::from_millis(5); /// Spawns a task for recording transaction metrics. From d7db10ce677cf95da461083dac5c532b1fe1d64e Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 12 Jun 2026 11:11:30 +0530 Subject: [PATCH 12/16] Add engine as dev-depency of core --- crates/core/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 83f44967d6e..b9c21843e59 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -156,6 +156,7 @@ spacetimedb-lib = { path = "../lib", features = ["proptest", "test"] } spacetimedb-sats = { path = "../sats", features = ["proptest"] } spacetimedb-commitlog = { path = "../commitlog", features = ["test"] } spacetimedb-datastore = { path = "../datastore/", features = ["test"] } +spacetimedb-engine = { workspace = true, features = ["test"] } criterion.workspace = true # Also as dev-dependencies for use in _this_ crate's tests. From fbae25eae3017d0066ed829dfb5319d628b9987e Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Mon, 15 Jun 2026 15:22:56 +0530 Subject: [PATCH 13/16] fmt --- crates/engine/src/update.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/update.rs b/crates/engine/src/update.rs index 646b5cd8782..6d420714a88 100644 --- a/crates/engine/src/update.rs +++ b/crates/engine/src/update.rs @@ -351,7 +351,10 @@ pub fn create_table_from_def( #[cfg(test)] mod test { use super::*; - use crate::relational_db::{open_snapshot_repo, tests_utils::{begin_mut_tx, insert, TestDB}}; + use crate::relational_db::{ + open_snapshot_repo, + tests_utils::{begin_mut_tx, insert, TestDB}, + }; use spacetimedb_datastore::locking_tx_datastore::PendingSchemaChange; use spacetimedb_datastore::system_tables::ST_EVENT_TABLE_ID; use spacetimedb_lib::{ From f44bb80b19ea1777ae15cb0d50e4ddd1fd145698 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Mon, 15 Jun 2026 15:49:15 +0530 Subject: [PATCH 14/16] merge ops --- crates/core/src/subscription/mod.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/crates/core/src/subscription/mod.rs b/crates/core/src/subscription/mod.rs index 856c9be3039..ec06dfc6f58 100644 --- a/crates/core/src/subscription/mod.rs +++ b/crates/core/src/subscription/mod.rs @@ -27,14 +27,7 @@ pub mod subscription; pub mod tx; pub mod websocket_building; -/// Execute a subscription query over a view. -/// -/// Specifically this utility is for queries that return rows from a view. -/// Unlike user tables, views have internal columns that should not be returned to clients. -/// The [`ViewProject`] operator implicitly drops these columns as part of its execution. -/// -/// NOTE: This method was largely copied from [`execute_plan`]. -/// TODO: Merge with [`execute_plan`]. +/// Execute subscription query fragments over a view. pub fn execute_plan_for_view<'p, F>( plan_fragments: impl IntoIterator, num_cols: usize, From 7bc0bac0e7789e13fd54865dfd2ece89bfa32443 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Mon, 15 Jun 2026 23:02:14 +0530 Subject: [PATCH 15/16] From d5dba08483f0325fa9406ad184cc4724996081a5 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Tue, 16 Jun 2026 08:15:04 -0400 Subject: [PATCH 16/16] Update engine crate lockfile version --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index f5029a41eaa..c48331b3b4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8228,7 +8228,7 @@ dependencies = [ [[package]] name = "spacetimedb-engine" -version = "2.5.0" +version = "2.6.0" dependencies = [ "anyhow", "async-trait",