diff --git a/documentation/configuration/configuration-utils/_http.config.json b/documentation/configuration/configuration-utils/_http.config.json
index b92ad0dce..de8ecc256 100644
--- a/documentation/configuration/configuration-utils/_http.config.json
+++ b/documentation/configuration/configuration-utils/_http.config.json
@@ -266,5 +266,9 @@
"http.redirect.1": {
"default": "/ -> /index.html",
"description": "Example redirect configuration. Format is 'source -> destination'."
+ },
+ "http.ingest.max.request.size": {
+ "default": "5M",
+ "description": "Maximum request body size for the `/ingest` endpoint used by [payload transforms](/docs/ingestion/payload-transforms/). The entire request body is held in memory during processing. Requests exceeding this limit receive an HTTP 413 (Payload Too Large) response."
}
}
diff --git a/documentation/ingestion/payload-transforms.md b/documentation/ingestion/payload-transforms.md
new file mode 100644
index 000000000..a30e5bc96
--- /dev/null
+++ b/documentation/ingestion/payload-transforms.md
@@ -0,0 +1,278 @@
+---
+title: Payload transforms
+sidebar_label: Payload Transforms
+description:
+ Guide to payload transforms in QuestDB, which parse and insert HTTP payloads
+ into tables using SQL expressions without middleware.
+---
+
+Payload transforms define how incoming HTTP payloads are parsed, transformed, and
+inserted into a QuestDB table. You define a transform once with a SQL `SELECT`
+expression, then POST data directly to QuestDB. The
+[`payload()`](/docs/query/functions/meta/#payload) function provides access to
+the raw HTTP request body within the transform. No middleware or intermediary
+service is required.
+
+Use cases include webhook ingestion, IoT device data, and external API responses
+where you want to skip building a dedicated ingestion service.
+
+For full SQL syntax, see
+[CREATE PAYLOAD TRANSFORM](/docs/query/sql/create-payload-transform/),
+[DROP PAYLOAD TRANSFORM](/docs/query/sql/drop-payload-transform/), and
+[SHOW PAYLOAD TRANSFORMS](/docs/query/sql/show/#show-payload-transforms).
+
+## Example: Coinbase order book snapshots
+
+Store order book snapshots from the
+[Coinbase book API](https://api.exchange.coinbase.com/products/BTC-USD/book?level=2).
+With `level=2`, the API returns up to 50 aggregated price levels per side:
+
+```json
+{
+ "sequence": 125688480181,
+ "bids": [["69678.77","0.00007525",2], ["69676.36","0.00000022",1], ...],
+ "asks": [["69678.78","0.35468555",6], ["69679.99","0.00071759",1], ...],
+ "time": "2026-04-06T11:52:14.454632476Z",
+ "auction_mode": false
+}
+```
+
+Each bid/ask entry contains the price, quantity, and number of orders at that
+level. The `time` field provides a nanosecond-precision exchange timestamp.
+
+Create a target table with full depth arrays plus top-of-book prices, and a
+transform that extracts them from the payload:
+
+```questdb-sql title="Table and transform definition"
+CREATE TABLE coinbase_order_book (
+ timestamp TIMESTAMP,
+ symbol SYMBOL,
+ bids DOUBLE[][],
+ asks DOUBLE[][],
+ best_bid DOUBLE,
+ best_ask DOUBLE
+) TIMESTAMP(timestamp) PARTITION BY DAY WAL;
+
+CREATE PAYLOAD TRANSFORM coinbase_book_api
+INTO coinbase_order_book
+DLQ dlq_errors PARTITION BY DAY TTL 7 DAYS
+AS DECLARE OVERRIDABLE @symbol := 'BTC-USD'
+SELECT
+ json_extract(payload(), '$.time')::TIMESTAMP AS timestamp,
+ @symbol AS symbol,
+ json_extract(payload(), '$.bids')::DOUBLE[][] AS bids,
+ json_extract(payload(), '$.asks')::DOUBLE[][] AS asks,
+ json_extract(payload(), '$.bids[0][0]')::DOUBLE AS best_bid,
+ json_extract(payload(), '$.asks[0][0]')::DOUBLE AS best_ask;
+```
+
+Fetch 50 levels of depth and ingest the snapshot:
+
+```shell title="POST a payload"
+curl -s "https://api.exchange.coinbase.com/products/BTC-USD/book?level=2" | \
+ curl -X POST "http://localhost:9000/ingest?transform=coinbase_book_api" -d @-
+```
+
+Response:
+
+```json
+{"status": "ok", "rows_inserted": 1}
+```
+
+### Overriding variables
+
+The `@symbol` variable is declared `OVERRIDABLE`, so you can override it per
+request via URL query parameters:
+
+```shell title="Override a variable"
+curl -s "https://api.exchange.coinbase.com/products/ETH-USD/book?level=2" | \
+ curl -X POST "http://localhost:9000/ingest?transform=coinbase_book_api&symbol=ETH-USD" -d @-
+```
+
+Any URL query parameter other than `transform` is matched to a
+`DECLARE OVERRIDABLE` variable by name. Variables not marked `OVERRIDABLE`
+cannot be overridden - attempting to do so returns an error.
+
+## Example: Coinbase trades with UNNEST
+
+The [Coinbase trades API](https://api.exchange.coinbase.com/products/BTC-USD/trades?limit=100)
+returns a JSON array of recent trades.
+
+```json
+[
+ {"trade_id": 994619709, "side": "sell", "size": "0.00000100",
+ "price": "69839.36000000", "time": "2026-04-06T10:32:55.517183Z"},
+ {"trade_id": 994619708, "side": "buy", "size": "0.00000006",
+ "price": "69839.35000000", "time": "2026-04-06T10:32:55.418434Z"},
+ ...
+]
+```
+
+The transform uses [JSON UNNEST](/docs/query/sql/unnest/#json-unnest) to expand
+the array into individual rows, one per trade. Each request may return trades
+already seen in a previous request, so the target table enables
+[deduplication](/docs/concepts/deduplication/) to handle overlapping results
+safely:
+
+```questdb-sql title="Table with deduplication and transform"
+CREATE TABLE coinbase_trades (
+ timestamp TIMESTAMP,
+ symbol SYMBOL,
+ trade_id LONG,
+ price DOUBLE,
+ size DOUBLE,
+ side SYMBOL
+) TIMESTAMP(timestamp) PARTITION BY DAY WAL
+DEDUP UPSERT KEYS(timestamp, symbol, side);
+
+CREATE PAYLOAD TRANSFORM coinbase_trades_api
+INTO coinbase_trades
+DLQ dlq_errors PARTITION BY DAY TTL 7 DAYS
+AS DECLARE OVERRIDABLE @symbol := 'BTC-USD'
+SELECT
+ u.time AS timestamp,
+ @symbol AS symbol,
+ u.trade_id,
+ u.price,
+ u.size,
+ u.side
+FROM UNNEST(
+ payload() COLUMNS(
+ trade_id LONG,
+ price DOUBLE,
+ size DOUBLE,
+ side VARCHAR,
+ time TIMESTAMP
+ )
+) u;
+```
+
+Fetch the latest 100 trades and ingest them:
+
+```shell title="Ingest trades"
+curl -s "https://api.exchange.coinbase.com/products/BTC-USD/trades?limit=100" | \
+ curl -X POST "http://localhost:9000/ingest?transform=coinbase_trades_api" -d @-
+```
+
+If any trades were already ingested from a previous request, deduplication
+discards the duplicates automatically.
+
+### Inspecting failed payloads
+
+When a payload fails (bad JSON, type mismatch, missing columns), QuestDB writes
+the original payload, the error stage, and the error message to the DLQ table
+configured in the transform:
+
+```questdb-sql title="Query the DLQ"
+SELECT ts, transform_name, stage, error FROM dlq_errors;
+```
+
+| ts | transform_name | stage | error |
+| :--- | :--- | :--- | :--- |
+| 2026-03-23T14:00:00.000000Z | coinbase_book_api | transform | column not found in target table [column=extra] |
+| 2026-03-23T14:01:00.000000Z | coinbase_trades_api | transform | bad JSON payload |
+
+Multiple transforms can share the same DLQ table. See
+[CREATE PAYLOAD TRANSFORM](/docs/query/sql/create-payload-transform/#dead-letter-queue-schema)
+for the full DLQ schema.
+
+## HTTP endpoint
+
+**POST** `/ingest`
+
+### Query parameters
+
+| Parameter | Required | Description |
+| :--- | :--- | :--- |
+| `transform` | Yes | Name of the payload transform to execute |
+| Any other | No | Overrides a `DECLARE OVERRIDABLE` variable by name |
+
+The request body is the raw payload, accessible via
+[`payload()`](/docs/query/functions/meta/#payload) in the transform SQL.
+
+### Responses
+
+Success:
+
+```json
+{"status": "ok", "rows_inserted": 1}
+```
+
+Error:
+
+```json
+{"status": "error", "message": "..."}
+```
+
+## Permissions
+
+In QuestDB Open Source, any user with access to the HTTP endpoint can create
+transforms and invoke `/ingest`.
+
+:::note Enterprise
+
+In [QuestDB Enterprise](/enterprise/) deployments with
+[RBAC](/docs/security/rbac/) enabled, the following grants are required:
+
+| Action | Required grants |
+| :----- | :-------------- |
+| Create a transform | `CREATE PAYLOAD TRANSFORM` and `INSERT` on the target table (and DLQ table, if configured) |
+| Replace a transform (`OR REPLACE`) | `CREATE PAYLOAD TRANSFORM` and `DROP PAYLOAD TRANSFORM` |
+| Drop a transform | `DROP PAYLOAD TRANSFORM` |
+| Invoke `/ingest` | `HTTP` endpoint grant and `INSERT` on the target table |
+
+```questdb-sql title="Typical Enterprise setup"
+-- Admin who manages transforms
+GRANT CREATE PAYLOAD TRANSFORM, DROP PAYLOAD TRANSFORM TO ingest_admin;
+GRANT INSERT ON coinbase_order_book, coinbase_trades, dlq_errors TO ingest_admin;
+
+-- Service account that calls /ingest
+GRANT HTTP TO ingest_service;
+GRANT INSERT ON coinbase_order_book, coinbase_trades TO ingest_service;
+```
+
+:::
+
+## Request size limit
+
+The `/ingest` endpoint rejects request bodies that exceed a configurable maximum
+size. The default limit is 5 MB. To change it, set the
+`http.ingest.max.request.size` property in `server.conf`:
+
+```ini title="server.conf"
+http.ingest.max.request.size=10M
+```
+
+Requests exceeding the limit receive an HTTP 413 (Payload Too Large) response.
+The entire request body is held in memory during processing, so set this limit
+based on available memory and expected payload sizes.
+
+## Limitations
+
+- **Single payload per request** - Each HTTP request executes the transform
+ once. That execution may produce multiple rows. Sending multiple
+ independent payload documents in a single request is not supported.
+- **Per-request SQL compilation** - Transform SQL is compiled on every request.
+ This is acceptable for low-rate ingestion workloads. Compiled-plan caching is
+ a planned optimization.
+- **No table references** - The transform SELECT must not reference existing
+ tables. It can only use functions and expressions, including CTEs.
+- **SELECT only** - Only `SELECT` statements are allowed. `INSERT`, `UPDATE`,
+ and other statements are rejected at creation time.
+- **Schema drift** - Column names and types are validated against the target
+ table at creation time. Schema changes to the target table after creating a
+ transform may cause runtime errors.
+- **Concurrent DDL** - `CREATE`, `DROP`, and `OR REPLACE` for the same
+ transform name are not serialized. If two sessions operate on the same
+ transform name concurrently, the outcome is last-writer-wins.
+
+:::info Related documentation
+- [CREATE PAYLOAD TRANSFORM](/docs/query/sql/create-payload-transform/)
+- [DROP PAYLOAD TRANSFORM](/docs/query/sql/drop-payload-transform/)
+- [SHOW PAYLOAD TRANSFORMS](/docs/query/sql/show/#show-payload-transforms)
+- [UNNEST](/docs/query/sql/unnest/)
+- [`payload()` function](/docs/query/functions/meta/#payload)
+- [JSON functions](/docs/query/functions/json/)
+- [REST API](/docs/query/rest-api/)
+- [Role-Based Access Control (RBAC)](/docs/security/rbac/)
+:::
diff --git a/documentation/query/functions/meta.md b/documentation/query/functions/meta.md
index 832f56bb2..0989f8c2f 100644
--- a/documentation/query/functions/meta.md
+++ b/documentation/query/functions/meta.md
@@ -34,6 +34,44 @@ SELECT build();
| -------------------------------------------------------------------------------------------------- |
| Build Information: QuestDB 7.3.5, JDK 17.0.7, Commit Hash 460b817b0a3705c5633619a8ef9efb5163f1569c |
+## current database, schema, or user
+
+`current_database()`, `current_schema()`, and `current_user()` are standard SQL
+functions that return information about the current database, schema, schemas,
+and user, respectively.
+
+```questdb-sql
+-- Get the current database
+SELECT current_database();
+
+-- Get the current schema
+SELECT current_schema();
+
+-- Get the current user
+SELECT current_user();
+```
+
+Each of these functions returns a single value, so you can use them in a SELECT
+statement without any arguments.
+
+## flush_query_cache()
+
+`flush_query_cache' invalidates cached query execution plans.
+
+**Arguments:**
+
+- `flush_query_cache()` does not require arguments.
+
+**Return value:**
+
+Returns `boolean`. `true` if successful, `false` if unsuccessful.
+
+**Examples:**
+
+```questdb-sql title="Flush cached query execution plans"
+SELECT flush_query_cache();
+```
+
## functions
**Arguments:**
@@ -56,38 +94,90 @@ functions();
| and | and(TT) | and(boolean, boolean) | FALSE | STANDARD |
| not | not(T) | not(boolean) | FALSE | STANDARD |
-## query_activity
+## hydrate_table_metadata('table1', 'table2' ...)
+
+`hydrate_table_metadata' re-reads table metadata from disk to update the static
+metadata cache.
+
+:::warning
+
+This function should only be used when directed by QuestDB support. Misuse could
+cause corruption of the metadata cache, requiring the database to be restarted.
+
+:::
**Arguments:**
-- `query_activity()` does not require arguments.
+A variable list of strings, corresponding to table names.
+
+Alternatively, a single asterisk, '\*', representing all tables.
**Return value:**
-Returns metadata on running SQL queries, including columns such as:
+Returns `boolean`. `true` if successful, `false` if unsuccessful.
-- query_id - identifier of the query that can be used with
- [cancel query](/docs/query/sql/cancel-query) command or
- [cancelQuery()](/docs/query/sql/cancel-query) function
-- worker_id - identifier of worker thread that initiated query processing. Note
- that many multithreaded queries also run on other workers
-- worker_pool - name of worker pool used to execute the query
-- username - name of user executing the query
-- query_start - timestamp of when query started
-- state_change - timestamp of latest query state change, such as a cancellation
-- state - state of running query, can be `active` or `cancelled`
-- query - text of sql query
+**Examples:**
+
+Simply pass table names as arguments to the function.
+
+```
+SELECT hydrate_table_metadata('trades', 'trips');
+```
+
+| hydrate_table_metadata |
+| ---------------------- |
+| true |
+
+If you want to re-read metadata for all user tables, simply use an asterisk:
+
+```
+SELECT hydrate_table_metadata('*');
+```
+
+## materialized_views
+
+`materialized_views()` returns the list of all materialized views in the
+database.
+
+**Arguments:**
+
+- `materialized_views()` does not require arguments.
+
+**Return value:**
+
+Returns a `table` including the following information:
+
+- `view_name` - materialized view name
+- `refresh_type` - refresh strategy type
+- `base_table_name` - base table name
+- `last_refresh_start_timestamp` - last time when an incremental refresh for the
+ view was started
+- `last_refresh_finish_timestamp` - last time when an incremental refresh for
+ the view finished
+- `view_sql` - query used to populate view data
+- `view_table_dir_name` - view directory name
+- `invalidation_reason` - message explaining why the view was marked as invalid
+- `view_status` - view status: 'valid', 'refreshing', or 'invalid'
+- `refresh_base_table_txn` - the last base table transaction used to refresh the
+ materialized view
+- `base_table_txn` - the last committed transaction in the base table
+- `refresh_limit_value` - how many units back in time the refresh limit goes
+- `refresh_limit_unit` - how long each unit is
+- `timer_start` - start date for the scheduled refresh timer
+- `timer_interval_value` - how many interval units between each refresh
+- `timer_interval_unit` - how long each unit is
**Examples:**
-```questdb-sql
-SELECT * FROM query_activity();
+```questdb-sql title="List all materialized views"
+materialized_views();
```
-| query_id | worker_id | worker_pool | username | query_start | state_change | state | query |
-| -------- | --------- | ----------- | -------- | --------------------------- | --------------------------- | ------ | --------------------------------------------------------- |
-| 62179 | 5 | shared | bob | 2024-01-09T10:03:05.557397Z | 2024-01-09T10:03:05.557397 | active | select \* from query_activity() |
-| 57777 | 6 | shared | bob | 2024-01-09T08:58:55.988017Z | 2024-01-09T08:58:55.988017Z | active | SELECT symbol,approx_percentile(price, 50, 2) from trades |
+| view_name | refresh_type | base_table_name | last_refresh_start_timestamp | last_refresh_finish_timestamp | view_sql | view_table_dir_name | invalidation_reason | view_status | refresh_base_table_txn | base_table_txn | refresh_limit_value | refresh_limit_unit | timer_start | timer_interval_value | timer_interval_unit |
+|------------------|--------------|-----------------|------------------------------|-------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|---------------------|-------------|------------------------|----------------|---------------------|--------------------|-------------|----------------------|---------------------|
+| trades_OHLC_15m | immediate | trades | 2025-05-30T16:40:37.562421Z | 2025-05-30T16:40:37.568800Z | SELECT timestamp, symbol, first(price) AS open, max(price) as high, min(price) as low, last(price) AS close, sum(amount) AS volume FROM trades SAMPLE BY 15m | trades_OHLC_15m~27 | null | valid | 55141609 | 55141609 | 0 | null | null | 0 | null |
+| trades_latest_1d | immediate | trades | 2025-05-30T16:40:37.554274Z | 2025-05-30T16:40:37.562049Z | SELECT timestamp, symbol, side, last(price) AS price, last(amount) AS amount, last(timestamp) as latest FROM trades SAMPLE BY 1d | trades_latest_1d~28 | null | valid | 55141609 | 55141609 | 0 | null | null | 0 | null |
+
## memory_metrics
@@ -114,6 +204,77 @@ memory_metrics();
| MMAP_O3 | 0 |
| NATIVE_O3 | 96 |
+## payload
+
+`payload()` returns the raw HTTP request body as a `VARCHAR`. It is only
+available inside
+[payload transform](/docs/ingestion/payload-transforms/) queries - calling it in
+any other context produces an error.
+
+**Arguments:**
+
+- `payload()` does not require arguments.
+
+**Return value:**
+
+Returns the HTTP request body as a `VARCHAR` value.
+
+**Examples:**
+
+Typically used with `json_extract()` to parse JSON payloads:
+
+```questdb-sql title="Extract fields from a JSON payload"
+CREATE PAYLOAD TRANSFORM sensor_ingest
+INTO sensor_data
+AS SELECT
+ now() AS ts,
+ json_extract(payload(), '$.device_id')::VARCHAR AS device_id,
+ json_extract(payload(), '$.temperature')::DOUBLE AS temperature;
+```
+
+Can also be stored directly as a raw string:
+
+```questdb-sql title="Store the raw payload"
+CREATE PAYLOAD TRANSFORM raw_events
+INTO event_log
+AS SELECT
+ now() AS ts,
+ payload() AS raw_body;
+```
+
+## query_activity
+
+**Arguments:**
+
+- `query_activity()` does not require arguments.
+
+**Return value:**
+
+Returns metadata on running SQL queries, including columns such as:
+
+- query_id - identifier of the query that can be used with
+ [cancel query](/docs/query/sql/cancel-query) command or
+ [cancelQuery()](/docs/query/sql/cancel-query) function
+- worker_id - identifier of worker thread that initiated query processing. Note
+ that many multithreaded queries also run on other workers
+- worker_pool - name of worker pool used to execute the query
+- username - name of user executing the query
+- query_start - timestamp of when query started
+- state_change - timestamp of latest query state change, such as a cancellation
+- state - state of running query, can be `active` or `cancelled`
+- query - text of sql query
+
+**Examples:**
+
+```questdb-sql
+SELECT * FROM query_activity();
+```
+
+| query_id | worker_id | worker_pool | username | query_start | state_change | state | query |
+| -------- | --------- | ----------- | -------- | --------------------------- | --------------------------- | ------ | --------------------------------------------------------- |
+| 62179 | 5 | shared | bob | 2024-01-09T10:03:05.557397Z | 2024-01-09T10:03:05.557397 | active | select \* from query_activity() |
+| 57777 | 6 | shared | bob | 2024-01-09T08:58:55.988017Z | 2024-01-09T08:58:55.988017Z | active | SELECT symbol,approx_percentile(price, 50, 2) from trades |
+
## reader_pool
**Arguments:**
@@ -139,78 +300,269 @@ SELECT * FROM reader_pool();
| ---------- | --------------- | --------------------------- | ----------- |
| sensors | null | 2023-12-01T19:28:14.311703Z | 1 |
-## writer_pool
+## reload_config()
+
+`reload_config' reloads server configuration file's contents (`server.conf`)
+without server restart. The list of reloadable settings can be found
+[here](/docs/configuration/overview/#reloadable-settings).
**Arguments:**
-- `writer_pool()` does not require arguments.
+- `reload_config()` does not require arguments.
**Return value:**
-Returns information about the current state of the writer pool in QuestDB. The
-writer pool is a cache of table writers that are kept open to speed up
-subsequent writes to the same table. The returned information includes the table
-name, the ID of the thread that currently owns the writer, the timestamp of the
-last time the writer was accessed, and the reason for the ownership.
+Returns `boolean`. `true` if any configuration properties were reloaded, `false`
+if none were reloaded.
**Examples:**
-```questdb-sql
-SELECT * FROM writer_pool();
-```
-
-| table_name | owner_thread_id | last_access_timestamp | ownership_reason |
-| ----------------------------- | --------------- | --------------------------- | ---------------- |
-| sys.column_versions_purge_log | 1 | 2023-12-01T18:50:03.412468Z | QuestDB system |
-| telemetry_config | 1 | 2023-12-01T18:50:03.470604Z | telemetryConfig |
-| telemetry | 1 | 2023-12-01T18:50:03.464501Z | telemetry |
-| sys.telemetry_wal | 1 | 2023-12-01T18:50:03.467924Z | telemetry |
-| example_table | null | 2023-12-01T20:33:33.270984Z | null |
-
-## current database, schema, or user
-
-`current_database()`, `current_schema()`, and `current_user()` are standard SQL
-functions that return information about the current database, schema, schemas,
-and user, respectively.
-
-```questdb-sql
--- Get the current database
-SELECT current_database();
-
--- Get the current schema
-SELECT current_schema();
+Edit `server.conf` and run `reload_config`:
--- Get the current user
-SELECT current_user();
+```questdb-sql title="Reload server configuration"
+SELECT reload_config();
```
-Each of these functions returns a single value, so you can use them in a SELECT
-statement without any arguments.
-
-## tables
+## table_columns
-`tables()` returns metadata and real-time statistics for all tables in the
-database, including write activity, WAL status, and performance metrics.
+`table_columns('tableName')` returns the schema of a table or a materialized
+view.
**Arguments:**
-- `tables()` does not require arguments.
+- `tableName` is the name of an existing table or materialized view as a string.
**Return value:**
Returns a `table` with the following columns:
-### Basic table information
+- `column` - name of the available columns in the table
+- `type` - type of the column
+- `indexed` - if indexing is applied to this column
+- `indexBlockCapacity` - how many row IDs to store in a single storage block on
+ disk
+- `symbolCached` - whether this `symbol` column is cached
+- `symbolCapacity` - how many distinct values this column of `symbol` type is
+ expected to have
+- `designated` - if this is set as the designated timestamp column for this
+ table
+- `upsertKey` - if this column is a part of UPSERT KEYS list for table
+ [deduplication](/docs/concepts/deduplication)
-| Column | Type | Description |
-|--------|------|-------------|
-| `id` | INT | Internal table ID |
-| `table_name` | STRING | Table name |
-| `designatedTimestamp` | STRING | Name of the designated timestamp column, or `null` |
-| `partitionBy` | STRING | Partition strategy: `NONE`, `HOUR`, `DAY`, `WEEK`, `MONTH`, `YEAR` |
-| `walEnabled` | BOOLEAN | Whether WAL (Write-Ahead Log) is enabled |
-| `dedup` | BOOLEAN | Whether deduplication is enabled |
-| `ttlValue` | INT | TTL (Time-To-Live) value |
+For more details on the meaning and use of these values, see the
+[CREATE TABLE](/docs/query/sql/create-table/) documentation.
+
+**Examples:**
+
+```questdb-sql title="Get all columns in a table"
+table_columns('my_table');
+```
+
+| column | type | indexed | indexBlockCapacity | symbolCached | symbolCapacity | designated | upsertKey |
+| ------ | --------- | ------- | ------------------ | ------------ | -------------- | ---------- | --------- |
+| symb | SYMBOL | true | 1048576 | false | 256 | false | false |
+| price | DOUBLE | false | 0 | false | 0 | false | false |
+| ts | TIMESTAMP | false | 0 | false | 0 | true | false |
+| s | VARCHAR | false | 0 | false | 0 | false | false |
+
+```questdb-sql title="Get designated timestamp column"
+SELECT "column", type, designated FROM table_columns('my_table') WHERE designated = true;
+```
+
+| column | type | designated |
+| ------ | --------- | ---------- |
+| ts | TIMESTAMP | true |
+
+```questdb-sql title="Get the count of column types"
+SELECT type, count() FROM table_columns('my_table');
+```
+
+| type | count |
+| --------- | ----- |
+| SYMBOL | 1 |
+| DOUBLE | 1 |
+| TIMESTAMP | 1 |
+| VARCHAR | 1 |
+
+## table_partitions
+
+`table_partitions('tableName')` returns information for the partitions of a
+table or a materialized view with the option to filter the partitions.
+
+**Arguments:**
+
+- `tableName` is the name of an existing table or materialized view as a string.
+
+**Return value:**
+
+Returns a table with the following columns:
+
+- `index` - _INTEGER_, index of the partition (_NaN_ when the partition is not
+ attached)
+- `partitionBy` - _STRING_, one of _NONE_, _HOUR_, _DAY_, _WEEK_, _MONTH_ and
+ _YEAR_
+- `name` - _STRING_, name of the partition, e.g. `2023-03-14`,
+ `2023-03-14.detached`, `2023-03-14.attachable`
+- `minTimestamp` - _LONG_, min timestamp of the partition (_NaN_ when the table
+ is not partitioned)
+- `maxTimestamp` - _LONG_, max timestamp of the partition (_NaN_ when the table
+ is not partitioned)
+- `numRows` - _LONG_, number of rows in the partition
+- `diskSize` - _LONG_, size of the partition in bytes
+- `diskSizeHuman` - _STRING_, size of the partition meant for humans to read
+ (same output as function
+ [size_pretty](/docs/query/functions/numeric/#size_pretty))
+- `readOnly` - _BOOLEAN_, true if the partition is
+ [attached via soft link](/docs/query/sql/alter-table-attach-partition/#symbolic-links)
+- `active` - _BOOLEAN_, true if the partition is the last partition, and whether
+ we are writing to it (at least one record)
+- `attached` - _BOOLEAN_, true if the partition is
+ [attached](/docs/query/sql/alter-table-attach-partition/)
+- `detached` - _BOOLEAN_, true if the partition is
+ [detached](/docs/query/sql/alter-table-detach-partition/) (`name` of the
+ partition will contain the `.detached` extension)
+- `attachable` - _BOOLEAN_, true if the partition is detached and can be
+ attached (`name` of the partition will contain the `.attachable` extension)
+
+**Examples:**
+
+```questdb-sql title="Create table my_table"
+CREATE TABLE my_table AS (
+ SELECT
+ rnd_symbol('EURO', 'USD', 'OTHER') symbol,
+ rnd_double() * 50.0 price,
+ rnd_double() * 20.0 amount,
+ to_timestamp('2023-01-01', 'yyyy-MM-dd') + x * 6 * 3600 * 100000L timestamp
+ FROM long_sequence(700)
+), INDEX(symbol capacity 32) TIMESTAMP(timestamp) PARTITION BY WEEK;
+```
+
+```questdb-sql title="Get all partitions from my_table"
+table_partitions('my_table');
+```
+
+| index | partitionBy | name | minTimestamp | maxTimestamp | numRows | diskSize | diskSizeHuman | readOnly | active | attached | detached | attachable |
+| ----- | ----------- | -------- | --------------------- | --------------------- | ------- | -------- | ------------- | -------- | ------ | -------- | -------- | ---------- |
+| 0 | WEEK | 2022-W52 | 2023-01-01 00:36:00.0 | 2023-01-01 23:24:00.0 | 39 | 98304 | 96.0 KiB | false | false | true | false | false |
+| 1 | WEEK | 2023-W01 | 2023-01-02 00:00:00.0 | 2023-01-08 23:24:00.0 | 280 | 98304 | 96.0 KiB | false | false | true | false | false |
+| 2 | WEEK | 2023-W02 | 2023-01-09 00:00:00.0 | 2023-01-15 23:24:00.0 | 280 | 98304 | 96.0 KiB | false | false | true | false | false |
+| 3 | WEEK | 2023-W03 | 2023-01-16 00:00:00.0 | 2023-01-18 12:00:00.0 | 101 | 83902464 | 80.0 MiB | false | true | true | false | false |
+
+```questdb-sql title="Get size of a table in disk"
+SELECT size_pretty(sum(diskSize)) FROM table_partitions('my_table');
+```
+
+| size_pretty |
+| ----------- |
+| 80.3 MB |
+
+```questdb-sql title="Get active partition of a table"
+SELECT * FROM table_partitions('my_table') WHERE active = true;
+```
+
+| index | partitionBy | name | minTimestamp | maxTimestamp | numRows | diskSize | diskSizeHuman | readOnly | active | attached | detached | attachable |
+| ----- | ----------- | -------- | --------------------- | --------------------- | ------- | -------- | ------------- | -------- | ------ | -------- | -------- | ---------- |
+| 3 | WEEK | 2023-W03 | 2023-01-16 00:00:00.0 | 2023-01-18 12:00:00.0 | 101 | 83902464 | 80.0 MiB | false | true | true | false | false |
+
+## table_storage
+
+`table_storage()` - Returns information about the storage and structure of all
+user tables and materialized views in the database.
+
+Provides detailed storage information about all user tables and materialized
+views within QuestDB. It returns one row per table, including information about
+partitioning, row counts, and disk usage.
+
+- The `table_storage()` function excludes system tables; it only lists
+ user-created tables.
+- The `diskSize` value represents the total size of all files associated with
+ the table on disk, including data, index, and metadata files.
+- The `partitionBy` column indicates the partitioning strategy used for the
+ table. It can be `NONE` if the table is not partitioned.
+
+**Return values:**
+
+The function returns the following columns:
+
+- `tableName` (`string`): The name of the table or materialized view.
+- `walEnabled` (`boolean`): Indicates whether Write-Ahead Logging (WAL) is
+ enabled for the table.
+- `partitionBy` (`string`): The partitioning type of the table (e.g., NONE, DAY,
+ MONTH, YEAR, etc.).
+- `partitionCount` (`long`): The number of partitions the table has.
+- `rowCount` (`long`): The total number of rows in the table.
+- `diskSize` (`long`): The total disk space used by the table, in bytes.
+
+**Examples:**
+
+Retrieve storage information for all tables.
+
+```questdb-sql title="Checking our demo tables" demo
+SELECT * FROM table_storage();
+```
+
+- The query retrieves storage details for all tables in the database.
+- The `diskSize` column shows the total disk space used by each table in bytes.
+
+| tableName | walEnabled | partitionBy | partitionCount | rowCount | diskSize |
+| -------------- | ---------- | ----------- | -------------- | ---------- | ------------ |
+| trips | true | MONTH | 126 | 1634599313 | 261536158948 |
+| AAPL_orderbook | true | HOUR | 16 | 3024878 | 2149403527 |
+| weather | false | NONE | 1 | 137627 | 9972598 |
+| trades | true | DAY | 954 | 1000848308 | 32764798760 |
+| ethblocks_json | true | DAY | 3328 | 20688364 | 28311960478 |
+
+
+
+Filter tables with WAL enabled.
+
+```questdb-sql title="WAL only tables" demo
+SELECT tableName, rowCount, diskSize
+FROM table_storage()
+WHERE walEnabled = true;
+```
+
+| tableName | rowCount | diskSize |
+| -------------- | ---------- | ------------ |
+| trips | 1634599313 | 261536158948 |
+| AAPL_orderbook | 3024878 | 2149403527 |
+| trades | 1000850255 | 32764804264 |
+| ethblocks_json | 20688364 | 28311960478 |
+
+
+
+Show tables partitioned by `HOUR`.
+
+```questdb-sql title="Show tables partitioned by hour" demo
+SELECT tableName, partitionCount, rowCount
+FROM table_storage()
+WHERE partitionBy = 'HOUR';
+```
+
+## tables
+
+`tables()` returns metadata and real-time statistics for all tables in the
+database, including write activity, WAL status, and performance metrics.
+
+**Arguments:**
+
+- `tables()` does not require arguments.
+
+**Return value:**
+
+Returns a `table` with the following columns:
+
+### Basic table information
+
+| Column | Type | Description |
+|--------|------|-------------|
+| `id` | INT | Internal table ID |
+| `table_name` | STRING | Table name |
+| `designatedTimestamp` | STRING | Name of the designated timestamp column, or `null` |
+| `partitionBy` | STRING | Partition strategy: `NONE`, `HOUR`, `DAY`, `WEEK`, `MONTH`, `YEAR` |
+| `walEnabled` | BOOLEAN | Whether WAL (Write-Ahead Log) is enabled |
+| `dedup` | BOOLEAN | Whether deduplication is enabled |
+| `ttlValue` | INT | TTL (Time-To-Live) value |
| `ttlUnit` | STRING | TTL unit: `HOUR`, `DAY`, `WEEK`, `MONTH`, `YEAR` |
| `matView` | BOOLEAN | Whether this is a materialized view |
| `directoryName` | STRING | Directory name on disk (includes ` (->)` suffix for symlinks) |
@@ -457,310 +809,32 @@ ORDER BY
wal_pending_row_count DESC;
```
-## table_storage
-
-`table_storage()` - Returns information about the storage and structure of all
-user tables and materialized views in the database.
+## version/pg_catalog.version
-Provides detailed storage information about all user tables and materialized
-views within QuestDB. It returns one row per table, including information about
-partitioning, row counts, and disk usage.
+`version()` or `pg_catalog.version()` returns the supported version of the
+PostgreSQL Wire Protocol.
-- The `table_storage()` function excludes system tables; it only lists
- user-created tables.
-- The `diskSize` value represents the total size of all files associated with
- the table on disk, including data, index, and metadata files.
-- The `partitionBy` column indicates the partitioning strategy used for the
- table. It can be `NONE` if the table is not partitioned.
+**Arguments:**
-**Return values:**
+- `version()` or `pg_catalog.version()` does not require arguments.
-The function returns the following columns:
+**Return value:**
-- `tableName` (`string`): The name of the table or materialized view.
-- `walEnabled` (`boolean`): Indicates whether Write-Ahead Logging (WAL) is
- enabled for the table.
-- `partitionBy` (`string`): The partitioning type of the table (e.g., NONE, DAY,
- MONTH, YEAR, etc.).
-- `partitionCount` (`long`): The number of partitions the table has.
-- `rowCount` (`long`): The total number of rows in the table.
-- `diskSize` (`long`): The total disk space used by the table, in bytes.
+Returns `string`.
**Examples:**
-Retrieve storage information for all tables.
+```questdb-sql
+SELECT version();
-```questdb-sql title="Checking our demo tables" demo
-SELECT * FROM table_storage();
-```
+--The above equals to:
-- The query retrieves storage details for all tables in the database.
-- The `diskSize` column shows the total disk space used by each table in bytes.
-
-| tableName | walEnabled | partitionBy | partitionCount | rowCount | diskSize |
-| -------------- | ---------- | ----------- | -------------- | ---------- | ------------ |
-| trips | true | MONTH | 126 | 1634599313 | 261536158948 |
-| AAPL_orderbook | true | HOUR | 16 | 3024878 | 2149403527 |
-| weather | false | NONE | 1 | 137627 | 9972598 |
-| trades | true | DAY | 954 | 1000848308 | 32764798760 |
-| ethblocks_json | true | DAY | 3328 | 20688364 | 28311960478 |
-
-
-
-Filter tables with WAL enabled.
-
-```questdb-sql title="WAL only tables" demo
-SELECT tableName, rowCount, diskSize
-FROM table_storage()
-WHERE walEnabled = true;
-```
-
-| tableName | rowCount | diskSize |
-| -------------- | ---------- | ------------ |
-| trips | 1634599313 | 261536158948 |
-| AAPL_orderbook | 3024878 | 2149403527 |
-| trades | 1000850255 | 32764804264 |
-| ethblocks_json | 20688364 | 28311960478 |
-
-
-
-Show tables partitioned by `HOUR`.
-
-```questdb-sql title="Show tables partitioned by hour" demo
-SELECT tableName, partitionCount, rowCount
-FROM table_storage()
-WHERE partitionBy = 'HOUR';
-```
-
-## wal_tables
-
-:::note
-
-For monitoring and observability, use [`tables()`](#tables) instead.
-`tables()` provides all the same information plus additional metrics
-(pending rows, memory pressure, deduplication stats, throughput histograms),
-and is fully in-memory. `wal_tables()` reads from disk and is less suitable
-for frequent polling.
-
-:::
-
-`wal_tables()` returns the WAL status for all
-[WAL tables](/docs/concepts/write-ahead-log/) or materialized views in the
-database.
-
-**Arguments:**
-
-- `wal_tables()` does not require arguments.
-
-**Return value:**
-
-Returns a `table` including the following information:
-
-- `name` - table or materialized view name
-- `suspended` - suspended status flag
-- `writerTxn` - the last committed transaction in TableWriter (equivalent to `table_txn` in `tables()`)
-- `writerLagTxnCount` - the number of transactions that are kept invisible when
- writing to the table; these transactions will be eventually moved to the table
- data and become visible for readers (equivalent to `wal_txn - table_txn`)
-- `sequencerTxn` - the last committed transaction in the sequencer (equivalent to `wal_txn` in `tables()`)
-
-**Examples:**
-
-```questdb-sql title="List all tables"
-wal_tables();
-```
-
-| name | suspended | writerTxn | writerLagTxnCount | sequencerTxn |
-| ----------- | --------- | --------- | ----------------- | ------------ |
-| sensor_wal | false | 2 | 1 | 4 |
-| weather_wal | false | 3 | 0 | 3 |
-| test_wal | true | 7 | 1 | 9 |
-
-## table_columns
-
-`table_columns('tableName')` returns the schema of a table or a materialized
-view.
-
-**Arguments:**
-
-- `tableName` is the name of an existing table or materialized view as a string.
-
-**Return value:**
-
-Returns a `table` with the following columns:
-
-- `column` - name of the available columns in the table
-- `type` - type of the column
-- `indexed` - if indexing is applied to this column
-- `indexBlockCapacity` - how many row IDs to store in a single storage block on
- disk
-- `symbolCached` - whether this `symbol` column is cached
-- `symbolCapacity` - how many distinct values this column of `symbol` type is
- expected to have
-- `designated` - if this is set as the designated timestamp column for this
- table
-- `upsertKey` - if this column is a part of UPSERT KEYS list for table
- [deduplication](/docs/concepts/deduplication)
-
-For more details on the meaning and use of these values, see the
-[CREATE TABLE](/docs/query/sql/create-table/) documentation.
-
-**Examples:**
-
-```questdb-sql title="Get all columns in a table"
-table_columns('my_table');
-```
-
-| column | type | indexed | indexBlockCapacity | symbolCached | symbolCapacity | designated | upsertKey |
-| ------ | --------- | ------- | ------------------ | ------------ | -------------- | ---------- | --------- |
-| symb | SYMBOL | true | 1048576 | false | 256 | false | false |
-| price | DOUBLE | false | 0 | false | 0 | false | false |
-| ts | TIMESTAMP | false | 0 | false | 0 | true | false |
-| s | VARCHAR | false | 0 | false | 0 | false | false |
-
-```questdb-sql title="Get designated timestamp column"
-SELECT "column", type, designated FROM table_columns('my_table') WHERE designated = true;
-```
-
-| column | type | designated |
-| ------ | --------- | ---------- |
-| ts | TIMESTAMP | true |
-
-```questdb-sql title="Get the count of column types"
-SELECT type, count() FROM table_columns('my_table');
-```
-
-| type | count |
-| --------- | ----- |
-| SYMBOL | 1 |
-| DOUBLE | 1 |
-| TIMESTAMP | 1 |
-| VARCHAR | 1 |
-
-## table_partitions
-
-`table_partitions('tableName')` returns information for the partitions of a
-table or a materialized view with the option to filter the partitions.
-
-**Arguments:**
-
-- `tableName` is the name of an existing table or materialized view as a string.
-
-**Return value:**
-
-Returns a table with the following columns:
-
-- `index` - _INTEGER_, index of the partition (_NaN_ when the partition is not
- attached)
-- `partitionBy` - _STRING_, one of _NONE_, _HOUR_, _DAY_, _WEEK_, _MONTH_ and
- _YEAR_
-- `name` - _STRING_, name of the partition, e.g. `2023-03-14`,
- `2023-03-14.detached`, `2023-03-14.attachable`
-- `minTimestamp` - _LONG_, min timestamp of the partition (_NaN_ when the table
- is not partitioned)
-- `maxTimestamp` - _LONG_, max timestamp of the partition (_NaN_ when the table
- is not partitioned)
-- `numRows` - _LONG_, number of rows in the partition
-- `diskSize` - _LONG_, size of the partition in bytes
-- `diskSizeHuman` - _STRING_, size of the partition meant for humans to read
- (same output as function
- [size_pretty](/docs/query/functions/numeric/#size_pretty))
-- `readOnly` - _BOOLEAN_, true if the partition is
- [attached via soft link](/docs/query/sql/alter-table-attach-partition/#symbolic-links)
-- `active` - _BOOLEAN_, true if the partition is the last partition, and whether
- we are writing to it (at least one record)
-- `attached` - _BOOLEAN_, true if the partition is
- [attached](/docs/query/sql/alter-table-attach-partition/)
-- `detached` - _BOOLEAN_, true if the partition is
- [detached](/docs/query/sql/alter-table-detach-partition/) (`name` of the
- partition will contain the `.detached` extension)
-- `attachable` - _BOOLEAN_, true if the partition is detached and can be
- attached (`name` of the partition will contain the `.attachable` extension)
-
-**Examples:**
-
-```questdb-sql title="Create table my_table"
-CREATE TABLE my_table AS (
- SELECT
- rnd_symbol('EURO', 'USD', 'OTHER') symbol,
- rnd_double() * 50.0 price,
- rnd_double() * 20.0 amount,
- to_timestamp('2023-01-01', 'yyyy-MM-dd') + x * 6 * 3600 * 100000L timestamp
- FROM long_sequence(700)
-), INDEX(symbol capacity 32) TIMESTAMP(timestamp) PARTITION BY WEEK;
-```
-
-```questdb-sql title="Get all partitions from my_table"
-table_partitions('my_table');
-```
-
-| index | partitionBy | name | minTimestamp | maxTimestamp | numRows | diskSize | diskSizeHuman | readOnly | active | attached | detached | attachable |
-| ----- | ----------- | -------- | --------------------- | --------------------- | ------- | -------- | ------------- | -------- | ------ | -------- | -------- | ---------- |
-| 0 | WEEK | 2022-W52 | 2023-01-01 00:36:00.0 | 2023-01-01 23:24:00.0 | 39 | 98304 | 96.0 KiB | false | false | true | false | false |
-| 1 | WEEK | 2023-W01 | 2023-01-02 00:00:00.0 | 2023-01-08 23:24:00.0 | 280 | 98304 | 96.0 KiB | false | false | true | false | false |
-| 2 | WEEK | 2023-W02 | 2023-01-09 00:00:00.0 | 2023-01-15 23:24:00.0 | 280 | 98304 | 96.0 KiB | false | false | true | false | false |
-| 3 | WEEK | 2023-W03 | 2023-01-16 00:00:00.0 | 2023-01-18 12:00:00.0 | 101 | 83902464 | 80.0 MiB | false | true | true | false | false |
-
-```questdb-sql title="Get size of a table in disk"
-SELECT size_pretty(sum(diskSize)) FROM table_partitions('my_table');
-```
-
-| size_pretty |
-| ----------- |
-| 80.3 MB |
-
-```questdb-sql title="Get active partition of a table"
-SELECT * FROM table_partitions('my_table') WHERE active = true;
-```
-
-| index | partitionBy | name | minTimestamp | maxTimestamp | numRows | diskSize | diskSizeHuman | readOnly | active | attached | detached | attachable |
-| ----- | ----------- | -------- | --------------------- | --------------------- | ------- | -------- | ------------- | -------- | ------ | -------- | -------- | ---------- |
-| 3 | WEEK | 2023-W03 | 2023-01-16 00:00:00.0 | 2023-01-18 12:00:00.0 | 101 | 83902464 | 80.0 MiB | false | true | true | false | false |
-
-## materialized_views
-
-`materialized_views()` returns the list of all materialized views in the
-database.
-
-**Arguments:**
-
-- `materialized_views()` does not require arguments.
-
-**Return value:**
-
-Returns a `table` including the following information:
-
-- `view_name` - materialized view name
-- `refresh_type` - refresh strategy type
-- `base_table_name` - base table name
-- `last_refresh_start_timestamp` - last time when an incremental refresh for the
- view was started
-- `last_refresh_finish_timestamp` - last time when an incremental refresh for
- the view finished
-- `view_sql` - query used to populate view data
-- `view_table_dir_name` - view directory name
-- `invalidation_reason` - message explaining why the view was marked as invalid
-- `view_status` - view status: 'valid', 'refreshing', or 'invalid'
-- `refresh_base_table_txn` - the last base table transaction used to refresh the
- materialized view
-- `base_table_txn` - the last committed transaction in the base table
-- `refresh_limit_value` - how many units back in time the refresh limit goes
-- `refresh_limit_unit` - how long each unit is
-- `timer_start` - start date for the scheduled refresh timer
-- `timer_interval_value` - how many interval units between each refresh
-- `timer_interval_unit` - how long each unit is
-
-**Examples:**
-
-```questdb-sql title="List all materialized views"
-materialized_views();
-```
-
-| view_name | refresh_type | base_table_name | last_refresh_start_timestamp | last_refresh_finish_timestamp | view_sql | view_table_dir_name | invalidation_reason | view_status | refresh_base_table_txn | base_table_txn | refresh_limit_value | refresh_limit_unit | timer_start | timer_interval_value | timer_interval_unit |
-|------------------|--------------|-----------------|------------------------------|-------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|---------------------|-------------|------------------------|----------------|---------------------|--------------------|-------------|----------------------|---------------------|
-| trades_OHLC_15m | immediate | trades | 2025-05-30T16:40:37.562421Z | 2025-05-30T16:40:37.568800Z | SELECT timestamp, symbol, first(price) AS open, max(price) as high, min(price) as low, last(price) AS close, sum(amount) AS volume FROM trades SAMPLE BY 15m | trades_OHLC_15m~27 | null | valid | 55141609 | 55141609 | 0 | null | null | 0 | null |
-| trades_latest_1d | immediate | trades | 2025-05-30T16:40:37.554274Z | 2025-05-30T16:40:37.562049Z | SELECT timestamp, symbol, side, last(price) AS price, last(amount) AS amount, last(timestamp) as latest FROM trades SAMPLE BY 1d | trades_latest_1d~28 | null | valid | 55141609 | 55141609 | 0 | null | null | 0 | null |
+SELECT pg_catalog.version();
+```
+| version |
+| ------------------------------------------------------------------- |
+| PostgreSQL 12.3, compiled by Visual C++ build 1914, 64-bit, QuestDB |
## views
@@ -803,110 +877,74 @@ WHERE view_status = 'invalid';
SELECT * FROM views() ORDER BY view_name;
```
-## version/pg_catalog.version
-
-`version()` or `pg_catalog.version()` returns the supported version of the
-PostgreSQL Wire Protocol.
-
-**Arguments:**
-
-- `version()` or `pg_catalog.version()` does not require arguments.
-
-**Return value:**
-
-Returns `string`.
-
-**Examples:**
-
-```questdb-sql
-SELECT version();
-
---The above equals to:
-
-SELECT pg_catalog.version();
-```
-
-| version |
-| ------------------------------------------------------------------- |
-| PostgreSQL 12.3, compiled by Visual C++ build 1914, 64-bit, QuestDB |
-
-## hydrate_table_metadata('table1', 'table2' ...)
-
-`hydrate_table_metadata' re-reads table metadata from disk to update the static
-metadata cache.
+## wal_tables
-:::warning
+:::note
-This function should only be used when directed by QuestDB support. Misuse could
-cause corruption of the metadata cache, requiring the database to be restarted.
+For monitoring and observability, use [`tables()`](#tables) instead.
+`tables()` provides all the same information plus additional metrics
+(pending rows, memory pressure, deduplication stats, throughput histograms),
+and is fully in-memory. `wal_tables()` reads from disk and is less suitable
+for frequent polling.
:::
-**Arguments:**
-
-A variable list of strings, corresponding to table names.
-
-Alternatively, a single asterisk, '\*', representing all tables.
-
-**Return value:**
-
-Returns `boolean`. `true` if successful, `false` if unsuccessful.
-
-**Examples:**
-
-Simply pass table names as arguments to the function.
-
-```
-SELECT hydrate_table_metadata('trades', 'trips');
-```
-
-| hydrate_table_metadata |
-| ---------------------- |
-| true |
-
-If you want to re-read metadata for all user tables, simply use an asterisk:
-
-```
-SELECT hydrate_table_metadata('*');
-```
-
-## flush_query_cache()
-
-`flush_query_cache' invalidates cached query execution plans.
+`wal_tables()` returns the WAL status for all
+[WAL tables](/docs/concepts/write-ahead-log/) or materialized views in the
+database.
**Arguments:**
-- `flush_query_cache()` does not require arguments.
+- `wal_tables()` does not require arguments.
**Return value:**
-Returns `boolean`. `true` if successful, `false` if unsuccessful.
+Returns a `table` including the following information:
+
+- `name` - table or materialized view name
+- `suspended` - suspended status flag
+- `writerTxn` - the last committed transaction in TableWriter (equivalent to `table_txn` in `tables()`)
+- `writerLagTxnCount` - the number of transactions that are kept invisible when
+ writing to the table; these transactions will be eventually moved to the table
+ data and become visible for readers (equivalent to `wal_txn - table_txn`)
+- `sequencerTxn` - the last committed transaction in the sequencer (equivalent to `wal_txn` in `tables()`)
**Examples:**
-```questdb-sql title="Flush cached query execution plans"
-SELECT flush_query_cache();
+```questdb-sql title="List all tables"
+wal_tables();
```
-## reload_config()
+| name | suspended | writerTxn | writerLagTxnCount | sequencerTxn |
+| ----------- | --------- | --------- | ----------------- | ------------ |
+| sensor_wal | false | 2 | 1 | 4 |
+| weather_wal | false | 3 | 0 | 3 |
+| test_wal | true | 7 | 1 | 9 |
-`reload_config' reloads server configuration file's contents (`server.conf`)
-without server restart. The list of reloadable settings can be found
-[here](/docs/configuration/overview/#reloadable-settings).
+## writer_pool
**Arguments:**
-- `reload_config()` does not require arguments.
+- `writer_pool()` does not require arguments.
**Return value:**
-Returns `boolean`. `true` if any configuration properties were reloaded, `false`
-if none were reloaded.
+Returns information about the current state of the writer pool in QuestDB. The
+writer pool is a cache of table writers that are kept open to speed up
+subsequent writes to the same table. The returned information includes the table
+name, the ID of the thread that currently owns the writer, the timestamp of the
+last time the writer was accessed, and the reason for the ownership.
**Examples:**
-Edit `server.conf` and run `reload_config`:
-
-```questdb-sql title="Reload server configuration"
-SELECT reload_config();
+```questdb-sql
+SELECT * FROM writer_pool();
```
+
+| table_name | owner_thread_id | last_access_timestamp | ownership_reason |
+| ----------------------------- | --------------- | --------------------------- | ---------------- |
+| sys.column_versions_purge_log | 1 | 2023-12-01T18:50:03.412468Z | QuestDB system |
+| telemetry_config | 1 | 2023-12-01T18:50:03.470604Z | telemetryConfig |
+| telemetry | 1 | 2023-12-01T18:50:03.464501Z | telemetry |
+| sys.telemetry_wal | 1 | 2023-12-01T18:50:03.467924Z | telemetry |
+| example_table | null | 2023-12-01T20:33:33.270984Z | null |
diff --git a/documentation/query/rest-api.md b/documentation/query/rest-api.md
index 5ba3d3b5c..6c2e2a1be 100644
--- a/documentation/query/rest-api.md
+++ b/documentation/query/rest-api.md
@@ -27,6 +27,7 @@ The [Web Console](/docs/getting-started/web-console/overview/) is the official W
- [`/imp`](#imp---import-data) for importing data from `.CSV` files
- [`/exec`](#exec---execute-queries) to execute a SQL statement
- [`/exp`](#exp---export-data) to export data
+- [`/ingest`](#ingest---payload-transforms) to ingest data via payload transforms
## Examples
@@ -38,6 +39,7 @@ insert-capable entrypoints:
| :----------------------------------------- | :---------- | :-------------------------------------- | :-------------------------------------------------------- |
| [`/imp`](#imp-uploading-tabular-data) | POST | Import CSV data | [Reference](/docs/query/rest-api/#imp---import-data) |
| [`/exec?query=..`](#exec-sql-insert-query) | GET | Run SQL Query returning JSON result set | [Reference](/docs/query/rest-api/#exec---execute-queries) |
+| [`/ingest?transform=..`](#ingest---payload-transforms) | POST | Execute a payload transform | [Reference](/docs/query/rest-api/#ingest---payload-transforms) |
For details such as content type, query parameters and more, refer to the
[REST API](/docs/query/rest-api/) docs.
@@ -696,6 +698,61 @@ curl -G \
http://localhost:9000/exp > trades_bloom.parquet
```
+## /ingest - Payload transforms
+
+`/ingest` executes a [payload transform](/docs/ingestion/payload-transforms/)
+against the raw HTTP request body and inserts the resulting rows into the
+transform's target table.
+
+### Overview
+
+`/ingest` expects an HTTP POST request with the raw payload as the request body.
+The `Content-Type` header is not enforced - the body is passed as-is to the
+transform's `payload()` function.
+
+#### Parameters
+
+| Parameter | Required | Description |
+| :--- | :--- | :--- |
+| `transform` | Yes | Name of the payload transform to execute |
+| Any other | No | Overrides a `DECLARE OVERRIDABLE` variable by name |
+
+#### Responses
+
+Success (HTTP 200):
+
+```json
+{"status": "ok", "rows_inserted": 1}
+```
+
+Error (HTTP 400 or 413):
+
+```json
+{"status": "error", "message": "..."}
+```
+
+Requests exceeding the configured `http.ingest.max.request.size` (default 5 MB)
+receive an HTTP 413 (Payload Too Large) response.
+
+### Example
+
+```shell title="Ingest a JSON payload"
+curl -s "https://api.exchange.coinbase.com/products/BTC-USD/book?level=2" | \
+ curl -X POST "http://localhost:9000/ingest?transform=coinbase_book_api" -d @-
+```
+
+For full documentation on creating and managing transforms, see
+[Payload transforms](/docs/ingestion/payload-transforms/).
+
+:::note Enterprise
+
+In [QuestDB Enterprise](/enterprise/) deployments with
+[RBAC](/docs/security/rbac/) enabled, the caller must hold the `HTTP` endpoint
+grant and `INSERT` permission on the target table.
+
+:::
+
+
## Error responses
### Malformed queries
diff --git a/documentation/query/sql/create-payload-transform.md b/documentation/query/sql/create-payload-transform.md
new file mode 100644
index 000000000..74c7a0241
--- /dev/null
+++ b/documentation/query/sql/create-payload-transform.md
@@ -0,0 +1,184 @@
+---
+title: CREATE PAYLOAD TRANSFORM
+sidebar_label: CREATE PAYLOAD TRANSFORM
+description:
+ Documentation for the CREATE PAYLOAD TRANSFORM SQL keyword in QuestDB.
+---
+
+Creates a payload transform that defines how incoming HTTP payloads are parsed,
+transformed, and inserted into a target table. Once created, data is ingested by
+POSTing to the [`/ingest` endpoint](/docs/ingestion/payload-transforms/#http-endpoint).
+
+## Syntax
+
+```
+CREATE [ OR REPLACE ] PAYLOAD TRANSFORM transformName
+INTO targetTable
+[ DLQ dlqTable [ PARTITION BY ( YEAR | MONTH | WEEK | DAY | HOUR ) ] [ TTL n timeUnit ] ]
+AS [ DECLARE [ OVERRIDABLE ] @var := value [, [ OVERRIDABLE ] @var2 := value2 ... ] ]
+SELECT ...
+```
+
+Where:
+- `timeUnit`: `HOURS | DAYS | WEEKS | MONTHS | YEARS`
+- The `SELECT` must not reference existing tables - it can only use functions
+ and expressions, including CTEs
+- Use [`payload()`](/docs/query/functions/meta/#payload) to access the raw HTTP
+ request body as a `VARCHAR`
+
+## Parameters
+
+| Parameter | Description |
+| --------- | ----------- |
+| `transformName` | Name for the payload transform |
+| `OR REPLACE` | Replace existing transform with the same name |
+| `targetTable` | Table to insert rows into |
+| `DLQ dlqTable` | Route failed payloads to a dead-letter queue table |
+| `PARTITION BY` | Partitioning unit for the DLQ table (if QuestDB creates it) |
+| `TTL` | Retention period for DLQ rows |
+| `DECLARE` | Define variables used in the SELECT |
+| `OVERRIDABLE` | Allow variable to be overridden via URL query parameters |
+
+## Column mapping
+
+SELECT output column names must match column names in the target table. Columns
+are matched by name, not position. You do not need to produce all columns - any
+columns not included in the SELECT receive their default values.
+
+## Examples
+
+### Basic transform
+
+```questdb-sql title="Create a table and a transform for Coinbase order book snapshots"
+CREATE TABLE coinbase_order_book (
+ timestamp TIMESTAMP,
+ symbol SYMBOL,
+ bids DOUBLE[][],
+ asks DOUBLE[][],
+ best_bid DOUBLE,
+ best_ask DOUBLE
+) TIMESTAMP(timestamp) PARTITION BY DAY WAL;
+
+CREATE PAYLOAD TRANSFORM coinbase_book_api
+INTO coinbase_order_book
+AS DECLARE OVERRIDABLE @symbol := 'BTC-USD'
+SELECT
+ json_extract(payload(), '$.time')::TIMESTAMP AS timestamp,
+ @symbol AS symbol,
+ json_extract(payload(), '$.bids')::DOUBLE[][] AS bids,
+ json_extract(payload(), '$.asks')::DOUBLE[][] AS asks,
+ json_extract(payload(), '$.bids[0][0]')::DOUBLE AS best_bid,
+ json_extract(payload(), '$.asks[0][0]')::DOUBLE AS best_ask;
+```
+
+### With dead-letter queue
+
+```questdb-sql title="Transform with DLQ and 7-day retention"
+CREATE PAYLOAD TRANSFORM coinbase_book_api
+INTO coinbase_order_book
+DLQ dlq_errors PARTITION BY DAY TTL 7 DAYS
+AS DECLARE OVERRIDABLE @symbol := 'BTC-USD'
+SELECT
+ json_extract(payload(), '$.time')::TIMESTAMP AS timestamp,
+ @symbol AS symbol,
+ json_extract(payload(), '$.bids')::DOUBLE[][] AS bids,
+ json_extract(payload(), '$.asks')::DOUBLE[][] AS asks,
+ json_extract(payload(), '$.bids[0][0]')::DOUBLE AS best_bid,
+ json_extract(payload(), '$.asks[0][0]')::DOUBLE AS best_ask;
+```
+
+### Replace an existing transform
+
+```questdb-sql title="Replace a transform definition"
+CREATE OR REPLACE PAYLOAD TRANSFORM coinbase_book_api
+INTO coinbase_order_book
+AS SELECT
+ json_extract(payload(), '$.time')::TIMESTAMP AS timestamp,
+ 'BTC-USD' AS symbol,
+ json_extract(payload(), '$.bids')::DOUBLE[][] AS bids,
+ json_extract(payload(), '$.asks')::DOUBLE[][] AS asks,
+ json_extract(payload(), '$.bids[0][0]')::DOUBLE AS best_bid,
+ json_extract(payload(), '$.asks[0][0]')::DOUBLE AS best_ask;
+```
+
+### Multiple overridable variables
+
+```questdb-sql title="Two overridable variables with defaults"
+CREATE PAYLOAD TRANSFORM sensor_ingest
+INTO sensor_data
+AS DECLARE OVERRIDABLE @source := 'default', OVERRIDABLE @region := 'us-east'
+SELECT
+ now() AS ts,
+ @source AS source,
+ @region AS region,
+ json_extract(payload(), '$.temperature')::DOUBLE AS temperature;
+```
+
+## Validation
+
+QuestDB validates the transform at creation time:
+
+| Check | Description |
+| ----- | ----------- |
+| Column names | Every SELECT output column must exist in the target table |
+| Column types | Each output type must be convertible to the target column type, following `INSERT AS SELECT` rules |
+| DLQ schema | If the DLQ table already exists, its schema must match the expected DLQ layout |
+
+Validation errors report the position of the offending column expression in the
+SELECT.
+
+## Dead-letter queue schema
+
+When a DLQ is configured and a transform error occurs, QuestDB writes a row with
+the following columns:
+
+| Column | Type | Description |
+| :----- | :--- | :---------- |
+| `ts` | TIMESTAMP | When the error occurred (designated timestamp) |
+| `transform_name` | SYMBOL | Name of the transform that failed |
+| `payload` | VARCHAR | The original HTTP body |
+| `query` | VARCHAR | The transform's SELECT SQL |
+| `stage` | SYMBOL | Processing stage where the error occurred |
+| `error` | VARCHAR | Error message |
+
+Multiple transforms can share the same DLQ table. The HTTP response still
+returns an error so the caller knows the request failed.
+
+## Permissions
+
+| Context | Requirement |
+| ------- | ----------- |
+| Target table | The `/ingest` caller must have INSERT permission, checked at request time |
+| DLQ table | The DDL caller must have INSERT permission, checked at creation time. Runtime DLQ writes use the system security context |
+
+:::note Enterprise
+
+In [QuestDB Enterprise](/enterprise/) deployments with
+[RBAC](/docs/security/rbac/) enabled, the user creating the transform must hold
+the `CREATE PAYLOAD TRANSFORM` grant. When using `OR REPLACE` on a transform
+that already exists, the user must also hold `DROP PAYLOAD TRANSFORM`. The
+`/ingest` caller must hold the `HTTP` endpoint grant and `INSERT` permission on
+the target table (and on the DLQ table, if configured).
+
+```questdb-sql
+-- Create-only
+GRANT CREATE PAYLOAD TRANSFORM TO ingest_admin;
+
+-- If using OR REPLACE
+GRANT CREATE PAYLOAD TRANSFORM, DROP PAYLOAD TRANSFORM TO ingest_admin;
+
+GRANT HTTP TO ingest_service;
+GRANT INSERT ON coinbase_order_book TO ingest_service;
+GRANT INSERT ON dlq_errors TO ingest_service;
+```
+
+:::
+
+:::info Related documentation
+- [DROP PAYLOAD TRANSFORM](/docs/query/sql/drop-payload-transform/)
+- [SHOW PAYLOAD TRANSFORMS](/docs/query/sql/show/#show-payload-transforms)
+- [Payload transforms overview](/docs/ingestion/payload-transforms/)
+- [`payload()` function](/docs/query/functions/meta/#payload)
+- [JSON functions](/docs/query/functions/json/)
+- [Role-Based Access Control (RBAC)](/docs/security/rbac/)
+:::
diff --git a/documentation/query/sql/drop-payload-transform.md b/documentation/query/sql/drop-payload-transform.md
new file mode 100644
index 000000000..509bffc06
--- /dev/null
+++ b/documentation/query/sql/drop-payload-transform.md
@@ -0,0 +1,61 @@
+---
+title: DROP PAYLOAD TRANSFORM
+sidebar_label: DROP PAYLOAD TRANSFORM
+description:
+ Documentation for the DROP PAYLOAD TRANSFORM SQL keyword in QuestDB.
+---
+
+Permanently deletes a payload transform definition. The target table and any DLQ
+table are not affected.
+
+## Syntax
+
+```
+DROP PAYLOAD TRANSFORM [ IF EXISTS ] transformName
+```
+
+## Parameters
+
+| Parameter | Description |
+| --------- | ----------- |
+| `transformName` | Name of the payload transform to drop |
+| `IF EXISTS` | Suppress error if the transform does not exist |
+
+## Examples
+
+```questdb-sql title="Drop a payload transform"
+DROP PAYLOAD TRANSFORM coinbase_book_api;
+```
+
+```questdb-sql title="Drop only if exists (no error if missing)"
+DROP PAYLOAD TRANSFORM IF EXISTS coinbase_book_api;
+```
+
+## Behavior
+
+| Aspect | Description |
+| ------ | ----------- |
+| Target table | Not affected - existing data remains |
+| DLQ table | Not affected - existing error rows remain |
+| Active requests | In-flight `/ingest` requests may still complete |
+
+## Permissions
+
+:::note Enterprise
+
+In [QuestDB Enterprise](/enterprise/) deployments with
+[RBAC](/docs/security/rbac/) enabled, the user must hold the
+`DROP PAYLOAD TRANSFORM` grant.
+
+```questdb-sql
+GRANT DROP PAYLOAD TRANSFORM TO ingest_admin;
+```
+
+:::
+
+:::info Related documentation
+- [CREATE PAYLOAD TRANSFORM](/docs/query/sql/create-payload-transform/)
+- [SHOW PAYLOAD TRANSFORMS](/docs/query/sql/show/#show-payload-transforms)
+- [Payload transforms overview](/docs/ingestion/payload-transforms/)
+- [Role-Based Access Control (RBAC)](/docs/security/rbac/)
+:::
diff --git a/documentation/query/sql/show.md b/documentation/query/sql/show.md
index 0d94883dd..556e23947 100644
--- a/documentation/query/sql/show.md
+++ b/documentation/query/sql/show.md
@@ -4,13 +4,10 @@ sidebar_label: SHOW
description: SHOW SQL keyword reference documentation.
---
-This keyword provides table, column, and partition information including
-metadata. The `SHOW` keyword is useful for checking the
-[designated timestamp setting](/docs/concepts/designated-timestamp/) column, the
-[partition attachment settings](/docs/query/sql/alter-table-attach-partition/),
-and partition storage size on disk.
+`SHOW` returns metadata about tables, columns, partitions, transforms,
+configuration, and users.
-## Syntax
+## Available statements
```questdb-sql
SHOW { TABLES
@@ -18,6 +15,7 @@ SHOW { TABLES
| PARTITIONS FROM tableName
| CREATE TABLE tableName
| CREATE VIEW viewName
+ | PAYLOAD TRANSFORMS
| USER [userName]
| USERS
| GROUPS [userName]
@@ -28,49 +26,37 @@ SHOW { TABLES
| PARAMETERS };
```
-## Description
-
-- `SHOW TABLES` returns all the tables.
-- `SHOW COLUMNS` returns all the columns and their metadata for the selected
- table.
-- `SHOW PARTITIONS` returns the partition information for the selected table.
-- `SHOW CREATE TABLE` returns a DDL query that allows you to recreate the table.
-- `SHOW CREATE VIEW` returns a DDL query that allows you to recreate a view.
-- `SHOW USER` shows user secret (enterprise-only)
-- `SHOW GROUPS` shows all groups the user belongs or all groups in the system
- (enterprise-only)
-- `SHOW USERS` shows all users (enterprise-only)
-- `SHOW SERVICE ACCOUNT` displays details of a service account (enterprise-only)
-- `SHOW SERVICE ACCOUNTS` displays all service accounts or those assigned to the
- user/group (enterprise-only)
-- `SHOW PERMISSIONS` displays permissions of user, group or service account
- (enterprise-only)
-- `SHOW SERVER_VERSION` displays PostgreSQL compatibility version
-- `SHOW PARAMETERS` shows configuration keys and their matching `env_var_name`,
- their values and the source of the value
-
-## Examples
-
-### SHOW TABLES
-
-```questdb-sql title="show tables" demo
-SHOW TABLES;
+- [`SHOW COLUMNS`](#show-columns) - column metadata for a table
+- [`SHOW CREATE TABLE`](#show-create-table) - DDL to recreate a table
+- [`SHOW CREATE VIEW`](#show-create-view) - DDL to recreate a view
+- [`SHOW GROUPS`](#show-groups) - groups a user belongs to, or all groups (enterprise)
+- [`SHOW PARAMETERS`](#show-parameters) - configuration keys, values, and sources
+- [`SHOW PARTITIONS`](#show-partitions) - partition information for a table
+- [`SHOW PAYLOAD TRANSFORMS`](#show-payload-transforms) - all defined payload transforms
+- [`SHOW PERMISSIONS`](#show-permissions) - permissions for a user, group, or service account (enterprise)
+- [`SHOW SERVER_VERSION`](#show-server_version) - PostgreSQL compatibility version
+- [`SHOW SERVICE ACCOUNT`](#show-service-account) - details of a service account (enterprise)
+- [`SHOW SERVICE ACCOUNTS`](#show-service-accounts) - all service accounts, or those assigned to a user/group (enterprise)
+- [`SHOW TABLES`](#show-tables) - all tables
+- [`SHOW USER`](#show-user) - user authentication details (enterprise)
+- [`SHOW USERS`](#show-users) - all users (enterprise)
+
+## SHOW COLUMNS
+
+### Syntax
+
+```
+SHOW COLUMNS FROM tableName
```
-| table_name |
-| --------------- |
-| ethblocks_json |
-| trades |
-| weather |
-| AAPL_orderbook |
-| trips |
+Returns all columns and their metadata for the selected table.
-### SHOW COLUMNS
+### Example
-```questdb-sql title="show columns" demo
+```questdb-sql title="Show columns" demo
SHOW COLUMNS FROM trades;
-
```
+
| column | type | indexed | indexBlockCapacity | symbolCached | symbolCapacity | symbolTableSize | designated | upsertKey |
| --------- | --------- | ------- | ------------------ | ------------ | -------------- | --------------- | ---------- | --------- |
| symbol | SYMBOL | false | 0 | true | 256 | 42 | false | false |
@@ -79,9 +65,19 @@ SHOW COLUMNS FROM trades;
| amount | DOUBLE | false | 0 | false | 0 | 0 | false | false |
| timestamp | TIMESTAMP | false | 0 | false | 0 | 0 | true | false |
-### SHOW CREATE TABLE
+## SHOW CREATE TABLE
+
+### Syntax
-```questdb-sql title="retrieving table ddl" demo
+```
+SHOW CREATE TABLE tableName
+```
+
+Returns a DDL query that allows you to recreate the table.
+
+### Example
+
+```questdb-sql title="Show create table" demo
SHOW CREATE TABLE trades;
```
@@ -140,9 +136,19 @@ This clause assigns permissions for the table to that user.
If permissions should be assigned to a different user,
please modify this clause appropriately.
-### SHOW CREATE VIEW
+## SHOW CREATE VIEW
+
+### Syntax
-```questdb-sql title="retrieving view ddl"
+```
+SHOW CREATE VIEW viewName
+```
+
+Returns a DDL query that allows you to recreate a view.
+
+### Example
+
+```questdb-sql title="Show create view"
SHOW CREATE VIEW my_view;
```
@@ -153,26 +159,48 @@ SHOW CREATE VIEW my_view;
This returns the `CREATE VIEW` statement that would recreate the view,
including any `DECLARE` parameters if the view is parameterized.
-### SHOW PARTITIONS
+## SHOW GROUPS
+
+### Syntax
+
+```
+SHOW GROUPS [ entityName ]
+```
+
+Shows all groups in the system, or all groups a user belongs to. Enterprise only.
+
+### Examples
```questdb-sql
-SHOW PARTITIONS FROM my_table;
+SHOW GROUPS;
```
-| index | partitionBy | name | minTimestamp | maxTimestamp | numRows | diskSize | diskSizeHuman | readOnly | active | attached | detached | attachable |
-| ----- | ----------- | -------- | --------------------- | --------------------- | ------- | -------- | ------------- | -------- | ------ | -------- | -------- | ---------- |
-| 0 | WEEK | 2022-W52 | 2023-01-01 00:36:00.0 | 2023-01-01 23:24:00.0 | 39 | 98304 | 96.0 KiB | false | false | true | false | false |
-| 1 | WEEK | 2023-W01 | 2023-01-02 00:00:00.0 | 2023-01-08 23:24:00.0 | 280 | 98304 | 96.0 KiB | false | false | true | false | false |
-| 2 | WEEK | 2023-W02 | 2023-01-09 00:00:00.0 | 2023-01-15 23:24:00.0 | 280 | 98304 | 96.0 KiB | false | false | true | false | false |
-| 3 | WEEK | 2023-W03 | 2023-01-16 00:00:00.0 | 2023-01-18 12:00:00.0 | 101 | 83902464 | 80.0 MiB | false | true | true | false | false |
+```questdb-sql
+SHOW GROUPS john;
+```
+
+| name |
+| ---------- |
+| management |
+
+## SHOW PARAMETERS
-### SHOW PARAMETERS
+### Syntax
+
+```
+SHOW PARAMETERS
+```
+
+Shows configuration keys and their matching `env_var_name`, their values, and
+the source of the value.
+
+### Example
```questdb-sql
SHOW PARAMETERS;
```
-The output demonstrates:
+The output columns:
- `property_path`: the configuration key
- `env_var_name`: the matching env var for the key
@@ -191,7 +219,6 @@ The output demonstrates:
| pg.readonly.password | QDB_PG_READONLY_PASSWORD | **** | default | true | true |
| http.password | QDB_HTTP_PASSWORD | **** | default | true | false |
-
You can optionally chain `SHOW PARAMETERS` with other clauses:
```questdb-sql
@@ -208,59 +235,136 @@ You can optionally chain `SHOW PARAMETERS` with other clauses:
(SHOW PARAMETERS) WHERE value_source <> 'default';
```
-### SHOW USER
+## SHOW PARTITIONS
-```questdb-sql
-SHOW USER; --as john
+### Syntax
+
+```
+SHOW PARTITIONS FROM tableName
```
-or
+Returns partition information for the selected table.
+
+### Example
```questdb-sql
-SHOW USER john;
+SHOW PARTITIONS FROM my_table;
```
-| auth_type | enabled |
-| ---------- | ------- |
-| Password | false |
-| JWK Token | false |
-| REST Token | false |
+| index | partitionBy | name | minTimestamp | maxTimestamp | numRows | diskSize | diskSizeHuman | readOnly | active | attached | detached | attachable |
+| ----- | ----------- | -------- | --------------------- | --------------------- | ------- | -------- | ------------- | -------- | ------ | -------- | -------- | ---------- |
+| 0 | WEEK | 2022-W52 | 2023-01-01 00:36:00.0 | 2023-01-01 23:24:00.0 | 39 | 98304 | 96.0 KiB | false | false | true | false | false |
+| 1 | WEEK | 2023-W01 | 2023-01-02 00:00:00.0 | 2023-01-08 23:24:00.0 | 280 | 98304 | 96.0 KiB | false | false | true | false | false |
+| 2 | WEEK | 2023-W02 | 2023-01-09 00:00:00.0 | 2023-01-15 23:24:00.0 | 280 | 98304 | 96.0 KiB | false | false | true | false | false |
+| 3 | WEEK | 2023-W03 | 2023-01-16 00:00:00.0 | 2023-01-18 12:00:00.0 | 101 | 83902464 | 80.0 MiB | false | true | true | false | false |
-### SHOW USERS
+## SHOW PAYLOAD TRANSFORMS
-```questdb-sql
-SHOW USERS;
+### Syntax
+
+```
+SHOW PAYLOAD TRANSFORMS
```
-| name |
-| ----- |
-| admin |
-| john |
+Lists all defined [payload transforms](/docs/ingestion/payload-transforms/).
-### SHOW GROUPS
+### Example
-```questdb-sql
-SHOW GROUPS;
+```questdb-sql title="List all payload transforms"
+SHOW PAYLOAD TRANSFORMS;
+```
+
+| name | target_table | dlq_table | query |
+| :--- | :--- | :--- | :--- |
+| coinbase_book_api | coinbase_order_book | dlq_errors | DECLARE OVERRIDABLE @symbol := 'BTC-USD' SELECT json_extract(payload(), '$.time')::TIMESTAMP AS timestamp, ... |
+| coinbase_trades_api | coinbase_trades | dlq_errors | DECLARE OVERRIDABLE @symbol := 'BTC-USD' SELECT u.time AS timestamp, ... FROM UNNEST(payload() COLUMNS(...)) u |
+
+## SHOW PERMISSIONS
+
+### Syntax
+
+```
+SHOW PERMISSIONS [ entityName ]
+```
+
+Displays permissions of a user, group, or service account. Enterprise only.
+
+Without an argument, shows permissions for the current user.
+
+### Examples
+
+```questdb-sql title="Current user"
+SHOW PERMISSIONS;
+```
+
+| permission | table_name | column_name | grant_option | origin |
+| ---------- | ---------- | ----------- | ------------ | ------ |
+| SELECT | | | t | G |
+
+```questdb-sql title="Specific user"
+SHOW PERMISSIONS admin;
+```
+
+| permission | table_name | column_name | grant_option | origin |
+| ---------- | ---------- | ----------- | ------------ | ------ |
+| SELECT | | | t | G |
+| INSERT | orders | | f | G |
+| UPDATE | order_itme | quantity | f | G |
+
+```questdb-sql title="Group"
+SHOW PERMISSIONS admin_group;
+```
+
+| permission | table_name | column_name | grant_option | origin |
+| ---------- | ---------- | ----------- | ------------ | ------ |
+| INSERT | orders | | f | G |
+
+```questdb-sql title="Service account"
+SHOW PERMISSIONS ilp_ingestion;
+```
+
+| permission | table_name | column_name | grant_option | origin |
+| ---------- | ---------- | ----------- | ------------ | ------ |
+| SELECT | | | t | G |
+| INSERT | | | f | G |
+| UPDATE | | | f | G |
+
+## SHOW SERVER_VERSION
+
+### Syntax
+
+```
+SHOW SERVER_VERSION
```
-or
+Shows PostgreSQL compatibility version.
+
+### Example
```questdb-sql
-SHOW GROUPS john;
+SHOW SERVER_VERSION;
```
-| name |
-| ---------- |
-| management |
+| server_version |
+| -------------- |
+| 12.3 (questdb) |
+
+## SHOW SERVICE ACCOUNT
-### SHOW SERVICE ACCOUNT
+### Syntax
+
+```
+SHOW SERVICE ACCOUNT [ accountName ]
+```
+
+Displays details of a service account. Enterprise only.
+
+### Examples
```questdb-sql
SHOW SERVICE ACCOUNT;
```
-or
-
```questdb-sql
SHOW SERVICE ACCOUNT ilp_ingestion;
```
@@ -271,7 +375,18 @@ SHOW SERVICE ACCOUNT ilp_ingestion;
| JWK Token | false |
| REST Token | false |
-### SHOW SERVICE ACCOUNTS
+## SHOW SERVICE ACCOUNTS
+
+### Syntax
+
+```
+SHOW SERVICE ACCOUNTS [ entityName ]
+```
+
+Displays all service accounts, or those assigned to a user or group. Enterprise
+only.
+
+### Examples
```questdb-sql
SHOW SERVICE ACCOUNTS;
@@ -298,63 +413,76 @@ SHOW SERVICE ACCOUNTS admin_group;
| ---------- |
| svc1_admin |
-### SHOW PERMISSIONS FOR CURRENT USER
+## SHOW TABLES
-```questdb-sql
-SHOW PERMISSIONS;
+### Syntax
+
+```
+SHOW TABLES
```
-| permission | table_name | column_name | grant_option | origin |
-| ---------- | ---------- | ----------- | ------------ | ------ |
-| SELECT | | | t | G |
+Returns all tables.
-### SHOW PERMISSIONS user
+### Example
-```questdb-sql
-SHOW PERMISSIONS admin;
+```questdb-sql title="Show tables" demo
+SHOW TABLES;
```
-| permission | table_name | column_name | grant_option | origin |
-| ---------- | ---------- | ----------- | ------------ | ------ |
-| SELECT | | | t | G |
-| INSERT | orders | | f | G |
-| UPDATE | order_itme | quantity | f | G |
+| table_name |
+| --------------- |
+| ethblocks_json |
+| trades |
+| weather |
+| AAPL_orderbook |
+| trips |
-### SHOW PERMISSIONS
+## SHOW USER
-#### For a group
+### Syntax
-```questdb-sql
-SHOW PERMISSIONS admin_group;
+```
+SHOW USER [ userName ]
```
-| permission | table_name | column_name | grant_option | origin |
-| ---------- | ---------- | ----------- | ------------ | ------ |
-| INSERT | orders | | f | G |
+Shows user authentication details. Enterprise only.
-#### For a service account
+### Examples
```questdb-sql
-SHOW PERMISSIONS ilp_ingestion;
+SHOW USER; --as john
```
-| permission | table_name | column_name | grant_option | origin |
-| ---------- | ---------- | ----------- | ------------ | ------ |
-| SELECT | | | t | G |
-| INSERT | | | f | G |
-| UPDATE | | | f | G |
+```questdb-sql
+SHOW USER john;
+```
-### SHOW SERVER_VERSION
+| auth_type | enabled |
+| ---------- | ------- |
+| Password | false |
+| JWK Token | false |
+| REST Token | false |
-Shows PostgreSQL compatibility version.
+## SHOW USERS
+
+### Syntax
+
+```
+SHOW USERS
+```
+
+Shows all users. Enterprise only.
+
+### Example
```questdb-sql
-SHOW SERVER_VERSION;
+SHOW USERS;
```
-| server_version |
-| -------------- |
-| 12.3 (questdb) |
+| name |
+| ----- |
+| admin |
+| john |
## See also
diff --git a/documentation/query/sql/unnest.md b/documentation/query/sql/unnest.md
index 1a0cf529a..5283bac7e 100644
--- a/documentation/query/sql/unnest.md
+++ b/documentation/query/sql/unnest.md
@@ -164,6 +164,16 @@ JSON `UNNEST` expands a JSON array (stored as `VARCHAR`) into rows with
explicitly typed columns. The `COLUMNS(...)` clause distinguishes JSON `UNNEST`
from array `UNNEST`.
+:::tip Automate API ingestion
+
+Combine JSON UNNEST with
+[payload transforms](/docs/ingestion/payload-transforms/) to ingest data from
+external APIs directly into QuestDB - no middleware required. See the
+[Coinbase trades example](/docs/ingestion/payload-transforms/#example-coinbase-trades-with-unnest)
+for a complete walkthrough.
+
+:::
+
### Syntax
```questdb-sql
diff --git a/documentation/security/rbac.md b/documentation/security/rbac.md
index 09101e276..9ecd05d18 100644
--- a/documentation/security/rbac.md
+++ b/documentation/security/rbac.md
@@ -566,16 +566,18 @@ SELECT * FROM all_permissions();
| ATTACH PARTITION | Database | Table | Attach partitions |
| BACKUP DATABASE | Database | Create database backups |
| CANCEL ANY COPY | Database | Cancel COPY operations |
-| CREATE TABLE | Database | Create tables |
| CREATE MATERIALIZED VIEW | Database | Create materialized views |
+| [CREATE PAYLOAD TRANSFORM](/docs/query/sql/create-payload-transform/) | Database | Create [payload transforms](/docs/ingestion/payload-transforms/) |
+| CREATE TABLE | Database | Create tables |
| DEDUP ENABLE | Database | Table | Enable deduplication |
| DEDUP DISABLE | Database | Table | Disable deduplication |
| DETACH PARTITION | Database | Table | Detach partitions |
| DROP COLUMN | Database | Table | Column | Drop columns |
| DROP INDEX | Database | Table | Column | Drop indexes |
| DROP PARTITION | Database | Table | Drop partitions |
-| DROP TABLE | Database | Table | Drop tables |
| DROP MATERIALIZED VIEW | Database | Table | Drop materialized views |
+| [DROP PAYLOAD TRANSFORM](/docs/query/sql/drop-payload-transform/) | Database | Drop [payload transforms](/docs/ingestion/payload-transforms/) |
+| DROP TABLE | Database | Table | Drop tables |
| INSERT | Database | Table | Insert data |
| REFRESH MATERIALIZED VIEW | Database | Table | Refresh materialized views |
| REINDEX | Database | Table | Column | Reindex columns |
@@ -650,4 +652,4 @@ SELECT * FROM all_permissions();
- [SHOW GROUPS](/docs/query/sql/show/#show-groups)
- [SHOW SERVICE ACCOUNT](/docs/query/sql/show/#show-service-account)
- [SHOW SERVICE ACCOUNTS](/docs/query/sql/show/#show-service-accounts)
-- [SHOW PERMISSIONS](/docs/query/sql/show/#show-permissions-for-current-user)
+- [SHOW PERMISSIONS](/docs/query/sql/show/#show-permissions)
diff --git a/documentation/sidebars.js b/documentation/sidebars.js
index 6a69796ad..f0cf0cb2e 100644
--- a/documentation/sidebars.js
+++ b/documentation/sidebars.js
@@ -153,6 +153,7 @@ module.exports = {
],
},
"ingestion/import-csv",
+ "ingestion/payload-transforms",
],
},
@@ -337,6 +338,7 @@ module.exports = {
id: "query/sql/acl/create-service-account",
type: "doc",
},
+ "query/sql/create-payload-transform",
"query/sql/create-table",
{
id: "query/sql/acl/create-user",
@@ -354,6 +356,7 @@ module.exports = {
type: "doc",
},
"query/sql/drop-mat-view",
+ "query/sql/drop-payload-transform",
{
id: "query/sql/acl/drop-service-account",
type: "doc",