Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `ManagedDatabaseClient.fetch_table` now carries the `X-Database-Id` scope header on the result poll, the query-run poll, and the Arrow fetch — not only on the query submit. Results of database-scoped queries are themselves database-scoped, so every read against an existing synced table (merge/append loads, dlt state restore) failed with `400: Bad Request` once the table had data (dlthubworker#70).

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

super nit: (not blocking) This "Fixed" entry still describes the scope as being carried "on the ... header," but with this PR shipping in the same Unreleased section, it now rides the native x_database_id parameters rather than a header. Consider rewording to "carries the database scope" so it stays consistent with the new "Changed" entry below.

- API error messages now include the response body (flattened, truncated to 500 chars). `400: Bad Request` alone hid the server's actual explanation.

### Changed

- The `hotdata` SDK dependency is now `>=0.6.0`, and the scope above rides its native `x_database_id` parameters (`get_result`, `get_query_run`, `get_result_arrow`). Note 0.6.0 made `x_database_id` **required** on `get_result_arrow`, so older framework releases cannot run on it.

## [0.6.0] - 2026-06-30

### Added
Expand Down
26 changes: 9 additions & 17 deletions hotdata_framework/managed_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,17 +118,13 @@ def _fetch_result_arrow(self, result_id: str, *, database_id: str) -> pa.Table:
"""Fetch a ready result as Arrow, carrying the database scope.

Results of database-scoped queries are themselves database-scoped —
the results endpoints reject requests without an ``X-Database-Id``
header. The Arrow helper takes no per-call headers, so the header is
set as a client default for the duration of the call and removed
after (this client is not shared across threads).
the results endpoints reject requests without the scope. The hotdata
0.6.0 SDK exposes (and requires) ``x_database_id`` on the Arrow
helper directly.
"""
api = self._runtime.api
api.set_default_header("X-Database-Id", database_id)
try:
return ArrowResultsApi(api).get_result_arrow(result_id)
finally:
api.default_headers.pop("X-Database-Id", None)
return ArrowResultsApi(self._runtime.api).get_result_arrow(
result_id, x_database_id=database_id
)

def _poll(
self,
Expand Down Expand Up @@ -172,9 +168,7 @@ def _await_query_run(self, query_run_id: str, *, database_id: str) -> str | None
runs = QueryRunsApi(self._runtime.api)
run = self._poll(
# Runs (like results) of database-scoped queries are database-scoped.
lambda: runs.get_query_run(
query_run_id, _headers={"X-Database-Id": database_id}
),
lambda: runs.get_query_run(query_run_id, x_database_id=database_id),
is_ready=lambda r: r.status == "succeeded",
describe="Query",
)
Expand All @@ -186,10 +180,8 @@ def _wait_result_ready(self, result_id: str | None, *, database_id: str) -> str
results = ResultsApi(self._runtime.api)
self._poll(
# The stored result of a database-scoped query 400s without the
# database scope header.
lambda: results.get_result(
result_id, _headers={"X-Database-Id": database_id}
),
# database scope.
lambda: results.get_result(result_id, x_database_id=database_id),
is_ready=lambda r: r.status == "ready",
describe=f"Result {result_id}",
)
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"hotdata>=0.5.0",
"hotdata>=0.6.0",
"pandas>=2.0",
"pyarrow>=14.0",
]
Expand Down
51 changes: 20 additions & 31 deletions tests/test_managed_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ class FakeArrowResultsApi:
def __init__(self, api: object) -> None:
pass

def get_result_arrow(self, result_id: str) -> pa.Table:
def get_result_arrow(self, result_id: str, **kwargs: Any) -> pa.Table:
calls.append("arrow")
return pa.table({"id": [1, 2]})

Expand Down Expand Up @@ -87,19 +87,9 @@ def get_result_arrow(self, result_id: str) -> pa.Table:
assert calls.index("arrow") > calls.index("get_result:ready")


class _FakeApiClient:
"""Just enough of the generated ApiClient's default-header surface."""

def __init__(self) -> None:
self.default_headers: dict[str, str] = {}

def set_default_header(self, name: str, value: str) -> None:
self.default_headers[name] = value


def _fake_runtime(api: object | None = None) -> SimpleNamespace:
def _fake_runtime() -> SimpleNamespace:
return SimpleNamespace(
api=api if api is not None else _FakeApiClient(),
api=object(),
resolve_managed_database=lambda name: SimpleNamespace(id="db1", default_connection_id="c"),
list_managed_tables=lambda database, schema=None: [
SimpleNamespace(table="orders", synced=True)
Expand All @@ -112,16 +102,16 @@ def test_fetch_table_carries_database_scope_on_result_reads(
) -> None:
"""Results (and runs) of a database-scoped query are database-scoped:
the results endpoints 400 with "X-Database-Id header is required" when
the header is missing. ``fetch_table`` must carry the database id on the
result poll and the Arrow fetch, not only on the query submit.
the scope is missing. ``fetch_table`` must carry the database id on the
result poll and the Arrow fetch, not only on the query submit — the
hotdata 0.6.0 SDK exposes ``x_database_id`` on all three.

Regression: reruns/append loads against an existing synced table failed
with an opaque ``400: Bad Request`` (dlthubworker#70) because both reads
omitted the header.
omitted the scope.
"""
seen_headers: list[dict[str, Any] | None] = []
arrow_headers: list[str | None] = []
fake_api = _FakeApiClient()
result_scopes: list[str | None] = []
arrow_scopes: list[str | None] = []

class FakeQueryApi:
def __init__(self, api: object) -> None:
Expand All @@ -135,17 +125,18 @@ class FakeResultsApi:
def __init__(self, api: object) -> None:
pass

def get_result(self, result_id: str, *, _headers: dict[str, Any] | None = None) -> Any:
seen_headers.append(_headers)
def get_result(self, result_id: str, *, x_database_id: str | None = None) -> Any:
result_scopes.append(x_database_id)
return SimpleNamespace(status="ready", result_id=result_id, error_message=None)

class FakeArrowResultsApi:
def __init__(self, api: _FakeApiClient) -> None:
self._api = api
def __init__(self, api: object) -> None:
pass

def get_result_arrow(self, result_id: str) -> pa.Table:
# The scoped default header must be present DURING the fetch.
arrow_headers.append(self._api.default_headers.get("X-Database-Id"))
# x_database_id is REQUIRED in the 0.6.0 SDK — mirroring that here
# makes this test fail if a caller ever drops the scope again.
def get_result_arrow(self, result_id: str, *, x_database_id: str) -> pa.Table:
arrow_scopes.append(x_database_id)
return pa.table({"id": [1]})

monkeypatch.setattr(mc, "QueryApi", FakeQueryApi)
Expand All @@ -159,12 +150,10 @@ def get_result_arrow(self, result_id: str) -> pa.Table:
max_retries=1,
retry_backoff_seconds=0.0,
)
client._runtime = _fake_runtime(fake_api)
client._runtime = _fake_runtime()

table = client.fetch_table(database="mydb", schema="public", table="orders")

assert table is not None
assert seen_headers == [{"X-Database-Id": "db1"}]
assert arrow_headers == ["db1"]
# The scoped header is removed once the fetch completes.
assert "X-Database-Id" not in fake_api.default_headers
assert result_scopes == ["db1"]
assert arrow_scopes == ["db1"]
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading