diff --git a/utils/test_validate_reusable_sweep_artifacts.py b/utils/test_validate_reusable_sweep_artifacts.py index e97516a71..69eb633cd 100644 --- a/utils/test_validate_reusable_sweep_artifacts.py +++ b/utils/test_validate_reusable_sweep_artifacts.py @@ -266,6 +266,40 @@ def test_multinode_agentic_identity_fields_match() -> None: assert identity({**default_row, field: 2}) != identity(default_row) +def test_agentic_identity_freezes_nested_kv_offload_backend() -> None: + row = { + **agentic_result(), + "kv_offloading": "dram", + "kv_offload_backend": { + "name": "native", + "options": {"layers": ["cpu", "gpu"]}, + }, + } + row.pop("offloading") + + identity = agentic_key(row) + + assert isinstance(hash(identity), int) + assert identity == agentic_key( + { + **row, + "kv_offload_backend": { + "options": {"layers": ["cpu", "gpu"]}, + "name": "native", + }, + } + ) + assert identity != agentic_key( + { + **row, + "kv_offload_backend": { + "name": "native", + "options": {"layers": ["cpu"]}, + }, + } + ) + + def write_agentic_artifacts( root: Path, conc: int = 16, @@ -519,6 +553,25 @@ def test_agentic_validation_rejects_duplicate_point_identity( assert "agentic point artifacts contain 1 duplicate row(s)" in errors +def test_agentic_validation_handles_mapping_kv_offload_backend( + tmp_path: Path, +) -> None: + row = { + **agentic_result(), + "kv_offloading": "dram", + "kv_offload_backend": {"name": "native"}, + } + row.pop("offloading") + point_dir = tmp_path / "bmk_agentic_native_offload" + point_dir.mkdir() + (point_dir / "result.json").write_text(json.dumps([row, row])) + (tmp_path / "agentic_native_offload").mkdir() + + errors = validate_agentic_artifacts(tmp_path) + + assert "agentic point artifacts contain 1 duplicate row(s)" in errors + + def test_eval_only_main_does_not_require_benchmark_artifacts( tmp_path: Path, monkeypatch, diff --git a/utils/validate_reusable_sweep_artifacts.py b/utils/validate_reusable_sweep_artifacts.py index 7602768bd..0cfd1d266 100644 --- a/utils/validate_reusable_sweep_artifacts.py +++ b/utils/validate_reusable_sweep_artifacts.py @@ -110,13 +110,27 @@ def actual_benchmark_keys(artifacts_dir: Path) -> set[tuple[Any, ...]]: return set(actual_benchmark_key_rows(artifacts_dir)) +def freeze_identity_value(value: Any) -> Any: + """Convert nested JSON values into deterministic, hashable identities.""" + if isinstance(value, dict): + return tuple( + sorted( + (key, freeze_identity_value(item)) + for key, item in value.items() + ) + ) + if isinstance(value, (list, tuple)): + return tuple(freeze_identity_value(item) for item in value) + return value + + def agentic_key(row: dict[str, Any]) -> tuple[Any, ...]: """Build an agentic identity from one point result.""" if "kv_offloading" in row: kv_offloading = row.get("kv_offloading") or "none" offload_key: Any = ( kv_offloading, - (row.get("kv_offload_backend") or "") + freeze_identity_value(row.get("kv_offload_backend") or "") if kv_offloading != "none" else "", )