diff --git a/changelog.d/us-bundle-17224.changed.md b/changelog.d/us-bundle-17224.changed.md new file mode 100644 index 00000000..3f424083 --- /dev/null +++ b/changelog.d/us-bundle-17224.changed.md @@ -0,0 +1 @@ +Refresh the bundled US release manifest to policyengine-us 1.722.4. diff --git a/pyproject.toml b/pyproject.toml index d07dd8c4..084a5ab5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ uk = [ ] us = [ "policyengine_core==3.26.1", - "policyengine-us==1.715.2", + "policyengine-us==1.722.4", ] dev = [ "pytest", @@ -64,7 +64,7 @@ dev = [ "ruff>=0.9.0", "policyengine_core==3.26.1", "policyengine-uk==2.88.20", - "policyengine-us==1.715.2", + "policyengine-us==1.722.4", "towncrier>=24.8.0", "mypy>=1.11.0", "pytest-cov>=5.0.0", diff --git a/src/policyengine/data/release_manifests/us.json b/src/policyengine/data/release_manifests/us.json index 0ceda4a3..b8abbd68 100644 --- a/src/policyengine/data/release_manifests/us.json +++ b/src/policyengine/data/release_manifests/us.json @@ -5,16 +5,16 @@ "policyengine_version": "4.14.2", "model_package": { "name": "policyengine-us", - "version": "1.715.2", - "sha256": "abf079828419762f5c4b0291a70f6e424744200f237e1ae0f06e25f10130c399", - "wheel_url": "https://files.pythonhosted.org/packages/45/a1/1d56bdbb69d7ce06bedd3892203a75ac3350a90c0b5fcea2fb50db46670f/policyengine_us-1.715.2-py3-none-any.whl" + "version": "1.722.4", + "sha256": "691d4aa9b1512d71d283b0b5805af5ba95ffed798fe471a97cf2195f4950b6e3", + "wheel_url": "https://files.pythonhosted.org/packages/20/fc/f6fa4ffbc79129a267d92782aba6889145fabfcb39023423fed1f468af3f/policyengine_us-1.722.4-py3-none-any.whl" }, "data_package": { "name": "policyengine-us-data", "version": "1.115.5", "repo_id": "policyengine/policyengine-us-data", - "release_manifest_path": "release_manifest.json", - "release_manifest_revision": "d47fb5475144260a75467d2f2e22b2d5d53d4d57" + "release_manifest_path": "releases/1.115.5/release_manifest.json", + "release_manifest_revision": "9531fe1d096244fe7eb45d791d52ef61b8a2a0a5" }, "certified_data_artifact": { "data_package": { @@ -23,14 +23,14 @@ }, "build_id": "policyengine-us-data-1.115.5", "dataset": "enhanced_cps_2024", - "uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@d47fb5475144260a75467d2f2e22b2d5d53d4d57", + "uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@9531fe1d096244fe7eb45d791d52ef61b8a2a0a5", "sha256": "0a6b961ad363a421bde99f2c8e5d8f20370bcba45fd303050537a25bdd805b14" }, "certification": { "compatibility_basis": "legacy_compatible_model_package", "data_build_id": "policyengine-us-data-1.115.5", "built_with_model_version": "1.700.0", - "certified_for_model_version": "1.715.2", + "certified_for_model_version": "1.722.4", "certified_by": "policyengine-us-data release manifest", "data_build_fingerprint": "sha256:b0862de383ffcbe45f4ba0aa9c6aaec286cd4c6688c6ccb33f939bc176f9a8a0" }, diff --git a/src/policyengine/data/release_manifests/us.trace.tro.jsonld b/src/policyengine/data/release_manifests/us.trace.tro.jsonld index 48e1cefc..ba644d05 100644 --- a/src/policyengine/data/release_manifests/us.trace.tro.jsonld +++ b/src/policyengine/data/release_manifests/us.trace.tro.jsonld @@ -45,7 +45,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/data_release_manifest" }, - "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/d47fb5475144260a75467d2f2e22b2d5d53d4d57/release_manifest.json" + "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/9531fe1d096244fe7eb45d791d52ef61b8a2a0a5/releases/1.115.5/release_manifest.json" }, { "@id": "arrangement/1/location/dataset", @@ -53,7 +53,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/dataset" }, - "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/d47fb5475144260a75467d2f2e22b2d5d53d4d57/enhanced_cps_2024.h5" + "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-us-data/resolve/9531fe1d096244fe7eb45d791d52ef61b8a2a0a5/enhanced_cps_2024.h5" }, { "@id": "arrangement/1/location/model_wheel", @@ -61,7 +61,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/model_wheel" }, - "trov:hasLocation": "https://files.pythonhosted.org/packages/45/a1/1d56bdbb69d7ce06bedd3892203a75ac3350a90c0b5fcea2fb50db46670f/policyengine_us-1.715.2-py3-none-any.whl" + "trov:hasLocation": "https://files.pythonhosted.org/packages/20/fc/f6fa4ffbc79129a267d92782aba6889145fabfcb39023423fed1f468af3f/policyengine_us-1.722.4-py3-none-any.whl" } ] } @@ -75,14 +75,14 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "c6d064defe7eafb2f2c69732d5b54f2df3477f933726beb38dc5396c0d8996d8" + "trov:sha256": "a920dc3f674025578ed0f8befae0af4f08d5aa197152ebd4b50bc70c20faf5e6" }, { "@id": "composition/1/artifact/data_release_manifest", "@type": "trov:ResearchArtifact", "schema:name": "policyengine-us-data release manifest 1.115.5", "trov:mimeType": "application/json", - "trov:sha256": "577fbf704da44f63d0432dc3e80f3686eb3a32020d13ca6b7b6cf7eb60b4742c" + "trov:sha256": "16ba5b9ea74b8f5aa996cc66f6d28f64d2def4ea4ce087aec8d35c04ac706fc2" }, { "@id": "composition/1/artifact/dataset", @@ -94,15 +94,15 @@ { "@id": "composition/1/artifact/model_wheel", "@type": "trov:ResearchArtifact", - "schema:name": "policyengine-us==1.715.2 wheel", + "schema:name": "policyengine-us==1.722.4 wheel", "trov:mimeType": "application/zip", - "trov:sha256": "abf079828419762f5c4b0291a70f6e424744200f237e1ae0f06e25f10130c399" + "trov:sha256": "691d4aa9b1512d71d283b0b5805af5ba95ffed798fe471a97cf2195f4950b6e3" } ], "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "f8fc7d3c16b969ad8ef1522661631163613053aa2328428a6e59302567381ab5" + "trov:sha256": "3e324b0ae99e0ce580e9dffae19e4643be5c74723f2caf8a3b139ee09085aa0b" } }, "trov:hasPerformance": { @@ -110,15 +110,12 @@ "@type": "trov:TransparentResearchPerformance", "pe:builtWithModelVersion": "1.700.0", "pe:certifiedBy": "policyengine-us-data release manifest", - "pe:certifiedForModelVersion": "1.715.2", - "pe:ciGitRef": "refs/heads/main", - "pe:ciGitSha": "3b5979918486892fff7465e03e99176532593cb3", - "pe:ciRunUrl": "https://github.com/PolicyEngine/policyengine.py/actions/runs/27094103103", + "pe:certifiedForModelVersion": "1.722.4", "pe:compatibilityBasis": "legacy_compatible_model_package", "pe:dataBuildFingerprint": "sha256:b0862de383ffcbe45f4ba0aa9c6aaec286cd4c6688c6ccb33f939bc176f9a8a0", "pe:dataBuildId": "policyengine-us-data-1.115.5", - "pe:emittedIn": "github-actions", - "rdfs:comment": "Certification of build policyengine-us-data-1.115.5 for policyengine-us 1.715.2.", + "pe:emittedIn": "local", + "rdfs:comment": "Certification of build policyengine-us-data-1.115.5 for policyengine-us 1.722.4.", "trov:accessedArrangement": { "@id": "arrangement/1" }, diff --git a/src/policyengine/provenance/bundle.py b/src/policyengine/provenance/bundle.py index 1a22642e..e54186fa 100644 --- a/src/policyengine/provenance/bundle.py +++ b/src/policyengine/provenance/bundle.py @@ -495,13 +495,17 @@ def refresh_release_bundle( data_package_json = manifest_json["data_package"] release_manifest_json = None new_release_manifest_revision = None + current_release_manifest_revision = data_package_json.get( + "release_manifest_revision" + ) + release_manifest_override = ( + release_manifest_path is not None or release_manifest_revision is not None + ) new_release_manifest_path = release_manifest_path or data_package_json.get( "release_manifest_path" ) should_fetch_release_manifest = new_release_manifest_path is not None and ( - new_data != old_data - or release_manifest_path is not None - or release_manifest_revision is not None + new_data != old_data or new_model != old_model or release_manifest_override ) if should_fetch_release_manifest: if release_manifest_path is None: @@ -510,7 +514,11 @@ def refresh_release_bundle( old_data=old_data, new_data=new_data, ) - fetch_revision = release_manifest_revision or new_data + fetch_revision = release_manifest_revision or ( + current_release_manifest_revision + if release_manifest_path is None and new_data == old_data + else new_data + ) release_manifest_fetch = _fetch_data_release_manifest( repo_id=repo_id, release_manifest_path=new_release_manifest_path, @@ -560,7 +568,12 @@ def refresh_release_bundle( ) dataset_repo_id = data_artifact_json.get("repo_id", repo_id) dataset_path = data_artifact_json.get("path", dataset_path) - dataset_revision = data_artifact_json.get("revision", new_data) + dataset_revision_default = ( + _old_revision + if new_data == old_data and not release_manifest_override + else new_data + ) + dataset_revision = data_artifact_json.get("revision", dataset_revision_default) if ( release_manifest_json is not None and new_release_manifest_revision is not None @@ -569,10 +582,6 @@ def refresh_release_bundle( ): dataset_revision = new_release_manifest_revision - release_manifest_override = ( - release_manifest_path is not None or release_manifest_revision is not None - ) - # Only hit HF if the data version or release manifest target changed. if new_data != old_data or release_manifest_override: new_dataset_sha256 = data_artifact_json.get("sha256") or _hf_dataset_sha256( diff --git a/tests/fixtures/household_calculator_snapshots/us_model_surface.json b/tests/fixtures/household_calculator_snapshots/us_model_surface.json index ff2f0972..e3461a25 100644 --- a/tests/fixtures/household_calculator_snapshots/us_model_surface.json +++ b/tests/fixtures/household_calculator_snapshots/us_model_surface.json @@ -5,7 +5,7 @@ "has_income_tax": true, "has_region_registry": true, "model_package_name": "policyengine-us", - "num_parameters_bucketed_100s": 883, - "num_variables_bucketed_100s": 51, + "num_parameters_bucketed_100s": 896, + "num_variables_bucketed_100s": 52, "region_registry_country": "us" } diff --git a/tests/test_bundle_refresh.py b/tests/test_bundle_refresh.py index c9cae63a..6c1d8338 100644 --- a/tests/test_bundle_refresh.py +++ b/tests/test_bundle_refresh.py @@ -165,7 +165,7 @@ def sandbox(tmp_path: Path) -> dict: }, "build_id": "policyengine-us-data-1.70.0", "dataset": "enhanced_cps_2024", - "uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@1.70.0", + "uri": "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@old-dataset-commit", "sha256": "d" * 64, }, "certification": { @@ -208,6 +208,25 @@ def fake_urlopen(request, *args, **kwargs): url = request.full_url if "pypi.org" in url: return _pypi_response("policyengine-us", "1.653.3") + if url.endswith("releases/1.70.0/release_manifest.json"): + return _data_release_manifest_response( + data_version="1.70.0", + dataset_sha256="d" * 64, + compatible_model_packages=[ + {"name": "policyengine-us", "specifier": "==1.600.0"}, + {"name": "policyengine-us", "specifier": "==1.653.3"}, + ], + extra_artifacts={ + "enhanced_cps_2024": { + "kind": "microdata", + "path": "enhanced_cps_2024.h5", + "repo_id": "policyengine/policyengine-us-data", + "revision": "old-dataset-commit", + "sha256": "d" * 64, + } + }, + headers={"x-repo-commit": "old-release-manifest-commit"}, + ) raise AssertionError(f"Unexpected URL fetched: {url}") with patch("policyengine.provenance.bundle.urlopen", side_effect=fake_urlopen): @@ -239,6 +258,41 @@ def fake_urlopen(request, *args, **kwargs): == "old-release-manifest-commit" ) assert written["certified_data_artifact"]["sha256"] == "d" * 64 + assert ( + written["certified_data_artifact"]["uri"] + == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@old-dataset-commit" + ) + + +def test__bump_model_only_requires_data_release_manifest_compatibility( + sandbox, +) -> None: + def fake_urlopen(request, *args, **kwargs): + url = request.full_url + if "pypi.org" in url: + return _pypi_response("policyengine-us", "1.653.3") + if url.endswith("releases/1.70.0/release_manifest.json"): + return _data_release_manifest_response( + data_version="1.70.0", + model_version="1.600.0", + compatible_model_packages=[ + {"name": "policyengine-us", "specifier": "==1.600.0"}, + ], + headers={"x-repo-commit": "old-release-manifest-commit"}, + ) + raise AssertionError(f"Unexpected URL fetched: {url}") + + with patch("policyengine.provenance.bundle.urlopen", side_effect=fake_urlopen): + with pytest.raises( + ValueError, + match="Data release manifest is not certified", + ): + refresh_release_bundle( + country="us", + model_version="1.653.3", + manifest_dir=sandbox["manifest_dir"], + pyproject_path=sandbox["pyproject_path"], + ) def test__bump_data_only_streams_hf_and_updates_uri(sandbox) -> None: @@ -871,8 +925,30 @@ def fake_urlopen(request, *args, **kwargs): def test__update_pyproject_false_leaves_pins_alone(sandbox) -> None: - def fake_urlopen(*args, **kwargs): - return _pypi_response("policyengine-us", "1.653.3") + def fake_urlopen(request, *args, **kwargs): + url = request.full_url + if "pypi.org" in url: + return _pypi_response("policyengine-us", "1.653.3") + if url.endswith("releases/1.70.0/release_manifest.json"): + return _data_release_manifest_response( + data_version="1.70.0", + dataset_sha256="d" * 64, + compatible_model_packages=[ + {"name": "policyengine-us", "specifier": "==1.600.0"}, + {"name": "policyengine-us", "specifier": "==1.653.3"}, + ], + extra_artifacts={ + "enhanced_cps_2024": { + "kind": "microdata", + "path": "enhanced_cps_2024.h5", + "repo_id": "policyengine/policyengine-us-data", + "revision": "old-dataset-commit", + "sha256": "d" * 64, + } + }, + headers={"x-repo-commit": "old-release-manifest-commit"}, + ) + raise AssertionError(f"Unexpected URL fetched: {url}") with patch("policyengine.provenance.bundle.urlopen", side_effect=fake_urlopen): result = refresh_release_bundle( diff --git a/tests/test_import_policyengine_bundle.py b/tests/test_import_policyengine_bundle.py index 785c13aa..79569ac4 100644 --- a/tests/test_import_policyengine_bundle.py +++ b/tests/test_import_policyengine_bundle.py @@ -47,16 +47,16 @@ def test_import_policyengine_bundle_verifies_and_vendors_release( us_manifest = json.loads((release_manifest_dir / "us.json").read_text()) assert us_manifest["bundle_id"] == "us-4.14.0" assert us_manifest["policyengine_version"] == "4.14.0" - assert us_manifest["model_package"]["version"] == "1.715.2" + assert us_manifest["model_package"]["version"] == "1.722.4" assert ( us_manifest["certified_data_artifact"]["uri"] == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@data-sha" ) - assert us_manifest["certification"]["certified_for_model_version"] == "1.715.2" + assert us_manifest["certification"]["certified_for_model_version"] == "1.722.4" pyproject = pyproject_path.read_text() assert '"policyengine_core==3.26.1"' in pyproject - assert '"policyengine-us==1.715.2"' in pyproject + assert '"policyengine-us==1.722.4"' in pyproject assert '"policyengine-uk==3.0.0"' in pyproject assert (changelog_dir / "policyengine-bundle-4.14.0.changed.md").exists() @@ -140,7 +140,7 @@ def _bundle_manifest(version: str) -> dict: }, "policyengine-us": { "name": "policyengine-us", - "version": "1.715.2", + "version": "1.722.4", "resolution_status": "pinned", }, "policyengine-uk": { @@ -167,7 +167,7 @@ def _bundle_manifest(version: str) -> dict: def _country_bundle(country_id: str, version: str) -> dict: model_package = "policyengine-us" if country_id == "us" else "policyengine-uk" - model_version = "1.715.2" if country_id == "us" else "3.0.0" + model_version = "1.722.4" if country_id == "us" else "3.0.0" data_package = ( "policyengine-us-data" if country_id == "us" else "policyengine-uk-data" ) diff --git a/tests/test_models.py b/tests/test_models.py index 4b220d74..8934004d 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -113,12 +113,12 @@ def test_has_release_manifest_metadata(self): assert us_latest.release_manifest is not None assert us_latest.release_manifest.country_id == "us" assert us_latest.model_package.name == "policyengine-us" - assert us_latest.model_package.version == "1.715.2" + assert us_latest.model_package.version == "1.722.4" assert us_latest.data_package.name == "policyengine-us-data" assert us_latest.data_package.version == "1.115.5" assert ( us_latest.default_dataset_uri - == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@d47fb5475144260a75467d2f2e22b2d5d53d4d57" + == "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5@9531fe1d096244fe7eb45d791d52ef61b8a2a0a5" ) def test_has_hundreds_of_parameters(self): diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 63d283ea..5d54d67a 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -41,11 +41,11 @@ PYPROJECT.read_text(), re.MULTILINE, ).group(1) -US_MODEL_VERSION = "1.715.2" +US_MODEL_VERSION = "1.722.4" US_BUILT_WITH_MODEL_VERSION = "1.700.0" US_DATA_RELEASE_VERSION = "1.115.5" -US_DATA_RELEASE_PATH = "release_manifest.json" -US_DATA_RELEASE_REVISION = "d47fb5475144260a75467d2f2e22b2d5d53d4d57" +US_DATA_RELEASE_PATH = "releases/1.115.5/release_manifest.json" +US_DATA_RELEASE_REVISION = "9531fe1d096244fe7eb45d791d52ef61b8a2a0a5" US_CERTIFICATION_SOURCE = "policyengine-us-data release manifest" US_DEFAULT_DATASET_URI = ( "hf://policyengine/policyengine-us-data/" diff --git a/tests/test_us_long_term_datasets.py b/tests/test_us_long_term_datasets.py index ceaae47a..3cdeb5b9 100644 --- a/tests/test_us_long_term_datasets.py +++ b/tests/test_us_long_term_datasets.py @@ -273,7 +273,7 @@ def test__load_managed_long_term_datasets__loads_bundled_local_mirror( "get_release_manifest", lambda country_id: _manifest_with_long_term_sha( _sha256(h5_path), - version="1.715.2", + version="1.722.4", certification=SimpleNamespace( compatibility_basis="legacy_compatible_model_package", built_with_model_version="1.700.0", @@ -283,7 +283,7 @@ def test__load_managed_long_term_datasets__loads_bundled_local_mirror( monkeypatch.setattr( us_datasets_module, "_runtime_policyengine_us_version", - lambda: "1.715.2", + lambda: "1.722.4", ) monkeypatch.setattr( us_datasets_module, diff --git a/uv.lock b/uv.lock index c4d5cc6b..510ba44f 100644 --- a/uv.lock +++ b/uv.lock @@ -2820,7 +2820,7 @@ wheels = [ [[package]] name = "policyengine" -version = "4.14.1" +version = "4.14.2" source = { editable = "." } dependencies = [ { name = "diskcache" }, @@ -2897,8 +2897,8 @@ requires-dist = [ { name = "policyengine-core", marker = "extra == 'us'", specifier = "==3.26.1" }, { name = "policyengine-uk", marker = "extra == 'dev'", specifier = "==2.88.20" }, { name = "policyengine-uk", marker = "extra == 'uk'", specifier = "==2.88.20" }, - { name = "policyengine-us", marker = "extra == 'dev'", specifier = "==1.715.2" }, - { name = "policyengine-us", marker = "extra == 'us'", specifier = "==1.715.2" }, + { name = "policyengine-us", marker = "extra == 'dev'", specifier = "==1.722.4" }, + { name = "policyengine-us", marker = "extra == 'us'", specifier = "==1.722.4" }, { name = "psutil", specifier = ">=5.9.0" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pytest", marker = "extra == 'dev'" }, @@ -2961,7 +2961,7 @@ wheels = [ [[package]] name = "policyengine-us" -version = "1.715.2" +version = "1.722.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -2973,9 +2973,9 @@ dependencies = [ { name = "tables", version = "3.11.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a7/ef/d87bb056084897932e083b0412976a386d29062834b0e697afa044642a75/policyengine_us-1.715.2.tar.gz", hash = "sha256:b3990ae9b7c694d2cbf497e6256850aca7be5a5a73ac98330682aba9edd61b61", size = 10014025, upload-time = "2026-05-29T02:48:39.527Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/21/da9e789e4abc7d2c8e42c6913042e2e371b9054c468f28f2212a0652f13a/policyengine_us-1.722.4.tar.gz", hash = "sha256:2a60f294a3cfa891a82c1eeee0c31e7868859ca233075a8f81ceb3f235662ae7", size = 10134261, upload-time = "2026-06-08T09:04:58.953Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/45/a1/1d56bdbb69d7ce06bedd3892203a75ac3350a90c0b5fcea2fb50db46670f/policyengine_us-1.715.2-py3-none-any.whl", hash = "sha256:abf079828419762f5c4b0291a70f6e424744200f237e1ae0f06e25f10130c399", size = 11035379, upload-time = "2026-05-29T02:48:35.193Z" }, + { url = "https://files.pythonhosted.org/packages/20/fc/f6fa4ffbc79129a267d92782aba6889145fabfcb39023423fed1f468af3f/policyengine_us-1.722.4-py3-none-any.whl", hash = "sha256:691d4aa9b1512d71d283b0b5805af5ba95ffed798fe471a97cf2195f4950b6e3", size = 11319909, upload-time = "2026-06-08T09:04:55.761Z" }, ] [[package]]