Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ canonicalize = "fromager.commands.canonicalize:canonicalize"
download-sequence = "fromager.commands.download_sequence:download_sequence"
wheel-server = "fromager.commands.server:wheel_server"
lint-requirements = "fromager.commands.lint_requirements:lint_requirements"
cache = "fromager.commands.cache_cmd:cache"

[tool.coverage.run]
branch = true
Expand Down
76 changes: 71 additions & 5 deletions src/fromager/bootstrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -655,15 +655,44 @@ def _find_cached_wheel(
) -> tuple[pathlib.Path | None, pathlib.Path | None]:
"""Look for cached wheel in 3 locations.

Checks for cached wheels in order:
1. wheels_build directory (previously built)
2. wheels_downloads directory (previously downloaded)
3. Cache server (remote cache)
When a CacheManager is configured on the context, delegates to it
for a unified hierarchical lookup across collections. Otherwise
falls back to the legacy per-directory search.

Returns:
Tuple of (cached_wheel_filename, unpacked_cached_wheel).
Both None if no cache hit.
"""
if self.ctx.cache is not None:
return self._find_cached_wheel_via_manager(req, resolved_version)
return self._find_cached_wheel_legacy(req, resolved_version)

def _find_cached_wheel_via_manager(
self,
req: Requirement,
resolved_version: Version,
) -> tuple[pathlib.Path | None, pathlib.Path | None]:
"""Cache lookup using the CacheManager."""
assert self.ctx.cache is not None
pbi = self.ctx.package_build_info(req)
build_tag = pbi.build_tag(resolved_version)

result = self.ctx.cache.lookup_wheel(req, resolved_version, build_tag)
if not result.hit:
return None, None

assert result.path is not None
metadata_dir = self._unpack_metadata_from_wheel(
req, resolved_version, result.path
)
return result.path, metadata_dir

def _find_cached_wheel_legacy(
self,
req: Requirement,
resolved_version: Version,
) -> tuple[pathlib.Path | None, pathlib.Path | None]:
"""Legacy cache lookup: check build dir, downloads dir, remote cache."""
# Check if we have previously built a wheel and still have it on the
# local filesystem.
cached_wheel, unpacked = self._look_for_existing_wheel(
Expand Down Expand Up @@ -1493,12 +1522,35 @@ def _phase_prepare_source(self, item: WorkItem) -> list[WorkItem]:
item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS
return [item]

# Source build path
# Source build path: try cache first
cached_wheel, unpacked = self._find_cached_wheel(
item.req, item.resolved_version
)
item.cached_wheel_filename = cached_wheel

# Short-circuit: when CacheManager provides a hit, skip directly to
# PROCESS_INSTALL_DEPS -- no source download, no build env, no build
# deps resolution needed. Install deps are extracted from the wheel.
if cached_wheel and self.ctx.cache is not None:
server.update_wheel_mirror(self.ctx)
# Route to the correct collection (e.g., variant dir for listed packages)
pbi = self.ctx.package_build_info(item.req)
build_tag = pbi.build_tag(item.resolved_version)
self.ctx.cache.store_wheel(
item.req, item.resolved_version, build_tag, cached_wheel
)
Comment on lines +1535 to +1541

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win

Update the wheel mirror after storing the routed cache hit.

store_wheel() can copy the hit into the routed collection, so updating the mirror first can leave the internal wheel index stale for that newly stored path.

Suggested fix
-            server.update_wheel_mirror(self.ctx)
             # Route to the correct collection (e.g., variant dir for listed packages)
             pbi = self.ctx.package_build_info(item.req)
             build_tag = pbi.build_tag(item.resolved_version)
-            self.ctx.cache.store_wheel(
+            cached_wheel = self.ctx.cache.store_wheel(
                 item.req, item.resolved_version, build_tag, cached_wheel
             )
+            server.update_wheel_mirror(self.ctx)
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
server.update_wheel_mirror(self.ctx)
# Route to the correct collection (e.g., variant dir for listed packages)
pbi = self.ctx.package_build_info(item.req)
build_tag = pbi.build_tag(item.resolved_version)
self.ctx.cache.store_wheel(
item.req, item.resolved_version, build_tag, cached_wheel
)
# Route to the correct collection (e.g., variant dir for listed packages)
pbi = self.ctx.package_build_info(item.req)
build_tag = pbi.build_tag(item.resolved_version)
cached_wheel = self.ctx.cache.store_wheel(
item.req, item.resolved_version, build_tag, cached_wheel
)
server.update_wheel_mirror(self.ctx)
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/fromager/bootstrapper.py` around lines 1535 - 1541, The wheel mirror
update is happening before the routed cache entry is stored, which can leave the
internal index out of sync for the newly copied path. In `bootstrapper.py`,
reorder the flow in the cache-hit handling around `server.update_wheel_mirror`,
`self.ctx.package_build_info`, and `self.ctx.cache.store_wheel` so the wheel is
stored in the routed collection first and the mirror is updated afterward. Keep
the existing routing logic intact, just move the mirror refresh to run after
`store_wheel()` completes.

unpack_dir = self._create_unpack_dir(item.req, item.resolved_version)
item.build_result = SourceBuildResult(
wheel_filename=cached_wheel,
sdist_filename=None,
unpack_dir=unpack_dir,
sdist_root_dir=None,
build_env=None,
source_type=SourceType.CACHED,
)
item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS
return [item]

if not unpacked:
logger.debug("no cached wheel, downloading sources")
source_filename = self._download_source(
Expand Down Expand Up @@ -1618,6 +1670,20 @@ def _phase_build(self, item: WorkItem) -> list[WorkItem]:
cached_wheel_filename=item.cached_wheel_filename,
)

# Route newly built wheels to the appropriate collection directory.
# This copies the wheel into the routed collection's storage while
# keeping the original in downloads/ for the internal wheel server.
if (
wheel_filename is not None
and self.ctx.cache is not None
and not item.cached_wheel_filename
):
pbi = self.ctx.package_build_info(item.req)
build_tag = pbi.build_tag(item.resolved_version)
self.ctx.cache.store_wheel(
item.req, item.resolved_version, build_tag, wheel_filename
)

source_type = sources.get_source_type(self.ctx, item.req)

item.build_result = SourceBuildResult(
Expand Down
Loading
Loading