diff --git a/docs/proposals/unified-cache-manager.rst b/docs/proposals/unified-cache-manager.rst new file mode 100644 index 00000000..9ca56652 --- /dev/null +++ b/docs/proposals/unified-cache-manager.rst @@ -0,0 +1,160 @@ +Proposal: Unified Cache Manager +================================ + +Problem +------- + +Fromager's caching logic is scattered across multiple modules with no central +coordination. Prebuilt wheels are checked separately from previously built +wheels, remote wheel servers are not consulted during local builds, and there +is no mechanism to share a cache across multiple backends. This leads to: + +- Redundant builds when wheels already exist in a remote cache. +- No visibility into cache hit rates, artifact integrity, or staleness. +- No short-circuit path — even a full cache hit still downloads source and + sets up a build environment before discovering the wheel exists. + +Proposed Solution +----------------- + +Introduce a unified ``CacheManager`` class that centralizes all cache +operations behind a prioritized multi-backend lookup strategy. + +Architecture +~~~~~~~~~~~~ + +.. code-block:: text + + CacheManager + ├── lookup_backends (searched in order): + │ ├── LocalDirectoryBackend(wheels-repo/build/) [recursive for parallel builds] + │ ├── LocalDirectoryBackend(wheels-repo/downloads/) + │ └── RemotePEP503Backend(https://cache-server/simple/) [optional] + └── store_backend: + └── LocalDirectoryBackend(wheels-repo/downloads/) + + Prebuilt wheels stay on the dedicated ``SourceType.PREBUILT`` path and are + not consulted by the general cache short-circuit. + +Key components: + +- ``WheelCacheKey`` — Content-addresses artifacts by canonicalized package + name, version, and numeric build tag. +- ``CacheBackend`` protocol — Abstract interface implemented by + ``LocalDirectoryBackend`` (filesystem) and ``RemotePEP503Backend`` + (PEP 503 simple repository). Thread-safe with internal locking. +- ``CacheManager`` — Orchestrates prioritized lookup across backends with + graceful fallback on fetch failures, and routes all stores to a single + designated local backend. + +Lookup and store +~~~~~~~~~~~~~~~~ + +On lookup, the manager iterates ``lookup_backends`` in order and returns the +first hit. Remote hits are downloaded to the store backend's directory and +registered in the local index for subsequent fast lookups. + +On store, newly built wheels are always placed in the single ``store_backend`` +(the downloads directory). This keeps the design simple and avoids +collection-routing complexity in core Fromager. Variant-specific routing +can be added as builder-level logic in a follow-up. + +Short-circuit optimization +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When a cache hit is found during the ``PREPARE_SOURCE`` phase and a +``CacheManager`` is active, ``PrepareSource.run()`` skips build environment +creation and build dependency resolution entirely — proceeding directly to +``ProcessInstallDeps``. Install dependencies are extracted from the cached +wheel's metadata. This eliminates the most expensive steps for packages that +do not need rebuilding. + +Remote cache with integrity +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``RemotePEP503Backend`` lazily fetches per-project package indices on first +access and maintains a session-scoped in-memory index. Lookups honor +interpreter wheel tags, skip yanked files (PEP 592), and filter +``data-requires-python`` the same way the resolver does. Downloads are +verified with streaming SHA256 checksums, use atomic temporary files, and +reject plaintext HTTP URLs that lack integrity hashes (unless +``--cache-allow-insecure`` is passed for development workflows). Filenames +are sanitized to prevent path traversal attacks. HTTP 400/404 (and empty +successful pages) are treated as definitive misses for the run; transport +and 5xx failures are retried on later lookups. + +Observability +~~~~~~~~~~~~~ + +A new ``fromager cache`` CLI command group provides: + +- ``cache list`` — Show all cached artifacts with versions and build tags. +- ``cache stats`` — Display on-disk inventory counts and sizes per backend. + (Hit/miss rates are process-local during bootstrap and are not persisted.) +- ``cache verify`` — Validate integrity of local cache contents. +- ``cache invalidate`` — Remove specific artifacts by name/version/tag. +- ``cache gc`` — Garbage-collect old build tags, keeping only the N most + recent per package+version. + +Scope +----- + +The new cache subsystem is opt-in via ``--use-cache-manager`` on the +``bootstrap`` command. When disabled, existing behavior is preserved +unchanged. + +Files added or modified: + +- ``src/fromager/cache.py`` — New module with all cache classes and factory. +- ``src/fromager/commands/cache_cmd.py`` — CLI commands. +- ``src/fromager/commands/bootstrap.py`` — Wiring ``--use-cache-manager`` and + ``--cache-allow-insecure`` options. +- ``src/fromager/bootstrapper/_cache.py`` — Short-circuit integration via + ``_find_cached_wheel_via_manager``. +- ``src/fromager/context.py`` — ``cache`` property on ``WorkContext``. +- ``tests/test_cache.py`` — Unit tests for the cache subsystem. + +Benefits +-------- + +- Eliminates redundant builds when wheels exist in a remote or local cache. +- Reduces bootstrap time by short-circuiting cached packages (skips source + download, build env setup, and build dep resolution). +- Provides cache observability through dedicated CLI commands. +- Enforces artifact integrity with SHA256 verification and atomic writes. +- Thread-safe design compatible with background I/O pre-fetching. + +Security considerations +----------------------- + +- Remote downloads are verified against SHA256 hashes declared in PEP 503 + index pages. Mismatched files are deleted and raise an error. +- Plaintext HTTP URLs without SHA256 hashes are rejected by default. + The ``--cache-allow-insecure`` flag explicitly opts in for internal or + development registries. +- Filenames from remote indices are sanitized to prevent directory traversal. +- Local cache writes use atomic ``tempfile`` + ``rename`` to prevent readers + from observing partial files. +- ``scan()`` skips symlinked wheels to prevent ``invalidate``/``gc`` from + deleting files outside the cache root. +- Fetch failures (network errors, hash mismatches) are caught and treated as + cache misses, falling through to the next backend or a fresh build. + +Verification +------------ + +- All existing unit and e2e tests pass unchanged (legacy path preserved). +- New tests cover cache components, short-circuit logic, concurrency safety, + CLI commands, and error handling. +- Linting (``ruff``), type checking (``mypy``), and formatting all pass. + +Future work +----------- + +- Integration with ``build_tag_hook`` (issue #1059) for platform-suffixed + cache keys. +- Variant/collection-based store routing as builder-level logic (separate PR). +- Automatic detection of accelerated packages via ELF inspection or wheel + tag analysis. +- Promotion of ``--use-cache-manager`` to default behavior once proven in + production. diff --git a/pyproject.toml b/pyproject.toml index ea3ce5f4..a706f454 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,6 +128,7 @@ canonicalize = "fromager.commands.canonicalize:canonicalize" download-sequence = "fromager.commands.download_sequence:download_sequence" wheel-server = "fromager.commands.server:wheel_server" lint-requirements = "fromager.commands.lint_requirements:lint_requirements" +cache = "fromager.commands.cache_cmd:cache_cli" [tool.coverage.run] branch = true diff --git a/src/fromager/bootstrapper/_build.py b/src/fromager/bootstrapper/_build.py index 250b99ac..78ee119f 100644 --- a/src/fromager/bootstrapper/_build.py +++ b/src/fromager/bootstrapper/_build.py @@ -130,6 +130,15 @@ def _build_wheel( server.update_wheel_mirror(ctx) # When we update the mirror, the built file moves to the downloads directory. wheel_filename = ctx.wheels_downloads / built_filename.name + # Keep the CacheManager index in sync for same-run lookups (e.g. + # --multiple-versions). Without this, initialize()-time scans miss + # wheels produced later in the bootstrap. + if ctx.cache is not None: + pbi = ctx.package_build_info(wi.req) + build_tag = pbi.build_tag(wi.resolved_version) + ctx.cache.store_wheel( + wi.req, wi.resolved_version, build_tag, wheel_filename + ) logger.info(f"built wheel for version {wi.resolved_version}: {wheel_filename}") return wheel_filename, sdist_filename diff --git a/src/fromager/bootstrapper/_cache.py b/src/fromager/bootstrapper/_cache.py index e8a3c683..a1a7f549 100644 --- a/src/fromager/bootstrapper/_cache.py +++ b/src/fromager/bootstrapper/_cache.py @@ -146,7 +146,8 @@ def _download_wheel_from_cache( req=req, wheel_url=wheel_url, output_directory=ctx.wheels_downloads ) if cache_wheel_server_url != ctx.wheel_server_url: - server.update_wheel_mirror(ctx) + # Index only this wheel — never sweep wheels_build from bg threads. + server.index_wheel(ctx, cached_wheel) logger.info("found built wheel on cache server") unpack_dir = _extract_build_reqs_from_wheel( ctx.work_dir, req, resolved_version, cached_wheel @@ -171,23 +172,71 @@ def _download_wheel_from_cache( return None, None +def _find_cached_wheel_via_manager( + ctx: context.WorkContext, + req: Requirement, + resolved_version: Version, +) -> tuple[pathlib.Path | None, pathlib.Path | None]: + """Cache lookup using the CacheManager (thread-safe, no Bootstrapper state). + + Delegates to ``ctx.cache.lookup_wheel()`` for a unified prioritized + lookup across backends. On hit, extracts build-requirement metadata + into a work_dir unpack directory (same contract as the legacy path). + + Returns: + Tuple of (cached_wheel_filename, unpack_dir). + Both None if no cache hit. ``unpack_dir`` is under ``ctx.work_dir``. + """ + assert ctx.cache is not None + pbi = ctx.package_build_info(req) + build_tag = pbi.build_tag(resolved_version) + + result = ctx.cache.lookup_wheel(req, resolved_version, build_tag) + if not result.hit: + return None, None + + assert result.path is not None + if result.was_downloaded: + # Index only the promoted wheel. Full update_wheel_mirror() would + # move every file out of the shared wheels_build directory and can + # steal an in-progress build from another package. + server.index_wheel(ctx, result.path) + + try: + unpack_dir = _extract_build_reqs_from_wheel( + ctx.work_dir, req, resolved_version, result.path + ) + except Exception as err: + logger.debug( + "could not extract build requirements from cached wheel %s: %s", + result.path.name, + err, + ) + unpack_dir = None + if unpack_dir is None: + unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) + return result.path, unpack_dir + + def find_cached_wheel( ctx: context.WorkContext, cache_wheel_server_url: str | None, req: Requirement, resolved_version: Version, ) -> tuple[pathlib.Path | None, pathlib.Path | None]: - """Look for cached wheel in 3 locations (thread-safe, no Bootstrapper state). + """Look for cached wheel (thread-safe, no Bootstrapper state). - Checks for cached wheels in order: - 1. wheels_build directory (previously built) - 2. wheels_downloads directory (previously downloaded) - 3. Cache server (remote cache) + When a CacheManager is configured on the context, delegates to it + for a unified prioritized lookup across backends. Otherwise falls + back to the legacy per-directory search. Returns: Tuple of (cached_wheel_filename, unpacked_cached_wheel). Both None if no cache hit. """ + if ctx.cache is not None: + return _find_cached_wheel_via_manager(ctx, req, resolved_version) + cached_wheel, unpacked = _look_for_existing_wheel( ctx, req, resolved_version, ctx.wheels_build ) @@ -218,9 +267,10 @@ def bg_prepare_prebuilt( ) -> PreparedSourceData: """Background-safe prebuilt download: no Bootstrapper state accessed.""" # Thread-safe: paths include {req.name}-{resolved_version} (unique per package), - # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked. + # mkdir uses exist_ok=True (atomic), and index_wheel() is locked. logger.info(f"using pre-built wheel for {req_type} requirement") wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) - server.update_wheel_mirror(ctx) + # Index only this prebuilt wheel so bg threads never sweep wheels_build. + server.index_wheel(ctx, wheel_filename) return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) diff --git a/src/fromager/bootstrapper/_prepare_source.py b/src/fromager/bootstrapper/_prepare_source.py index fd4c761b..83a037e0 100644 --- a/src/fromager/bootstrapper/_prepare_source.py +++ b/src/fromager/bootstrapper/_prepare_source.py @@ -6,7 +6,7 @@ from packaging.requirements import Requirement from packaging.version import Version -from .. import build_environment, dependencies, sources +from .. import build_environment, dependencies, server, sources from ..log import req_ctxvar_context from ..requirements_file import RequirementType, SourceType from . import _cache @@ -40,7 +40,7 @@ def _bg_prepare_source( ) if unpacked is not None: return PreparedSourceData( - sdist_root_dir=unpacked / unpacked.stem, + sdist_root_dir=unpacked / unpacked.name, cached_wheel_filename=cached_wheel, ) source_filename = sources.download_source( @@ -146,6 +146,35 @@ def run(self, bt: Bootstrapper) -> list[Phase]: sdist_root_dir = prepared.sdist_root_dir wi.cached_wheel_filename = prepared.cached_wheel_filename + # Short-circuit: when CacheManager provides a hit, skip build env + # creation and build-dep resolution. Install deps come from the wheel. + if prepared.cached_wheel_filename is not None and bt.ctx.cache is not None: + pbi = bt.ctx.package_build_info(wi.req) + build_tag = pbi.build_tag(wi.resolved_version) + bt.ctx.cache.store_wheel( + wi.req, + wi.resolved_version, + build_tag, + prepared.cached_wheel_filename, + ) + # Wheel is already in downloads; index it without sweeping wheels_build. + server.index_wheel(bt.ctx, prepared.cached_wheel_filename) + # Prefer the unpack dir produced by find_cached_wheel (work_dir/-). + unpack_dir = prepared.sdist_root_dir.parent + if unpack_dir.parent != bt.ctx.work_dir: + unpack_dir = _cache._create_unpack_dir( + bt.ctx.work_dir, wi.req, wi.resolved_version + ) + wi.build_result = SourceBuildResult( + wheel_filename=prepared.cached_wheel_filename, + sdist_filename=None, + unpack_dir=unpack_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.CACHED, + ) + return [ProcessInstallDeps(wi)] + assert sdist_root_dir is not None if sdist_root_dir.parent.parent != bt.ctx.work_dir: diff --git a/src/fromager/cache.py b/src/fromager/cache.py new file mode 100644 index 00000000..821623f2 --- /dev/null +++ b/src/fromager/cache.py @@ -0,0 +1,1084 @@ +"""Unified cache subsystem for Fromager artifact management. + +Provides a layered cache with prioritized multi-backend lookup. Backends +are searched in order (local directories first, then remote PEP 503 +repositories). Stores always go to a single designated local backend. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import logging +import pathlib +import re +import shutil +import tempfile +import threading +import time +import typing +from html import unescape +from urllib.parse import unquote, urlparse + +from packaging.requirements import Requirement +from packaging.specifiers import InvalidSpecifier +from packaging.utils import ( + BuildTag, + InvalidWheelFilename, + NormalizedName, + canonicalize_name, + parse_wheel_filename, +) +from packaging.version import Version + +from .request_session import session +from .resolver import SUPPORTED_TAGS, match_py_req + +if typing.TYPE_CHECKING: + from . import context + +logger = logging.getLogger(__name__) + + +def _wheel_compatible(filename: str) -> bool: + """Return True if the wheel's tags intersect the current interpreter tags.""" + try: + _, _, _, tags = parse_wheel_filename(filename) + except InvalidWheelFilename: + return False + return bool(SUPPORTED_TAGS.intersection(tags)) + + +# Quote-aware matching so attributes like data-requires-python=">=3.8" +# do not truncate the tag at the embedded '>'. +_ANCHOR_RE = re.compile( + r"\"']|\"[^\"]*\"|'[^']*')*)>(.*?)", + re.IGNORECASE | re.DOTALL, +) +_HREF_RE = re.compile(r"""\bhref\s*=\s*["']([^"']+)["']""", re.IGNORECASE) +_YANKED_RE = re.compile(r"\bdata-yanked\b", re.IGNORECASE) +_REQUIRES_PYTHON_RE = re.compile( + r"""\bdata-requires-python\s*=\s*["']([^"']*)["']""", + re.IGNORECASE, +) + + +def _iter_anchor_links(html: str) -> typing.Iterator[tuple[str, str, str]]: + """Yield ``(href, link_text, attrs)`` triples from HTML anchor tags.""" + for match in _ANCHOR_RE.finditer(html): + attrs, text = match.group(1), match.group(2) + href_match = _HREF_RE.search(attrs) + if href_match is None: + continue + yield href_match.group(1), re.sub(r"\s+", " ", text).strip(), attrs + + +# --------------------------------------------------------------------------- +# Cache Keys +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class WheelCacheKey: + """Identifies a cached wheel artifact. + + The key uses package name, version, and build tag for precise matching. + """ + + package: NormalizedName + version: Version + build_tag: BuildTag # (int, str) from changelog; () if untagged + + def __str__(self) -> str: + tag_str = f"-{self.build_tag[0]}{self.build_tag[1]}" if self.build_tag else "" + return f"{self.package}=={self.version}{tag_str}" + + +# --------------------------------------------------------------------------- +# Artifact Metadata +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class ArtifactInfo: + """Lightweight metadata for a cached artifact. + + Produced by scanning backends. For local backends, ``url_or_path`` is + an absolute filesystem path. For remote backends, it is a download URL. + """ + + filename: str + url_or_path: str + size_bytes: int | None = None + sha256: str | None = None + + +# --------------------------------------------------------------------------- +# Cache Result +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class CacheResult: + """Result of a cache lookup operation. + + ``was_downloaded`` is True when the artifact came from a backend other + than the store backend (for example build-dir or remote) and was + registered into the store directory, so the caller should refresh the + local wheel mirror. ``path`` always points at the store copy after + promotion. + """ + + hit: bool + path: pathlib.Path | None = None + backend_name: str = "" + build_tag: BuildTag = () + was_downloaded: bool = False + + @property + def miss(self) -> bool: + return not self.hit + + +# --------------------------------------------------------------------------- +# Observability +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class CacheEvent: + """A single cache interaction event.""" + + timestamp: float + action: typing.Literal["hit", "miss", "store"] + artifact_type: typing.Literal["wheel", "sdist"] + package: str + version: str + backend: str + duration_ms: float | None = None + + +@dataclasses.dataclass +class CacheStats: + """Accumulates cache events for a single run. + + Thread-safe: all mutations and reads of ``events`` are protected by a lock + so background I/O threads can record hits/misses concurrently. + """ + + events: list[CacheEvent] = dataclasses.field(default_factory=list) + _lock: threading.Lock = dataclasses.field( + default_factory=threading.Lock, repr=False, compare=False + ) + + def record_hit( + self, + req: Requirement, + version: Version, + backend: str, + artifact_type: typing.Literal["wheel", "sdist"] = "wheel", + duration_ms: float | None = None, + ) -> None: + event = CacheEvent( + timestamp=time.time(), + action="hit", + artifact_type=artifact_type, + package=str(req.name), + version=str(version), + backend=backend, + duration_ms=duration_ms, + ) + with self._lock: + self.events.append(event) + + def record_miss( + self, + req: Requirement, + version: Version, + reason: str, + artifact_type: typing.Literal["wheel", "sdist"] = "wheel", + ) -> None: + event = CacheEvent( + timestamp=time.time(), + action="miss", + artifact_type=artifact_type, + package=str(req.name), + version=str(version), + backend=reason, + ) + with self._lock: + self.events.append(event) + + def record_store( + self, + req: Requirement, + version: Version, + backend: str, + artifact_type: typing.Literal["wheel", "sdist"] = "wheel", + ) -> None: + event = CacheEvent( + timestamp=time.time(), + action="store", + artifact_type=artifact_type, + package=str(req.name), + version=str(version), + backend=backend, + ) + with self._lock: + self.events.append(event) + + @property + def hits(self) -> int: + with self._lock: + return sum(1 for e in self.events if e.action == "hit") + + @property + def misses(self) -> int: + with self._lock: + return sum(1 for e in self.events if e.action == "miss") + + @property + def stores(self) -> int: + with self._lock: + return sum(1 for e in self.events if e.action == "store") + + @property + def hit_rate(self) -> float: + with self._lock: + hits = sum(1 for e in self.events if e.action == "hit") + misses = sum(1 for e in self.events if e.action == "miss") + total = hits + misses + if total == 0: + return 0.0 + return hits / total + + def summary(self) -> dict[str, typing.Any]: + """Return a structured summary suitable for JSON serialization.""" + with self._lock: + events = list(self.events) + hits_by_backend: dict[str, int] = {} + hits = 0 + misses = 0 + stores = 0 + for e in events: + if e.action == "hit": + hits += 1 + hits_by_backend[e.backend] = hits_by_backend.get(e.backend, 0) + 1 + elif e.action == "miss": + misses += 1 + elif e.action == "store": + stores += 1 + total = hits + misses + return { + "hits": { + "total": hits, + "by_backend": hits_by_backend, + }, + "misses": misses, + "stores": stores, + "hit_rate": round(hits / total, 4) if total else 0.0, + } + + +# --------------------------------------------------------------------------- +# Cache Backend Protocol +# --------------------------------------------------------------------------- + + +class CacheBackend(typing.Protocol): + """Protocol for a single storage location that can find and store artifacts.""" + + @property + def name(self) -> str: + """Human-readable identifier (e.g., 'local:downloads', 'remote:https://...').""" + ... + + @property + def writable(self) -> bool: + """Whether this backend supports store operations.""" + ... + + def scan(self) -> dict[WheelCacheKey, ArtifactInfo]: + """Bulk index at startup. Local backends return full inventory; + remote backends fetch the top-level package list only and return empty. + """ + ... + + def lookup(self, key: WheelCacheKey) -> ArtifactInfo | None: + """Find a specific artifact by key. + + For local backends, checks the in-memory index. + For remote backends, lazily fetches the project page on first access. + """ + ... + + def fetch( + self, key: WheelCacheKey, info: ArtifactInfo, dest: pathlib.Path + ) -> pathlib.Path: + """Retrieve artifact to a local path. + + For local backends, returns the existing path (no-op). + For remote backends, downloads the file to ``dest``. + """ + ... + + def store(self, key: WheelCacheKey, artifact: pathlib.Path) -> ArtifactInfo: + """Store a newly built artifact. Only valid if ``writable`` is True.""" + ... + + def items(self) -> typing.Iterable[tuple[WheelCacheKey, ArtifactInfo]]: + """Iterate over all indexed artifacts in this backend.""" + ... + + +# --------------------------------------------------------------------------- +# Local Directory Backend +# --------------------------------------------------------------------------- + + +class LocalDirectoryBackend: + """Cache backend backed by a local filesystem directory. + + Scans at startup to populate an in-memory index from existing wheel files. + New stores are reflected immediately in the index. All public methods are + thread-safe. + """ + + def __init__( + self, + directory: pathlib.Path, + backend_name: str = "local", + *, + recursive: bool = False, + ) -> None: + self._directory = directory + self._backend_name = backend_name + self._recursive = recursive + self._index: dict[WheelCacheKey, ArtifactInfo] = {} + self._lock = threading.Lock() + + @property + def name(self) -> str: + return self._backend_name + + @property + def writable(self) -> bool: + return True + + @property + def directory(self) -> pathlib.Path: + return self._directory + + def _iter_wheel_files(self) -> typing.Iterator[pathlib.Path]: + """Yield wheel files in this backend's directory.""" + yield from self._directory.glob("*.whl") + if self._recursive: + # Parallel builds land wheels in wheels_build_base//. + yield from self._directory.glob("*/*.whl") + + def scan(self) -> dict[WheelCacheKey, ArtifactInfo]: + """Scan the directory for wheel files and populate the index. + + Only indexes wheels compatible with the current interpreter tags. + """ + with self._lock: + self._index.clear() + if not self._directory.exists(): + return dict(self._index) + + for wheel_file in self._iter_wheel_files(): + if wheel_file.is_symlink(): + continue + if not _wheel_compatible(wheel_file.name): + logger.debug( + "skipping incompatible wheel for this platform: %s", + wheel_file.name, + ) + continue + try: + name, version, build_tag, _ = parse_wheel_filename(wheel_file.name) + key = WheelCacheKey( + package=name, + version=version, + build_tag=build_tag, + ) + # Keep the first compatible wheel for a key. + if key in self._index: + continue + info = ArtifactInfo( + filename=wheel_file.name, + url_or_path=str(wheel_file.resolve()), + size_bytes=wheel_file.stat().st_size, + ) + self._index[key] = info + except InvalidWheelFilename: + logger.debug("skipping unparseable wheel file: %s", wheel_file.name) + logger.debug("scanned %d wheels in %s", len(self._index), self._directory) + return dict(self._index) + + def lookup(self, key: WheelCacheKey) -> ArtifactInfo | None: + """Look up artifact in the in-memory index. + + When ``key.build_tag`` is empty, matches any build tag for the same + package+version (legacy ``find_wheel`` semantics). Prefers the highest + build tag number when multiple candidates exist. Only returns wheels + compatible with the current interpreter tags. + """ + with self._lock: + if key.build_tag: + info = self._index.get(key) + if info is None: + return None + file_path = pathlib.Path(info.url_or_path) + if not file_path.exists(): + del self._index[key] + return None + if not _wheel_compatible(info.filename): + return None + return info + + # Empty build tag: accept any matching package+version. + candidates: list[tuple[WheelCacheKey, ArtifactInfo]] = [] + stale: list[WheelCacheKey] = [] + for indexed_key, info in self._index.items(): + if ( + indexed_key.package != key.package + or indexed_key.version != key.version + ): + continue + if not pathlib.Path(info.url_or_path).exists(): + stale.append(indexed_key) + continue + if not _wheel_compatible(info.filename): + continue + candidates.append((indexed_key, info)) + for stale_key in stale: + del self._index[stale_key] + if not candidates: + return None + candidates.sort( + key=lambda item: item[0].build_tag[0] if item[0].build_tag else -1, + reverse=True, + ) + return candidates[0][1] + + def fetch( + self, key: WheelCacheKey, info: ArtifactInfo, dest: pathlib.Path + ) -> pathlib.Path: + """Return the existing local path (no-op for local backends).""" + return pathlib.Path(info.url_or_path) + + def store(self, key: WheelCacheKey, artifact: pathlib.Path) -> ArtifactInfo: + """Register an artifact in this backend's directory. + + If the artifact is not already in the directory, it is atomically + copied there (preserving the original for the internal wheel server + index). Updates the in-memory index. + """ + dest = self._directory / artifact.name + self._directory.mkdir(parents=True, exist_ok=True) + if not dest.exists() or not artifact.samefile(dest): + fd = tempfile.NamedTemporaryFile( + dir=self._directory, + prefix=f".{dest.name}.", + suffix=".tmp", + delete=False, + ) + tmp_dest = pathlib.Path(fd.name) + fd.close() + try: + shutil.copy2(str(artifact), str(tmp_dest)) + tmp_dest.replace(dest) + except BaseException: + tmp_dest.unlink(missing_ok=True) + raise + + info = ArtifactInfo( + filename=dest.name, + url_or_path=str(dest.resolve()), + size_bytes=dest.stat().st_size, + ) + with self._lock: + self._index[key] = info + return info + + def items(self) -> typing.Iterable[tuple[WheelCacheKey, ArtifactInfo]]: + """Iterate over all indexed artifacts.""" + with self._lock: + return list(self._index.items()) + + +# --------------------------------------------------------------------------- +# Remote PEP 503 Backend +# --------------------------------------------------------------------------- + + +class RemotePEP503Backend: + """Cache backend backed by a remote PEP 503 (Simple Repository API) server. + + At startup, fetches the top-level package list. Individual project pages + are fetched lazily on first lookup per package and memoized for the run. + All public methods are thread-safe. + """ + + def __init__( + self, + server_url: str, + download_dir: pathlib.Path, + backend_name: str | None = None, + allow_insecure: bool = False, + ) -> None: + self._server_url = server_url.rstrip("/") + self._download_dir = download_dir + self._backend_name = backend_name or f"remote:{self._server_url}" + self._allow_insecure = allow_insecure + self._available_packages: set[NormalizedName] | None = None + self._project_cache: dict[NormalizedName, list[ArtifactInfo]] = {} + self._lock = threading.Lock() + + @property + def name(self) -> str: + return self._backend_name + + @property + def writable(self) -> bool: + return False + + def scan(self) -> dict[WheelCacheKey, ArtifactInfo]: + """Fetch top-level index to learn which packages exist. + + On index fetch failure, ``_available_packages`` stays ``None`` so + lookups still attempt per-project pages instead of treating every + package as unknown. + """ + self._available_packages = self._fetch_package_list() + if self._available_packages is None: + logger.warning( + "remote index %s unavailable; will probe project pages lazily", + self._server_url, + ) + else: + logger.debug( + "remote %s has %d packages available", + self._server_url, + len(self._available_packages), + ) + return {} + + def lookup(self, key: WheelCacheKey) -> ArtifactInfo | None: + """Lazy per-package lookup with short-circuit for unknown packages. + + Only returns wheels compatible with the current interpreter tags. + Transient project-page failures are not memoized. + """ + if ( + self._available_packages is not None + and key.package not in self._available_packages + ): + return None + + # Double-checked locking: never hold the lock during network I/O so + # concurrent lookups for other packages are not blocked. + with self._lock: + cached = self._project_cache.get(key.package) + if cached is None: + fetched = self._fetch_project_page(key.package) + if fetched is None: + # Transient failure — do not memoize; try again next lookup. + return None + with self._lock: + # Another thread may have filled the cache while we fetched. + cached = self._project_cache.setdefault(key.package, fetched) + + matches: list[tuple[BuildTag, ArtifactInfo]] = [] + for info in cached: + try: + name, version, build_tag, _ = parse_wheel_filename(info.filename) + except InvalidWheelFilename: + continue + if name != key.package or version != key.version: + continue + if not _wheel_compatible(info.filename): + continue + if key.build_tag and build_tag != key.build_tag: + continue + if key.build_tag: + return info + matches.append((build_tag, info)) + + if not matches: + return None + # Empty expected build tag: prefer highest available build tag. + matches.sort(key=lambda item: item[0][0] if item[0] else -1, reverse=True) + return matches[0][1] + + def fetch( + self, key: WheelCacheKey, info: ArtifactInfo, dest: pathlib.Path + ) -> pathlib.Path: + """Download the wheel from the remote server with SHA256 verification.""" + dest.mkdir(parents=True, exist_ok=True) + target = dest / info.filename + + if target.exists(): + if info.sha256: + verify_hash = hashlib.sha256() + with open(target, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + verify_hash.update(chunk) + if verify_hash.hexdigest().lower() == info.sha256.lower(): + return target + logger.warning( + "existing %s has wrong sha256, re-downloading", info.filename + ) + target.unlink() + else: + return target + + url = info.url_or_path + logger.info( + "downloading cached wheel %s from %s", info.filename, self._server_url + ) + resp = session.get(url, stream=True) + resp.raise_for_status() + hasher = hashlib.sha256() if info.sha256 else None + fd = tempfile.NamedTemporaryFile( + dir=dest, + prefix=f".{target.name}.", + suffix=".tmp", + delete=False, + ) + tmp_target = pathlib.Path(fd.name) + fd.close() + try: + with open(tmp_target, "wb") as f: + for chunk in resp.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + if hasher is not None: + hasher.update(chunk) + f.write(chunk) + expected_sha256 = info.sha256 + if ( + hasher is not None + and expected_sha256 is not None + and hasher.hexdigest().lower() != expected_sha256.lower() + ): + raise ValueError( + f"sha256 mismatch for {info.filename}: " + f"expected {expected_sha256}, got {hasher.hexdigest()}" + ) + tmp_target.replace(target) + except BaseException: + tmp_target.unlink(missing_ok=True) + raise + return target + + def store(self, key: WheelCacheKey, artifact: pathlib.Path) -> ArtifactInfo: + """Not supported for remote backends.""" + raise NotImplementedError("Remote backends are read-only") + + def items(self) -> typing.Iterable[tuple[WheelCacheKey, ArtifactInfo]]: + """Iterate over cached project page data (may be incomplete).""" + with self._lock: + results: list[tuple[WheelCacheKey, ArtifactInfo]] = [] + for artifacts in self._project_cache.values(): + for info in artifacts: + try: + name, version, build_tag, _ = parse_wheel_filename( + info.filename + ) + key = WheelCacheKey( + package=name, version=version, build_tag=build_tag + ) + results.append((key, info)) + except InvalidWheelFilename: + continue + return results + + def _fetch_package_list(self) -> set[NormalizedName] | None: + """Fetch the top-level /simple/ index and extract package names. + + Returns an empty set for definitive "no packages" responses (HTTP 400/404 + or a successful empty page). Returns ``None`` for transport/5xx failures + so callers can still fall back to lazy project-page probes. + """ + url = f"{self._server_url}/" + try: + resp = session.get(url) + except Exception as err: + logger.warning("failed to fetch remote index %s: %s", url, err) + return None + + if resp.status_code in {400, 404}: + logger.warning( + "remote index %s returned %s; treating as empty for this run", + url, + resp.status_code, + ) + return set() + + try: + resp.raise_for_status() + except Exception as err: + logger.warning("failed to fetch remote index %s: %s", url, err) + return None + + return self._parse_index_page(resp.text) + + def _fetch_project_page(self, package: NormalizedName) -> list[ArtifactInfo] | None: + """Fetch a project's page and extract wheel artifact info. + + Returns an empty list for definitive misses (HTTP 400/404 or a + successful page with no usable wheels) so callers can memoize the + negative result for the run. Returns ``None`` for transport/5xx + failures (not memoized; retried on later lookups). + """ + url = f"{self._server_url}/{package}/" + try: + resp = session.get(url) + except Exception as err: + logger.debug("failed to fetch project page %s: %s", url, err) + return None + + if resp.status_code in {400, 404}: + logger.debug( + "project page %s returned %s; treating as empty for this run", + url, + resp.status_code, + ) + return [] + + try: + resp.raise_for_status() + except Exception as err: + logger.debug("failed to fetch project page %s: %s", url, err) + return None + + return self._parse_project_page(resp.text, url, self._allow_insecure) + + @staticmethod + def _parse_index_page(html: str) -> set[NormalizedName]: + """Extract package names from a PEP 503 index page.""" + names: set[NormalizedName] = set() + for _href, text, _attrs in _iter_anchor_links(html): + name = text.rstrip("/") + if name: + names.add(canonicalize_name(name)) + return names + + @staticmethod + def _parse_project_page( + html: str, base_url: str, allow_insecure: bool = False + ) -> list[ArtifactInfo]: + """Extract wheel artifact info from a PEP 503 project page. + + Skips yanked files (PEP 592) and wheels whose ``data-requires-python`` + does not match the current interpreter, matching resolver behavior. + """ + artifacts: list[ArtifactInfo] = [] + for href, filename, attrs in _iter_anchor_links(html): + # Simple indexes (including Fromager's) percent-encode filenames + # in link text/hrefs (e.g. local versions with '+'). + filename = unquote(filename) + if not filename.endswith(".whl"): + continue + + if _YANKED_RE.search(attrs): + logger.debug("skipping yanked remote artifact %r", filename) + continue + + requires_python_match = _REQUIRES_PYTHON_RE.search(attrs) + if requires_python_match: + # Warehouse escapes attribute values (e.g. ">=3.8"). + requires_python = unescape(requires_python_match.group(1)) + try: + if not match_py_req(requires_python): + logger.debug( + "skipping remote artifact %r due to requires-python %r", + filename, + requires_python, + ) + continue + except InvalidSpecifier: + logger.debug( + "skipping remote artifact %r due to invalid requires-python %r", + filename, + requires_python, + ) + continue + + # Reject filenames with path components to prevent directory traversal + safe_filename = pathlib.PurePosixPath(filename).name + if safe_filename != filename: + logger.warning( + "skipping remote artifact with unsafe filename %r", filename + ) + continue + filename = safe_filename + + # Resolve relative URLs (schemes are case-insensitive per RFC 3986). + href_scheme = urlparse(href).scheme.lower() + if href_scheme in {"http", "https"}: + url = href + elif href.startswith("/"): + parsed = urlparse(base_url) + url = f"{parsed.scheme}://{parsed.netloc}{href}" + else: + url = base_url.rstrip("/") + "/" + href + + # Strip hash fragment for the URL but extract sha256 if present + sha256 = None + if "#" in url: + url_part, fragment = url.rsplit("#", 1) + if fragment.lower().startswith("sha256="): + sha256 = fragment.split("=", 1)[1] + url = url_part + + # Reject plaintext HTTP URLs that lack integrity metadata + if ( + urlparse(url).scheme.lower() == "http" + and not sha256 + and not allow_insecure + ): + logger.warning( + "skipping insecure artifact %r (http without sha256)", filename + ) + continue + + artifacts.append( + ArtifactInfo( + filename=filename, + url_or_path=url, + sha256=sha256, + ) + ) + return artifacts + + +# --------------------------------------------------------------------------- +# Cache Manager +# --------------------------------------------------------------------------- + + +class CacheManager: + """Unified entry point for all cache operations. + + Searches multiple backends in priority order for lookups. Stores always + go to a single designated local backend. + """ + + def __init__( + self, + lookup_backends: list[CacheBackend], + store_backend: LocalDirectoryBackend, + force: bool = False, + ) -> None: + self._lookup_backends = lookup_backends + self._store_backend = store_backend + self._force = force + self._stats = CacheStats() + + @property + def lookup_backends(self) -> list[CacheBackend]: + """Ordered list of backends searched during lookups.""" + return list(self._lookup_backends) + + @property + def store_backend(self) -> LocalDirectoryBackend: + """The single backend that receives stored artifacts.""" + return self._store_backend + + def initialize(self) -> None: + """Scan all backends at build start. + + Local backends populate their in-memory index from disk. + Remote backends fetch the top-level package list. + """ + for backend in self._lookup_backends: + backend.scan() + + def lookup_wheel( + self, + req: Requirement, + version: Version, + build_tag: BuildTag, + ) -> CacheResult: + """Search backends in priority order for a matching wheel. + + Returns the first hit found. On a remote hit, the wheel is + downloaded to the store backend's directory. + """ + if self._force: + self._stats.record_miss(req, version, "forced") + return CacheResult(hit=False) + + key = WheelCacheKey( + package=canonicalize_name(req.name), + version=version, + build_tag=build_tag, + ) + + for backend in self._lookup_backends: + t0 = time.monotonic() + info = backend.lookup(key) + if info is None: + continue + + # Hit -- fetch the artifact to a local path + try: + local_path = backend.fetch(key, info, self._store_backend.directory) + # Register in the store index so subsequent lookups + # find it locally without hitting the remote again. + # was_downloaded means "came from a non-store backend" so the + # caller knows the local mirror may need updating. + was_downloaded = backend is not self._store_backend + if was_downloaded: + stored = self._store_backend.store(key, local_path) + local_path = pathlib.Path(stored.url_or_path) + except Exception as err: + logger.warning( + "cache hit for %s==%s in %s could not be fetched: %s", + req.name, + version, + backend.name, + err, + ) + continue + duration_ms = (time.monotonic() - t0) * 1000 + + self._stats.record_hit( + req, + version, + backend.name, + duration_ms=duration_ms, + ) + logger.info( + "cache hit for %s==%s in %s", + req.name, + version, + backend.name, + ) + result_build_tag = build_tag + if not result_build_tag: + try: + _, _, parsed_tag, _ = parse_wheel_filename(local_path.name) + result_build_tag = parsed_tag + except InvalidWheelFilename: + pass + return CacheResult( + hit=True, + path=local_path, + backend_name=backend.name, + build_tag=result_build_tag, + was_downloaded=was_downloaded, + ) + + self._stats.record_miss(req, version, "not_found") + logger.debug("cache miss for %s==%s", req.name, version) + return CacheResult(hit=False) + + def store_wheel( + self, + req: Requirement, + version: Version, + build_tag: BuildTag, + wheel_path: pathlib.Path, + ) -> pathlib.Path: + """Store a newly built wheel in the store backend. + + When ``build_tag`` is empty, derive it from the wheel filename so the + index key matches what ``scan()`` would produce. + """ + if not build_tag: + try: + _, _, parsed_tag, _ = parse_wheel_filename(wheel_path.name) + build_tag = parsed_tag + except InvalidWheelFilename: + pass + + key = WheelCacheKey( + package=canonicalize_name(req.name), + version=version, + build_tag=build_tag, + ) + + info = self._store_backend.store(key, wheel_path) + self._stats.record_store(req, version, self._store_backend.name) + logger.info("stored %s in %s", info.filename, self._store_backend.name) + return pathlib.Path(info.url_or_path) + + @property + def stats(self) -> CacheStats: + """Cache statistics for the current run.""" + return self._stats + + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + + +def build_cache_manager( + wkctx: context.WorkContext, + cache_url: str | None = None, + allow_insecure: bool = False, +) -> CacheManager: + """Construct a CacheManager from the WorkContext configuration. + + If the context already has a cache configured, return it. + Otherwise, build one from the standard filesystem layout. + + Lookup order (matches legacy ``find_cached_wheel``): + 1. ``wheels_build`` (freshly built, not yet mirrored; scanned recursively + for parallel-build thread subdirectories) + 2. ``wheels_downloads`` (previously downloaded/built wheels) + 3. Remote PEP 503 server (if ``cache_url`` is provided) + + ``wheels_prebuilt`` is intentionally omitted: prebuilt packages use the + dedicated prebuilt bootstrap path (``SourceType.PREBUILT``), not the + general cache short-circuit. + + Store destination: ``wheels_downloads``. + + Args: + wkctx: The work context providing local paths. + cache_url: Optional URL to a remote PEP 503 cache server. + allow_insecure: Allow HTTP URLs without SHA256 hashes. + """ + if wkctx.cache is not None: + return wkctx.cache + + # Match legacy find_cached_wheel order: build dir, downloads, then remote. + # Use wheels_build_base (not the thread-local wheels_build property) so the + # index is shared across threads. recursive=True covers parallel-build + # subdirectories under wheels_build_base//. + build_backend = LocalDirectoryBackend( + wkctx.wheels_build_base, + backend_name="local:build", + recursive=True, + ) + downloads_backend = LocalDirectoryBackend( + wkctx.wheels_downloads, backend_name="local:downloads" + ) + + lookup_backends: list[CacheBackend] = [ + build_backend, + downloads_backend, + ] + + if cache_url: + remote_backend = RemotePEP503Backend( + server_url=cache_url, + download_dir=wkctx.wheels_downloads, + backend_name=f"remote:{cache_url}", + allow_insecure=allow_insecure, + ) + lookup_backends.append(remote_backend) + + manager = CacheManager( + lookup_backends=lookup_backends, + store_backend=downloads_backend, + ) + manager.initialize() + wkctx.cache = manager + return manager diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index 287335d9..a285cdf4 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -12,6 +12,7 @@ from .. import ( bootstrapper, + cache, context, dependency_graph, metrics, @@ -124,6 +125,19 @@ def _get_requirements_from_args( show_default=True, help="Number of background threads for parallel I/O pre-fetching (min 1).", ) +@click.option( + "--use-cache-manager/--no-cache-manager", + "use_cache_manager", + default=False, + help="Enable the unified cache manager for prioritized multi-backend lookup.", +) +@click.option( + "--cache-allow-insecure", + "cache_allow_insecure", + is_flag=True, + default=False, + help="Allow HTTP cache registries without SHA256 hashes (dev/internal use).", +) @click.argument("toplevel", nargs=-1) @click.pass_obj def bootstrap( @@ -137,6 +151,8 @@ def bootstrap( multiple_versions: bool, max_release_age: int | None, num_bg_threads: int, + use_cache_manager: bool, + cache_allow_insecure: bool, toplevel: list[str], ) -> None: """Compute and build the dependencies of a set of requirements recursively @@ -199,6 +215,16 @@ def bootstrap( server.start_wheel_server(wkctx) + if use_cache_manager: + wkctx.cache = cache.build_cache_manager( + wkctx, + cache_url=cache_wheel_server_url, + allow_insecure=cache_allow_insecure, + ) + logger.info( + "cache manager enabled with %d backends", len(wkctx.cache.lookup_backends) + ) + with progress.progress_context(total=len(to_build * 2)) as progressbar: with bootstrapper.Bootstrapper( wkctx, @@ -515,6 +541,19 @@ def write_constraints_file( show_default=True, help="Number of background threads for parallel I/O pre-fetching (min 1).", ) +@click.option( + "--use-cache-manager/--no-cache-manager", + "use_cache_manager", + default=False, + help="Enable the unified cache manager for prioritized multi-backend lookup.", +) +@click.option( + "--cache-allow-insecure", + "cache_allow_insecure", + is_flag=True, + default=False, + help="Allow HTTP cache registries without SHA256 hashes (dev/internal use).", +) @click.argument("toplevel", nargs=-1) @click.pass_obj @click.pass_context @@ -531,6 +570,8 @@ def bootstrap_parallel( multiple_versions: bool, max_release_age: int | None, num_bg_threads: int, + use_cache_manager: bool, + cache_allow_insecure: bool, toplevel: list[str], ) -> None: """Bootstrap and build-parallel @@ -560,6 +601,8 @@ def bootstrap_parallel( multiple_versions=multiple_versions, max_release_age=max_release_age, num_bg_threads=num_bg_threads, + use_cache_manager=use_cache_manager, + cache_allow_insecure=cache_allow_insecure, toplevel=toplevel, ) diff --git a/src/fromager/commands/cache_cmd.py b/src/fromager/commands/cache_cmd.py new file mode 100644 index 00000000..576028da --- /dev/null +++ b/src/fromager/commands/cache_cmd.py @@ -0,0 +1,313 @@ +"""CLI commands for cache management and observability.""" + +import json +import logging +import pathlib + +import click +import rich +import rich.box +from packaging.utils import canonicalize_name +from rich.table import Table + +from fromager import cache, context + +logger = logging.getLogger(__name__) + + +@click.group(name="cache") +def cache_cli() -> None: + """Manage the fromager wheel cache.""" + pass + + +@cache_cli.command(name="list") +@click.option( + "--format", + "output_format", + type=click.Choice(["table", "json"], case_sensitive=False), + default="table", + help="Output format (default: table).", +) +@click.pass_obj +def cache_list( + wkctx: context.WorkContext, + output_format: str, +) -> None: + """List all cached wheel artifacts.""" + manager = cache.build_cache_manager(wkctx) + + entries: list[dict[str, str | int]] = [] + for backend in manager.lookup_backends: + for key, info in backend.items(): + entries.append( + { + "backend": backend.name, + "package": str(key.package), + "version": str(key.version), + "build_tag": ( + f"{key.build_tag[0]}{key.build_tag[1]}" if key.build_tag else "" + ), + "filename": info.filename, + "size_bytes": info.size_bytes or 0, + } + ) + + entries.sort( + key=lambda e: (str(e["backend"]), str(e["package"]), str(e["version"])) + ) + + if output_format == "json": + click.echo(json.dumps(entries, indent=2)) + return + + if not entries: + click.echo("No cached wheels found.") + return + + table = Table(title="Cached Wheels", box=rich.box.SIMPLE) + table.add_column("Package", no_wrap=True) + table.add_column("Version", no_wrap=True) + table.add_column("Build Tag", no_wrap=True) + table.add_column("Size", justify="right", no_wrap=True) + table.add_column("Backend", no_wrap=True) + + for entry in entries: + size = _format_size(int(entry["size_bytes"])) + table.add_row( + str(entry["package"]), + str(entry["version"]), + str(entry["build_tag"]), + size, + str(entry["backend"]), + ) + + console = rich.get_console() + console.print(table) + console.print(f"\nTotal: {len(entries)} wheel(s)") + + +@cache_cli.command(name="stats") +@click.pass_obj +def cache_stats(wkctx: context.WorkContext) -> None: + """Show on-disk cache inventory by backend. + + Hit/miss rates are recorded in-memory during a bootstrap run and are not + persisted, so this command reports backend inventory only. + """ + manager = cache.build_cache_manager(wkctx) + + table = Table(title="Cache Inventory", box=rich.box.SIMPLE) + table.add_column("Metric", no_wrap=True) + table.add_column("Value", justify="right", no_wrap=True) + + total_wheels = 0 + total_size = 0 + for backend in manager.lookup_backends: + backend_count = 0 + backend_size = 0 + for _key, info in backend.items(): + backend_count += 1 + backend_size += info.size_bytes or 0 + table.add_row(f"{backend.name} wheels", str(backend_count)) + table.add_row(f"{backend.name} size", _format_size(backend_size)) + total_wheels += backend_count + total_size += backend_size + + table.add_section() + table.add_row("Total wheels on disk", str(total_wheels)) + table.add_row("Total size on disk", _format_size(total_size)) + + console = rich.get_console() + console.print(table) + + +@cache_cli.command() +@click.option( + "--remove-missing", + is_flag=True, + default=False, + help="Remove index entries for files that no longer exist on disk.", +) +@click.pass_obj +def verify(wkctx: context.WorkContext, remove_missing: bool) -> None: + """Verify cache integrity: check that indexed files exist on disk.""" + manager = cache.build_cache_manager(wkctx) + + missing: list[dict[str, str]] = [] + checked = 0 + backends_to_rescan: list[cache.LocalDirectoryBackend] = [] + + for backend in manager.lookup_backends: + if not isinstance(backend, cache.LocalDirectoryBackend): + continue + backend_had_missing = False + for key, info in backend.items(): + checked += 1 + file_path = pathlib.Path(info.url_or_path) + if not file_path.exists(): + missing.append( + { + "backend": backend.name, + "key": str(key), + "path": str(file_path), + } + ) + backend_had_missing = True + if remove_missing and backend_had_missing: + backends_to_rescan.append(backend) + + for backend in backends_to_rescan: + backend.scan() + + if not missing: + click.echo(f"All {checked} cached artifacts verified OK.") + return + + click.echo(f"Found {len(missing)} missing artifact(s) out of {checked} checked:") + for m in missing: + action = " [removed from index]" if remove_missing else "" + click.echo(f" {m['backend']}/{m['key']}: {m['path']}{action}") + + +@cache_cli.command() +@click.argument("packages", nargs=-1) +@click.option( + "--all", + "invalidate_all", + is_flag=True, + default=False, + help="Invalidate the entire cache.", +) +@click.pass_obj +def invalidate( + wkctx: context.WorkContext, + packages: tuple[str, ...], + invalidate_all: bool, +) -> None: + """Invalidate (remove) cached wheels for specific packages. + + Pass package names as arguments, or use --all to clear everything. + """ + if not packages and not invalidate_all: + raise click.UsageError("Specify package names or use --all.") + + manager = cache.build_cache_manager(wkctx) + removed = 0 + + target_packages = {canonicalize_name(p) for p in packages} if packages else None + + # Only mutate the store backend. Lookup backends may include + # in-progress build wheels and user-supplied prebuilts. + backend = manager.store_backend + keys_to_remove = [] + for key, info in backend.items(): + if target_packages and key.package not in target_packages: + continue + keys_to_remove.append((key, info)) + + for _key, info in keys_to_remove: + file_path = pathlib.Path(info.url_or_path) + if file_path.exists(): + file_path.unlink() + logger.info("removed %s", file_path) + removed += 1 + + # Re-scan after removing files to update the index + if keys_to_remove: + backend.scan() + + click.echo(f"Invalidated {removed} cached artifact(s).") + + +@cache_cli.command() +@click.option( + "--dry-run", + is_flag=True, + default=False, + help="Show what would be removed without actually deleting.", +) +@click.option( + "--keep-latest", + type=click.IntRange(min=0), + default=1, + help="Keep this many build tags per package+version (default: 1).", +) +@click.pass_obj +def gc( + wkctx: context.WorkContext, + dry_run: bool, + keep_latest: int, +) -> None: + """Garbage-collect old builds, keeping only the latest build tags. + + For each package+version, removes all but the --keep-latest most + recent builds (highest build tag number). + """ + manager = cache.build_cache_manager(wkctx) + removed = 0 + freed_bytes = 0 + + # Only mutate the store backend. Lookup backends may include + # in-progress build wheels and user-supplied prebuilts. + backend = manager.store_backend + + # Group by (package, version) + groups: dict[ + tuple[str, str], list[tuple[cache.WheelCacheKey, cache.ArtifactInfo]] + ] = {} + for key, info in backend.items(): + group_key = (str(key.package), str(key.version)) + groups.setdefault(group_key, []).append((key, info)) + + needs_rescan = False + for _group_key, entries in groups.items(): + if len(entries) <= keep_latest: + continue + + # Prefer higher build tags; break ties (including untagged) by mtime + # so --keep-latest retains the newest file on disk. + def _gc_sort_key( + entry: tuple[cache.WheelCacheKey, cache.ArtifactInfo], + ) -> tuple[int, float]: + key, info = entry + tag_num = key.build_tag[0] if key.build_tag else 0 + try: + mtime = pathlib.Path(info.url_or_path).stat().st_mtime + except OSError: + mtime = 0.0 + return (tag_num, mtime) + + entries.sort(key=_gc_sort_key, reverse=True) + to_remove = entries[keep_latest:] + + for _key, info in to_remove: + file_path = pathlib.Path(info.url_or_path) + size = info.size_bytes or 0 + if dry_run: + click.echo(f" would remove: {info.filename} ({_format_size(size)})") + else: + if file_path.exists(): + file_path.unlink() + needs_rescan = True + logger.info("gc removed %s", file_path) + removed += 1 + freed_bytes += size + + if needs_rescan: + backend.scan() + + verb = "Would remove" if dry_run else "Removed" + click.echo(f"{verb} {removed} old build(s), freeing {_format_size(freed_bytes)}.") + + +def _format_size(size_bytes: int) -> str: + """Format bytes as human-readable size.""" + if size_bytes == 0: + return "0 B" + size = float(size_bytes) + for unit in ("B", "KB", "MB", "GB"): + if abs(size) < 1024: + return f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} TB" diff --git a/src/fromager/context.py b/src/fromager/context.py index 8ce9dd71..a93169a2 100644 --- a/src/fromager/context.py +++ b/src/fromager/context.py @@ -22,7 +22,7 @@ ) if typing.TYPE_CHECKING: - from . import build_environment, candidate + from . import build_environment, cache, candidate logger = logging.getLogger(__name__) @@ -97,6 +97,7 @@ def __init__( self.cooldown: candidate.Cooldown | None = cooldown self._max_release_age: datetime.timedelta | None = max_release_age + self._cache: cache.CacheManager | None = None @property def max_release_age(self) -> datetime.timedelta | None: @@ -108,6 +109,15 @@ def set_max_release_age(self, days: int) -> None: raise ValueError(f"max_release_age must be positive, got {days}") self._max_release_age = datetime.timedelta(days=days) + @property + def cache(self) -> cache.CacheManager | None: + """The unified cache manager, if configured.""" + return self._cache + + @cache.setter + def cache(self, value: cache.CacheManager | None) -> None: + self._cache = value + def enable_parallel_builds(self) -> None: self._parallel_builds = True diff --git a/src/fromager/requirements_file.py b/src/fromager/requirements_file.py index c7beed39..d4c4f3d9 100644 --- a/src/fromager/requirements_file.py +++ b/src/fromager/requirements_file.py @@ -33,6 +33,7 @@ class SourceType(StrEnum): PREBUILT = "prebuilt" SDIST = "sdist" OVERRIDE = "override" + CACHED = "cached" def parse_requirements_file( diff --git a/src/fromager/server.py b/src/fromager/server.py index 7d282f87..0cc823c6 100644 --- a/src/fromager/server.py +++ b/src/fromager/server.py @@ -20,13 +20,15 @@ from starlette.responses import FileResponse, HTMLResponse, RedirectResponse, Response from starlette.routing import Route -from .threading_utils import with_thread_lock - if typing.TYPE_CHECKING: from . import context logger = logging.getLogger(__name__) +# Shared by index_wheel() and update_wheel_mirror() so they cannot race on the +# same symlink path (with_thread_lock() would give each function its own lock). +_mirror_lock = threading.Lock() + def start_wheel_server(ctx: context.WorkContext) -> None: update_wheel_mirror(ctx) @@ -58,35 +60,65 @@ def run_wheel_server( return server, sock, thread -@with_thread_lock() -def update_wheel_mirror(ctx: context.WorkContext) -> None: - for wheel in ctx.wheels_build.glob("*.whl"): - logger.info("adding %s to local wheel server", wheel.name) - downloads_dest_filename = ctx.wheels_downloads / wheel.name - # Always move the file so the code managing the timer for the - # wheels does not find more than one wheel in the build - # directory. - shutil.move(wheel, downloads_dest_filename) - - wheels: list[pathlib.Path] = [] - wheels.extend(ctx.wheels_downloads.glob("*.whl")) - wheels.extend(ctx.wheels_prebuilt.glob("*.whl")) - - for wheel in wheels: - # Now also symlink the files into the simple hierarchy. We always - # process all files to be safe. - (normalized_name, _, _, _) = parse_wheel_filename(wheel.name) - simple_dest_filename = ctx.wheel_server_dir / normalized_name / wheel.name - - if simple_dest_filename.is_symlink() and not simple_dest_filename.is_file(): - logger.debug("remove dangling symlink %s", simple_dest_filename) - simple_dest_filename.unlink() - - if not simple_dest_filename.is_file(): - relpath = os.path.relpath(wheel, simple_dest_filename.parent) - logger.debug("linking %s -> %s into local index", wheel.name, relpath) - simple_dest_filename.parent.mkdir(parents=True, exist_ok=True) +def _link_wheel_into_index(ctx: context.WorkContext, wheel: pathlib.Path) -> None: + """Symlink a single wheel into the local simple index hierarchy. + + Must be called while holding ``_mirror_lock``. Treats an existing + destination as success so concurrent indexers cannot raise + ``FileExistsError``. + """ + (normalized_name, _, _, _) = parse_wheel_filename(wheel.name) + simple_dest_filename = ctx.wheel_server_dir / normalized_name / wheel.name + + if simple_dest_filename.is_symlink() and not simple_dest_filename.is_file(): + logger.debug("remove dangling symlink %s", simple_dest_filename) + simple_dest_filename.unlink() + + if not simple_dest_filename.is_file(): + relpath = os.path.relpath(wheel, simple_dest_filename.parent) + logger.debug("linking %s -> %s into local index", wheel.name, relpath) + simple_dest_filename.parent.mkdir(parents=True, exist_ok=True) + try: simple_dest_filename.symlink_to(relpath) + except FileExistsError: + # Another caller created the link between the is_file() check and + # symlink_to(); accept the existing destination. + if not simple_dest_filename.is_file(): + raise + + +def index_wheel(ctx: context.WorkContext, wheel: pathlib.Path) -> None: + """Index one existing wheel into the local simple server. + + Unlike :func:`update_wheel_mirror`, this never scans or moves files out of + ``wheels_build``. Use it from background cache-hit paths so concurrent + builds are not disrupted. + """ + if not wheel.is_file(): + raise ValueError(f"cannot index missing wheel: {wheel}") + logger.info("adding %s to local wheel server", wheel.name) + with _mirror_lock: + _link_wheel_into_index(ctx, wheel) + + +def update_wheel_mirror(ctx: context.WorkContext) -> None: + with _mirror_lock: + for wheel in ctx.wheels_build.glob("*.whl"): + logger.info("adding %s to local wheel server", wheel.name) + downloads_dest_filename = ctx.wheels_downloads / wheel.name + # Always move the file so the code managing the timer for the + # wheels does not find more than one wheel in the build + # directory. + shutil.move(wheel, downloads_dest_filename) + + wheels: list[pathlib.Path] = [] + wheels.extend(ctx.wheels_downloads.glob("*.whl")) + wheels.extend(ctx.wheels_prebuilt.glob("*.whl")) + + for wheel in wheels: + # Now also symlink the files into the simple hierarchy. We always + # process all files to be safe. + _link_wheel_into_index(ctx, wheel) class SimpleHTMLIndex: diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 47c89402..9e447916 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -1023,7 +1023,7 @@ def test_bg_prepare_prebuilt_log_prefix_includes_version( "fromager.wheels.download_wheel", return_value=pathlib.Path("mypkg-1.2.3-py3-none-any.whl"), ), - patch("fromager.server.update_wheel_mirror"), + patch("fromager.server.index_wheel"), log.req_ctxvar_context(req, version), ): bg_prepare_prebuilt( @@ -1136,6 +1136,40 @@ def test_build_item_build_wheel(tmp_context: WorkContext) -> None: assert sdist_filename == built_sdist +def test_build_item_build_wheel_registers_with_cache_manager( + tmp_context: WorkContext, +) -> None: + """When CacheManager is active, builds are registered for same-run lookups.""" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = Build(wi) + built_wheel = tmp_context.wheels_build / "testpkg-1.0-py3-none-any.whl" + built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + cache = Mock() + tmp_context.cache = cache + + with ( + patch.object(item, "_build_sdist", return_value=built_sdist), + patch("fromager.wheels.build_wheel", return_value=built_wheel), + patch("fromager.server.update_wheel_mirror"), + ): + wheel_filename, _sdist_filename = item._build_wheel(tmp_context) + + cache.store_wheel.assert_called_once() + args = cache.store_wheel.call_args.args + assert args[0] == wi.req + assert args[1] == Version("1.0") + assert args[3] == wheel_filename + + def test_build_item_do_build_returns_cached_wheel( tmp_context: WorkContext, ) -> None: diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 925f8035..6e6c3954 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -30,6 +30,7 @@ from fromager import bootstrapper, build_environment from fromager.bootstrapper._build import Build +from fromager.bootstrapper._cache import _find_cached_wheel_via_manager from fromager.bootstrapper._complete import Complete from fromager.bootstrapper._phase import Phase from fromager.bootstrapper._prepare_build import PrepareBuild @@ -43,6 +44,7 @@ SourceBuildResult, ) from fromager.bootstrapper._work_item import WorkItem +from fromager.cache import CacheManager, LocalDirectoryBackend from fromager.context import WorkContext from fromager.requirements_file import RequirementType, SourceType @@ -1234,7 +1236,7 @@ def test_source_no_cache_uses_background_result( def test_source_cached_wheel_uses_background_result( self, tmp_context: WorkContext ) -> None: - """Cached wheel background result sets cached_wheel_filename and sdist_root.""" + """Legacy cache hit (no CacheManager) still proceeds through PrepareBuild.""" bt = bootstrapper.Bootstrapper(tmp_context) item = _make_build_item(phase=BootstrapPhase.PREPARE_SOURCE) @@ -1269,6 +1271,80 @@ def test_source_cached_wheel_uses_background_result( assert isinstance(result[0], PrepareBuild) assert len(result) == 1 + def test_cache_manager_hit_short_circuits_to_install_deps( + self, tmp_context: WorkContext + ) -> None: + """CacheManager hit skips build env and goes to ProcessInstallDeps.""" + downloads = tmp_context.wheels_downloads + store = LocalDirectoryBackend(downloads, backend_name="local:downloads") + manager = CacheManager(lookup_backends=[store], store_backend=store) + tmp_context.cache = manager + + bt = bootstrapper.Bootstrapper(tmp_context) + item = _make_build_item(phase=BootstrapPhase.PREPARE_SOURCE) + + unpack_dir = tmp_context.work_dir / "testpkg-1.0" + unpack_dir.mkdir(parents=True) + cached_wheel = downloads / "testpkg-1.0-py3-none-any.whl" + cached_wheel.write_bytes(b"wheel") + item.bg_future = _make_resolved_future( + PreparedSourceData( + sdist_root_dir=unpack_dir / unpack_dir.stem, + cached_wheel_filename=cached_wheel, + ) + ) + + mock_pbi = Mock() + mock_pbi.build_tag.return_value = () + with ( + patch.object(tmp_context.constraints, "get_constraint", return_value=None), + patch.object(tmp_context, "package_build_info", return_value=mock_pbi), + patch( + "fromager.bootstrapper._prepare_source.server.index_wheel" + ) as mock_index, + patch("fromager.build_environment.BuildEnvironment") as mock_build_env_cls, + ): + result = item.run(bt) + + assert len(result) == 1 + assert isinstance(result[0], ProcessInstallDeps) + wi = result[0].work_item + assert wi.build_result is not None + assert wi.build_result.source_type == SourceType.CACHED + assert wi.build_result.wheel_filename == cached_wheel + assert wi.build_result.sdist_root_dir is None + assert wi.build_result.build_env is None + assert wi.build_result.unpack_dir == unpack_dir + mock_index.assert_called_once_with(bt.ctx, cached_wheel) + mock_build_env_cls.assert_not_called() + + +class TestFindCachedWheelViaManager: + """Tests for CacheManager dispatch in find_cached_wheel.""" + + def test_returns_unpack_dir_under_work_dir(self, tmp_context: WorkContext) -> None: + """Unpack dir must be under work_dir so PrepareSource path checks pass.""" + downloads = tmp_context.wheels_downloads + whl = downloads / "testpkg-1.0-py3-none-any.whl" + whl.write_bytes(b"wheel") + + store = LocalDirectoryBackend(downloads, backend_name="local:downloads") + store.scan() + manager = CacheManager(lookup_backends=[store], store_backend=store) + tmp_context.cache = manager + + mock_pbi = Mock() + mock_pbi.build_tag.return_value = () + with patch.object(tmp_context, "package_build_info", return_value=mock_pbi): + path, unpack_dir = _find_cached_wheel_via_manager( + tmp_context, Requirement("testpkg"), Version("1.0") + ) + + assert path == whl.resolve() + assert unpack_dir is not None + assert unpack_dir.parent == tmp_context.work_dir + assert unpack_dir.name == "testpkg-1.0" + def test_bad_sdist_root_raises_valueerror(self, tmp_context: WorkContext) -> None: """ValueError raised when sdist_root_dir.parent.parent != work_dir.""" bt = bootstrapper.Bootstrapper(tmp_context) diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 00000000..3b717267 --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,1225 @@ +"""Unit tests for the fromager.cache module.""" + +import hashlib +import os +import pathlib +import re +import threading +import typing +from unittest.mock import MagicMock, patch + +import pytest +import requests_mock +from click.testing import CliRunner +from packaging.requirements import Requirement +from packaging.tags import Tag +from packaging.utils import canonicalize_name +from packaging.version import Version + +from fromager.cache import ( + ArtifactInfo, + CacheManager, + CacheStats, + LocalDirectoryBackend, + RemotePEP503Backend, + WheelCacheKey, + build_cache_manager, +) +from fromager.commands.cache_cmd import cache_cli + +# --------------------------------------------------------------------------- +# WheelCacheKey +# --------------------------------------------------------------------------- + + +class TestWheelCacheKey: + def test_str_with_build_tag(self) -> None: + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + assert str(key) == "requests==2.31.0-1" + + def test_str_without_build_tag(self) -> None: + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(), + ) + assert str(key) == "requests==2.31.0" + + def test_equality(self) -> None: + key1 = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + key2 = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + assert key1 == key2 + + +# --------------------------------------------------------------------------- +# LocalDirectoryBackend +# --------------------------------------------------------------------------- + + +class TestLocalDirectoryBackend: + def test_scan_empty_directory(self, tmp_path: pathlib.Path) -> None: + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + result = backend.scan() + assert result == {} + + def test_scan_nonexistent_directory(self, tmp_path: pathlib.Path) -> None: + backend = LocalDirectoryBackend(tmp_path / "nonexistent", backend_name="test") + result = backend.scan() + assert result == {} + + def test_scan_finds_wheels(self, tmp_path: pathlib.Path) -> None: + whl = tmp_path / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"fake wheel data") + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + result = backend.scan() + assert len(result) == 1 + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + assert key in result + + def test_scan_skips_symlinks(self, tmp_path: pathlib.Path) -> None: + real = tmp_path / "real" / "requests-2.31.0-1-py3-none-any.whl" + real.parent.mkdir() + real.write_bytes(b"fake") + link = tmp_path / "requests-2.31.0-1-py3-none-any.whl" + link.symlink_to(real) + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + result = backend.scan() + assert len(result) == 0 + + def test_scan_skips_invalid_filenames(self, tmp_path: pathlib.Path) -> None: + bad = tmp_path / "not_a_valid_wheel.whl" + bad.write_bytes(b"bad") + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + result = backend.scan() + assert result == {} + + def test_lookup_found(self, tmp_path: pathlib.Path) -> None: + whl = tmp_path / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = backend.lookup(key) + assert info is not None + assert info.filename == "requests-2.31.0-1-py3-none-any.whl" + + def test_lookup_not_found(self, tmp_path: pathlib.Path) -> None: + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + assert backend.lookup(key) is None + + def test_lookup_evicts_deleted_file(self, tmp_path: pathlib.Path) -> None: + whl = tmp_path / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + backend.scan() + whl.unlink() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + assert backend.lookup(key) is None + + def test_store_copies_file(self, tmp_path: pathlib.Path) -> None: + source_dir = tmp_path / "source" + source_dir.mkdir() + whl = source_dir / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"wheel content") + + dest_dir = tmp_path / "dest" + dest_dir.mkdir() + backend = LocalDirectoryBackend(dest_dir, backend_name="test") + + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = backend.store(key, whl) + + # Original still exists + assert whl.exists() + # Stored in dest + stored = dest_dir / whl.name + assert stored.exists() + assert stored.read_bytes() == b"wheel content" + assert info.filename == whl.name + + def test_store_same_file_noop(self, tmp_path: pathlib.Path) -> None: + whl = tmp_path / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = backend.store(key, whl) + assert info.filename == whl.name + + def test_store_overwrites_stale_same_named_wheel( + self, tmp_path: pathlib.Path + ) -> None: + """store() replaces an existing same-named wheel that is a different file.""" + dest_dir = tmp_path / "dest" + dest_dir.mkdir() + stale = dest_dir / "requests-2.31.0-1-py3-none-any.whl" + stale.write_bytes(b"stale content") + + source_dir = tmp_path / "source" + source_dir.mkdir() + fresh = source_dir / "requests-2.31.0-1-py3-none-any.whl" + fresh.write_bytes(b"fresh content") + + backend = LocalDirectoryBackend(dest_dir, backend_name="test") + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = backend.store(key, fresh) + + assert stale.read_bytes() == b"fresh content" + assert info.size_bytes == len(b"fresh content") + assert backend.lookup(key) is not None + + def test_fetch_returns_local_path(self, tmp_path: pathlib.Path) -> None: + whl = tmp_path / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = backend.lookup(key) + assert info is not None + path = backend.fetch(key, info, tmp_path / "other") + assert path == pathlib.Path(info.url_or_path) + + def test_items_returns_all(self, tmp_path: pathlib.Path) -> None: + whl1 = tmp_path / "requests-2.31.0-1-py3-none-any.whl" + whl1.write_bytes(b"a") + whl2 = tmp_path / "urllib3-2.0.0-1-py3-none-any.whl" + whl2.write_bytes(b"b") + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + backend.scan() + items = list(backend.items()) + assert len(items) == 2 + + def test_scan_skips_incompatible_platform_wheels( + self, tmp_path: pathlib.Path + ) -> None: + compatible = tmp_path / "requests-2.31.0-1-py3-none-any.whl" + incompatible = tmp_path / "requests-2.31.0-1-cp39-cp39-win_amd64.whl" + compatible.write_bytes(b"ok") + incompatible.write_bytes(b"nope") + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + with patch( + "fromager.cache.SUPPORTED_TAGS", + frozenset({Tag("py3", "none", "any")}), + ): + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = backend.lookup(key) + assert info is not None + assert info.filename == compatible.name + + def test_scan_recursive_finds_thread_subdir_wheels( + self, tmp_path: pathlib.Path + ) -> None: + nested = tmp_path / "12345" + nested.mkdir() + whl = nested / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + backend = LocalDirectoryBackend( + tmp_path, backend_name="local:build", recursive=True + ) + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + assert backend.lookup(key) is not None + + +# --------------------------------------------------------------------------- +# RemotePEP503Backend +# --------------------------------------------------------------------------- + + +class TestRemotePEP503Backend: + def test_parse_index_page(self) -> None: + html = """ + + requests + urllib3 + + """ + names = RemotePEP503Backend._parse_index_page(html) + assert canonicalize_name("requests") in names + assert canonicalize_name("urllib3") in names + + def test_parse_index_page_allows_extra_attributes(self) -> None: + html = """ + requests + """ + names = RemotePEP503Backend._parse_index_page(html) + assert canonicalize_name("requests") in names + + def test_parse_project_page(self) -> None: + html = """ + requests-2.31.0-1-py3-none-any.whl + requests-2.31.0.tar.gz + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "https://cache.test/simple/requests/" + ) + assert len(artifacts) == 1 + assert artifacts[0].filename == "requests-2.31.0-1-py3-none-any.whl" + assert artifacts[0].sha256 == "abc123" + + def test_parse_project_page_rejects_path_traversal(self) -> None: + # Link text must end in .whl to pass the suffix check and reach the + # safe_filename != filename traversal guard. + html = """ + ../../../evil-1.0.0-py3-none-any.whl + requests-2.31.0-1-py3-none-any.whl + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "https://cache.test/simple/requests/" + ) + assert len(artifacts) == 1 + assert artifacts[0].filename == "requests-2.31.0-1-py3-none-any.whl" + + def test_parse_project_page_rejects_http_without_sha256(self) -> None: + html = """ + requests-2.31.0-1-py3-none-any.whl + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "http://cache.test/simple/requests/" + ) + assert len(artifacts) == 0 + + def test_parse_project_page_allows_http_with_sha256(self) -> None: + html = """ + requests-2.31.0-1-py3-none-any.whl + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "http://cache.test/simple/requests/" + ) + assert len(artifacts) == 1 + + def test_parse_project_page_resolves_uppercase_absolute_scheme(self) -> None: + """Absolute href schemes are case-insensitive (RFC 3986).""" + html = """ + requests-2.31.0-1-py3-none-any.whl + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "https://cache.test/simple/requests/" + ) + assert len(artifacts) == 1 + assert artifacts[0].url_or_path.startswith("HTTPS://CACHE.TEST/") + + def test_parse_project_page_allows_http_insecure(self) -> None: + html = """ + requests-2.31.0-1-py3-none-any.whl + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "http://cache.test/simple/requests/", allow_insecure=True + ) + assert len(artifacts) == 1 + + def test_scan_fetches_package_list( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get( + "https://cache.test/simple/", + text='requests', + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + backend.scan() + assert backend._available_packages is not None + assert canonicalize_name("requests") in backend._available_packages + + def test_lookup_fetches_project_page( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get( + "https://cache.test/simple/", + text='requests', + ) + requests_mock.get( + "https://cache.test/simple/requests/", + text='requests-2.31.0-1-py3-none-any.whl', + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = backend.lookup(key) + assert info is not None + assert info.sha256 == "abc" + + def test_lookup_short_circuits_unknown_package( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get( + "https://cache.test/simple/", + text='requests', + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("nonexistent"), + version=Version("1.0.0"), + build_tag=(), + ) + assert backend.lookup(key) is None + + def test_scan_index_failure_still_allows_project_lookup( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get("https://cache.test/simple/", status_code=500) + requests_mock.get( + "https://cache.test/simple/requests/", + text='requests-2.31.0-1-py3-none-any.whl', + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + backend.scan() + assert backend._available_packages is None + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + assert backend.lookup(key) is not None + + def test_project_page_5xx_is_not_memoized( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get( + "https://cache.test/simple/", + text='requests', + ) + project_mock = requests_mock.get( + "https://cache.test/simple/requests/", + [ + {"status_code": 500}, + { + "text": ( + '' + "requests-2.31.0-1-py3-none-any.whl" + ) + }, + ], + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + assert backend.lookup(key) is None + assert backend.lookup(key) is not None + assert project_mock.call_count == 2 + + def test_project_page_404_is_memoized_empty( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get( + "https://cache.test/simple/", + text='requests', + ) + project_mock = requests_mock.get( + "https://cache.test/simple/requests/", + status_code=404, + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + assert backend.lookup(key) is None + assert backend.lookup(key) is None + assert project_mock.call_count == 1 + + def test_scan_index_400_treated_as_empty( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get("https://cache.test/simple/", status_code=400) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + backend.scan() + assert backend._available_packages == set() + + def test_parse_project_page_skips_yanked(self) -> None: + html = """ + requests-2.31.0-1-py3-none-any.whl + requests-2.32.0-1-py3-none-any.whl + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "https://cache.test/simple/requests/" + ) + assert len(artifacts) == 1 + assert artifacts[0].filename == "requests-2.32.0-1-py3-none-any.whl" + + def test_parse_project_page_skips_incompatible_requires_python(self) -> None: + html = """ + requests-2.31.0-1-py3-none-any.whl + requests-2.32.0-1-py3-none-any.whl + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "https://cache.test/simple/requests/" + ) + assert len(artifacts) == 1 + assert artifacts[0].filename == "requests-2.32.0-1-py3-none-any.whl" + + def test_parse_project_page_unescapes_requires_python_entities(self) -> None: + """Warehouse emits HTML-escaped requires-python (e.g. >=3.8).""" + html = """ + requests-2.32.0-1-py3-none-any.whl + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "https://cache.test/simple/requests/" + ) + assert len(artifacts) == 1 + assert artifacts[0].filename == "requests-2.32.0-1-py3-none-any.whl" + + def test_parse_project_page_unquotes_percent_encoded_filenames(self) -> None: + """Local versions use '+' which simple indexes percent-encode as %2B.""" + html = """ + torch-2.0.0%2Bcpu-py3-none-any.whl + """ + artifacts = RemotePEP503Backend._parse_project_page( + html, "https://cache.test/simple/torch/" + ) + assert len(artifacts) == 1 + assert artifacts[0].filename == "torch-2.0.0+cpu-py3-none-any.whl" + + def test_lookup_accepts_percent_encoded_local_version( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get( + "https://cache.test/simple/", + text='torch', + ) + requests_mock.get( + "https://cache.test/simple/torch/", + text=( + '' + "torch-2.0.0%2Bcpu-py3-none-any.whl" + ), + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("torch"), + version=Version("2.0.0+cpu"), + build_tag=(), + ) + info = backend.lookup(key) + assert info is not None + assert info.filename == "torch-2.0.0+cpu-py3-none-any.whl" + + def test_lookup_skips_incompatible_platform_wheels( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get( + "https://cache.test/simple/", + text='requests', + ) + requests_mock.get( + "https://cache.test/simple/requests/", + text=( + '' + "requests-2.31.0-1-cp39-cp39-win_amd64.whl" + '' + "requests-2.31.0-1-py3-none-any.whl" + ), + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + backend.scan() + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + with patch( + "fromager.cache.SUPPORTED_TAGS", + frozenset({Tag("py3", "none", "any")}), + ): + info = backend.lookup(key) + assert info is not None + assert info.filename == "requests-2.31.0-1-py3-none-any.whl" + assert info.sha256 == "good" + + def test_fetch_downloads_wheel( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + wheel_data = b"fake wheel content" + sha256 = hashlib.sha256(wheel_data).hexdigest() + requests_mock.get( + "https://cache.test/simple/requests/requests-2.31.0-1-py3-none-any.whl", + content=wheel_data, + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = ArtifactInfo( + filename="requests-2.31.0-1-py3-none-any.whl", + url_or_path="https://cache.test/simple/requests/requests-2.31.0-1-py3-none-any.whl", + sha256=sha256, + ) + dest = tmp_path / "downloads" + dest.mkdir() + result = backend.fetch(key, info, dest) + assert result.exists() + assert result.read_bytes() == wheel_data + + def test_fetch_rejects_sha256_mismatch( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + requests_mock.get( + "https://cache.test/simple/requests/requests-2.31.0-1-py3-none-any.whl", + content=b"corrupted content", + ) + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = ArtifactInfo( + filename="requests-2.31.0-1-py3-none-any.whl", + url_or_path="https://cache.test/simple/requests/requests-2.31.0-1-py3-none-any.whl", + sha256="expected_sha256_that_wont_match", + ) + dest = tmp_path / "downloads" + dest.mkdir() + with pytest.raises(ValueError, match="sha256 mismatch"): + backend.fetch(key, info, dest) + + def test_fetch_uses_cached_file_if_sha256_matches( + self, tmp_path: pathlib.Path + ) -> None: + wheel_data = b"cached wheel" + sha256 = hashlib.sha256(wheel_data).hexdigest() + dest = tmp_path / "downloads" + dest.mkdir() + existing = dest / "requests-2.31.0-1-py3-none-any.whl" + existing.write_bytes(wheel_data) + + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = ArtifactInfo( + filename="requests-2.31.0-1-py3-none-any.whl", + url_or_path="https://cache.test/simple/requests/requests-2.31.0-1-py3-none-any.whl", + sha256=sha256, + ) + result = backend.fetch(key, info, dest) + assert result == existing + + def test_fetch_accepts_uppercase_sha256(self, tmp_path: pathlib.Path) -> None: + """Index fragments may advertise uppercase hex digests.""" + wheel_data = b"cached wheel" + sha256 = hashlib.sha256(wheel_data).hexdigest().upper() + dest = tmp_path / "downloads" + dest.mkdir() + existing = dest / "requests-2.31.0-1-py3-none-any.whl" + existing.write_bytes(wheel_data) + + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = ArtifactInfo( + filename="requests-2.31.0-1-py3-none-any.whl", + url_or_path="https://cache.test/simple/requests/requests-2.31.0-1-py3-none-any.whl", + sha256=sha256, + ) + result = backend.fetch(key, info, dest) + assert result == existing + + def test_fetch_redownloads_when_existing_sha256_mismatches( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + wheel_data = b"good wheel" + sha256 = hashlib.sha256(wheel_data).hexdigest() + url = "https://cache.test/simple/requests/requests-2.31.0-1-py3-none-any.whl" + requests_mock.get(url, content=wheel_data) + dest = tmp_path / "downloads" + dest.mkdir() + existing = dest / "requests-2.31.0-1-py3-none-any.whl" + existing.write_bytes(b"stale") + + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + info = ArtifactInfo( + filename="requests-2.31.0-1-py3-none-any.whl", + url_or_path=url, + sha256=sha256, + ) + result = backend.fetch(key, info, dest) + assert result.read_bytes() == wheel_data + + def test_store_raises(self, tmp_path: pathlib.Path) -> None: + backend = RemotePEP503Backend( + server_url="https://cache.test/simple", + download_dir=tmp_path, + ) + key = WheelCacheKey( + package=canonicalize_name("requests"), + version=Version("2.31.0"), + build_tag=(1, ""), + ) + with pytest.raises(NotImplementedError): + backend.store(key, tmp_path / "fake.whl") + + +# --------------------------------------------------------------------------- +# CacheManager +# --------------------------------------------------------------------------- + + +class TestCacheManager: + def _make_manager( + self, tmp_path: pathlib.Path + ) -> tuple[CacheManager, LocalDirectoryBackend]: + store = LocalDirectoryBackend(tmp_path / "downloads", backend_name="store") + build = LocalDirectoryBackend(tmp_path / "build", backend_name="build") + manager = CacheManager( + lookup_backends=[build, store], + store_backend=store, + ) + return manager, store + + def test_lookup_miss(self, tmp_path: pathlib.Path) -> None: + manager, _ = self._make_manager(tmp_path) + (tmp_path / "downloads").mkdir() + (tmp_path / "build").mkdir() + manager.initialize() + result = manager.lookup_wheel( + Requirement("requests"), Version("2.31.0"), (1, "") + ) + assert result.miss + + def test_lookup_hit_in_store(self, tmp_path: pathlib.Path) -> None: + downloads = tmp_path / "downloads" + downloads.mkdir() + whl = downloads / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + (tmp_path / "build").mkdir() + + manager, _ = self._make_manager(tmp_path) + manager.initialize() + result = manager.lookup_wheel( + Requirement("requests"), Version("2.31.0"), (1, "") + ) + assert result.hit + assert result.was_downloaded is False + assert result.path == whl.resolve() + + def test_lookup_empty_build_tag_matches_tagged_wheel( + self, tmp_path: pathlib.Path + ) -> None: + """Empty expected build tag matches wheels with tag 0 (legacy semantics).""" + downloads = tmp_path / "downloads" + downloads.mkdir() + whl = downloads / "requests-2.31.0-0-py3-none-any.whl" + whl.write_bytes(b"data") + (tmp_path / "build").mkdir() + + manager, _ = self._make_manager(tmp_path) + manager.initialize() + result = manager.lookup_wheel(Requirement("requests"), Version("2.31.0"), ()) + assert result.hit + assert result.path == whl.resolve() + assert result.build_tag == (0, "") + + def test_lookup_hit_in_build_copies_to_store(self, tmp_path: pathlib.Path) -> None: + downloads = tmp_path / "downloads" + downloads.mkdir() + build = tmp_path / "build" + build.mkdir() + whl = build / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"build data") + + manager, _store = self._make_manager(tmp_path) + manager.initialize() + result = manager.lookup_wheel( + Requirement("requests"), Version("2.31.0"), (1, "") + ) + assert result.hit + assert result.was_downloaded is True + stored = downloads / "requests-2.31.0-1-py3-none-any.whl" + assert stored.exists() + # Promoted hits expose the store path, not the build-dir source. + assert result.path == stored.resolve() + + def test_lookup_forced_miss(self, tmp_path: pathlib.Path) -> None: + downloads = tmp_path / "downloads" + downloads.mkdir() + whl = downloads / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + (tmp_path / "build").mkdir() + + store = LocalDirectoryBackend(downloads, backend_name="store") + manager = CacheManager( + lookup_backends=[store], + store_backend=store, + force=True, + ) + manager.initialize() + result = manager.lookup_wheel( + Requirement("requests"), Version("2.31.0"), (1, "") + ) + assert result.miss + + def test_store_wheel(self, tmp_path: pathlib.Path) -> None: + downloads = tmp_path / "downloads" + downloads.mkdir() + (tmp_path / "build").mkdir() + + source = tmp_path / "built" / "requests-2.31.0-1-py3-none-any.whl" + source.parent.mkdir() + source.write_bytes(b"built wheel") + + manager, _ = self._make_manager(tmp_path) + manager.initialize() + stored_path = manager.store_wheel( + Requirement("requests"), Version("2.31.0"), (1, ""), source + ) + assert pathlib.Path(stored_path).exists() + assert (downloads / source.name).exists() + + def test_stats_recording(self, tmp_path: pathlib.Path) -> None: + downloads = tmp_path / "downloads" + downloads.mkdir() + whl = downloads / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + (tmp_path / "build").mkdir() + + manager, _ = self._make_manager(tmp_path) + manager.initialize() + + manager.lookup_wheel(Requirement("requests"), Version("2.31.0"), (1, "")) + manager.lookup_wheel(Requirement("nonexist"), Version("1.0.0"), ()) + + assert manager.stats.hits == 1 + assert manager.stats.misses == 1 + + def test_lookup_continues_on_fetch_failure(self, tmp_path: pathlib.Path) -> None: + """If a backend's fetch() fails, lookup continues to next backend.""" + downloads = tmp_path / "downloads" + downloads.mkdir() + build = tmp_path / "build" + build.mkdir() + whl = build / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"fallback data") + + # Create a failing backend mock + failing_backend = MagicMock() + failing_backend.name = "failing" + failing_backend.writable = True + failing_backend.lookup.return_value = ArtifactInfo( + filename="requests-2.31.0-1-py3-none-any.whl", + url_or_path="/nonexistent", + ) + failing_backend.fetch.side_effect = OSError("disk error") + + build_backend = LocalDirectoryBackend(build, backend_name="build") + build_backend.scan() + + store = LocalDirectoryBackend(downloads, backend_name="store") + manager = CacheManager( + lookup_backends=[failing_backend, build_backend], + store_backend=store, + ) + result = manager.lookup_wheel( + Requirement("requests"), Version("2.31.0"), (1, "") + ) + assert result.hit + assert result.path == (downloads / whl.name).resolve() + + +# --------------------------------------------------------------------------- +# CacheStats +# --------------------------------------------------------------------------- + + +class TestCacheStats: + def test_hit_rate(self) -> None: + stats = CacheStats() + req = Requirement("requests") + ver = Version("2.31.0") + stats.record_hit(req, ver, "local") + stats.record_miss(req, ver, "not_found") + assert stats.hit_rate == 0.5 + + def test_summary(self) -> None: + stats = CacheStats() + req = Requirement("requests") + ver = Version("2.31.0") + stats.record_hit(req, ver, "local:downloads") + stats.record_store(req, ver, "local:downloads") + summary = stats.summary() + assert summary["hits"]["total"] == 1 + assert summary["stores"] == 1 + assert "local:downloads" in summary["hits"]["by_backend"] + + +# --------------------------------------------------------------------------- +# Concurrency +# --------------------------------------------------------------------------- + + +class TestConcurrency: + def test_concurrent_store_and_lookup(self, tmp_path: pathlib.Path) -> None: + """Multiple threads can safely contend on the same cache key.""" + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + errors: list[Exception] = [] + key = WheelCacheKey( + package=canonicalize_name("pkg"), + version=Version("1.0.0"), + build_tag=(1, ""), + ) + source_dir = tmp_path / "sources" + source_dir.mkdir() + + def store_worker(idx: int) -> None: + try: + dest_name = "pkg-1.0.0-1-py3-none-any.whl" + unique = source_dir / f"t{idx}" / dest_name + unique.parent.mkdir() + unique.write_bytes(f"data{idx}".encode()) + backend.store(key, unique) + except Exception as e: + errors.append(e) + + def lookup_worker() -> None: + try: + for _ in range(20): + backend.lookup(key) + except Exception as e: + errors.append(e) + + threads = [threading.Thread(target=store_worker, args=(i,)) for i in range(10)] + threads.extend(threading.Thread(target=lookup_worker) for _ in range(10)) + + for t in threads: + t.start() + for t in threads: + t.join() + + assert errors == [] + info = backend.lookup(key) + assert info is not None + assert pathlib.Path(info.url_or_path).exists() + assert pathlib.Path(info.url_or_path).read_bytes().startswith(b"data") + + def test_concurrent_scan_and_lookup(self, tmp_path: pathlib.Path) -> None: + """Scanning while lookups are happening doesn't crash.""" + for i in range(5): + whl = tmp_path / f"pkg{i}-1.0.0-1-py3-none-any.whl" + whl.write_bytes(f"data{i}".encode()) + + backend = LocalDirectoryBackend(tmp_path, backend_name="test") + backend.scan() + errors: list[Exception] = [] + + def scan_worker() -> None: + try: + backend.scan() + except Exception as e: + errors.append(e) + + def lookup_worker() -> None: + try: + for i in range(5): + key = WheelCacheKey( + package=canonicalize_name(f"pkg{i}"), + version=Version("1.0.0"), + build_tag=(1, ""), + ) + backend.lookup(key) + except Exception as e: + errors.append(e) + + threads = [ + threading.Thread(target=scan_worker), + threading.Thread(target=lookup_worker), + threading.Thread(target=lookup_worker), + ] + for t in threads: + t.start() + for t in threads: + t.join() + + assert errors == [] + + +# --------------------------------------------------------------------------- +# CLI commands (via CliRunner) +# --------------------------------------------------------------------------- + + +class TestCLICommands: + @pytest.fixture() + def wkctx(self, tmp_path: pathlib.Path) -> typing.Any: + """Minimal WorkContext-like object for CLI tests.""" + wkctx = MagicMock() + wkctx.wheels_build_base = tmp_path / "build" + wkctx.wheels_build_base.mkdir() + wkctx.wheels_downloads = tmp_path / "downloads" + wkctx.wheels_downloads.mkdir() + wkctx.wheels_prebuilt = tmp_path / "prebuilt" + wkctx.wheels_prebuilt.mkdir() + wkctx.cache = None + return wkctx + + def test_cache_list_empty(self, wkctx: typing.Any) -> None: + runner = CliRunner() + result = runner.invoke(cache_cli, ["list"], obj=wkctx) + assert result.exit_code == 0 + assert "No cached wheels found" in result.output + + def test_cache_list_with_wheels(self, wkctx: typing.Any) -> None: + whl = wkctx.wheels_downloads / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"fake") + + runner = CliRunner() + result = runner.invoke(cache_cli, ["list", "--format", "json"], obj=wkctx) + assert result.exit_code == 0 + assert "requests" in result.output + + def test_cache_stats(self, wkctx: typing.Any) -> None: + whl = wkctx.wheels_downloads / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + + runner = CliRunner() + result = runner.invoke(cache_cli, ["stats"], obj=wkctx) + assert result.exit_code == 0 + assert "Cache Inventory" in result.output + assert "local:downloads wheels" in result.output + # Rich table cells are padded; match the metric/value pair loosely. + assert re.search(r"Total wheels on disk\s+1\b", result.output) + + def test_cache_gc_keeps_latest_by_mtime(self, wkctx: typing.Any) -> None: + """Untagged wheels are ordered by mtime so --keep-latest keeps newest.""" + older = wkctx.wheels_downloads / "requests-2.31.0-py3-none-any.whl" + newer = wkctx.wheels_downloads / "requests-2.31.0-0-py3-none-any.whl" + older.write_bytes(b"older") + newer.write_bytes(b"newer") + # Force deterministic recency regardless of write order. + older_mtime = 1_700_000_000 + newer_mtime = 1_700_000_100 + os.utime(older, (older_mtime, older_mtime)) + os.utime(newer, (newer_mtime, newer_mtime)) + + runner = CliRunner() + result = runner.invoke(cache_cli, ["gc", "--keep-latest", "1"], obj=wkctx) + assert result.exit_code == 0 + assert newer.exists() + assert not older.exists() + + def test_cache_verify_ok(self, wkctx: typing.Any) -> None: + runner = CliRunner() + result = runner.invoke(cache_cli, ["verify"], obj=wkctx) + assert result.exit_code == 0 + assert "verified OK" in result.output + + def test_cache_gc_nothing_to_remove(self, wkctx: typing.Any) -> None: + runner = CliRunner() + result = runner.invoke(cache_cli, ["gc"], obj=wkctx) + assert result.exit_code == 0 + assert "Removed 0" in result.output + + def test_cache_invalidate_requires_args(self, wkctx: typing.Any) -> None: + runner = CliRunner() + result = runner.invoke(cache_cli, ["invalidate"], obj=wkctx) + assert result.exit_code != 0 + + def test_cache_invalidate_all(self, wkctx: typing.Any) -> None: + whl = wkctx.wheels_downloads / "requests-2.31.0-1-py3-none-any.whl" + whl.write_bytes(b"data") + + runner = CliRunner() + result = runner.invoke(cache_cli, ["invalidate", "--all"], obj=wkctx) + assert result.exit_code == 0 + assert "Invalidated 1" in result.output + assert not whl.exists() + + def test_cache_invalidate_preserves_prebuilt_and_build( + self, wkctx: typing.Any + ) -> None: + """Destructive ops only touch the downloads store backend.""" + downloads_whl = wkctx.wheels_downloads / "requests-2.31.0-1-py3-none-any.whl" + downloads_whl.write_bytes(b"downloads") + prebuilt_whl = wkctx.wheels_prebuilt / "urllib3-2.0.0-1-py3-none-any.whl" + prebuilt_whl.write_bytes(b"prebuilt") + build_whl = wkctx.wheels_build_base / "idna-3.0-1-py3-none-any.whl" + build_whl.write_bytes(b"build") + + runner = CliRunner() + result = runner.invoke(cache_cli, ["invalidate", "--all"], obj=wkctx) + assert result.exit_code == 0 + assert not downloads_whl.exists() + assert prebuilt_whl.exists() + assert build_whl.exists() + + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + + +class TestBuildCacheManager: + def test_lookup_order_includes_build_dir(self, tmp_path: pathlib.Path) -> None: + """Factory searches build then downloads; prebuilt is intentionally omitted.""" + wkctx = MagicMock() + wkctx.wheels_build_base = tmp_path / "build" + wkctx.wheels_build_base.mkdir() + wkctx.wheels_downloads = tmp_path / "downloads" + wkctx.wheels_downloads.mkdir() + wkctx.wheels_prebuilt = tmp_path / "prebuilt" + wkctx.wheels_prebuilt.mkdir() + wkctx.cache = None + + manager = build_cache_manager(wkctx) + names = [b.name for b in manager.lookup_backends] + assert names == ["local:build", "local:downloads"] + assert manager.store_backend.name == "local:downloads" + assert wkctx.cache is manager + assert build_cache_manager(wkctx) is manager + build_backend = manager.lookup_backends[0] + assert isinstance(build_backend, LocalDirectoryBackend) + assert build_backend._recursive is True + + def test_returns_existing_cache(self, tmp_path: pathlib.Path) -> None: + existing = MagicMock() + wkctx = MagicMock() + wkctx.cache = existing + assert build_cache_manager(wkctx) is existing + + def test_adds_remote_backend_with_allow_insecure( + self, tmp_path: pathlib.Path, requests_mock: requests_mock.Mocker + ) -> None: + wkctx = MagicMock() + wkctx.wheels_build_base = tmp_path / "build" + wkctx.wheels_build_base.mkdir() + wkctx.wheels_downloads = tmp_path / "downloads" + wkctx.wheels_downloads.mkdir() + wkctx.wheels_prebuilt = tmp_path / "prebuilt" + wkctx.wheels_prebuilt.mkdir() + wkctx.cache = None + + cache_url = "https://cache.test/simple" + requests_mock.get( + f"{cache_url}/", + text='requests', + ) + manager = build_cache_manager(wkctx, cache_url=cache_url, allow_insecure=True) + names = [b.name for b in manager.lookup_backends] + assert names == [ + "local:build", + "local:downloads", + f"remote:{cache_url}", + ] + assert isinstance(manager.lookup_backends[-1], RemotePEP503Backend) diff --git a/tests/test_cli.py b/tests/test_cli.py index 85d97506..c3ec1928 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -138,6 +138,7 @@ def test_multiple_constraints_files( "build-order", "build-parallel", "build-sequence", + "cache", "canonicalize", "download-sequence", "find-updates", diff --git a/tests/test_commands.py b/tests/test_commands.py index 5678d3d9..d2eb3fc3 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -20,3 +20,9 @@ def test_bootstrap_parallel_options() -> None: expected.discard("test_mode") assert set(get_option_names(bootstrap.bootstrap_parallel)) == expected + + +def test_bootstrap_has_cache_manager_options() -> None: + option_names = set(get_option_names(bootstrap.bootstrap)) + assert "use_cache_manager" in option_names + assert "cache_allow_insecure" in option_names diff --git a/tests/test_server.py b/tests/test_server.py index d507682e..eecb497c 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -60,6 +60,39 @@ def test_update_wheel_mirror_moves_to_downloads( assert tmp_context.wheels_downloads.joinpath("foo-1.0-py3-none-any.whl").exists() +def test_index_wheel_links_without_touching_build_dir( + tmp_context: context.WorkContext, +) -> None: + """index_wheel must not move unrelated in-progress build wheels.""" + in_progress = _create_fake_wheel( + tmp_context.wheels_build, "other-1.0-py3-none-any.whl" + ) + cached = _create_fake_wheel( + tmp_context.wheels_downloads, "foo-1.0-py3-none-any.whl" + ) + + server.index_wheel(tmp_context, cached) + + assert in_progress.exists() + symlink = tmp_context.wheel_server_dir.joinpath("foo", "foo-1.0-py3-none-any.whl") + assert symlink.is_symlink() + assert symlink.is_file() + + +def test_index_wheel_idempotent_when_symlink_exists( + tmp_context: context.WorkContext, +) -> None: + """Re-indexing an already-linked wheel must not raise FileExistsError.""" + cached = _create_fake_wheel( + tmp_context.wheels_downloads, "foo-1.0-py3-none-any.whl" + ) + server.index_wheel(tmp_context, cached) + server.index_wheel(tmp_context, cached) + symlink = tmp_context.wheel_server_dir.joinpath("foo", "foo-1.0-py3-none-any.whl") + assert symlink.is_symlink() + assert symlink.is_file() + + def test_update_wheel_mirror_creates_symlink( tmp_context: context.WorkContext, ) -> None: