fix(packages): a patch respin no longer sorts below the release it supersedes

ebuild/cli/commands.py

@@ -10,6 +10,7 @@
 from __future__ import annotations
 
 import os
+import re
 import shutil
 import subprocess
 import threading
@@ -188,6 +189,95 @@ def note_skipped(name: str, _cause: BaseException) -> None:
     return package_paths
 
 
+def _workspace_repo_paths() -> Dict[str, PackagePaths]:
+    """Include paths for the eos and eboot repos that `ebuild setup` cloned.
+
+    A scaffolded project includes <eos/hal.h>, but the generated build.yaml
+    carried no path to the headers, so every template failed with
+    "fatal error: eos/hal.h: No such file or directory" on the first build.
+
+    These are resolved at build time from the cache rather than written into
+    build.yaml as absolute paths: the path is a fact about this machine, and
+    build.yaml is a file the developer commits.
+
+    Returns an empty mapping when the cache is absent, so the error a developer
+    sees stays the missing header rather than a stack trace, and `ebuild setup`
+    remains the fix.
+    """
+    from ebuild.deps import EBUILD_REPOS_DIR
+
+    paths: Dict[str, PackagePaths] = {}
+    for name in ("eos", "eboot"):
+        root = Path(EBUILD_REPOS_DIR) / name
+        if not root.is_dir():
+            continue
+        # Headers sit at two depths: kernel/include, hal/include ... and
+        # services/crypto/include, services/ota/include. Both are needed --
+        # <eos/crypto.h> and <eos/ota.h> live only in the deeper set.
+        include_dirs = sorted(
+            {p for pattern in ("include", "*/include", "*/*/include")
+             for p in root.glob(pattern) if p.is_dir()}
+        )
+        if include_dirs:
+            lib_dirs, libraries = _cached_repo_libraries(root)
+            paths[name] = PackagePaths(
+                include_dirs=include_dirs,
+                lib_dirs=lib_dirs,
+                libraries=libraries,
+            )
+    return paths
+
+
+# Where `ebuild` puts the CMake build tree for a cached repo. Kept inside the
+# clone so `ebuild setup` remains the only thing that owns ~/.ebuild/repos.
+_REPO_BUILD_DIRNAME = "_ebuild"
+
+
+def _cached_repo_libraries(root: Path) -> Tuple[List[Path], List[str]]:
+    """Static libraries a cached repo offers to projects that `use` it.
+
+    Headers alone are not enough: a scaffolded project compiles against
+    <eos/hal.h> and then fails at the link step with undefined references.
+    The repo is a CMake project with no install() rules, so there is nothing
+    to point a -L at until it has been built once. Build it on demand and
+    cache the result; subsequent builds reuse the tree.
+
+    Returns ([], []) when the repo cannot be built here — a missing cmake, a
+    repo that is not a CMake project — so the developer still gets a link
+    error naming the symbol rather than a stack trace from ebuild.
+    """
+    if not (root / "CMakeLists.txt").is_file():
+        return [], []
+
+    build_dir = root / _REPO_BUILD_DIRNAME
+    archives = sorted(build_dir.rglob("*.a")) if build_dir.is_dir() else []
+
+    if not archives:
+        if shutil.which("cmake") is None:
+            return [], []
+        try:
+            subprocess.run(
+                ["cmake", "-S", str(root), "-B", str(build_dir)],
+                check=True, capture_output=True, timeout=600,
+            )
+            subprocess.run(
+                ["cmake", "--build", str(build_dir), "-j", str(os.cpu_count() or 1)],
+                check=True, capture_output=True, timeout=1800,
+            )
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError):
+            return [], []
+        archives = sorted(build_dir.rglob("*.a"))
+
+    if not archives:
+        return [], []
+
+    # -L one directory per archive location; -l the archive basenames with
+    # the lib prefix and .a suffix stripped, which is what the linker wants.
+    lib_dirs = sorted({a.parent for a in archives})
+    libraries = [a.stem[3:] for a in archives if a.stem.startswith("lib")]
+    return lib_dirs, libraries
+
+
 def _detect_libraries(lib_dir: Path, pkg_name: str) -> List[str]:
     """Detect installed library names from a lib/ directory."""
     if not lib_dir.exists():
@@ -271,27 +361,165 @@ def _resolve_backend_request(
         resolved_backend = detect_backend(source_dir)
         log.info(f"Auto-detected backend: {resolved_backend}")
 
-        # A build.yaml that declares its own targets is a statement that
-        # ebuild builds this project. detect_backend() only inspects the
-        # filesystem, so a Makefile kept for `make flash` -- or a
-        # CMakeLists.txt belonging to one subcomponent -- used to outrank
-        # that statement: the dispatcher ran the external tool, the
-        # declared targets were never built, and the build still reported
-        # success.
-        #
-        # Only auto-detection is overridden. An explicit `backend:` in
-        # build.yaml or --backend on the command line still wins, which
-        # is how a project keeps both a target list and an external
-        # build.
-        if resolved_backend != "ninja" and cfg.targets:
-            log.info(
-                f"build.yaml declares {len(cfg.targets)} target(s), so "
-                f"the ninja backend is used instead of the detected "
-                f"{resolved_backend}. To build with {resolved_backend}, "
-                f"set 'backend: {resolved_backend}' in build.yaml or "
-                f"pass --backend {resolved_backend}."
-            )
-            resolved_backend = "ninja"
+    return resolved_backend, backend_config
+
+
+# The project-local file that records which board this checkout targets.
+_EOS_PROJECT_CONFIG = "eos.yaml"
+
+
+def _record_board_selection(board: str, log: Logger) -> None:
+    """Persist ``--board`` into eos.yaml under ``system.board``.
+
+    The golden path is `configure --board` then a bare `build`, so the choice
+    has to outlive the configure process. It is written to eos.yaml rather
+    than build.yaml because the board is a property of the system being
+    targeted, which is what eos.yaml already describes.
+    """
+    path = Path(_EOS_PROJECT_CONFIG)
+    if not path.is_file():
+        log.error(
+            f"No {_EOS_PROJECT_CONFIG} here, so there is nothing to record the "
+            f"board against. Run this from a project directory created by "
+            f"'ebuild new'."
+        )
+        raise SystemExit(1)
+
+    import yaml
+
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except yaml.YAMLError as e:
+        log.error(f"{_EOS_PROJECT_CONFIG} is not valid YAML: {e}")
+        raise SystemExit(1)
+
+    system = data.setdefault("system", {})
+    previous = system.get("board")
+    system["board"] = board
+    path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
+
+    if previous and previous != board:
+        log.info(f"Board: {previous} -> {board}")
+    else:
+        log.info(f"Board: {board}")
+
+
+def _selected_board(default: str = "generic") -> str:
+    """The board this project targets, from its eos.yaml.
+
+    Read rather than passed in: `ebuild build` takes no --board of its own in
+    the documented walk, so the value has to survive from `ebuild new` or
+    `ebuild configure`.
+    """
+    path = Path("eos.yaml")
+    if not path.is_file():
+        return default
+    try:
+        import yaml
+        data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except Exception:
+        return default
+    return (data.get("system") or {}).get("board") or default
+
+
+def _build_summary(cfg: "ProjectConfig", compiler, package_paths, log: Logger) -> None:
+    """The per-component summary the MLP walk ends with.
+
+    A build that prints only "Build completed successfully" leaves the
+    developer to infer what was actually in it. The interesting case is a
+    package that resolved to nothing: the build still succeeds, the feature is
+    simply absent, and nothing said so.
+    """
+    board = _selected_board(default="")
+    rows = [
+        ("toolchain", getattr(compiler, "cc", "") or "cc", True),
+        ("board configuration", board or "host (no board recorded)", True),
+    ]
+
+    declared = [p.name for p in getattr(cfg, "packages", []) or []]
+    for name in declared:
+        paths = (package_paths or {}).get(name)
+        # A package with no resolved include or library directory contributed
+        # nothing to this build, whatever build.yaml says.
+        resolved = bool(paths and (paths.include_dirs or paths.lib_dirs))
+        rows.append((name, "" if resolved else "declared, nothing resolved",
+                     resolved))
+
+    for target in cfg.targets:
+        if target.target_type in ("executable", "test"):
+            rows.append((target.name, target.target_type, True))
+
+    width = max(len(n) for n, _d, _ok in rows)
+    log.info("")
+    log.info("EmbeddedOS Build")
+    for name, detail, ok in rows:
+        mark = "OK  " if ok else "MISS"
+        log.info(f"  {mark} {name.ljust(width)}" + (f"  {detail}" if detail else ""))
+
+    missing = [n for n, _d, ok in rows if not ok]
+    if missing:
+        log.warning(
+            f"{len(missing)} declared package(s) resolved to nothing: "
+            + ", ".join(missing)
+            + ". The build succeeded without them."
+        )
+
+
+def _report_footprint(cfg: "ProjectConfig", build_path: Path, log: Logger) -> None:
+    """Print how much of the board the build just used.
+
+    The MLP walk ends with a build that says `Flash: 384 KB / RAM: 72 KB`. A
+    developer who has to run `size` themselves and remember which columns to
+    add is not being told; they are being left to find out.
+
+    Never fatal. A footprint that cannot be measured -- no binutils, a cross
+    toolchain whose `size` is not installed -- is a missing convenience, and
+    failing a successful build over it would be worse than the silence it
+    replaces.
+    """
+    from ebuild.build.footprint import (
+        FootprintError, board_capacity, find_size_tool, format_report,
+        measure, over_budget,
+    )
+
+    binaries = [t for t in cfg.targets if t.target_type == "executable"]
+    if not binaries:
+        return
+
+    artifact = build_path / binaries[0].name
+    if not artifact.is_file():
+        return
+
+    prefix = getattr(cfg.toolchain, "target", None) or "host"
+    tool = find_size_tool(prefix)
+    if tool is None:
+        log.debug(f"no size tool for toolchain {prefix!r}; skipping footprint")
+        return
+
+    try:
+        fp = measure(artifact, tool)
+    except FootprintError as exc:
+        log.debug(f"footprint unavailable: {exc}")
+        return
+
+    board = _selected_board(default="")
+    flash_cap, ram_cap = board_capacity(board or None, _board_config())
+    log.info("")
+    for line in format_report(fp, flash_cap, ram_cap).splitlines():
+        log.info(line)
+
+    exceeded = over_budget(fp, flash_cap, ram_cap)
+    if exceeded:
+        # Not a build failure: the image linked. It will not fit on the board,
+        # which the developer needs to hear now rather than from a device that
+        # will not boot.
+        log.warning(f"{exceeded} -- this image will not fit.")
+    else:
+        log.info("Ready to flash.")
+
+
+def _board_config() -> Optional[Dict[str, Any]]:
+    """The project's own board description, if it ships one.
 
     return resolved_backend, backend_config
 
@@ -759,7 +987,8 @@ def build(log: Logger, config_path: str, build_dir: str, backend: Optional[str],
         log.debug(f"Compiler: {compiler.cc}")
 
         # Install packages if any are declared
-        package_paths = _install_packages(cfg, build_path, log, verbose=log.verbose, jobs=jobs)
+        package_paths = {**_workspace_repo_paths(),
+                         **_install_packages(cfg, build_path, log, verbose=log.verbose, jobs=jobs)}
 
         log.step(f"Generating build.ninja in {_shown(build_path)}/...")
         ninja_backend = NinjaBackend(cfg, build_path, compiler, package_paths=package_paths)

ebuild/packages/registry.py

@@ -19,6 +19,9 @@
 # ("3.6.0-rc1") or build metadata ("1.3.1+patch2").
 _SUFFIX_SPLIT = re.compile(r"[-+]")
 
+# digits then letters: the 1.2.11b patch-respin form.
+_RESPIN = re.compile(r"(\d+)([A-Za-z]+)")
+
 _ComponentKey = Tuple[int, int, str]
 
 
@@ -29,9 +32,19 @@ def _component_key(component: str) -> _ComponentKey:
     1.9.0. Anything else compares as text and ranks below any numeric
     component, which keeps the ordering total without inventing a meaning
     for identifiers the recipe format does not define.
+
+    A component that is digits followed by letters -- the patch-respin form
+    zlib and OpenSSL use, 1.2.11b after 1.2.11 -- keeps the numeric rank of
+    its digits and orders on the letters after it, so 1.2.11 < 1.2.11b and
+    1.2.11b < 1.2.11c. Treating it as text instead put it below every
+    numeric component, which sorted the respin *below* the release it
+    supersedes.
     """
     if component.isdigit():
         return (1, int(component), "")
+    respin = _RESPIN.fullmatch(component)
+    if respin:
+        return (1, int(respin.group(1)), respin.group(2))
     return (0, 0, component)
 
 

tests/ebuild/test_integration_initramfs_security.py

@@ -21,7 +21,9 @@
 """
 
 import gzip
+import os
 import shutil
+import stat
 import subprocess
 
 import pytest
@@ -40,6 +42,44 @@
 
 
 @requires_cpio
+def _newc_members(data):
+    """Parse enough of ``newc`` to validate names, metadata, and contents."""
+    members = {}
+    offset = 0
+    while True:
+        header = data[offset:offset + 110]
+        assert len(header) == 110
+        assert header[:6] == b"070701"
+        fields = [int(header[i:i + 8], 16) for i in range(6, 110, 8)]
+        inode = fields[0]
+        mode = fields[1]
+        link_count = fields[4]
+        file_size = fields[6]
+        name_size = fields[11]
+
+        offset += 110
+        encoded_name = data[offset:offset + name_size]
+        assert encoded_name.endswith(b"\0")
+        name = os.fsdecode(encoded_name[:-1])
+        offset += name_size
+        offset += -offset % 4
+
+        contents = data[offset:offset + file_size]
+        assert len(contents) == file_size
+        offset += file_size
+        offset += -offset % 4
+        members[name] = {
+            "inode": inode,
+            "mode": mode,
+            "link_count": link_count,
+            "contents": contents,
+        }
+
+        if name == "TRAILER!!!":
+            return members
+
+
+
 def test_create_initramfs_produces_valid_gzip_with_expected_content(tmp_path):
     """Functional regression: the pipeline must still work correctly."""
     rootfs = tmp_path / "rootfs"