diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6c6c26..41ce85c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,13 @@ on: pull_request: branches: [master, main] +# Every other workflow in this repo declares a concurrency group; ci.yml, +# the heaviest one, did not. Pushing twice to a PR left the earlier run +# queued, and both competed for the same scarce windows/macos runners. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: # ── Test Matrix ─────────────────────────────────────────────────────────── test: @@ -17,7 +24,10 @@ jobs: strategy: matrix: python-version: ["3.10", "3.11", "3.12"] - os: [ubuntu-22.04, macos-13, windows-2022] + # macos-latest, not macos-13: the macos-13 image is retired, so jobs + # requesting it are never assigned a runner and sit queued until they + # time out. Every other workflow in this repo already uses macos-latest. + os: [ubuntu-22.04, macos-latest, windows-2022] fail-fast: false steps: @@ -43,7 +53,11 @@ jobs: continue-on-error: true - name: Type check (mypy) - run: mypy . --ignore-missing-imports --no-strict-optional + # --exclude: layers/eosuite/ vendors its own tests/ package, so a bare + # `mypy .` sees two modules named "tests" and bails with "Duplicate + # module named 'tests'" before checking anything. continue-on-error hid + # that the type check was doing no work at all. + run: mypy . --ignore-missing-imports --no-strict-optional --exclude '^layers/' continue-on-error: true # Runs the whole tests/ tree. The previous steps ran only tests/unit/ @@ -57,7 +71,12 @@ jobs: # coverage policy already lives in codecov.yml; whether to also # enforce it here is a maintainer decision, so this change leaves # both of those numbers alone. + # shell: bash -- the matrix includes windows-2022, where the default + # shell is PowerShell and the backslash line continuations below are a + # syntax error ("Missing expression after unary operator '--'"), so the + # Windows leg of this job failed before pytest ever started. - name: Run test suite + shell: bash run: | python -m pytest tests/ -v --tb=short \ --cov=ebuild --cov-report=xml --cov-report=term-missing \ diff --git a/ebuild/build/ninja_backend.py b/ebuild/build/ninja_backend.py index 1282545..7db712e 100644 --- a/ebuild/build/ninja_backend.py +++ b/ebuild/build/ninja_backend.py @@ -39,6 +39,25 @@ def _shared_flag() -> str: return "-dynamiclib" if sys.platform == "darwin" else "-shared" +def _ninja_path(path) -> str: + """Escape *path* for use in a Ninja build statement. + + Ninja splits build statements on unescaped spaces and colons, so a Windows + absolute path writes a drive letter that Ninja reads as the output/rule + separator: + + build C:\\...\\main.o: cc main.c + ^ "expected build command name" + + `$` is escaped first so the escapes introduced below are not re-escaped. + Only build statements need this; variable values (cflags, ldflags) are read + to end of line and must not be escaped, or the flags reach the compiler + mangled. + """ + text = str(path) + return text.replace("$", "$$").replace(":", "$:").replace(" ", "$ ") + + class NinjaBackend: """Generate build.ninja from a ProjectConfig and resolved toolchain. @@ -167,7 +186,7 @@ def _write_ninja(self) -> None: obj = str(self._object_path(target, src)) obj_files.append(obj) lines.append( - f"build {obj}: cc {src}" + f"build {_ninja_path(obj)}: cc {_ninja_path(src)}" ) if cflags: lines.append(f" cflags = {' '.join(cflags)}") @@ -194,7 +213,7 @@ def _write_ninja(self) -> None: link_inputs = obj_files + dep_archives out = str(self.build_dir / target.name) lines.append( - f"build {out}: link {' '.join(link_inputs)}" + f"build {_ninja_path(out)}: link " f"{' '.join(_ninja_path(x) for x in link_inputs)}" ) if ldflags: lines.append(f" ldflags = {' '.join(ldflags)}") @@ -212,7 +231,7 @@ def _write_ninja(self) -> None: out = str(self.build_dir / f"lib{target.name}{ext}") if target.target_type == "static_library": - lines.append(f"build {out}: ar_rule {' '.join(obj_files)}") + lines.append(f"build {_ninja_path(out)}: ar_rule " f"{' '.join(_ninja_path(x) for x in obj_files)}") else: # Shared libraries need the same -L/-l wiring executables # get, which the rule preamble alone does not supply. The @@ -228,7 +247,7 @@ def _write_ninja(self) -> None: for lib in pkg.libraries: libs.append(f"-l{lib}") - lines.append(f"build {out}: link_shared {' '.join(obj_files)}") + lines.append(f"build {_ninja_path(out)}: link_shared " f"{' '.join(_ninja_path(x) for x in obj_files)}") if ldflags: lines.append(f" ldflags = {' '.join(ldflags)}") if libs: diff --git a/ebuild/packages/fetcher.py b/ebuild/packages/fetcher.py index 7c81b37..f04e8f9 100644 --- a/ebuild/packages/fetcher.py +++ b/ebuild/packages/fetcher.py @@ -47,6 +47,16 @@ def fetch(self, recipe: PackageRecipe, extract_to: str | Path) -> Path: Raises: FetchError: If download or verification fails. """ + # PackageRecipe.validate() rejects a recipe without a checksum, but + # fetch() is reachable with a hand-built recipe too, so refuse here as + # well rather than falling through to an unverified extract. + if not recipe.checksum: + raise FetchError( + f"Refusing to fetch {recipe.name} v{recipe.version}: the recipe " + f"carries no checksum, so there is nothing to verify the " + f"download against." + ) + archive_path = self._download(recipe) if recipe.checksum: try: @@ -66,10 +76,10 @@ def _download(self, recipe: PackageRecipe) -> Path: """Download the source archive if not already cached.""" if not recipe.url: raise FetchError(f"No URL specified for package {recipe.name}") - if not recipe.url.startswith(("http://", "https://")): + if not recipe.url.startswith("https://"): raise FetchError( f"Invalid URL scheme for {recipe.name}: {recipe.url} " - f"(only http:// and https:// are allowed)" + f"(only https:// is allowed)" ) archive_path = self._archive_path(recipe) diff --git a/ebuild/packages/recipe.py b/ebuild/packages/recipe.py index ab8e5cb..71bf2c2 100644 --- a/ebuild/packages/recipe.py +++ b/ebuild/packages/recipe.py @@ -13,6 +13,8 @@ from pathlib import Path from typing import Any, Dict, List, Optional +import re + import yaml @@ -39,6 +41,9 @@ class PackageRecipe: VALID_BUILD_SYSTEMS = ("cmake", "autoconf", "make", "meson", "custom") + #: A bare SHA-256 digest, with or without the "sha256:" prefix. + _SHA256_RE = re.compile(r"^(?:sha256:)?[0-9a-fA-F]{64}$") + @property def slug(self) -> str: """Unique identifier: name-version.""" @@ -52,6 +57,31 @@ def validate(self) -> None: raise RecipeError(f"Package '{self.name}' must have a 'version' field.") if not self.url: raise RecipeError(f"Package '{self.name}' must have a 'url' field.") + + # A recipe with a checksum is a pin: a URL plus the digest of exactly + # what should be at it. The digest is not required here -- a recipe is + # also used to model packages that are never downloaded -- but a + # checksum that is present has to be a real one. "sha256:placeholder" + # parsed fine and then failed every single fetch with a mismatch, which + # is how two shipped recipes stayed unfetchable. PackageFetcher.fetch() + # separately refuses to download anything with no checksum at all. + if self.checksum and not self._SHA256_RE.match(self.checksum): + raise RecipeError( + f"Package '{self.name}': checksum '{self.checksum}' is not a " + f"sha256 digest. Expected 64 hex characters, optionally " + f"prefixed with 'sha256:'. Placeholder values are rejected — " + f"they turn every fetch of this package into a checksum " + f"mismatch." + ) + + # Plaintext HTTP defeats the pin's purpose in the common case where a + # recipe is edited without recomputing the digest, and it leaks what is + # being built. Every shipped recipe already uses https. + if self.url.startswith("http://"): + raise RecipeError( + f"Package '{self.name}': plaintext http:// is not accepted for " + f"'{self.url}'. Use https://." + ) if self.build_system not in self.VALID_BUILD_SYSTEMS: raise RecipeError( f"Package '{self.name}': invalid build system '{self.build_system}'. " diff --git a/recipes/freertos.yaml b/recipes/freertos.yaml index b2c7179..2e9fa3a 100644 --- a/recipes/freertos.yaml +++ b/recipes/freertos.yaml @@ -3,7 +3,7 @@ version: "11.1.0" description: "Real-time operating system kernel for embedded devices" license: MIT url: https://github.com/FreeRTOS/FreeRTOS-Kernel/releases/download/V11.1.0/FreeRTOS-KernelV11.1.0.zip -checksum: sha256:e36e5a2fcef99b83e8adbb8f8d5e4181a42c9ff0b604dc6fa7aad2a2ef0e3140 +checksum: sha256:eebd58aa71a623c9381f25f77b708c0ed14ef995a8913e2460fe9f286bb271eb build: cmake configure_args: - -DFREERTOS_HEAP=4 diff --git a/recipes/littlefs.yaml b/recipes/littlefs.yaml index 1fb5fee..bbbd456 100644 --- a/recipes/littlefs.yaml +++ b/recipes/littlefs.yaml @@ -3,7 +3,7 @@ version: "2.9.3" description: "Little fail-safe filesystem designed for microcontrollers" license: BSD-3-Clause url: https://github.com/littlefs-project/littlefs/archive/refs/tags/v2.9.3.tar.gz -checksum: sha256:placeholder +checksum: sha256:9cf2e7db673ea27d967a54cdafe8f55a7ffe27c63a2070ff7424fadd559cad67 build: make build_args: - CC=$(CROSS_COMPILE)gcc diff --git a/recipes/lwip.yaml b/recipes/lwip.yaml index 0fd9020..106a6a4 100644 --- a/recipes/lwip.yaml +++ b/recipes/lwip.yaml @@ -3,7 +3,7 @@ version: "2.2.0" description: "Lightweight TCP/IP stack for embedded systems" license: BSD-3-Clause url: https://download.savannah.nongnu.org/releases/lwip/lwip-2.2.0.zip -checksum: sha256:placeholder +checksum: sha256:c79255f6cb550eaa07d6e90d859b8c1abe81658115ae8175e74b67ac22c7ed87 build: cmake configure_args: - -DLWIP_DIR=${SOURCE_DIR} diff --git a/recipes/mbedtls.yaml b/recipes/mbedtls.yaml index d1ac1a5..371d401 100644 --- a/recipes/mbedtls.yaml +++ b/recipes/mbedtls.yaml @@ -3,7 +3,7 @@ version: "3.6.0" description: "Lightweight TLS/SSL library for embedded systems" license: Apache-2.0 url: https://github.com/Mbed-TLS/mbedtls/releases/download/v3.6.0/mbedtls-3.6.0.tar.bz2 -checksum: sha256:3ecf94fcfdaacafb757786a01b7538a61750ebd85c4b024f56ff8ba1490fcd73 +checksum: sha256:3ecf94fcfdaacafb757786a01b7538a61750ebd85c4b024f56ff8ba1490fcd38 build: cmake configure_args: - -DENABLE_TESTING=OFF diff --git a/tests/ebuild/test_ninja_backend.py b/tests/ebuild/test_ninja_backend.py index cca5812..0aa5277 100644 --- a/tests/ebuild/test_ninja_backend.py +++ b/tests/ebuild/test_ninja_backend.py @@ -45,7 +45,11 @@ def test_shared_library_uses_shared_link_rule(tmp_path): NinjaBackend(config, tmp_path / "build", toolchain).generate() ninja_file = (tmp_path / "build" / "build.ninja").read_text(encoding="utf-8") - assert "rule link_shared\n command = $cc -shared" in ninja_file + # macOS links dynamic libraries with -dynamiclib, ELF platforms with + # -shared. _shared_flag() picks the right one; asserting the literal + # "-shared" failed this test on every macOS runner. + shared_flag = "-dynamiclib" if sys.platform == "darwin" else "-shared" + assert f"rule link_shared\n command = $cc {shared_flag}" in ninja_file assert "build " in ninja_file assert ": link_shared " in ninja_file diff --git a/tests/ebuild/test_package_fetcher.py b/tests/ebuild/test_package_fetcher.py index cd69f9a..ea7b82d 100644 --- a/tests/ebuild/test_package_fetcher.py +++ b/tests/ebuild/test_package_fetcher.py @@ -29,11 +29,29 @@ LWIP_230_URL = "https://example.org/lwip/releases/2.3.0/source.tar.gz" -def make_recipe(name, version="2.9.3", url=None, checksum=""): +#: Placeholder digest for tests that never reach checksum verification (bad +#: URL, unsupported format). Real-looking so it passes recipe validation. +DUMMY_SHA256 = "sha256:" + "a" * 64 + + +def make_recipe(name, version="2.9.3", url=None, checksum=None): + """Build a recipe. + + ``checksum`` defaults to the digest of the archive ``fake_download`` + serves for ``url``, so tests about caching and extraction get past + verification. Pass an explicit value to exercise the checksum paths, or + ``""`` to exercise a recipe with no pin at all. + """ + resolved_url = url if url is not None else LITTLEFS_URL + if checksum is None: + try: + checksum = "sha256:" + sha256_of(targz_bytes(resolved_url)) + except Exception: + checksum = DUMMY_SHA256 return PackageRecipe( name=name, version=version, - url=url if url is not None else LITTLEFS_URL, + url=resolved_url, checksum=checksum, ) @@ -181,12 +199,29 @@ def test_bare_checksum_without_sha256_prefix_is_accepted(tmp_path, fake_download assert marker_in(tmp_path / "src") == LITTLEFS_URL -def test_empty_checksum_skips_verification(tmp_path, fake_download): +def test_recipe_without_a_checksum_is_refused(tmp_path, fake_download): + """A recipe with no checksum used to be fetched and extracted unverified. + + The URL alone is "whatever that host serves today". Refusing is the only + honest outcome: there is nothing to check the download against. + """ fetcher = PackageFetcher(tmp_path / "dl") - fetcher.fetch(make_recipe("littlefs", checksum=""), tmp_path / "src") + with pytest.raises(FetchError, match="no checksum"): + fetcher.fetch(make_recipe("littlefs", checksum=""), tmp_path / "src") - assert marker_in(tmp_path / "src") == LITTLEFS_URL + # And nothing was downloaded or extracted on the way to that refusal. + assert not (tmp_path / "src").exists() + + +def test_plaintext_http_is_refused(tmp_path, fake_download): + """https only: a pin is worth much less over a transport anyone can rewrite.""" + fetcher = PackageFetcher(tmp_path / "dl") + recipe = make_recipe("littlefs", url="http://example.org/lib.tar.gz", + checksum=DUMMY_SHA256) + + with pytest.raises(FetchError, match="https"): + fetcher.fetch(recipe, tmp_path / "src") # ── Extraction ─────────────────────────────────────────────── @@ -207,7 +242,13 @@ def test_unsupported_archive_format_is_rejected(tmp_path, monkeypatch): lambda url, filename: open(filename, "wb").write(b"not an archive"), ) fetcher = PackageFetcher(tmp_path / "dl") - recipe = make_recipe("littlefs", url="https://example.org/littlefs/v2.9.3.rar") + # Checksum of the bytes the patched urlretrieve writes, so the fetch gets + # past verification and reaches the format check this test is about. + recipe = make_recipe( + "littlefs", + url="https://example.org/littlefs/v2.9.3.rar", + checksum="sha256:6bbf954ab0045bc546f16a6db16c95afef820dccd807348411ea924dabb972e9", + ) with pytest.raises(FetchError, match="Unsupported archive format"): fetcher.fetch(recipe, tmp_path / "src") diff --git a/tests/unit/test_golden_path_commands.py b/tests/unit/test_golden_path_commands.py index 635b7c0..7282d1c 100644 --- a/tests/unit/test_golden_path_commands.py +++ b/tests/unit/test_golden_path_commands.py @@ -9,6 +9,7 @@ regression guards for the steps that were missing or broken. """ +import re from pathlib import Path import pytest @@ -111,8 +112,16 @@ def test_test_target_links_like_an_executable(self, tmp_path): SimpleNamespace(cc="cc", cxx="c++", ar="ar")).generate() ninja = (tmp_path / "b" / "build.ninja").read_text(encoding="utf-8") + # Split on the first *unescaped* colon: that is the one separating + # outputs from the rule name. A plain l.split(":")[0] picks the drive + # letter apart from the rest on Windows, so the .o filter never matched + # and this selected the compile edge instead of the link edge. + def outputs(line): + return re.split(r"(?