diff --git a/dirac.cfg b/dirac.cfg index 417c740a4ab..1f0920227de 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -643,7 +643,17 @@ Resources Singularity { - # The root image location for the container to use + # OCI image reference for multiarch layout (e.g. registry.hub.docker.com/library/alma9:latest) + # The resolver builds: // + # No default -- if unset, multiarch resolution is skipped and ContainerRoot is used. + # ImageReference = registry.hub.docker.com/library/alma9:latest + + # Base path for the CVMFS multiarch image repository + # Default: /cvmfs/unpacked.cern.ch/.multiarch + # BasePath = /cvmfs/unpacked.cern.ch/.multiarch + + # (Deprecated) Legacy root image location for the container + # Used as fallback when the multiarch image is not found # Default: /cvmfs/cernvm-prod.cern.ch/cvm4 ContainerRoot = /cvmfs/cernvm-prod.cern.ch/cvm4 diff --git a/src/DIRAC/Core/Utilities/ContainerImageResolver.py b/src/DIRAC/Core/Utilities/ContainerImageResolver.py new file mode 100644 index 00000000000..25d4566cb29 --- /dev/null +++ b/src/DIRAC/Core/Utilities/ContainerImageResolver.py @@ -0,0 +1,127 @@ +"""Utilities to resolve container images for Apptainer/Singularity +based on machine architecture and CVMFS multiarch layout. +""" +from __future__ import annotations + +import platform +import warnings +from pathlib import Path + +from DIRAC import gConfig, gLogger + +BASE_PATH_DEFAULT = "/cvmfs/unpacked.cern.ch/.multiarch" +# Legacy default used before multiarch support +CONTAINER_DEFROOT = "/cvmfs/cernvm-prod.cern.ch/cvm4" + +# Normalisation map to OCI image spec architecture names (GOARCH values). +# Accepts both platform.machine() output (e.g. "x86_64", "aarch64") and +# OCI names themselves (e.g. "amd64", "arm64") as input. +# See https://github.com/opencontainers/image-spec/blob/main/image-index.md +_OCI_ARCH_MAP = { + "x86_64": "amd64", + "amd64": "amd64", + "aarch64": "arm64", + "arm64": "arm64", + "armv7l": "arm", + "armv6l": "arm", + "ppc64le": "ppc64le", + "s390x": "s390x", +} + + +def _normalise_arch(arch: str) -> str: + """Normalise an architecture string to its OCI image spec name (GOARCH). + + Accepts ``platform.machine()`` values (e.g. ``x86_64``, ``aarch64``) as + well as OCI names (e.g. ``amd64``, ``arm64``). Unknown values are + returned as-is, lowercased. + """ + return _OCI_ARCH_MAP.get(arch.lower(), arch.lower()) + + +def _resolve_multiarch_path(image_ref: str, base_path: str, arch: str | None = None) -> Path: + """Build the CVMFS multiarch path for the given OCI image reference. + + :param image_ref: Full OCI reference (e.g. ``registry.hub.docker.com/library/alpine:latest``) + :param base_path: Multiarch base directory (e.g. ``/cvmfs/unpacked.cern.ch/.multiarch``) + :param arch: Override architecture (default: autodetect via ``platform.machine()``) + :returns: constructed path ``//`` + + Example:: + + >>> _resolve_multiarch_path("registry.hub.docker.com/library/alpine:latest", + ... "/cvmfs/unpacked.cern.ch/.multiarch", arch="x86_64") + PosixPath('/cvmfs/unpacked.cern.ch/.multiarch/amd64/registry.hub.docker.com/library/alpine:latest') + """ + arch_dir = _normalise_arch(arch or platform.machine()) + return Path(base_path) / arch_dir / image_ref + + +def resolve_image_path( + image_ref: str | None = None, + base_path: str | None = None, + container_root: str | None = None, +) -> Path | None: + """Resolve the container image path to use for Apptainer/Singularity. + + Resolution order: + + 1. **Multiarch** ``//`` -- only attempted when + an ``image_ref`` is explicitly provided (via parameter or CS config). + 2. **Legacy** ``container_root`` parameter, ``ContainerRoot`` CS option, or + ``CONTAINER_DEFROOT`` -- emits a :class:`DeprecationWarning` when used + and an ``image_ref`` was configured (i.e. multiarch was attempted but + the path did not exist). + 3. ``None`` if nothing is found. + + :param image_ref: OCI image reference + (e.g. ``registry.hub.docker.com/library/alma9:latest``) + :param base_path: Base directory for the multiarch layout + (e.g. ``/cvmfs/unpacked.cern.ch/.multiarch``) + :param container_root: Legacy container root path for backward compatibility + :returns: resolved :class:`~pathlib.Path` or ``None`` + """ + image_ref = image_ref or gConfig.getValue("/Resources/Computing/Singularity/ImageReference") or None + base_path = base_path or gConfig.getValue("/Resources/Computing/Singularity/BasePath") or BASE_PATH_DEFAULT + + # 1) Try CVMFS multiarch path (only if an image reference is configured) + multiarch_candidate = None + if image_ref: + multiarch_candidate = _resolve_multiarch_path(image_ref, base_path=base_path) + if multiarch_candidate.exists(): + gLogger.debug("Resolved multiarch image path", str(multiarch_candidate)) + return multiarch_candidate + + # 2) Fall back to legacy ContainerRoot + legacy_root = ( + container_root or gConfig.getValue("/Resources/Computing/Singularity/ContainerRoot") or CONTAINER_DEFROOT + ) + legacy_path = Path(legacy_root) + if legacy_path.exists(): + if multiarch_candidate is not None: + # Multiarch was attempted but not found + warnings.warn( + f"Multiarch image not found at {multiarch_candidate}. " + f"Falling back to legacy ContainerRoot '{legacy_root}'. " + "Please verify your ImageReference and BasePath settings.", + DeprecationWarning, + stacklevel=2, + ) + gLogger.warn( + "Multiarch image not found, falling back to legacy ContainerRoot", + f"{multiarch_candidate} -> {legacy_root}", + ) + else: + # No ImageReference configured, pure legacy usage + warnings.warn( + f"Using legacy ContainerRoot '{legacy_root}'. " + "ContainerRoot is deprecated and will be removed in a future release. " + "Please configure ImageReference and BasePath for the multiarch layout.", + DeprecationWarning, + stacklevel=2, + ) + gLogger.warn("Using deprecated ContainerRoot", legacy_root) + return legacy_path + + gLogger.warn("No container image found", f"multiarch={multiarch_candidate}, legacy={legacy_root}") + return None diff --git a/src/DIRAC/Core/Utilities/test/Test_ContainerImageResolver.py b/src/DIRAC/Core/Utilities/test/Test_ContainerImageResolver.py new file mode 100644 index 00000000000..f8fb7f20417 --- /dev/null +++ b/src/DIRAC/Core/Utilities/test/Test_ContainerImageResolver.py @@ -0,0 +1,220 @@ +"""Tests for ContainerImageResolver.""" +from pathlib import Path + +import pytest + +import DIRAC.Core.Utilities.ContainerImageResolver as cir + + +@pytest.mark.parametrize( + "arch,expected", + [ + ("x86_64", "amd64"), + ("amd64", "amd64"), + ("aarch64", "arm64"), + ("arm64", "arm64"), + ("armv7l", "arm"), + ("armv6l", "arm"), + ("ppc64le", "ppc64le"), + ("s390x", "s390x"), + ("weirdarch", "weirdarch"), # passthrough for unknown (lowercased) + ("WeirdArch", "weirdarch"), # passthrough is lowercased + ("X86_64", "amd64"), # lowercase + ], +) +def test_normalise_arch(arch, expected): + assert cir._normalise_arch(arch) == expected + + +def test_explicit_arch(): + path = cir._resolve_multiarch_path( + "registry.hub.docker.com/library/alpine:latest", + base_path="/cvmfs/unpacked.cern.ch/.multiarch", + arch="x86_64", + ) + assert path == Path("/cvmfs/unpacked.cern.ch/.multiarch/amd64/registry.hub.docker.com/library/alpine:latest") + + +def test_inferred_arch(monkeypatch): + monkeypatch.setattr(cir.platform, "machine", lambda: "aarch64") + path = cir._resolve_multiarch_path( + "registry.hub.docker.com/library/alpine:latest", + base_path="/cvmfs/unpacked.cern.ch/.multiarch", + ) + assert path == Path("/cvmfs/unpacked.cern.ch/.multiarch/arm64/registry.hub.docker.com/library/alpine:latest") + + +def test_trailing_slash_stripped(): + path = cir._resolve_multiarch_path( + # Note the image reference ending with "/" + "registry.hub.docker.com/library/alpine:latest/", + base_path="/cvmfs/unpacked.cern.ch/.multiarch", + arch="x86_64", + ) + assert path == Path("/cvmfs/unpacked.cern.ch/.multiarch/amd64/registry.hub.docker.com/library/alpine:latest") + + +def test_base_with_trailing_slash(): + path = cir._resolve_multiarch_path( + "alpine:latest", + # Note the base path ending with "/" + base_path="/cvmfs/unpacked.cern.ch/.multiarch/", + arch="x86_64", + ) + assert path == Path("/cvmfs/unpacked.cern.ch/.multiarch/amd64/alpine:latest") + + +def _make_multiarch_image(tmp_path, arch, image_ref): + """Helper to create a fake multiarch image directory.""" + image_dir = tmp_path / ".multiarch" / arch / image_ref + image_dir.mkdir(parents=True) + return image_dir + + +def _make_legacy_root(tmp_path, name="cvm4"): + """Helper to create a fake legacy container root.""" + legacy = tmp_path / name + legacy.mkdir(parents=True) + return legacy + + +def test_multiarch_found_as_directory(tmp_path, monkeypatch): + monkeypatch.setattr(cir.platform, "machine", lambda: "x86_64") + expected = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/alma9:latest") + + result = cir.resolve_image_path( + image_ref="registry.hub.docker.com/library/alma9:latest", + base_path=str(tmp_path / ".multiarch"), + ) + assert result == expected + + +def test_multiarch_found_aarch64(tmp_path, monkeypatch): + monkeypatch.setattr(cir.platform, "machine", lambda: "aarch64") + expected = _make_multiarch_image(tmp_path, "arm64", "registry.hub.docker.com/library/alma9:latest") + + result = cir.resolve_image_path( + image_ref="registry.hub.docker.com/library/alma9:latest", + base_path=str(tmp_path / ".multiarch"), + ) + assert result == expected + + +def test_multiarch_preferred_over_legacy(tmp_path, monkeypatch): + """When both multiarch and legacy exist, multiarch wins.""" + monkeypatch.setattr(cir.platform, "machine", lambda: "x86_64") + multiarch = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/alma9:latest") + legacy = _make_legacy_root(tmp_path) + + result = cir.resolve_image_path( + image_ref="registry.hub.docker.com/library/alma9:latest", + base_path=str(tmp_path / ".multiarch"), + container_root=str(legacy), + ) + assert result == multiarch + + +def test_multiarch_not_found_falls_back_to_container_root(tmp_path, monkeypatch): + monkeypatch.setattr(cir.platform, "machine", lambda: "x86_64") + legacy = _make_legacy_root(tmp_path) + + with pytest.warns(DeprecationWarning, match="Falling back to legacy ContainerRoot"): + result = cir.resolve_image_path( + image_ref="nonexistent:latest", + base_path=str(tmp_path / ".multiarch"), + container_root=str(legacy), + ) + assert result == legacy + + +def test_multiarch_not_found_falls_back_to_config_container_root(tmp_path, monkeypatch): + monkeypatch.setattr(cir.platform, "machine", lambda: "x86_64") + legacy = _make_legacy_root(tmp_path) + + class FakeGConfig: + @staticmethod + def getValue(key, default=None): + if key == "/Resources/Computing/Singularity/ContainerRoot": + return str(legacy) + return "" + + monkeypatch.setattr(cir, "gConfig", FakeGConfig) + + with pytest.warns(DeprecationWarning): + result = cir.resolve_image_path( + image_ref="nonexistent:latest", + base_path=str(tmp_path / ".multiarch"), + ) + assert result == legacy + + +def test_no_image_ref_skips_multiarch_uses_legacy_with_deprecation(tmp_path, monkeypatch): + """When no ImageReference is configured, legacy works but emits a deprecation warning.""" + monkeypatch.setattr(cir.platform, "machine", lambda: "x86_64") + legacy = _make_legacy_root(tmp_path) + + with pytest.warns(DeprecationWarning, match="ContainerRoot is deprecated"): + result = cir.resolve_image_path( + container_root=str(legacy), + ) + assert result == legacy + + +def test_no_image_ref_no_legacy_returns_none(tmp_path, monkeypatch): + """When nothing is configured and defaults don't exist, return None.""" + monkeypatch.setattr(cir.platform, "machine", lambda: "x86_64") + result = cir.resolve_image_path( + container_root=str(tmp_path / "no_such_root"), + ) + assert result is None + + +def test_config_provides_image_ref_and_base(tmp_path, monkeypatch): + monkeypatch.setattr(cir.platform, "machine", lambda: "x86_64") + expected = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/centos:7") + + class FakeGConfig: + @staticmethod + def getValue(key, default=None): + if key == "/Resources/Computing/Singularity/BasePath": + return str(tmp_path / ".multiarch") + if key == "/Resources/Computing/Singularity/ImageReference": + return "registry.hub.docker.com/library/centos:7" + return "" + + monkeypatch.setattr(cir, "gConfig", FakeGConfig) + + result = cir.resolve_image_path() + assert result == expected + + +def test_explicit_args_override_config(tmp_path, monkeypatch): + monkeypatch.setattr(cir.platform, "machine", lambda: "x86_64") + expected = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/myimage:v1") + + class FakeGConfig: + @staticmethod + def getValue(key, default=None): + if key == "/Resources/Computing/Singularity/BasePath": + return "/should/not/be/used" + if key == "/Resources/Computing/Singularity/ImageReference": + return "should_not_be_used:latest" + return "" + + monkeypatch.setattr(cir, "gConfig", FakeGConfig) + + result = cir.resolve_image_path( + image_ref="registry.hub.docker.com/library/myimage:v1", + base_path=str(tmp_path / ".multiarch"), + ) + assert result == expected + + +def test_nothing_found_returns_none(tmp_path, monkeypatch): + monkeypatch.setattr(cir.platform, "machine", lambda: "x86_64") + result = cir.resolve_image_path( + image_ref="nonexistent:latest", + base_path=str(tmp_path / ".multiarch"), + container_root=str(tmp_path / "no_such_root"), + ) + assert result is None diff --git a/src/DIRAC/Core/scripts/dirac_apptainer_exec.py b/src/DIRAC/Core/scripts/dirac_apptainer_exec.py index a3f939e70a5..0b872ce1f72 100644 --- a/src/DIRAC/Core/scripts/dirac_apptainer_exec.py +++ b/src/DIRAC/Core/scripts/dirac_apptainer_exec.py @@ -9,6 +9,7 @@ from DIRAC import S_ERROR, gConfig, gLogger from DIRAC.Core.Base.Script import Script from DIRAC.Core.Security.Locations import getCAsLocation, getProxyLocation, getVOMSLocation +from DIRAC.Core.Utilities.ContainerImageResolver import resolve_image_path from DIRAC.Core.Utilities.Os import safe_listdir from DIRAC.Core.Utilities.Subprocess import systemCall @@ -37,15 +38,12 @@ def generate_container_wrapper(dirac_env_var, diracos_env_var, etc_dir, rc_scrip return "\n".join(lines) -CONTAINER_DEFROOT = "" # Should add something like "/cvmfs/dirac.egi.eu/container/apptainer/alma9/x86_64" - - @Script() def main(): command = sys.argv[1] user_image = None - Script.registerSwitch("i:", "image=", " apptainer image to use") + Script.registerSwitch("i:", "image=", " Container image: OCI ref or local path") Script.parseCommandLine(ignoreErrors=False) for switch in Script.getUnprocessedSwitches(): if switch[0].lower() == "i" or switch[0].lower() == "image": @@ -69,7 +67,18 @@ def main(): fd.write(script) os.chmod("dirac_container.sh", 0o755) - # Now let's construct the apptainer command + # Resolve the container image: the user value is passed as both image_ref + # (for multiarch lookup) and container_root (for legacy local path). + # The resolver tries multiarch first, then falls back to the local path. + image_path = resolve_image_path( + image_ref=user_image, + container_root=user_image, + ) + if not image_path: + gLogger.error("No container image could be resolved") + return S_ERROR("Failed to find Apptainer image to exec") + + # Build the apptainer command cmd = ["apptainer", "exec"] cmd.extend(["--contain"]) # use minimal /dev and empty other directories (e.g. /tmp and $HOME) cmd.extend(["--ipc"]) # run container in a new IPC namespace @@ -92,14 +101,7 @@ def main(): gLogger.warn(f"Bind path {bind_path} does not exist, skipping") cmd.extend(["--cwd", cwd]) # set working directory - rootImage = user_image or gConfig.getValue("/Resources/Computing/Singularity/ContainerRoot") or CONTAINER_DEFROOT - - if os.path.isdir(rootImage) or os.path.isfile(rootImage): - cmd.extend([rootImage, f"{cwd}/dirac_container.sh"]) - else: - # if we are here is because there's no image, or it is not accessible (e.g. not on CVMFS) - gLogger.error("Apptainer image to exec not found: ", rootImage) - return S_ERROR("Failed to find Apptainer image to exec") + cmd.extend([str(image_path), f"{cwd}/dirac_container.sh"]) gLogger.debug(f"Execute Apptainer command: {' '.join(cmd)}") result = systemCall(0, cmd) diff --git a/src/DIRAC/Resources/Computing/SingularityComputingElement.py b/src/DIRAC/Resources/Computing/SingularityComputingElement.py index f3cfed99b90..8b972d24883 100644 --- a/src/DIRAC/Resources/Computing/SingularityComputingElement.py +++ b/src/DIRAC/Resources/Computing/SingularityComputingElement.py @@ -3,8 +3,8 @@ A computing element class using singularity containers, where Singularity is supposed to be found on the WN. -The goal of this CE is to start the job in the container set by -the "ContainerRoot" config option. +The goal of this CE is to start the job in a container image resolved by +:py:func:`~DIRAC.Core.Utilities.ContainerImageResolver.resolve_image_path`. DIRAC can be re-installed within the container. @@ -24,13 +24,12 @@ from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers import Operations from DIRAC.Core.Utilities.CGroups2 import CG2Manager +from DIRAC.Core.Utilities.ContainerImageResolver import resolve_image_path from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler from DIRAC.Resources.Computing.ComputingElement import ComputingElement from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.WorkloadManagementSystem.Utilities.Utils import createJobWrapper -# Default container to use if it isn't specified in the CE options -CONTAINER_DEFROOT = "/cvmfs/cernvm-prod.cern.ch/cvm4" CONTAINER_WORKDIR = "DIRAC_containers" CONTAINER_INNERDIR = "/tmp" @@ -103,9 +102,10 @@ def __init__(self, ceUniqueID): super().__init__(ceUniqueID) self.__submittedJobs = 0 self.__runningJobs = 0 - self.__root = CONTAINER_DEFROOT - if "ContainerRoot" in self.ceParameters: - self.__root = self.ceParameters["ContainerRoot"] + # Container image resolution parameters (see ContainerImageResolver) + self.__imageReference = self.ceParameters.get("ImageReference") + self.__basePath = self.ceParameters.get("BasePath") + self.__containerRoot = self.ceParameters.get("ContainerRoot") self.__workdir = CONTAINER_WORKDIR self.__innerdir = CONTAINER_INNERDIR self.__installDIRACInContainer = self.ceParameters.get("InstallDIRACInContainer", False) @@ -312,7 +312,15 @@ def submitJob(self, executableFile, proxy=None, **kwargs): :return: S_OK(payload exit code) / S_ERROR() if submission issue """ - rootImage = self.__root + imagePath = resolve_image_path( + image_ref=self.__imageReference, + base_path=self.__basePath, + container_root=self.__containerRoot, + ) + if not imagePath: + self.log.error("No container image could be resolved") + return S_ERROR("Failed to find singularity image to exec") + renewTask = None self.log.info("Creating singularity container") @@ -408,11 +416,8 @@ def submitJob(self, executableFile, proxy=None, **kwargs): containerOpts = self.ceParameters["ContainerOptions"].split(",") for opt in containerOpts: outerCmd.extend([opt.strip()]) - if not (os.path.isdir(rootImage) or os.path.isfile(rootImage)): - # if we are here is because there's no image, or it is not accessible (e.g. not on CVMFS) - self.log.error("Singularity image to exec not found: ", rootImage) - return S_ERROR("Failed to find singularity image to exec") - outerCmd.append(rootImage) + + outerCmd.append(str(imagePath)) cmd = outerCmd + [innerCmd] self.log.debug(f"Execute singularity command: {cmd}")