feat: add @latest source form resolving the newest release tag

.github/workflows/lxc-test.yml

@@ -54,7 +54,7 @@ jobs:
   lxc-test:
     needs: plan
     runs-on: ubuntu-24.04
-    timeout-minutes: 30
+    timeout-minutes: 60
     steps:
       - name: Checkout caller repository
         uses: actions/checkout@v7

src/cmlxc/driver_base.py

@@ -20,6 +20,7 @@
     DNS_CONTAINER_NAME,
     BuilderContainer,
     DNSContainer,
+    SetupError,
 )
 from cmlxc.incus import Incus
 
@@ -46,6 +47,7 @@ def parse_source(value: str, default_url: str) -> SourceSpec:
 
     Accepted forms:
       @ref           -- branch/tag on the default remote
+      @latest        -- newest release tag
       /path or ./path -- local directory
       URL@ref        -- custom remote at a given ref
     """
@@ -64,6 +66,19 @@ def parse_source(value: str, default_url: str) -> SourceSpec:
     raise ValueError(f"Invalid SOURCE: {value!r}. Use @ref, /path, ./path, or URL@ref.")
 
 
+# Don't match pre-release and non-semver tags
+_RELEASE_TAG_RE = re.compile(r"^v?\d+\.\d+\.\d+$")
+
+
+def latest_release_tag(tag_output):
+    """Pick the first matching and thus newest release tag from
+    ``git tag -l --sort=-v:refname`` output."""
+    for tag in (tag_output or "").splitlines():
+        if _RELEASE_TAG_RE.match(tag):
+            return tag
+    return None
+
+
 _RELAY_NAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9-]*$")
 
 
@@ -146,7 +161,10 @@ def add_cli_options(cls, parser, completer=None):
             "--source",
             default=f"@{cls.DEFAULT_REF}",
             metavar="SOURCE",
-            help=f"Driver source: @ref, /path, ./path, or URL@ref (default: @{cls.DEFAULT_REF}).",
+            help=(
+                "Driver source: @ref (branch, tag), @latest (newest release tag),"
+                f" /path, ./path, or URL@ref (default: @{cls.DEFAULT_REF})."
+            ),
         )
         action = parser.add_argument(
             "name",
@@ -192,7 +210,7 @@ def on_prep_builder(cls, out, bld_ct, tmp_dest):
         """Hook called by ``prep_builder`` after the git-main checkout is ready."""
         pass
 
-    def on_init_relay(self, repo_path):
+    def on_init_relay(self, repo_path, tag):
         """Hook called by ``init_builder`` after a relay checkout is ready."""
         pass
 
@@ -221,7 +239,9 @@ def prep_builder(cls, ix, out, bld_ct):
 
         tmp_dest = f"/root/{cls.REPO_NAME}-git-main"
         if bld_ct.bash(f"test -d {tmp_dest}", check=False) is None:
-            source = parse_source(f"@{cls.DEFAULT_REF}", cls.DEFAULT_SOURCE_URL)
+            # Always main: this is the shared cache clone, and init_builder
+            # checks the requested ref out of its copy
+            source = parse_source("@main", cls.DEFAULT_SOURCE_URL)
             bld_ct.setup_repo(tmp_dest, out, source)
         else:
             out.print(f"  Fetching {cls.REPO_NAME}-git-main from upstream ...")
@@ -241,6 +261,15 @@ def init_builder(self, source):
                 f"  Copying {self.REPO_NAME}-git-main to {repo_path} on builder"
             )
             self.bld_ct.bash(f"rm -rf {repo_path} && cp -a {tmp_dest} {repo_path}")
+            if source.ref == "latest":
+                # prep_builder already fetched --tags, update source to replace
+                # "latest" with the tag
+                source.ref = latest_release_tag(
+                    self.bld_ct.bash(f"cd {repo_path} && git tag -l --sort=-v:refname")
+                )
+                if not source.ref:
+                    raise SetupError(f"No release tag found for {self.REPO_NAME}")
+                self.out.print(f"  Resolved @latest to {source.ref}")
             if source.ref != "main":
                 self.out.print(f"  Checking out {source.ref!r} ...")
             self.bld_ct.bash(f"""
@@ -260,7 +289,8 @@ def init_builder(self, source):
             self.bld_ct.sync_to(source.path, repo_path)
 
         # Relay-specific preparation (e.g. build binary, init venv)
-        self.on_init_relay(repo_path)
+        tag = source.ref if source.kind == "remote" else None
+        self.on_init_relay(repo_path, tag)
 
     def run_deploy(self, *, source, ipv4_only):
         """Perform the driver-specific deployment.

src/cmlxc/driver_cmdeploy.py

@@ -60,7 +60,7 @@ def get_test_domain_or_ip(self):
             case _:
                 return self.ct.domain
 
-    def on_init_relay(self, repo_path):
+    def on_init_relay(self, repo_path, tag):
         """Hook called by ``init_builder`` to run initenv.sh for the relay."""
         self.out.print(f"  Running scripts/initenv.sh for {self.ct.shortname} ...")
         self.bld_ct.bash(f"cd {repo_path} && bash scripts/initenv.sh")

src/cmlxc/driver_madmail.py

@@ -1,17 +1,28 @@
 """madmail-based deployment driver for cmlxc.
 
-The maddy binary is built inside the builder container
+The madmail binary is built inside the builder container
 and transferred to relay containers via SCP.
 Madmail relays run on IP addresses
 and do not require DNS entries.
 """
 
+import re
 import time
 
 from cmlxc.container import SetupError
 from cmlxc.driver_base import Driver
 
 MADMAIL = "madmail"
+RELEASED_TAG_RE = re.compile(r"^v\d+\.\d+\.\d+$")
+RELEASE_ARCHES = ("amd64", "arm64")
+
+
+def release_asset_url(tag, arch):
+    """Return the musl release asset URL, or None if tag/arch has no asset."""
+    if not RELEASED_TAG_RE.match(tag or "") or arch not in RELEASE_ARCHES:
+        return None
+    asset = f"madmail-linux-{arch}-musl"
+    return f"https://github.com/themadorg/madmail/releases/download/{tag}/{asset}"
 
 
 class MadmailDriver(Driver):
@@ -21,8 +32,8 @@ class MadmailDriver(Driver):
     CLI_DOC = "Deploy a madmail relay service into a container."
     DEFAULT_SOURCE_URL = "https://github.com/themadorg/madmail.git"
     REPO_NAME = MADMAIL
-    REQUIRED_SOURCE_PATHS = ["go.mod", "Makefile"]
-    DEFAULT_REF = "v0.47.1"
+    REQUIRED_SOURCE_PATHS = ["Cargo.toml", "Makefile"]
+    DEFAULT_REF = "v2.20.0"
     type = "ipv4"
 
     @classmethod
@@ -43,72 +54,80 @@ def configure_from_args(self, args):
 
     @classmethod
     def on_prep_builder(cls, out, bld_ct, tmp_dest):
-        """Hook called by ``prep_builder`` to ensure the Go toolchain is ready."""
-        out.print("  Ensuring build environment (Go) ...")
+        """Hook called by ``prep_builder`` to ensure the Rust toolchain is ready."""
+        out.print("  Ensuring build environment (Rust) ...")
         prepare_build_container(bld_ct, tmp_dest)
 
-    def on_init_relay(self, repo_path):
-        """Hook called by ``init_builder`` to build the maddy binary."""
+    def on_init_relay(self, repo_path, tag):
+        arch = self.bld_ct.bash("dpkg --print-architecture")
+        url = release_asset_url(tag, arch)
+        if url:
+            with self.out.section(
+                f"Fetching madmail {tag} release binary for {self.ct.shortname}"
+            ):
+                if self._download_release_binary(repo_path, url):
+                    return
+                self.out.print("  Download failed, falling back to source build ...")
+
+        self._build_from_source(repo_path)
+
+    def _download_release_binary(self, repo_path, url):
+        self.bld_ct.bash(f"mkdir -p {repo_path}/target/release")
+        ret = self.bld_ct.bash(
+            f"curl -fsSL -o {repo_path}/target/release/madmail '{url}'",
+            check=False,
+        )
+        if ret is None:
+            return False
+        self.bld_ct.bash(f"chmod +x {repo_path}/target/release/madmail")
+        self.out.print(f"  Downloaded {url.rsplit('/', 1)[-1]}")
+        return True
+
+    def _build_from_source(self, repo_path):
         mode = "with admin web UI" if self.with_admin else "without admin web UI"
         with self.out.section(
-            f"Building maddy binary for {self.ct.shortname} ({mode})"
+            f"Building madmail binary for {self.ct.shortname} ({mode})"
         ):
+            self.bld_ct.bash(f"cd '{repo_path}' && make init")
+
             if self.with_admin:
-                # Ensure admin-web submodule is populated and dependencies installed;
-                # build.sh copy_admin_web() handles the actual SPA build.
-                self.bld_ct.bash(f"""
-                    if [ ! -f '{repo_path}/admin-web/package.json' ]; then
-                        cd '{repo_path}' && git submodule update --init admin-web
-                    fi
-                    cd '{repo_path}/admin-web'
-                    if command -v bun >/dev/null 2>&1; then
-                        bun install
-                    elif command -v npm >/dev/null 2>&1; then
-                        npm install
-                    fi
-                """)
+                # v2's `build-admin-web` Makefile target runs
+                # `git submodule update --init` itself and no separate
+                # submodule/bun-install step is needed
+                ret = self.out.shell(
+                    f"incus exec {self.bld_ct.name} -- bash -c "
+                    f"'cd {repo_path} && make build-release'"
+                )
             else:
-                # Hide package.json so build.sh creates a placeholder instead.
-                self.bld_ct.bash(f"""
-                    PKG='{repo_path}/admin-web/package.json'
-                    BAK='{repo_path}/admin-web/package.json.cmlxc-disabled'
-                    if [ -f "$PKG" ]; then mv "$PKG" "$BAK"; fi
-                """)
-
-            try:
+                # Clear any stale embed
+                self.bld_ct.bash(f"rm -rf {repo_path}/crates/chatmail-admin-web/embed")
                 ret = self.out.shell(
                     f"incus exec {self.bld_ct.name} -- bash -c "
-                    f"'cd {repo_path} && make clean build'"
+                    f"'cd {repo_path} && cargo build -p chatmail --release'"
                 )
-            finally:
-                # Restore package.json if we hid it.
-                self.bld_ct.bash(f"""
-                    BAK='{repo_path}/admin-web/package.json.cmlxc-disabled'
-                    PKG='{repo_path}/admin-web/package.json'
-                    if [ -f "$BAK" ] && [ ! -f "$PKG" ]; then mv "$BAK" "$PKG"; fi
-                """)
 
             if ret:
-                raise SetupError(f"maddy build failed in {repo_path} (exit {ret})")
+                raise SetupError(f"madmail build failed in {repo_path} (exit {ret})")
 
             if self.with_admin:
                 check = self.bld_ct.bash(
-                    f"test -f {repo_path}/internal/adminweb/build/index.html",
+                    f"test -f {repo_path}/crates/chatmail-admin-web/embed/index.html",
                     check=False,
                 )
                 if check is None:
                     raise SetupError("admin-web build produced no index.html")
 
     def run_tests(self, second_domain=None, cool=False, simple=False):
         """Execute the madmail E2E test suite against relays."""
-        test_src = f"{self.get_git_main_path()}/tests/deltachat-test"
+        test_src = f"{self.repo_path}/tests/deltachat-test"
 
         with self.out.section("test-madmail"):
-            # Symlink the built maddy binary into the test directory so
+            # Symlink the built madmail binary into the test directory so
             # tests that spawn a local server can find it at build/maddy.
             self.bld_ct.bash(
                 f"mkdir -p {test_src}/build"
-                f" && ln -sf {self.repo_path}/build/maddy {test_src}/build/maddy"
+                f" && ln -sf {self.repo_path}/target/release/madmail"
+                f" {test_src}/build/maddy"
             )
 
             relay1 = self.get_test_domain_or_ip()
@@ -172,17 +191,15 @@ def deploy(self, source=None):
             self.out.print("Pushing madmail binary via SCP ...")
             self.ct.bash("rm -f /tmp/madmail")
             self.bld_ct.scp_to_relay(
-                f"{self.repo_path}/build/maddy",
+                f"{self.repo_path}/target/release/madmail",
                 ip,
                 "/tmp/madmail",
             )
             self.ct.bash("chmod +x /tmp/madmail")
 
             install_flags = (
-                f"--simple --ip {ip}"
-                " --tls-mode self_signed"
-                " --enable-chatmail"
-                " --non-interactive"
+                f"--simple --ip {ip} --tls-mode self_signed"
+                " --enable-chatmail --enable-iroh --non-interactive"
             )
             self.out.print(f"Running madmail install {install_flags} ...")
             self.ct.bash("systemctl stop madmail || true")
@@ -204,6 +221,7 @@ def deploy(self, source=None):
                 # Path changes are applied at startup.
                 self.ct.bash("systemctl restart madmail")
             else:
+                # Release binaries always embed admin-web; only disable it here.
                 self.out.print("Disabling admin web interface ...")
                 self.ct.bash("madmail admin-web disable")
 
@@ -247,8 +265,8 @@ def _parse_admin_web_status(status):
     return enabled, path
 
 
-def prepare_build_container(bld_ct, go_mod_path):
-    """Install or update Go inside the builder according to go.mod."""
+def prepare_build_container(bld_ct, cargo_toml_path):
+    """Install or update Rust inside the builder according to Cargo.toml."""
     bld_ct.bash("""
         if ! command -v node >/dev/null 2>&1 || [ "$(node -v | cut -d. -f1 | tr -d v)" -lt 22 ]; then
             apt-get -o DPkg::Lock::Timeout=60 update
@@ -262,21 +280,24 @@ def prepare_build_container(bld_ct, go_mod_path):
             ln -sf /root/.bun/bin/bun /usr/local/bin/bun
         fi
     """)
+    bld_ct.bash("""
+        if ! command -v rustc >/dev/null 2>&1; then
+            apt-get -o DPkg::Lock::Timeout=60 update
+            DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl build-essential
+            apt-get clean && rm -rf /var/lib/apt/lists/*
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \\
+                | bash -s -- -y --default-toolchain stable --profile minimal
+        fi
+        ln -sf /root/.cargo/bin/cargo /usr/local/bin/cargo
+        ln -sf /root/.cargo/bin/rustc /usr/local/bin/rustc
+        ln -sf /root/.cargo/bin/rustup /usr/local/bin/rustup
+    """)
     bld_ct.bash(f"""
-        NEED=$(awk '/^go / {{print $2}}' {go_mod_path}/go.mod)
-        ARCH=$(dpkg --print-architecture)
-        case "$ARCH" in
-            amd64) GOARCH=amd64 ;;
-            arm64) GOARCH=arm64 ;;
-            *) echo "unsupported arch: $ARCH" >&2; exit 1 ;;
-        esac
-        if [ -x /usr/local/go/bin/go ]; then
-            HAVE=$(/usr/local/go/bin/go version | awk '{{print $3}}' | sed 's/^go//')
-            [ "$HAVE" = "$NEED" ] && exit 0
+        NEED=$(awk -F'"' '/^rust-version/ {{print $2}}' {cargo_toml_path}/Cargo.toml)
+        HAVE=$(rustc --version | awk '{{print $2}}')
+        OLDEST=$(printf '%s\\n%s\\n' "$HAVE" "$NEED" | sort -V | head -n1)
+        if [ "$OLDEST" = "$HAVE" ] && [ "$HAVE" != "$NEED" ]; then
+            echo "Updating Rust: $HAVE < required $NEED ..."
+            rustup update stable
         fi
-        URL="https://go.dev/dl/go${{NEED}}.linux-${{GOARCH}}.tar.gz"
-        echo "Installing Go ${{NEED}} from ${{URL}} ..."
-        curl -fsSL "$URL" | tar -C /usr/local -xzf -
-        ln -sf /usr/local/go/bin/go /usr/local/bin/go
-        ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt
     """)