From 2126604108e977a7dd424d6958cd2a4da0f6ff40 Mon Sep 17 00:00:00 2001 From: j4n Date: Wed, 29 Jul 2026 16:57:51 +0200 Subject: [PATCH 1/5] feat(madmail): port deploy-madmail driver to v2 Rust build - Drop v1 support entirely rather, set DEFAULT_REF to v2.20.0, latest release. - Replace Go toolchain block with rustup, installed if missing or below Cargo.toml's requirement. - Web admin UI build is handled by the main build process now - driver_base.py: pass tag through on_init_relay() for driver consumption - madmail's on_init_relay now tries a download first for a semver release tags, falls back to the source build. --- src/cmlxc/driver_base.py | 5 +- src/cmlxc/driver_cmdeploy.py | 2 +- src/cmlxc/driver_madmail.py | 147 ++++++++++++++++++++--------------- tests/test_cli.py | 22 ++++++ 4 files changed, 110 insertions(+), 66 deletions(-) diff --git a/src/cmlxc/driver_base.py b/src/cmlxc/driver_base.py index c6f7ea2..101e786 100644 --- a/src/cmlxc/driver_base.py +++ b/src/cmlxc/driver_base.py @@ -192,7 +192,7 @@ def on_prep_builder(cls, out, bld_ct, tmp_dest): """Hook called by ``prep_builder`` after the git-main checkout is ready.""" pass - def on_init_relay(self, repo_path): + def on_init_relay(self, repo_path, tag): """Hook called by ``init_builder`` after a relay checkout is ready.""" pass @@ -260,7 +260,8 @@ def init_builder(self, source): self.bld_ct.sync_to(source.path, repo_path) # Relay-specific preparation (e.g. build binary, init venv) - self.on_init_relay(repo_path) + tag = source.ref if source.kind == "remote" else None + self.on_init_relay(repo_path, tag) def run_deploy(self, *, source, ipv4_only): """Perform the driver-specific deployment. diff --git a/src/cmlxc/driver_cmdeploy.py b/src/cmlxc/driver_cmdeploy.py index 696a676..9b87779 100644 --- a/src/cmlxc/driver_cmdeploy.py +++ b/src/cmlxc/driver_cmdeploy.py @@ -60,7 +60,7 @@ def get_test_domain_or_ip(self): case _: return self.ct.domain - def on_init_relay(self, repo_path): + def on_init_relay(self, repo_path, tag): """Hook called by ``init_builder`` to run initenv.sh for the relay.""" self.out.print(f" Running scripts/initenv.sh for {self.ct.shortname} ...") self.bld_ct.bash(f"cd {repo_path} && bash scripts/initenv.sh") diff --git a/src/cmlxc/driver_madmail.py b/src/cmlxc/driver_madmail.py index 6327f5b..3a5c2db 100644 --- a/src/cmlxc/driver_madmail.py +++ b/src/cmlxc/driver_madmail.py @@ -1,17 +1,28 @@ """madmail-based deployment driver for cmlxc. -The maddy binary is built inside the builder container +The madmail binary is built inside the builder container and transferred to relay containers via SCP. Madmail relays run on IP addresses and do not require DNS entries. """ +import re import time from cmlxc.container import SetupError from cmlxc.driver_base import Driver MADMAIL = "madmail" +RELEASED_TAG_RE = re.compile(r"^v\d+\.\d+\.\d+$") +RELEASE_ARCHES = ("amd64", "arm64") + + +def release_asset_url(tag, arch): + """Return the musl release asset URL, or None if tag/arch has no asset.""" + if not RELEASED_TAG_RE.match(tag or "") or arch not in RELEASE_ARCHES: + return None + asset = f"madmail-linux-{arch}-musl" + return f"https://github.com/themadorg/madmail/releases/download/{tag}/{asset}" class MadmailDriver(Driver): @@ -21,8 +32,8 @@ class MadmailDriver(Driver): CLI_DOC = "Deploy a madmail relay service into a container." DEFAULT_SOURCE_URL = "https://github.com/themadorg/madmail.git" REPO_NAME = MADMAIL - REQUIRED_SOURCE_PATHS = ["go.mod", "Makefile"] - DEFAULT_REF = "v0.47.1" + REQUIRED_SOURCE_PATHS = ["Cargo.toml", "Makefile"] + DEFAULT_REF = "v2.20.0" type = "ipv4" @classmethod @@ -43,57 +54,64 @@ def configure_from_args(self, args): @classmethod def on_prep_builder(cls, out, bld_ct, tmp_dest): - """Hook called by ``prep_builder`` to ensure the Go toolchain is ready.""" - out.print(" Ensuring build environment (Go) ...") + """Hook called by ``prep_builder`` to ensure the Rust toolchain is ready.""" + out.print(" Ensuring build environment (Rust) ...") prepare_build_container(bld_ct, tmp_dest) - def on_init_relay(self, repo_path): - """Hook called by ``init_builder`` to build the maddy binary.""" + def on_init_relay(self, repo_path, tag): + arch = self.bld_ct.bash("dpkg --print-architecture") + url = release_asset_url(tag, arch) + if url: + with self.out.section( + f"Fetching madmail {tag} release binary for {self.ct.shortname}" + ): + if self._download_release_binary(repo_path, url): + return + self.out.print(" Download failed, falling back to source build ...") + + self._build_from_source(repo_path) + + def _download_release_binary(self, repo_path, url): + self.bld_ct.bash(f"mkdir -p {repo_path}/target/release") + ret = self.bld_ct.bash( + f"curl -fsSL -o {repo_path}/target/release/madmail '{url}'", + check=False, + ) + if ret is None: + return False + self.bld_ct.bash(f"chmod +x {repo_path}/target/release/madmail") + self.out.print(f" Downloaded {url.rsplit('/', 1)[-1]}") + return True + + def _build_from_source(self, repo_path): mode = "with admin web UI" if self.with_admin else "without admin web UI" with self.out.section( - f"Building maddy binary for {self.ct.shortname} ({mode})" + f"Building madmail binary for {self.ct.shortname} ({mode})" ): + self.bld_ct.bash(f"cd '{repo_path}' && make init") + if self.with_admin: - # Ensure admin-web submodule is populated and dependencies installed; - # build.sh copy_admin_web() handles the actual SPA build. - self.bld_ct.bash(f""" - if [ ! -f '{repo_path}/admin-web/package.json' ]; then - cd '{repo_path}' && git submodule update --init admin-web - fi - cd '{repo_path}/admin-web' - if command -v bun >/dev/null 2>&1; then - bun install - elif command -v npm >/dev/null 2>&1; then - npm install - fi - """) + # v2's `build-admin-web` Makefile target runs + # `git submodule update --init` itself and no separate + # submodule/bun-install step is needed + ret = self.out.shell( + f"incus exec {self.bld_ct.name} -- bash -c " + f"'cd {repo_path} && make build-release'" + ) else: - # Hide package.json so build.sh creates a placeholder instead. - self.bld_ct.bash(f""" - PKG='{repo_path}/admin-web/package.json' - BAK='{repo_path}/admin-web/package.json.cmlxc-disabled' - if [ -f "$PKG" ]; then mv "$PKG" "$BAK"; fi - """) - - try: + # Clear any stale embed + self.bld_ct.bash(f"rm -rf {repo_path}/crates/chatmail-admin-web/embed") ret = self.out.shell( f"incus exec {self.bld_ct.name} -- bash -c " - f"'cd {repo_path} && make clean build'" + f"'cd {repo_path} && cargo build -p chatmail --release'" ) - finally: - # Restore package.json if we hid it. - self.bld_ct.bash(f""" - BAK='{repo_path}/admin-web/package.json.cmlxc-disabled' - PKG='{repo_path}/admin-web/package.json' - if [ -f "$BAK" ] && [ ! -f "$PKG" ]; then mv "$BAK" "$PKG"; fi - """) if ret: - raise SetupError(f"maddy build failed in {repo_path} (exit {ret})") + raise SetupError(f"madmail build failed in {repo_path} (exit {ret})") if self.with_admin: check = self.bld_ct.bash( - f"test -f {repo_path}/internal/adminweb/build/index.html", + f"test -f {repo_path}/crates/chatmail-admin-web/embed/index.html", check=False, ) if check is None: @@ -104,11 +122,12 @@ def run_tests(self, second_domain=None, cool=False, simple=False): test_src = f"{self.get_git_main_path()}/tests/deltachat-test" with self.out.section("test-madmail"): - # Symlink the built maddy binary into the test directory so + # Symlink the built madmail binary into the test directory so # tests that spawn a local server can find it at build/maddy. self.bld_ct.bash( f"mkdir -p {test_src}/build" - f" && ln -sf {self.repo_path}/build/maddy {test_src}/build/maddy" + f" && ln -sf {self.repo_path}/target/release/madmail" + f" {test_src}/build/maddy" ) relay1 = self.get_test_domain_or_ip() @@ -172,17 +191,15 @@ def deploy(self, source=None): self.out.print("Pushing madmail binary via SCP ...") self.ct.bash("rm -f /tmp/madmail") self.bld_ct.scp_to_relay( - f"{self.repo_path}/build/maddy", + f"{self.repo_path}/target/release/madmail", ip, "/tmp/madmail", ) self.ct.bash("chmod +x /tmp/madmail") install_flags = ( - f"--simple --ip {ip}" - " --tls-mode self_signed" - " --enable-chatmail" - " --non-interactive" + f"--simple --ip {ip} --tls-mode self_signed" + " --enable-chatmail --enable-iroh --non-interactive" ) self.out.print(f"Running madmail install {install_flags} ...") self.ct.bash("systemctl stop madmail || true") @@ -204,6 +221,7 @@ def deploy(self, source=None): # Path changes are applied at startup. self.ct.bash("systemctl restart madmail") else: + # Release binaries always embed admin-web; only disable it here. self.out.print("Disabling admin web interface ...") self.ct.bash("madmail admin-web disable") @@ -247,8 +265,8 @@ def _parse_admin_web_status(status): return enabled, path -def prepare_build_container(bld_ct, go_mod_path): - """Install or update Go inside the builder according to go.mod.""" +def prepare_build_container(bld_ct, cargo_toml_path): + """Install or update Rust inside the builder according to Cargo.toml.""" bld_ct.bash(""" if ! command -v node >/dev/null 2>&1 || [ "$(node -v | cut -d. -f1 | tr -d v)" -lt 22 ]; then apt-get -o DPkg::Lock::Timeout=60 update @@ -262,21 +280,24 @@ def prepare_build_container(bld_ct, go_mod_path): ln -sf /root/.bun/bin/bun /usr/local/bin/bun fi """) + bld_ct.bash(""" + if ! command -v rustc >/dev/null 2>&1; then + apt-get -o DPkg::Lock::Timeout=60 update + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl build-essential + apt-get clean && rm -rf /var/lib/apt/lists/* + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \\ + | bash -s -- -y --default-toolchain stable --profile minimal + fi + ln -sf /root/.cargo/bin/cargo /usr/local/bin/cargo + ln -sf /root/.cargo/bin/rustc /usr/local/bin/rustc + ln -sf /root/.cargo/bin/rustup /usr/local/bin/rustup + """) bld_ct.bash(f""" - NEED=$(awk '/^go / {{print $2}}' {go_mod_path}/go.mod) - ARCH=$(dpkg --print-architecture) - case "$ARCH" in - amd64) GOARCH=amd64 ;; - arm64) GOARCH=arm64 ;; - *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; - esac - if [ -x /usr/local/go/bin/go ]; then - HAVE=$(/usr/local/go/bin/go version | awk '{{print $3}}' | sed 's/^go//') - [ "$HAVE" = "$NEED" ] && exit 0 + NEED=$(awk -F'"' '/^rust-version/ {{print $2}}' {cargo_toml_path}/Cargo.toml) + HAVE=$(rustc --version | awk '{{print $2}}') + OLDEST=$(printf '%s\\n%s\\n' "$HAVE" "$NEED" | sort -V | head -n1) + if [ "$OLDEST" = "$HAVE" ] && [ "$HAVE" != "$NEED" ]; then + echo "Updating Rust: $HAVE < required $NEED ..." + rustup update stable fi - URL="https://go.dev/dl/go${{NEED}}.linux-${{GOARCH}}.tar.gz" - echo "Installing Go ${{NEED}} from ${{URL}} ..." - curl -fsSL "$URL" | tar -C /usr/local -xzf - - ln -sf /usr/local/go/bin/go /usr/local/bin/go - ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt """) diff --git a/tests/test_cli.py b/tests/test_cli.py index d245c14..c954ee7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,6 +6,7 @@ from cmlxc.driver_base import SourceSpec, parse_source, validate_relay_name from cmlxc.driver_cmdeploy import get_ini_overrides +from cmlxc.driver_madmail import release_asset_url URL = "https://github.com/chatmail/relay.git" @@ -61,3 +62,24 @@ def test_ini_overrides_disable_ipv6(): assert "disable_ipv6" not in get_ini_overrides("cm0.localchat") overrides = get_ini_overrides("cm0.localchat", disable_ipv6=True) assert overrides["disable_ipv6"] == "True" + + +@pytest.mark.parametrize( + "tag, arch, expected_asset", + [ + ("v2.23.0", "amd64", "madmail-linux-amd64-musl"), + ("v2.23.0", "arm64", "madmail-linux-arm64-musl"), + # not a plain release tag -> build from source + ("v2.23.0-dirty", "amd64", None), + ("v2.23.0-4-gabc1234", "amd64", None), + (None, "amd64", None), + ("", "amd64", None), + ("v2.23.0", "riscv64", None), + ], +) +def test_release_asset_url(tag, arch, expected_asset): + url = release_asset_url(tag, arch) + if expected_asset is None: + assert url is None + else: + assert url.endswith(f"/v2.23.0/{expected_asset}") From 7cbd658c617dc38e7695b3e18dd992d34206d55b Mon Sep 17 00:00:00 2001 From: j4n Date: Thu, 13 Aug 2026 13:35:16 +0200 Subject: [PATCH 2/5] fix: align minitest suite's assertions with madmail v2's behaviour test_delivery_port_blocked: extend grep patterns to current madmail variants. test_hide_senders_ip_address: Use FETCH instead of UID SEARCH unsupported in madmail and scan through all fetched messages. --- src/relay_minitest/test_relay.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/relay_minitest/test_relay.py b/src/relay_minitest/test_relay.py index 8ac12e4..9587445 100644 --- a/src/relay_minitest/test_relay.py +++ b/src/relay_minitest/test_relay.py @@ -1,10 +1,10 @@ +import email import imaplib import ipaddress import smtplib import ssl from email.mime.text import MIMEText -import imap_tools import pytest import requests @@ -75,8 +75,10 @@ def test_delivery_port_blocked( relayadmin2.block_port(443) relayadmin2.block_port(80) chat.send_text("should not arrive") - # Postfix logs "status=deferred", madmail logs "delivery attempt failed". - lines = relayadmin.wait_for_journal_match("deferred|delivery attempt failed") + # Postfix logs "status=deferred"; madmail logs one of: + lines = relayadmin.wait_for_journal_match( + "deferred|outbound delivery (failed, requeued|permanent failure|exceeded max_tries)" + ) lp.indent(lines.splitlines()[-1]) def test_one_on_one_http_only(self, cmfactory, cmfactory2, relayadmin2, lp): @@ -115,11 +117,17 @@ def test_hide_senders_ip_address(cmfactory, ssl_context): addr = user2.get_config("addr") host = addr.split("@")[1].strip("[]") pw = user2.get_config("mail_pw") - mailbox = imap_tools.MailBox(host, ssl_context=ssl_context) - mailbox.login(addr, pw) - msgs = list(mailbox.fetch(mark_seen=False)) - assert msgs, "expected at least one message" - assert public_ip not in msgs[0].obj.as_string() + + # madmail's IMAP server doesn't implement SEARCH/UID SEARCH, fetch by range instead. + imap = imaplib.IMAP4_SSL(host, ssl_context=ssl_context) + imap.login(addr, pw) + imap.select("INBOX") + typ, data = imap.fetch("1:*", "(BODY.PEEK[])") + assert typ == "OK" + raw_messages = [item[1] for item in data if isinstance(item, tuple)] + assert raw_messages, "expected at least one message" + for raw in raw_messages: + assert public_ip not in email.message_from_bytes(raw).as_string() def test_unencrypted_rejection(cmsetup, lp): From 0da26125d2c567eadce0e90d4d21c041a80549d2 Mon Sep 17 00:00:00 2001 From: j4n Date: Tue, 11 Aug 2026 16:55:20 +0200 Subject: [PATCH 3/5] fix(madmail): run E2E tests from the deployed checkout, not the cached clone --- src/cmlxc/driver_madmail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmlxc/driver_madmail.py b/src/cmlxc/driver_madmail.py index 3a5c2db..16c4647 100644 --- a/src/cmlxc/driver_madmail.py +++ b/src/cmlxc/driver_madmail.py @@ -119,7 +119,7 @@ def _build_from_source(self, repo_path): def run_tests(self, second_domain=None, cool=False, simple=False): """Execute the madmail E2E test suite against relays.""" - test_src = f"{self.get_git_main_path()}/tests/deltachat-test" + test_src = f"{self.repo_path}/tests/deltachat-test" with self.out.section("test-madmail"): # Symlink the built madmail binary into the test directory so From a172fbac635a3dd7db02da4976bb5c037021dbb0 Mon Sep 17 00:00:00 2001 From: j4n Date: Tue, 11 Aug 2026 16:55:26 +0200 Subject: [PATCH 4/5] ci: raise lxc-test job timeout to 60 minutes to fit potential madmail source build --- .github/workflows/lxc-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lxc-test.yml b/.github/workflows/lxc-test.yml index cfa3240..a544257 100644 --- a/.github/workflows/lxc-test.yml +++ b/.github/workflows/lxc-test.yml @@ -54,7 +54,7 @@ jobs: lxc-test: needs: plan runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 60 steps: - name: Checkout caller repository uses: actions/checkout@v7 From 7b140b7ddbd7fdcc8a7776da940040e48249b461 Mon Sep 17 00:00:00 2001 From: j4n Date: Wed, 12 Aug 2026 07:56:29 +0200 Subject: [PATCH 5/5] feat: add @latest source form resolving the newest release tag - Resolve highest semver from the git main clone that prep_builder already fetched and update the source spec to record the tag and replace "latest". - prep_builder:parse_source(): change from DEFAULT_REF to @main, a "latest" value would fail to clone and the cached main clone does not need the target ref anyway. --- src/cmlxc/driver_base.py | 33 +++++++++++++++++++++++++++++++-- tests/test_cli.py | 26 +++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/cmlxc/driver_base.py b/src/cmlxc/driver_base.py index 101e786..9ebe60b 100644 --- a/src/cmlxc/driver_base.py +++ b/src/cmlxc/driver_base.py @@ -20,6 +20,7 @@ DNS_CONTAINER_NAME, BuilderContainer, DNSContainer, + SetupError, ) from cmlxc.incus import Incus @@ -46,6 +47,7 @@ def parse_source(value: str, default_url: str) -> SourceSpec: Accepted forms: @ref -- branch/tag on the default remote + @latest -- newest release tag /path or ./path -- local directory URL@ref -- custom remote at a given ref """ @@ -64,6 +66,19 @@ def parse_source(value: str, default_url: str) -> SourceSpec: raise ValueError(f"Invalid SOURCE: {value!r}. Use @ref, /path, ./path, or URL@ref.") +# Don't match pre-release and non-semver tags +_RELEASE_TAG_RE = re.compile(r"^v?\d+\.\d+\.\d+$") + + +def latest_release_tag(tag_output): + """Pick the first matching and thus newest release tag from + ``git tag -l --sort=-v:refname`` output.""" + for tag in (tag_output or "").splitlines(): + if _RELEASE_TAG_RE.match(tag): + return tag + return None + + _RELAY_NAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9-]*$") @@ -146,7 +161,10 @@ def add_cli_options(cls, parser, completer=None): "--source", default=f"@{cls.DEFAULT_REF}", metavar="SOURCE", - help=f"Driver source: @ref, /path, ./path, or URL@ref (default: @{cls.DEFAULT_REF}).", + help=( + "Driver source: @ref (branch, tag), @latest (newest release tag)," + f" /path, ./path, or URL@ref (default: @{cls.DEFAULT_REF})." + ), ) action = parser.add_argument( "name", @@ -221,7 +239,9 @@ def prep_builder(cls, ix, out, bld_ct): tmp_dest = f"/root/{cls.REPO_NAME}-git-main" if bld_ct.bash(f"test -d {tmp_dest}", check=False) is None: - source = parse_source(f"@{cls.DEFAULT_REF}", cls.DEFAULT_SOURCE_URL) + # Always main: this is the shared cache clone, and init_builder + # checks the requested ref out of its copy + source = parse_source("@main", cls.DEFAULT_SOURCE_URL) bld_ct.setup_repo(tmp_dest, out, source) else: out.print(f" Fetching {cls.REPO_NAME}-git-main from upstream ...") @@ -241,6 +261,15 @@ def init_builder(self, source): f" Copying {self.REPO_NAME}-git-main to {repo_path} on builder" ) self.bld_ct.bash(f"rm -rf {repo_path} && cp -a {tmp_dest} {repo_path}") + if source.ref == "latest": + # prep_builder already fetched --tags, update source to replace + # "latest" with the tag + source.ref = latest_release_tag( + self.bld_ct.bash(f"cd {repo_path} && git tag -l --sort=-v:refname") + ) + if not source.ref: + raise SetupError(f"No release tag found for {self.REPO_NAME}") + self.out.print(f" Resolved @latest to {source.ref}") if source.ref != "main": self.out.print(f" Checking out {source.ref!r} ...") self.bld_ct.bash(f""" diff --git a/tests/test_cli.py b/tests/test_cli.py index c954ee7..5c96729 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,12 @@ import pytest -from cmlxc.driver_base import SourceSpec, parse_source, validate_relay_name +from cmlxc.driver_base import ( + SourceSpec, + latest_release_tag, + parse_source, + validate_relay_name, +) from cmlxc.driver_cmdeploy import get_ini_overrides from cmlxc.driver_madmail import release_asset_url @@ -17,6 +22,7 @@ ("@main", SourceSpec("remote", url=URL, ref="main")), ("@fix-dovecot", SourceSpec("remote", url=URL, ref="fix-dovecot")), ("@v2.1", SourceSpec("remote", url=URL, ref="v2.1")), + ("@latest", SourceSpec("remote", url=URL, ref="latest")), ("/home/me/relay", SourceSpec("local", path=Path("/home/me/relay"))), ("./relay", SourceSpec("local", path=Path("./relay"))), ("../relay", SourceSpec("local", path=Path("../relay"))), @@ -83,3 +89,21 @@ def test_release_asset_url(tag, arch, expected_asset): assert url is None else: assert url.endswith(f"/v2.23.0/{expected_asset}") + + +@pytest.mark.parametrize( + "tag_output, expected", + [ + # `git tag -l --sort=-v:refname` output, newest first + ("v2.23.0\nv2.22.1\nv2.2.2\n", "v2.23.0"), + # pre-releases and non-semver tags are skipped + ("v2.24.0-rc1\nv2.23.0\n", "v2.23.0"), + ("test\nlatest\nv1.0.0\n", "v1.0.0"), + # tags without the v prefix still count + ("2.23.0\n", "2.23.0"), + ("", None), + (None, None), + ], +) +def test_latest_release_tag(tag_output, expected): + assert latest_release_tag(tag_output) == expected