Save and restore a running container on macOS

Sources/Containerization/CHVirtualMachineManager.swift

@@ -103,7 +103,11 @@ public struct CHVirtualMachineManager: VirtualMachineManager {
         self.logger = logger
     }
 
-    public func create(config: some VMCreationConfig) async throws -> any VirtualMachineInstance {
+    public func create(config: some VMCreationConfig, from savedState: URL? = nil) async throws -> any VirtualMachineInstance {
+        guard savedState == nil else {
+            throw ContainerizationError(.unsupported, message: "restoring from saved state")
+        }
+
         let vmConfig = config.configuration
 
         var instanceConfig = CHVirtualMachineInstance.Configuration()

Sources/Containerization/LinuxContainer+Checkpoint.swift

@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2026 Apple Inc. and the Containerization project authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+extension LinuxContainer {
+    /// The container-level data stored in a checkpoint directory as `container.json`.
+    ///
+    /// Alongside it, `state/` holds the virtual machine's saved state bundle.
+    public struct Checkpoint: Codable, Sendable, Equatable {
+        public static let currentVersion = 1
+        public static let manifestName = "container.json"
+        public static let stateDirectoryName = "state"
+
+        public var version: Int
+        /// The identifier of the container that was checkpointed.
+        public var id: String
+
+        public init(id: String) {
+            self.version = Self.currentVersion
+            self.id = id
+        }
+
+        /// Read a checkpoint written by ``save(to:)``.
+        public static func load(from url: URL) throws -> Checkpoint {
+            let data = try Data(contentsOf: url)
+            return try JSONDecoder().decode(Checkpoint.self, from: data)
+        }
+
+        /// Write the checkpoint to the file at `url` as JSON.
+        public func save(to url: URL) throws {
+            let encoder = JSONEncoder()
+            encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+            try encoder.encode(self).write(to: url)
+        }
+    }
+}

Sources/Containerization/LinuxContainer.swift

@@ -763,7 +763,14 @@ extension LinuxContainer {
     /// Create and start the underlying container's virtual machine
     /// and set up the runtime environment. The container's init process
     /// is NOT running afterwards.
-    public func create() async throws {
+    ///
+    /// - Parameter checkpoint: A directory written by ``checkpoint(to:)``. The virtual machine is
+    ///   then restored from the checkpoint's saved state instead of booted, and none of the
+    ///   guest setup below is redone: it is already part of the saved state. The container's
+    ///   init process is already running in the restored guest, so the container is started
+    ///   afterwards (do not call ``start()``): its stdio is connected to the
+    ///   `config.process` streams again.
+    public func create(from checkpoint: URL? = nil) async throws {
         try await self.state.withLock { state in
             try state.validateForCreate()
 
@@ -800,9 +807,54 @@ extension LinuxContainer {
                 nestedVirtualization: self.config.virtualization
             )
             let creationConfig = StandardVMConfig(configuration: vmConfig)
-            let vm = try await self.vmm.create(config: creationConfig)
+            let savedState = checkpoint?.appending(path: Checkpoint.stateDirectoryName)
+            let vm = try await self.vmm.create(config: creationConfig, from: savedState)
             let relayManager = UnixSocketRelayManager(vm: vm, log: self.logger)
 
+            if let savedState {
+                let createdState = State.CreatedState(
+                    vm: vm,
+                    relayManager: relayManager,
+                    fileMountContext: fileMountContextHolder.withLock { $0 }
+                )
+                var agent: (any VirtualMachineAgent)?
+                do {
+                    try await vm.restore(from: savedState)
+
+                    // The init process is already running in the guest. Give it a LinuxProcess
+                    // and connect its stdio again, rather than starting it.
+                    let restoredAgent = try await vm.dialAgent()
+                    agent = restoredAgent
+                    let stdio = IOUtil.setup(
+                        portAllocator: self.hostVsockPorts,
+                        stdin: self.config.process.stdin,
+                        stdout: self.config.process.stdout,
+                        stderr: self.config.process.stderr
+                    )
+                    let process = LinuxProcess(
+                        self.id,
+                        containerID: self.id,
+                        spec: try self.generateRuntimeSpec(for: .containerInit),
+                        io: stdio,
+                        portAllocator: self.hostVsockPorts,
+                        ociRuntimePath: self.config.ociRuntimePath,
+                        agent: restoredAgent,
+                        vm: vm,
+                        logger: self.logger
+                    )
+                    try await process.attachIO()
+
+                    state = .started(.init(createdState, process: process))
+                } catch {
+                    try? await agent?.close()
+                    try? await relayManager.stopAll()
+                    try? await vm.stop()
+                    state.setErrored(error: error)
+                    throw error
+                }
+                return
+            }
+
             do {
                 try await vm.start()
                 let mountsForAgent = containerMounts
@@ -1007,6 +1059,100 @@ extension LinuxContainer {
         }
     }
 
+    /// Checkpoint the running container to a new directory at `url`, then keep it running.
+    ///
+    /// The directory holds `container.json` (see ``Checkpoint``) and `state/`, the virtual
+    /// machine's saved state bundle. The container is paused for the duration of the checkpoint
+    /// and resumed afterwards, whether or not the checkpoint succeeded.
+    ///
+    /// Only memory and board are captured, not disks. To make the checkpoint restorable, preserve
+    /// the backing disk images (e.g. clone them) in `whilePaused`.
+    ///
+    /// - Parameters:
+    ///   - url: The directory to create.
+    ///   - whilePaused: Called after the state is saved, before the container resumes. If it
+    ///     throws, the checkpoint is removed and the error is rethrown. The guest is frozen
+    ///     until it returns, and the container's state is locked, so keep it short and don't
+    ///     call back into this container.
+    ///
+    /// - Throws: ``ContainerizationError`` with `.invalidState` if the container is not running,
+    ///   `.exists` if `url` already exists, and `.unsupported` if the container has processes
+    ///   started through ``exec(_:configuration:)`` or the virtual machine cannot be saved.
+    public func checkpoint(
+        to url: URL,
+        whilePaused: @escaping @Sendable () async throws -> Void = {}
+    ) async throws {
+        try await self.state.withLock { state in
+            let startedState = try state.startedState("checkpoint")
+
+            // Exec'd processes would have to be reconstructed on restore.
+            guard startedState.vendedProcesses.isEmpty else {
+                throw ContainerizationError(
+                    .unsupported,
+                    message: "failed to checkpoint: container has \(startedState.vendedProcesses.count) exec'd process(es) running"
+                )
+            }
+
+            let fm = FileManager.default
+            if fm.fileExists(atPath: url.path) {
+                throw ContainerizationError(.exists, message: "\(url.path) already exists")
+            }
+
+            let vm = startedState.vm
+
+            // The bundle holds a dump of guest memory.
+            try fm.createDirectory(
+                at: url,
+                withIntermediateDirectories: true,
+                attributes: [.posixPermissions: 0o700]
+            )
+
+            // The connections that carry the process's stdio to the host do not survive a restore,
+            // and a save must not catch them half-way through a transfer. Take them down first.
+            // If this fails the process's stdio is partly gone, so give up.
+            let process = startedState.process
+            do {
+                try await process.detachIO()
+            } catch {
+                try? fm.removeItem(at: url)
+                throw error
+            }
+
+            var failure: Error?
+            var pausedState: State.PausedState?
+            do {
+                try await vm.pause()
+                let paused = State.PausedState(startedState)
+                state = .paused(paused)
+                pausedState = paused
+
+                try await vm.save(to: url.appending(path: Checkpoint.stateDirectoryName))
+                try await whilePaused()
+                try Checkpoint(id: self.id).save(to: url.appending(path: Checkpoint.manifestName))
+            } catch {
+                failure = error
+                try? fm.removeItem(at: url)
+            }
+
+            // Whether or not the save worked, do not leave the container frozen.
+            if let pausedState {
+                try await vm.resume()
+                state = .started(.init(pausedState))
+            }
+
+            // And connect the process's stdio again, so the container carries on.
+            do {
+                try await process.attachIO()
+            } catch {
+                failure = failure ?? error
+            }
+
+            if let failure {
+                throw failure
+            }
+        }
+    }
+
     /// Stop the container from executing. This MUST be called even if wait() has returned
     /// as their are additional resources to free.
     public func stop() async throws {

Sources/Containerization/LinuxProcess.swift

@@ -400,6 +400,152 @@ extension LinuxProcess {
         }
     }
 
+    /// Take the process's stdio down without disturbing the process, so the virtual machine can
+    /// be paused. Call ``attachIO()`` afterwards to connect it again.
+    ///
+    /// Anything the process writes in between waits in its pipes, and what the host had sent but
+    /// the process had not read is kept in the guest. The guest closes its stdout and stderr
+    /// connections itself, so this reads them to EOF first: nothing already sent is lost.
+    ///
+    /// Stdin forwarding stops and its connection is closed, without telling the process: the
+    /// process is not given an EOF. Anything typed before ``attachIO()`` is not forwarded, and
+    /// may be lost.
+    ///
+    /// If this throws, the process's stdio is left partly detached. stdin in particular is gone,
+    /// so treat it as fatal for the process's I/O.
+    func detachIO() async throws {
+        // Stop forwarding first. A cancelled relay does not close stdin in the guest.
+        let relay = self.state.withLock { state -> Task<(), Never>? in
+            let relay = state.stdinRelay
+            state.stdinRelay = nil
+            relay?.cancel()
+            return relay
+        }
+        await relay?.value
+
+        // Closing the connection is the EOF the guest reads stdin up to.
+        try self.state.withLock { state in
+            if let stdin = state.stdio.stdin {
+                stdin.readabilityHandler = nil
+                state.stdio.stdin = nil
+                try stdin.close()
+            }
+        }
+
+        do {
+            try await self.agent.detachProcessIO(id: self.id, containerID: self.owningContainer)
+        } catch {
+            throw ContainerizationError(
+                .internalError,
+                message: "failed to detach IO of process \(self.id)",
+                cause: error
+            )
+        }
+
+        // The guest closed its end, so stdout and stderr end once everything it sent is read.
+        try await self.waitForOutputEOF()
+        try self.state.withLock { try $0.stdio.close() }
+    }
+
+    /// Connect the process's stdio to the guest again, after ``detachIO()`` or after the virtual
+    /// machine was restored. The guest delivers what it kept first.
+    func attachIO() async throws {
+        var pending = [VsockListener?](repeating: nil, count: 3)
+        do {
+            if let stdin = self.ioSetup.stdin {
+                pending[0] = try self.vm.listen(stdin.port)
+            }
+            if let stdout = self.ioSetup.stdout {
+                pending[1] = try self.vm.listen(stdout.port)
+            }
+            if let stderr = self.ioSetup.stderr {
+                pending[2] = try self.vm.listen(stderr.port)
+            }
+            let listeners = pending
+
+            // As in start(), the guest dials back once the request has reached it.
+            let handles = try await withThrowingTaskGroup(of: StartStep.self, returning: [FileHandle?].self) { group in
+                group.addTask {
+                    let handles = try await self.setupIO(listeners: listeners)
+                    return .stdio(handles)
+                }
+
+                group.addTask {
+                    try await self.agent.attachProcessIO(
+                        id: self.id,
+                        containerID: self.owningContainer,
+                        stdinPort: self.ioSetup.stdin?.port,
+                        stdoutPort: self.ioSetup.stdout?.port,
+                        stderrPort: self.ioSetup.stderr?.port
+                    )
+                    return .processCreated
+                }
+
+                var handles = [FileHandle?](repeating: nil, count: 3)
+                for try await step in group {
+                    if case .stdio(let stdio) = step {
+                        handles = stdio
+                    }
+                }
+                return handles
+            }
+
+            if let stdinHandle = handles[0] {
+                self.startStdinRelay(handle: stdinHandle)
+            }
+
+            self.state.withLock {
+                $0.stdio = StdioHandles(
+                    stdin: handles[0],
+                    stdout: handles[1],
+                    stderr: handles[2]
+                )
+            }
+        } catch {
+            for listener in pending {
+                try? listener?.finish()
+            }
+            if let err = error as? ContainerizationError {
+                throw err
+            }
+            throw ContainerizationError(
+                .internalError,
+                message: "failed to attach IO of process \(self.id)",
+                cause: error
+            )
+        }
+    }
+
+    /// Wait for stdout and stderr to reach EOF, which `setupIO` counts. Unlike
+    /// `waitIoComplete`, running out of time is an error: the output may not all be read.
+    private func waitForOutputEOF() async throws {
+        let ioTracker = self.state.withLock { $0.ioTracker }
+        guard let ioTracker else {
+            return
+        }
+        do {
+            try await Timeout.run(seconds: self.stdioTimeout) {
+                var counter = ioTracker.configuredStreams
+                for await _ in ioTracker.stream {
+                    counter -= 1
+                    if counter == 0 {
+                        ioTracker.cont.finish()
+                        break
+                    }
+                }
+            }
+        } catch {
+            throw ContainerizationError(
+                .timeout,
+                message: "timed out waiting for stdout and stderr of process \(self.id) to reach EOF",
+                cause: error
+            )
+        }
+        self.state.withLock {
+            $0.ioTracker = nil
+        }
+    }
+
     /// Kill the process with the specified signal.
     public func kill(_ signal: Signal) async throws {
         do {