diff --git a/Sources/Containerization/CHVirtualMachineManager.swift b/Sources/Containerization/CHVirtualMachineManager.swift index 85708f266..88e71a25e 100644 --- a/Sources/Containerization/CHVirtualMachineManager.swift +++ b/Sources/Containerization/CHVirtualMachineManager.swift @@ -103,7 +103,11 @@ public struct CHVirtualMachineManager: VirtualMachineManager { self.logger = logger } - public func create(config: some VMCreationConfig) async throws -> any VirtualMachineInstance { + public func create(config: some VMCreationConfig, from savedState: URL? = nil) async throws -> any VirtualMachineInstance { + guard savedState == nil else { + throw ContainerizationError(.unsupported, message: "restoring from saved state") + } + let vmConfig = config.configuration var instanceConfig = CHVirtualMachineInstance.Configuration() diff --git a/Sources/Containerization/LinuxContainer+Checkpoint.swift b/Sources/Containerization/LinuxContainer+Checkpoint.swift new file mode 100644 index 000000000..6f76aaf05 --- /dev/null +++ b/Sources/Containerization/LinuxContainer+Checkpoint.swift @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +extension LinuxContainer { + /// The container-level data stored in a checkpoint directory as `container.json`. + /// + /// Alongside it, `state/` holds the virtual machine's saved state bundle. + public struct Checkpoint: Codable, Sendable, Equatable { + public static let currentVersion = 1 + public static let manifestName = "container.json" + public static let stateDirectoryName = "state" + + public var version: Int + /// The identifier of the container that was checkpointed. + public var id: String + + public init(id: String) { + self.version = Self.currentVersion + self.id = id + } + + /// Read a checkpoint written by ``save(to:)``. + public static func load(from url: URL) throws -> Checkpoint { + let data = try Data(contentsOf: url) + return try JSONDecoder().decode(Checkpoint.self, from: data) + } + + /// Write the checkpoint to the file at `url` as JSON. + public func save(to url: URL) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(self).write(to: url) + } + } +} diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 639abeefe..7a2901097 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -763,7 +763,14 @@ extension LinuxContainer { /// Create and start the underlying container's virtual machine /// and set up the runtime environment. The container's init process /// is NOT running afterwards. - public func create() async throws { + /// + /// - Parameter checkpoint: A directory written by ``checkpoint(to:)``. The virtual machine is + /// then restored from the checkpoint's saved state instead of booted, and none of the + /// guest setup below is redone: it is already part of the saved state. The container's + /// init process is already running in the restored guest, so the container is started + /// afterwards (do not call ``start()``): its stdio is connected to the + /// `config.process` streams again. + public func create(from checkpoint: URL? = nil) async throws { try await self.state.withLock { state in try state.validateForCreate() @@ -800,9 +807,54 @@ extension LinuxContainer { nestedVirtualization: self.config.virtualization ) let creationConfig = StandardVMConfig(configuration: vmConfig) - let vm = try await self.vmm.create(config: creationConfig) + let savedState = checkpoint?.appending(path: Checkpoint.stateDirectoryName) + let vm = try await self.vmm.create(config: creationConfig, from: savedState) let relayManager = UnixSocketRelayManager(vm: vm, log: self.logger) + if let savedState { + let createdState = State.CreatedState( + vm: vm, + relayManager: relayManager, + fileMountContext: fileMountContextHolder.withLock { $0 } + ) + var agent: (any VirtualMachineAgent)? + do { + try await vm.restore(from: savedState) + + // The init process is already running in the guest. Give it a LinuxProcess + // and connect its stdio again, rather than starting it. + let restoredAgent = try await vm.dialAgent() + agent = restoredAgent + let stdio = IOUtil.setup( + portAllocator: self.hostVsockPorts, + stdin: self.config.process.stdin, + stdout: self.config.process.stdout, + stderr: self.config.process.stderr + ) + let process = LinuxProcess( + self.id, + containerID: self.id, + spec: try self.generateRuntimeSpec(for: .containerInit), + io: stdio, + portAllocator: self.hostVsockPorts, + ociRuntimePath: self.config.ociRuntimePath, + agent: restoredAgent, + vm: vm, + logger: self.logger + ) + try await process.attachIO() + + state = .started(.init(createdState, process: process)) + } catch { + try? await agent?.close() + try? await relayManager.stopAll() + try? await vm.stop() + state.setErrored(error: error) + throw error + } + return + } + do { try await vm.start() let mountsForAgent = containerMounts @@ -1007,6 +1059,100 @@ extension LinuxContainer { } } + /// Checkpoint the running container to a new directory at `url`, then keep it running. + /// + /// The directory holds `container.json` (see ``Checkpoint``) and `state/`, the virtual + /// machine's saved state bundle. The container is paused for the duration of the checkpoint + /// and resumed afterwards, whether or not the checkpoint succeeded. + /// + /// Only memory and board are captured, not disks. To make the checkpoint restorable, preserve + /// the backing disk images (e.g. clone them) in `whilePaused`. + /// + /// - Parameters: + /// - url: The directory to create. + /// - whilePaused: Called after the state is saved, before the container resumes. If it + /// throws, the checkpoint is removed and the error is rethrown. The guest is frozen + /// until it returns, and the container's state is locked, so keep it short and don't + /// call back into this container. + /// + /// - Throws: ``ContainerizationError`` with `.invalidState` if the container is not running, + /// `.exists` if `url` already exists, and `.unsupported` if the container has processes + /// started through ``exec(_:configuration:)`` or the virtual machine cannot be saved. + public func checkpoint( + to url: URL, + whilePaused: @escaping @Sendable () async throws -> Void = {} + ) async throws { + try await self.state.withLock { state in + let startedState = try state.startedState("checkpoint") + + // Exec'd processes would have to be reconstructed on restore. + guard startedState.vendedProcesses.isEmpty else { + throw ContainerizationError( + .unsupported, + message: "failed to checkpoint: container has \(startedState.vendedProcesses.count) exec'd process(es) running" + ) + } + + let fm = FileManager.default + if fm.fileExists(atPath: url.path) { + throw ContainerizationError(.exists, message: "\(url.path) already exists") + } + + let vm = startedState.vm + + // The bundle holds a dump of guest memory. + try fm.createDirectory( + at: url, + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700] + ) + + // The connections that carry the process's stdio to the host do not survive a restore, + // and a save must not catch them half-way through a transfer. Take them down first. + // If this fails the process's stdio is partly gone, so give up. + let process = startedState.process + do { + try await process.detachIO() + } catch { + try? fm.removeItem(at: url) + throw error + } + + var failure: Error? + var pausedState: State.PausedState? + do { + try await vm.pause() + let paused = State.PausedState(startedState) + state = .paused(paused) + pausedState = paused + + try await vm.save(to: url.appending(path: Checkpoint.stateDirectoryName)) + try await whilePaused() + try Checkpoint(id: self.id).save(to: url.appending(path: Checkpoint.manifestName)) + } catch { + failure = error + try? fm.removeItem(at: url) + } + + // Whether or not the save worked, do not leave the container frozen. + if let pausedState { + try await vm.resume() + state = .started(.init(pausedState)) + } + + // And connect the process's stdio again, so the container carries on. + do { + try await process.attachIO() + } catch { + failure = failure ?? error + } + + if let failure { + throw failure + } + } + } + /// Stop the container from executing. This MUST be called even if wait() has returned /// as their are additional resources to free. public func stop() async throws { diff --git a/Sources/Containerization/LinuxProcess.swift b/Sources/Containerization/LinuxProcess.swift index 4e54b1d84..016e8b9f2 100644 --- a/Sources/Containerization/LinuxProcess.swift +++ b/Sources/Containerization/LinuxProcess.swift @@ -400,6 +400,152 @@ extension LinuxProcess { } } + /// Take the process's stdio down without disturbing the process, so the virtual machine can + /// be paused. Call ``attachIO()`` afterwards to connect it again. + /// + /// Anything the process writes in between waits in its pipes, and what the host had sent but + /// the process had not read is kept in the guest. The guest closes its stdout and stderr + /// connections itself, so this reads them to EOF first: nothing already sent is lost. + /// + /// Stdin forwarding stops and its connection is closed, without telling the process: the + /// process is not given an EOF. Anything typed before ``attachIO()`` is not forwarded, and + /// may be lost. + /// + /// If this throws, the process's stdio is left partly detached. stdin in particular is gone, + /// so treat it as fatal for the process's I/O. + func detachIO() async throws { + // Stop forwarding first. A cancelled relay does not close stdin in the guest. + let relay = self.state.withLock { state -> Task<(), Never>? in + let relay = state.stdinRelay + state.stdinRelay = nil + relay?.cancel() + return relay + } + await relay?.value + + // Closing the connection is the EOF the guest reads stdin up to. + try self.state.withLock { state in + if let stdin = state.stdio.stdin { + stdin.readabilityHandler = nil + state.stdio.stdin = nil + try stdin.close() + } + } + + do { + try await self.agent.detachProcessIO(id: self.id, containerID: self.owningContainer) + } catch { + throw ContainerizationError( + .internalError, + message: "failed to detach IO of process \(self.id)", + cause: error + ) + } + + // The guest closed its end, so stdout and stderr end once everything it sent is read. + try await self.waitForOutputEOF() + try self.state.withLock { try $0.stdio.close() } + } + + /// Connect the process's stdio to the guest again, after ``detachIO()`` or after the virtual + /// machine was restored. The guest delivers what it kept first. + func attachIO() async throws { + var pending = [VsockListener?](repeating: nil, count: 3) + do { + if let stdin = self.ioSetup.stdin { + pending[0] = try self.vm.listen(stdin.port) + } + if let stdout = self.ioSetup.stdout { + pending[1] = try self.vm.listen(stdout.port) + } + if let stderr = self.ioSetup.stderr { + pending[2] = try self.vm.listen(stderr.port) + } + let listeners = pending + + // As in start(), the guest dials back once the request has reached it. + let handles = try await withThrowingTaskGroup(of: StartStep.self, returning: [FileHandle?].self) { group in + group.addTask { + let handles = try await self.setupIO(listeners: listeners) + return .stdio(handles) + } + + group.addTask { + try await self.agent.attachProcessIO( + id: self.id, + containerID: self.owningContainer, + stdinPort: self.ioSetup.stdin?.port, + stdoutPort: self.ioSetup.stdout?.port, + stderrPort: self.ioSetup.stderr?.port + ) + return .processCreated + } + + var handles = [FileHandle?](repeating: nil, count: 3) + for try await step in group { + if case .stdio(let stdio) = step { + handles = stdio + } + } + return handles + } + + if let stdinHandle = handles[0] { + self.startStdinRelay(handle: stdinHandle) + } + + self.state.withLock { + $0.stdio = StdioHandles( + stdin: handles[0], + stdout: handles[1], + stderr: handles[2] + ) + } + } catch { + for listener in pending { + try? listener?.finish() + } + if let err = error as? ContainerizationError { + throw err + } + throw ContainerizationError( + .internalError, + message: "failed to attach IO of process \(self.id)", + cause: error + ) + } + } + + /// Wait for stdout and stderr to reach EOF, which `setupIO` counts. Unlike + /// `waitIoComplete`, running out of time is an error: the output may not all be read. + private func waitForOutputEOF() async throws { + let ioTracker = self.state.withLock { $0.ioTracker } + guard let ioTracker else { + return + } + do { + try await Timeout.run(seconds: self.stdioTimeout) { + var counter = ioTracker.configuredStreams + for await _ in ioTracker.stream { + counter -= 1 + if counter == 0 { + ioTracker.cont.finish() + break + } + } + } + } catch { + throw ContainerizationError( + .timeout, + message: "timed out waiting for stdout and stderr of process \(self.id) to reach EOF", + cause: error + ) + } + self.state.withLock { + $0.ioTracker = nil + } + } + /// Kill the process with the specified signal. public func kill(_ signal: Signal) async throws { do { diff --git a/Sources/Containerization/SandboxContext/SandboxContext.grpc.swift b/Sources/Containerization/SandboxContext/SandboxContext.grpc.swift index 6776ed1b3..5c0264879 100644 --- a/Sources/Containerization/SandboxContext/SandboxContext.grpc.swift +++ b/Sources/Containerization/SandboxContext/SandboxContext.grpc.swift @@ -283,6 +283,32 @@ public enum Com_Apple_Containerization_Sandbox_V3_SandboxContext: Sendable { type: .unary ) } + /// Namespace for "DetachProcessIO" metadata. + public enum DetachProcessIO: Sendable { + /// Request type for "DetachProcessIO". + public typealias Input = Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest + /// Response type for "DetachProcessIO". + public typealias Output = Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse + /// Descriptor for "DetachProcessIO". + public static let descriptor = GRPCCore.MethodDescriptor( + service: GRPCCore.ServiceDescriptor(fullyQualifiedService: "com.apple.containerization.sandbox.v3.SandboxContext"), + method: "DetachProcessIO", + type: .unary + ) + } + /// Namespace for "AttachProcessIO" metadata. + public enum AttachProcessIO: Sendable { + /// Request type for "AttachProcessIO". + public typealias Input = Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest + /// Response type for "AttachProcessIO". + public typealias Output = Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse + /// Descriptor for "AttachProcessIO". + public static let descriptor = GRPCCore.MethodDescriptor( + service: GRPCCore.ServiceDescriptor(fullyQualifiedService: "com.apple.containerization.sandbox.v3.SandboxContext"), + method: "AttachProcessIO", + type: .unary + ) + } /// Namespace for "ContainerStatistics" metadata. public enum ContainerStatistics: Sendable { /// Request type for "ContainerStatistics". @@ -447,6 +473,8 @@ public enum Com_Apple_Containerization_Sandbox_V3_SandboxContext: Sendable { WaitProcess.descriptor, ResizeProcess.descriptor, CloseProcessStdin.descriptor, + DetachProcessIO.descriptor, + AttachProcessIO.descriptor, ContainerStatistics.descriptor, ProxyVsock.descriptor, StopVsockProxy.descriptor, @@ -832,6 +860,48 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext { context: GRPCCore.ServerContext ) async throws -> GRPCCore.StreamingServerResponse + /// Handle the "DetachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Take down the relays that carry a process's stdio to the host, without + /// > disturbing the process, so the virtual machine can be paused. What the + /// > process writes meanwhile waits in its pipes, and what the host had sent but + /// > the process had not read is kept in the guest. The host must have closed its + /// > stdin connection first and must read stdout and stderr until EOF. + /// + /// - Parameters: + /// - request: A streaming request of `Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest` messages. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A streaming response of `Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse` messages. + func detachProcessIO( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse + + /// Handle the "AttachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Connect a process whose stdio was detached to the host again, over new vsock + /// > ports the host is listening on. Anything kept by DetachProcessIO is + /// > delivered first. + /// + /// - Parameters: + /// - request: A streaming request of `Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest` messages. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A streaming response of `Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse` messages. + func attachProcessIO( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse + /// Handle the "ContainerStatistics" method. /// /// > Source IDL Documentation: @@ -1388,6 +1458,48 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext { context: GRPCCore.ServerContext ) async throws -> GRPCCore.ServerResponse + /// Handle the "DetachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Take down the relays that carry a process's stdio to the host, without + /// > disturbing the process, so the virtual machine can be paused. What the + /// > process writes meanwhile waits in its pipes, and what the host had sent but + /// > the process had not read is kept in the guest. The host must have closed its + /// > stdin connection first and must read stdout and stderr until EOF. + /// + /// - Parameters: + /// - request: A request containing a single `Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest` message. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A response containing a single `Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse` message. + func detachProcessIO( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse + + /// Handle the "AttachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Connect a process whose stdio was detached to the host again, over new vsock + /// > ports the host is listening on. Anything kept by DetachProcessIO is + /// > delivered first. + /// + /// - Parameters: + /// - request: A request containing a single `Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest` message. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A response containing a single `Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse` message. + func attachProcessIO( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse + /// Handle the "ContainerStatistics" method. /// /// > Source IDL Documentation: @@ -1943,6 +2055,48 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext { context: GRPCCore.ServerContext ) async throws -> Com_Apple_Containerization_Sandbox_V3_CloseProcessStdinResponse + /// Handle the "DetachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Take down the relays that carry a process's stdio to the host, without + /// > disturbing the process, so the virtual machine can be paused. What the + /// > process writes meanwhile waits in its pipes, and what the host had sent but + /// > the process had not read is kept in the guest. The host must have closed its + /// > stdin connection first and must read stdout and stderr until EOF. + /// + /// - Parameters: + /// - request: A `Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest` message. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A `Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse` to respond with. + func detachProcessIO( + request: Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest, + context: GRPCCore.ServerContext + ) async throws -> Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse + + /// Handle the "AttachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Connect a process whose stdio was detached to the host again, over new vsock + /// > ports the host is listening on. Anything kept by DetachProcessIO is + /// > delivered first. + /// + /// - Parameters: + /// - request: A `Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest` message. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A `Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse` to respond with. + func attachProcessIO( + request: Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest, + context: GRPCCore.ServerContext + ) async throws -> Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse + /// Handle the "ContainerStatistics" method. /// /// > Source IDL Documentation: @@ -2356,6 +2510,28 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext.StreamingServiceP ) } ) + router.registerHandler( + forMethod: Com_Apple_Containerization_Sandbox_V3_SandboxContext.Method.DetachProcessIO.descriptor, + deserializer: GRPCProtobuf.ProtobufDeserializer(), + serializer: GRPCProtobuf.ProtobufSerializer(), + handler: { request, context in + try await self.detachProcessIO( + request: request, + context: context + ) + } + ) + router.registerHandler( + forMethod: Com_Apple_Containerization_Sandbox_V3_SandboxContext.Method.AttachProcessIO.descriptor, + deserializer: GRPCProtobuf.ProtobufDeserializer(), + serializer: GRPCProtobuf.ProtobufSerializer(), + handler: { request, context in + try await self.attachProcessIO( + request: request, + context: context + ) + } + ) router.registerHandler( forMethod: Com_Apple_Containerization_Sandbox_V3_SandboxContext.Method.ContainerStatistics.descriptor, deserializer: GRPCProtobuf.ProtobufDeserializer(), @@ -2692,6 +2868,28 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext.ServiceProtocol { return GRPCCore.StreamingServerResponse(single: response) } + public func detachProcessIO( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse { + let response = try await self.detachProcessIO( + request: GRPCCore.ServerRequest(stream: request), + context: context + ) + return GRPCCore.StreamingServerResponse(single: response) + } + + public func attachProcessIO( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse { + let response = try await self.attachProcessIO( + request: GRPCCore.ServerRequest(stream: request), + context: context + ) + return GRPCCore.StreamingServerResponse(single: response) + } + public func containerStatistics( request: GRPCCore.StreamingServerRequest, context: GRPCCore.ServerContext @@ -3068,6 +3266,32 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServiceProt ) } + public func detachProcessIO( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse { + return GRPCCore.ServerResponse( + message: try await self.detachProcessIO( + request: request.message, + context: context + ), + metadata: [:] + ) + } + + public func attachProcessIO( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse { + return GRPCCore.ServerResponse( + message: try await self.attachProcessIO( + request: request.message, + context: context + ), + metadata: [:] + ) + } + public func containerStatistics( request: GRPCCore.ServerRequest, context: GRPCCore.ServerContext @@ -3665,6 +3889,58 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext { onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result ) async throws -> Result where Result: Sendable + /// Call the "DetachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Take down the relays that carry a process's stdio to the host, without + /// > disturbing the process, so the virtual machine can be paused. What the + /// > process writes meanwhile waits in its pipes, and what the host had sent but + /// > the process had not read is kept in the guest. The host must have closed its + /// > stdin connection first and must read stdout and stderr until EOF. + /// + /// - Parameters: + /// - request: A request containing a single `Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest` message. + /// - serializer: A serializer for `Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest` messages. + /// - deserializer: A deserializer for `Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + func detachProcessIO( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result + ) async throws -> Result where Result: Sendable + + /// Call the "AttachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Connect a process whose stdio was detached to the host again, over new vsock + /// > ports the host is listening on. Anything kept by DetachProcessIO is + /// > delivered first. + /// + /// - Parameters: + /// - request: A request containing a single `Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest` message. + /// - serializer: A serializer for `Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest` messages. + /// - deserializer: A deserializer for `Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + func attachProcessIO( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result + ) async throws -> Result where Result: Sendable + /// Call the "ContainerStatistics" method. /// /// > Source IDL Documentation: @@ -4586,6 +4862,80 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext { ) } + /// Call the "DetachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Take down the relays that carry a process's stdio to the host, without + /// > disturbing the process, so the virtual machine can be paused. What the + /// > process writes meanwhile waits in its pipes, and what the host had sent but + /// > the process had not read is kept in the guest. The host must have closed its + /// > stdin connection first and must read stdout and stderr until EOF. + /// + /// - Parameters: + /// - request: A request containing a single `Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest` message. + /// - serializer: A serializer for `Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest` messages. + /// - deserializer: A deserializer for `Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func detachProcessIO( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + try await self.client.unary( + request: request, + descriptor: Com_Apple_Containerization_Sandbox_V3_SandboxContext.Method.DetachProcessIO.descriptor, + serializer: serializer, + deserializer: deserializer, + options: options, + onResponse: handleResponse + ) + } + + /// Call the "AttachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Connect a process whose stdio was detached to the host again, over new vsock + /// > ports the host is listening on. Anything kept by DetachProcessIO is + /// > delivered first. + /// + /// - Parameters: + /// - request: A request containing a single `Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest` message. + /// - serializer: A serializer for `Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest` messages. + /// - deserializer: A deserializer for `Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func attachProcessIO( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + try await self.client.unary( + request: request, + descriptor: Com_Apple_Containerization_Sandbox_V3_SandboxContext.Method.AttachProcessIO.descriptor, + serializer: serializer, + deserializer: deserializer, + options: options, + onResponse: handleResponse + ) + } + /// Call the "ContainerStatistics" method. /// /// > Source IDL Documentation: @@ -5517,6 +5867,70 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext.ClientProtocol { ) } + /// Call the "DetachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Take down the relays that carry a process's stdio to the host, without + /// > disturbing the process, so the virtual machine can be paused. What the + /// > process writes meanwhile waits in its pipes, and what the host had sent but + /// > the process had not read is kept in the guest. The host must have closed its + /// > stdin connection first and must read stdout and stderr until EOF. + /// + /// - Parameters: + /// - request: A request containing a single `Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest` message. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func detachProcessIO( + request: GRPCCore.ClientRequest, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + try await self.detachProcessIO( + request: request, + serializer: GRPCProtobuf.ProtobufSerializer(), + deserializer: GRPCProtobuf.ProtobufDeserializer(), + options: options, + onResponse: handleResponse + ) + } + + /// Call the "AttachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Connect a process whose stdio was detached to the host again, over new vsock + /// > ports the host is listening on. Anything kept by DetachProcessIO is + /// > delivered first. + /// + /// - Parameters: + /// - request: A request containing a single `Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest` message. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func attachProcessIO( + request: GRPCCore.ClientRequest, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + try await self.attachProcessIO( + request: request, + serializer: GRPCProtobuf.ProtobufSerializer(), + deserializer: GRPCProtobuf.ProtobufDeserializer(), + options: options, + onResponse: handleResponse + ) + } + /// Call the "ContainerStatistics" method. /// /// > Source IDL Documentation: @@ -6468,6 +6882,78 @@ extension Com_Apple_Containerization_Sandbox_V3_SandboxContext.ClientProtocol { ) } + /// Call the "DetachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Take down the relays that carry a process's stdio to the host, without + /// > disturbing the process, so the virtual machine can be paused. What the + /// > process writes meanwhile waits in its pipes, and what the host had sent but + /// > the process had not read is kept in the guest. The host must have closed its + /// > stdin connection first and must read stdout and stderr until EOF. + /// + /// - Parameters: + /// - message: request message to send. + /// - metadata: Additional metadata to send, defaults to empty. + /// - options: Options to apply to this RPC, defaults to `.defaults`. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func detachProcessIO( + _ message: Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest, + metadata: GRPCCore.Metadata = [:], + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + let request = GRPCCore.ClientRequest( + message: message, + metadata: metadata + ) + return try await self.detachProcessIO( + request: request, + options: options, + onResponse: handleResponse + ) + } + + /// Call the "AttachProcessIO" method. + /// + /// > Source IDL Documentation: + /// > + /// > Connect a process whose stdio was detached to the host again, over new vsock + /// > ports the host is listening on. Anything kept by DetachProcessIO is + /// > delivered first. + /// + /// - Parameters: + /// - message: request message to send. + /// - metadata: Additional metadata to send, defaults to empty. + /// - options: Options to apply to this RPC, defaults to `.defaults`. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func attachProcessIO( + _ message: Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest, + metadata: GRPCCore.Metadata = [:], + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + let request = GRPCCore.ClientRequest( + message: message, + metadata: metadata + ) + return try await self.attachProcessIO( + request: request, + options: options, + onResponse: handleResponse + ) + } + /// Call the "ContainerStatistics" method. /// /// > Source IDL Documentation: diff --git a/Sources/Containerization/SandboxContext/SandboxContext.pb.swift b/Sources/Containerization/SandboxContext/SandboxContext.pb.swift index a80a2509b..7b42b4480 100644 --- a/Sources/Containerization/SandboxContext/SandboxContext.pb.swift +++ b/Sources/Containerization/SandboxContext/SandboxContext.pb.swift @@ -740,6 +740,102 @@ public nonisolated struct Com_Apple_Containerization_Sandbox_V3_CloseProcessStdi public init() {} } +public nonisolated struct Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var id: String = String() + + public var containerID: String { + get {_containerID ?? String()} + set {_containerID = newValue} + } + /// Returns true if `containerID` has been explicitly set. + public var hasContainerID: Bool {self._containerID != nil} + /// Clears the value of `containerID`. Subsequent reads from it will return its default value. + public mutating func clearContainerID() {self._containerID = nil} + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} + + fileprivate var _containerID: String? = nil +} + +public nonisolated struct Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} +} + +public nonisolated struct Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var id: String = String() + + public var containerID: String { + get {_containerID ?? String()} + set {_containerID = newValue} + } + /// Returns true if `containerID` has been explicitly set. + public var hasContainerID: Bool {self._containerID != nil} + /// Clears the value of `containerID`. Subsequent reads from it will return its default value. + public mutating func clearContainerID() {self._containerID = nil} + + public var stdin: UInt32 { + get {_stdin ?? 0} + set {_stdin = newValue} + } + /// Returns true if `stdin` has been explicitly set. + public var hasStdin: Bool {self._stdin != nil} + /// Clears the value of `stdin`. Subsequent reads from it will return its default value. + public mutating func clearStdin() {self._stdin = nil} + + public var stdout: UInt32 { + get {_stdout ?? 0} + set {_stdout = newValue} + } + /// Returns true if `stdout` has been explicitly set. + public var hasStdout: Bool {self._stdout != nil} + /// Clears the value of `stdout`. Subsequent reads from it will return its default value. + public mutating func clearStdout() {self._stdout = nil} + + public var stderr: UInt32 { + get {_stderr ?? 0} + set {_stderr = newValue} + } + /// Returns true if `stderr` has been explicitly set. + public var hasStderr: Bool {self._stderr != nil} + /// Clears the value of `stderr`. Subsequent reads from it will return its default value. + public mutating func clearStderr() {self._stderr = nil} + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} + + fileprivate var _containerID: String? = nil + fileprivate var _stdin: UInt32? = nil + fileprivate var _stdout: UInt32? = nil + fileprivate var _stderr: UInt32? = nil +} + +public nonisolated struct Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} +} + public nonisolated struct Com_Apple_Containerization_Sandbox_V3_MkdirRequest: Sendable { // SwiftProtobuf.Message conformance is added in an extension below. See the // `Message` and `Message+*Additions` files in the SwiftProtobuf library for @@ -3095,6 +3191,137 @@ nonisolated extension Com_Apple_Containerization_Sandbox_V3_CloseProcessStdinRes } } +nonisolated extension Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".DetachProcessIORequest" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}id\0\u{1}containerID\0") + + public mutating func decodeMessage(decoder: inout D) throws { + while let fieldNumber = try decoder.nextFieldNumber() { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every case branch when no optimizations are + // enabled. https://github.com/apple/swift-protobuf/issues/1034 + switch fieldNumber { + case 1: try { try decoder.decodeSingularStringField(value: &self.id) }() + case 2: try { try decoder.decodeSingularStringField(value: &self._containerID) }() + default: break + } + } + } + + public func traverse(visitor: inout V) throws { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every if/case branch local when no optimizations + // are enabled. https://github.com/apple/swift-protobuf/issues/1034 and + // https://github.com/apple/swift-protobuf/issues/1182 + if !self.id.isEmpty { + try visitor.visitSingularStringField(value: self.id, fieldNumber: 1) + } + try { if let v = self._containerID { + try visitor.visitSingularStringField(value: v, fieldNumber: 2) + } }() + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest, rhs: Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest) -> Bool { + if lhs.id != rhs.id {return false} + if lhs._containerID != rhs._containerID {return false} + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + +nonisolated extension Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".DetachProcessIOResponse" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap() + + public mutating func decodeMessage(decoder: inout D) throws { + // Load everything into unknown fields + while try decoder.nextFieldNumber() != nil {} + } + + public func traverse(visitor: inout V) throws { + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse, rhs: Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse) -> Bool { + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + +nonisolated extension Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".AttachProcessIORequest" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}id\0\u{1}containerID\0\u{1}stdin\0\u{1}stdout\0\u{1}stderr\0") + + public mutating func decodeMessage(decoder: inout D) throws { + while let fieldNumber = try decoder.nextFieldNumber() { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every case branch when no optimizations are + // enabled. https://github.com/apple/swift-protobuf/issues/1034 + switch fieldNumber { + case 1: try { try decoder.decodeSingularStringField(value: &self.id) }() + case 2: try { try decoder.decodeSingularStringField(value: &self._containerID) }() + case 3: try { try decoder.decodeSingularUInt32Field(value: &self._stdin) }() + case 4: try { try decoder.decodeSingularUInt32Field(value: &self._stdout) }() + case 5: try { try decoder.decodeSingularUInt32Field(value: &self._stderr) }() + default: break + } + } + } + + public func traverse(visitor: inout V) throws { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every if/case branch local when no optimizations + // are enabled. https://github.com/apple/swift-protobuf/issues/1034 and + // https://github.com/apple/swift-protobuf/issues/1182 + if !self.id.isEmpty { + try visitor.visitSingularStringField(value: self.id, fieldNumber: 1) + } + try { if let v = self._containerID { + try visitor.visitSingularStringField(value: v, fieldNumber: 2) + } }() + try { if let v = self._stdin { + try visitor.visitSingularUInt32Field(value: v, fieldNumber: 3) + } }() + try { if let v = self._stdout { + try visitor.visitSingularUInt32Field(value: v, fieldNumber: 4) + } }() + try { if let v = self._stderr { + try visitor.visitSingularUInt32Field(value: v, fieldNumber: 5) + } }() + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest, rhs: Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest) -> Bool { + if lhs.id != rhs.id {return false} + if lhs._containerID != rhs._containerID {return false} + if lhs._stdin != rhs._stdin {return false} + if lhs._stdout != rhs._stdout {return false} + if lhs._stderr != rhs._stderr {return false} + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + +nonisolated extension Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".AttachProcessIOResponse" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap() + + public mutating func decodeMessage(decoder: inout D) throws { + // Load everything into unknown fields + while try decoder.nextFieldNumber() != nil {} + } + + public func traverse(visitor: inout V) throws { + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse, rhs: Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse) -> Bool { + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + nonisolated extension Com_Apple_Containerization_Sandbox_V3_MkdirRequest: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { public static let protoMessageName: String = _protobuf_package + ".MkdirRequest" public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}path\0\u{1}all\0\u{1}perms\0") diff --git a/Sources/Containerization/SandboxContext/SandboxContext.proto b/Sources/Containerization/SandboxContext/SandboxContext.proto index 21bb4a22d..cf6b362d2 100644 --- a/Sources/Containerization/SandboxContext/SandboxContext.proto +++ b/Sources/Containerization/SandboxContext/SandboxContext.proto @@ -48,6 +48,16 @@ service SandboxContext { rpc ResizeProcess(ResizeProcessRequest) returns (ResizeProcessResponse); // Close IO for a given process. rpc CloseProcessStdin(CloseProcessStdinRequest) returns (CloseProcessStdinResponse); + // Take down the relays that carry a process's stdio to the host, without + // disturbing the process, so the virtual machine can be paused. What the + // process writes meanwhile waits in its pipes, and what the host had sent but + // the process had not read is kept in the guest. The host must have closed its + // stdin connection first and must read stdout and stderr until EOF. + rpc DetachProcessIO(DetachProcessIORequest) returns (DetachProcessIOResponse); + // Connect a process whose stdio was detached to the host again, over new vsock + // ports the host is listening on. Anything kept by DetachProcessIO is + // delivered first. + rpc AttachProcessIO(AttachProcessIORequest) returns (AttachProcessIOResponse); // Get statistics for containers. rpc ContainerStatistics(ContainerStatisticsRequest) returns (ContainerStatisticsResponse); @@ -211,6 +221,23 @@ message CloseProcessStdinRequest { message CloseProcessStdinResponse {} +message DetachProcessIORequest { + string id = 1; + optional string containerID = 2; +} + +message DetachProcessIOResponse {} + +message AttachProcessIORequest { + string id = 1; + optional string containerID = 2; + optional uint32 stdin = 3; + optional uint32 stdout = 4; + optional uint32 stderr = 5; +} + +message AttachProcessIOResponse {} + message MkdirRequest { string path = 1; bool all = 2; diff --git a/Sources/Containerization/VZVirtualMachine+Helpers.swift b/Sources/Containerization/VZVirtualMachine+Helpers.swift index 2cbadb1c8..bbdc1f617 100644 --- a/Sources/Containerization/VZVirtualMachine+Helpers.swift +++ b/Sources/Containerization/VZVirtualMachine+Helpers.swift @@ -106,6 +106,34 @@ extension VZVirtualMachine { } } + func saveMachineState(to url: URL, queue: DispatchQueue) async throws { + try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in + queue.sync { + self.saveMachineStateTo(url: url) { error in + if let error { + cont.resume(throwing: error) + return + } + cont.resume() + } + } + } + } + + func restoreMachineState(from url: URL, queue: DispatchQueue) async throws { + try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in + queue.sync { + self.restoreMachineStateFrom(url: url) { error in + if let error { + cont.resume(throwing: error) + return + } + cont.resume() + } + } + } + } + func resume(queue: DispatchQueue) async throws { try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in queue.sync { diff --git a/Sources/Containerization/VZVirtualMachineConfigurationSnapshot.swift b/Sources/Containerization/VZVirtualMachineConfigurationSnapshot.swift new file mode 100644 index 000000000..5135c2248 --- /dev/null +++ b/Sources/Containerization/VZVirtualMachineConfigurationSnapshot.swift @@ -0,0 +1,393 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +#if os(macOS) +import ContainerizationError +import CryptoKit +import Foundation +@preconcurrency import Virtualization + +/// A persistable description of a virtual machine's board: its platform and +/// peripherals. It does not describe in-memory state (RAM, registers, device +/// internals); that lives in the file written by `saveMachineStateTo`. +/// +/// The snapshot is built by walking a final, validated +/// `VZVirtualMachineConfiguration`, so it covers everything `toVZ()` and any +/// `VZInstanceExtension` produced. It is a description, not a recipe: +/// `VZVirtualMachineConfiguration` is always built by `toVZ()`, and the +/// snapshot is only used to verify that a newly built configuration matches +/// the one a saved machine state depends on, and to tell the caller which +/// backing state (disk images, network identity) it must preserve. +/// +/// Building a snapshot is also the sanitizing step: any device or attachment +/// outside the allowlist in `docs/vm-save-restore.md` throws +/// `ContainerizationError(.unsupported)`. +public struct VZVirtualMachineConfigurationSnapshot: Codable, Sendable, Equatable { + /// The current manifest format version. + public static let currentVersion = 1 + + /// A file that is identified by content, because its bits live in the + /// guest's memory state and the path alone does not identify them. + public struct HashedFile: Codable, Sendable, Equatable { + public var path: String + public var sha256: String + } + + public struct Host: Codable, Sendable, Equatable { + /// `ProcessInfo.operatingSystemVersionString`. + public var osVersion: String + /// The `hw.model` sysctl value. + public var model: String + } + + public struct Platform: Codable, Sendable, Equatable { + /// `VZGenericMachineIdentifier.dataRepresentation`. VZ generates a new + /// identifier per configuration unless one is set, so it must be + /// persisted and reapplied. + public var machineIdentifier: Data + public var nestedVirtualization: Bool + } + + public struct BootLoader: Codable, Sendable, Equatable { + public var kernel: HashedFile + public var commandLine: String + /// Set when the configuration carries a separate initial ramdisk. + public var initialRamdisk: HashedFile? + } + + public enum CachingMode: String, Codable, Sendable { + case automatic + case cached + case uncached + } + + public enum SynchronizationMode: String, Codable, Sendable { + case full + case fsync + case none + } + + /// A virtio block device backed by a disk image. + /// + /// The image is its own source of truth, so it is not hashed. The caller is + /// responsible for preserving its contents and presenting it again. + /// `sha256` is only set for the initial filesystem, whose contents the + /// guest has already loaded into memory. + public struct Storage: Codable, Sendable, Equatable { + public var path: String + public var sizeInBytes: UInt64 + public var readOnly: Bool + public var cachingMode: CachingMode + public var synchronizationMode: SynchronizationMode + public var sha256: String? + } + + public enum NetworkAttachment: String, Codable, Sendable { + case nat + /// A vmnet network. The host side (subnet, gateway) is owned by the caller and must be + /// recreated before a restore. + case vmnet + } + + /// A virtio network device. `macAddress` is always the effective address, + /// including one VZ generated because none was requested. + public struct Network: Codable, Sendable, Equatable { + public var macAddress: String + public var attachment: NetworkAttachment + } + + /// A virtio-fs device. Only an empty `VZMultipleDirectoryShare` is allowed. + public struct DirectorySharing: Codable, Sendable, Equatable { + public var tag: String + } + + public enum SerialAttachment: String, Codable, Sendable { + case file + case fileHandle + } + + public struct SerialPort: Codable, Sendable, Equatable { + public var attachment: SerialAttachment + } + + public var version: Int + public var host: Host + public var cpuCount: Int + public var memorySize: UInt64 + public var platform: Platform + public var bootLoader: BootLoader + public var storage: [Storage] + public var network: [Network] + public var directorySharing: [DirectorySharing] + public var socketDeviceCount: Int + public var entropyDeviceCount: Int + public var serialPorts: [SerialPort] + + /// Build a snapshot from a validated VZ configuration. + /// + /// - Parameters: + /// - configuration: The configuration the machine is (or will be) built from. + /// - initialFilesystem: The initial filesystem image. It is attached as a + /// block device but is identified by content like the kernel. + /// - Throws: `ContainerizationError(.unsupported)` for anything that is not + /// explicitly allowed. + public init(configuration: VZVirtualMachineConfiguration, initialFilesystem: URL? = nil) throws { + self.version = Self.currentVersion + self.host = Host( + osVersion: ProcessInfo.processInfo.operatingSystemVersionString, + model: Self.sysctlString("hw.model") ?? "unknown" + ) + self.cpuCount = configuration.cpuCount + self.memorySize = configuration.memorySize + + guard let platform = configuration.platform as? VZGenericPlatformConfiguration else { + throw Self.unsupported("platform \(type(of: configuration.platform))") + } + self.platform = Platform( + machineIdentifier: platform.machineIdentifier.dataRepresentation, + nestedVirtualization: platform.isNestedVirtualizationEnabled + ) + // Nested virtualization state is not supported in v1. + if platform.isNestedVirtualizationEnabled { + throw Self.unsupported("nested virtualization") + } + + guard let loader = configuration.bootLoader as? VZLinuxBootLoader else { + throw Self.unsupported("boot loader \(configuration.bootLoader.map { "\(type(of: $0))" } ?? "nil")") + } + self.bootLoader = BootLoader( + kernel: try Self.hashedFile(loader.kernelURL), + commandLine: loader.commandLine, + initialRamdisk: try loader.initialRamdiskURL.map { try Self.hashedFile($0) } + ) + + self.storage = try configuration.storageDevices.map { device in + guard + let block = device as? VZVirtioBlockDeviceConfiguration, + let attachment = device.attachment as? VZDiskImageStorageDeviceAttachment + else { + throw Self.unsupported("storage device \(type(of: device)) with attachment \(type(of: device.attachment))") + } + // The identifier is guest-visible (virtio-blk serial, /dev/disk/by-id) and is not + // recorded, so only the default (empty) value is allowed. + guard block.blockDeviceIdentifier.isEmpty else { + throw Self.unsupported("storage device with block device identifier '\(block.blockDeviceIdentifier)'") + } + return try Self.storage(attachment, initialFilesystem: initialFilesystem) + } + + self.network = try configuration.networkDevices.map { device in + guard let device = device as? VZVirtioNetworkDeviceConfiguration else { + throw Self.unsupported("network device \(type(of: device))") + } + let attachment: NetworkAttachment + if device.attachment is VZNATNetworkDeviceAttachment { + attachment = .nat + } else if #available(macOS 26, *), device.attachment is VZVmnetNetworkDeviceAttachment { + attachment = .vmnet + } else { + throw Self.unsupported("network attachment \(device.attachment.map { "\(type(of: $0))" } ?? "nil")") + } + return Network(macAddress: device.macAddress.string, attachment: attachment) + } + + self.directorySharing = try configuration.directorySharingDevices.map { device in + guard + let device = device as? VZVirtioFileSystemDeviceConfiguration, + let share = device.share as? VZMultipleDirectoryShare + else { + throw Self.unsupported("directory sharing device \(type(of: device))") + } + // Host files cannot be snapshotted, so no share may be present. + guard share.directories.isEmpty else { + throw Self.unsupported("virtiofs device '\(device.tag)' with \(share.directories.count) share(s)") + } + return DirectorySharing(tag: device.tag) + } + + guard configuration.socketDevices.allSatisfy({ $0 is VZVirtioSocketDeviceConfiguration }) else { + throw Self.unsupported("socket device") + } + self.socketDeviceCount = configuration.socketDevices.count + + guard configuration.entropyDevices.allSatisfy({ $0 is VZVirtioEntropyDeviceConfiguration }) else { + throw Self.unsupported("entropy device") + } + self.entropyDeviceCount = configuration.entropyDevices.count + + self.serialPorts = try configuration.serialPorts.map { port in + guard port is VZVirtioConsoleDeviceSerialPortConfiguration else { + throw Self.unsupported("serial port \(type(of: port))") + } + switch port.attachment { + case is VZFileSerialPortAttachment: + return SerialPort(attachment: .file) + case is VZFileHandleSerialPortAttachment: + return SerialPort(attachment: .fileHandle) + default: + throw Self.unsupported("serial attachment \(port.attachment.map { "\(type(of: $0))" } ?? "nil")") + } + } + + // Default-deny: every remaining device category must be empty. + let denied: [(String, Int)] = [ + ("console devices", configuration.consoleDevices.count), + ("memory balloon devices", configuration.memoryBalloonDevices.count), + ("graphics devices", configuration.graphicsDevices.count), + ("keyboards", configuration.keyboards.count), + ("pointing devices", configuration.pointingDevices.count), + ("audio devices", configuration.audioDevices.count), + ("USB controllers", configuration.usbControllers.count), + ] + for (name, count) in denied where count > 0 { + throw Self.unsupported("\(count) \(name)") + } + } + + /// Read a snapshot written by ``save(to:)``. + /// + /// Use this before creating the instance that will restore a saved state, e.g. to learn the + /// `platform.machineIdentifier` it must be created with. + /// + /// - Parameter url: The snapshot file. + public static func load(from url: URL) throws -> VZVirtualMachineConfigurationSnapshot { + let data = try Data(contentsOf: url) + return try JSONDecoder().decode(VZVirtualMachineConfigurationSnapshot.self, from: data) + } + + /// Write the snapshot to the file at `url` as JSON. + /// + /// - Parameter url: The snapshot file. + public func save(to url: URL) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(self).write(to: url) + } + + /// Describe every way `current` differs from this (saved) snapshot in a way + /// that makes restoring this snapshot's machine state unsafe. + /// + /// File paths are not compared where content is identified by hash, and + /// disk image paths are never compared: a caller restoring from a cloned + /// image legitimately presents a different path. An empty result means the + /// two boards match. + public func differences(from current: VZVirtualMachineConfigurationSnapshot) -> [String] { + var out: [String] = [] + func check(_ name: String, _ saved: T, _ now: T) { + if saved != now { out.append("\(name): saved \(saved), current \(now)") } + } + + check("snapshot version", version, current.version) + check("host os version", host.osVersion, current.host.osVersion) + check("host model", host.model, current.host.model) + check("cpu count", cpuCount, current.cpuCount) + check("memory size", memorySize, current.memorySize) + check("machine identifier", platform.machineIdentifier, current.platform.machineIdentifier) + check("nested virtualization", platform.nestedVirtualization, current.platform.nestedVirtualization) + check("kernel sha256", bootLoader.kernel.sha256, current.bootLoader.kernel.sha256) + check("kernel command line", bootLoader.commandLine, current.bootLoader.commandLine) + check("initial ramdisk sha256", bootLoader.initialRamdisk?.sha256, current.bootLoader.initialRamdisk?.sha256) + + check("storage device count", storage.count, current.storage.count) + for (index, (saved, now)) in zip(storage, current.storage).enumerated() { + check("storage[\(index)] size", saved.sizeInBytes, now.sizeInBytes) + check("storage[\(index)] read-only", saved.readOnly, now.readOnly) + check("storage[\(index)] caching mode", saved.cachingMode, now.cachingMode) + check("storage[\(index)] synchronization mode", saved.synchronizationMode, now.synchronizationMode) + check("storage[\(index)] sha256", saved.sha256, now.sha256) + } + + check("network device count", network.count, current.network.count) + for (index, (saved, now)) in zip(network, current.network).enumerated() { + check("network[\(index)] mac address", saved.macAddress, now.macAddress) + check("network[\(index)] attachment", saved.attachment, now.attachment) + } + + check("directory sharing devices", directorySharing, current.directorySharing) + check("socket device count", socketDeviceCount, current.socketDeviceCount) + check("entropy device count", entropyDeviceCount, current.entropyDeviceCount) + check("serial ports", serialPorts, current.serialPorts) + return out + } +} + +extension VZVirtualMachineConfigurationSnapshot { + private static func unsupported(_ what: String) -> ContainerizationError { + ContainerizationError(.unsupported, message: "\(what) is not supported for save/restore") + } + + private static func storage( + _ attachment: VZDiskImageStorageDeviceAttachment, + initialFilesystem: URL? + ) throws -> Storage { + let url = attachment.url + let size = try FileManager.default.attributesOfItem(atPath: url.path)[.size] as? UInt64 ?? 0 + + let caching: CachingMode + switch attachment.cachingMode { + case .automatic: caching = .automatic + case .cached: caching = .cached + case .uncached: caching = .uncached + @unknown default: throw unsupported("disk caching mode \(attachment.cachingMode)") + } + + let sync: SynchronizationMode + switch attachment.synchronizationMode { + case .full: sync = .full + case .fsync: sync = .fsync + case .none: sync = .none + @unknown default: throw unsupported("disk synchronization mode \(attachment.synchronizationMode)") + } + + let isInitialFilesystem = initialFilesystem.map { $0.standardizedFileURL == url.standardizedFileURL } ?? false + return Storage( + path: url.path, + sizeInBytes: size, + readOnly: attachment.isReadOnly, + cachingMode: caching, + synchronizationMode: sync, + sha256: isInitialFilesystem ? try sha256(of: url) : nil + ) + } + + private static func hashedFile(_ url: URL) throws -> HashedFile { + HashedFile(path: url.path, sha256: try sha256(of: url)) + } + + /// Hex SHA-256 of a file, read in chunks so large kernels and images are not loaded whole. + private static func sha256(of url: URL) throws -> String { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + var hasher = SHA256() + while let chunk = try handle.read(upToCount: 1 << 20), !chunk.isEmpty { + hasher.update(data: chunk) + } + return hasher.finalize().map { String(format: "%02x", $0) }.joined() + } + + private static func sysctlString(_ name: String) -> String? { + var size = 0 + guard sysctlbyname(name, nil, &size, nil, 0) == 0, size > 0 else { + return nil + } + var buffer = [CChar](repeating: 0, count: size) + guard sysctlbyname(name, &buffer, &size, nil, 0) == 0 else { + return nil + } + return String(decoding: buffer.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) }, as: UTF8.self) + } +} +#endif diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 160c50267..3f8712ded 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -98,6 +98,8 @@ public final class VZVirtualMachineInstance: Sendable { // `vm` isn't used concurrently. private nonisolated(unsafe) let vm: VZVirtualMachine + // The configuration `vm` was created from. Kept so the board can be snapshotted on save. + private nonisolated(unsafe) let vzConfig: VZVirtualMachineConfiguration private let queue: DispatchQueue private let lock: AsyncLock private let group: EventLoopGroup @@ -105,17 +107,22 @@ public final class VZVirtualMachineInstance: Sendable { private let timeSyncer: TimeSyncer private let logger: Logger? + /// - Parameter checkpoint: The directory a virtual machine's state was saved to by + /// ``save(to:)``, when this instance is going to be restored from it with + /// ``restore(from:)``. VZ requires a restored machine to carry the saved machine's + /// identifier, so it is read from the saved configuration snapshot here. public convenience init( group: EventLoopGroup? = nil, logger: Logger? = nil, + checkpoint: URL? = nil, with: (inout Configuration) throws -> Void ) throws { var config = Configuration() try with(&config) - try self.init(group: group, config: config, logger: logger) + try self.init(group: group, config: config, logger: logger, checkpoint: checkpoint) } - init(group: EventLoopGroup?, config: Configuration, logger: Logger?) throws { + init(group: EventLoopGroup?, config: Configuration, logger: Logger?, checkpoint: URL? = nil) throws { if let group { self.ownsGroup = false self.group = group @@ -134,8 +141,23 @@ public final class VZVirtualMachineInstance: Sendable { let (mountAttachments, _) = try config.mountAttachments(allocator: allocator) self._mounts = Mutex(mountAttachments) + let vzConfig = try config.toVZ(allocator: allocator) + if let checkpoint { + let saved = try VZVirtualMachineConfigurationSnapshot.load( + from: checkpoint.appending(path: Self.bundleConfigurationName) + ) + guard + let platform = vzConfig.platform as? VZGenericPlatformConfiguration, + let identifier = VZGenericMachineIdentifier(dataRepresentation: saved.platform.machineIdentifier) + else { + throw ContainerizationError(.invalidArgument, message: "invalid machine identifier in \(checkpoint.path)") + } + platform.machineIdentifier = identifier + vzConfig.platform = platform + } + self.vzConfig = vzConfig self.vm = VZVirtualMachine( - configuration: try config.toVZ(allocator: allocator), + configuration: vzConfig, queue: self.queue ) @@ -252,6 +274,133 @@ extension VZVirtualMachineInstance: VirtualMachineInstance { } } + /// File names inside a saved state bundle. + static let bundleConfigurationName = "configuration.json" + static let bundleMachineStateName = "vm.state" + + /// Save the paused virtual machine's state to a new directory bundle at `url`. + /// + /// The bundle holds the machine state (`vm.state`) and a + /// ``VZVirtualMachineConfigurationSnapshot`` (`configuration.json`) describing the board the + /// state depends on. + /// + /// The caller owns the pause/resume lifecycle: call ``pause()`` first, preserve the backing + /// peripheral state (e.g. clone disk images) while the guest is quiescent, and then call + /// ``resume()`` or ``stop()``. This method does not change the virtual machine's run state. + /// + /// - Throws: ``ContainerizationError`` with `.exists` if `url` already exists, `.invalidState` + /// if the virtual machine is not paused, and `.unsupported` if the configuration contains + /// anything that cannot be saved. On failure no bundle is left behind. + public func save(to url: URL) async throws { + try await lock.withLock { _ in + let fm = FileManager.default + if fm.fileExists(atPath: url.path) { + throw ContainerizationError(.exists, message: "\(url.path) already exists") + } + guard self.vm.state == .paused else { + throw ContainerizationError(.invalidState, message: "vm must be paused to save its state") + } + + // Everything that can fail without touching the bundle goes first, so an + // unsupported configuration never leaves a directory behind. + try self.vzConfig.validateSaveRestoreSupport() + let snapshot = try VZVirtualMachineConfigurationSnapshot( + configuration: self.vzConfig, + initialFilesystem: self.config.initialFilesystem.map { URL(fileURLWithPath: $0.source) } + ) + + // The state file is a dump of guest memory. + try fm.createDirectory( + at: url, + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700] + ) + + do { + try await self.vm.saveMachineState( + to: url.appending(path: Self.bundleMachineStateName), + queue: self.queue + ) + try snapshot.save(to: url.appending(path: Self.bundleConfigurationName)) + } catch { + try? fm.removeItem(at: url) + throw error + } + } + } + + /// Restore the virtual machine from a state bundle written by ``save(to:)`` and run it. + /// + /// This is the counterpart of ``start()`` for a saved machine: the guest does not boot, it + /// continues from where it was saved. The instance must be freshly created (stopped) with + /// `checkpoint:` set to this bundle, from a configuration whose board matches the bundle's + /// snapshot — the same kernel, initial filesystem and device layout — and + /// the caller is responsible for the backing peripheral state (disk image contents, network + /// identity). Every difference found in the board is reported before VZ is asked to restore. + /// + /// Once the guest is running again the agent connection is re-established and the guest clock + /// is synchronized. Connections the host held across the save (vsock connections, listeners) + /// do not survive and must be re-established by the caller. + /// + /// - Throws: ``ContainerizationError`` with `.invalidState` if the virtual machine is not + /// stopped, `.notFound` if the bundle is incomplete, `.invalidArgument` if the board does + /// not match, and `.unsupported` if the configuration cannot be restored. If the restore + /// itself fails the virtual machine is left stopped. + public func restore(from url: URL) async throws { + try await lock.withLock { _ in + guard self.vm.state == .stopped else { + throw ContainerizationError( + .invalidState, + message: "virtual machine must be stopped to restore its state" + ) + } + + let stateURL = url.appending(path: Self.bundleMachineStateName) + guard FileManager.default.fileExists(atPath: stateURL.path) else { + throw ContainerizationError(.notFound, message: "no machine state at \(stateURL.path)") + } + let saved = try VZVirtualMachineConfigurationSnapshot.load( + from: url.appending(path: Self.bundleConfigurationName) + ) + + try self.vzConfig.validateSaveRestoreSupport() + let current = try VZVirtualMachineConfigurationSnapshot( + configuration: self.vzConfig, + initialFilesystem: self.config.initialFilesystem.map { URL(fileURLWithPath: $0.source) } + ) + let differences = saved.differences(from: current) + guard differences.isEmpty else { + throw ContainerizationError( + .invalidArgument, + message: "virtual machine does not match the saved state:\n" + differences.map { " - \($0)" }.joined(separator: "\n") + ) + } + + // VZ leaves the restored machine paused. + try await self.vm.restoreMachineState(from: stateURL, queue: self.queue) + try await self.vm.resume(queue: self.queue) + + let agent = try await Vminitd( + connection: try await self.vm.waitForAgent(queue: self.queue), + group: self.group + ) + + // The guest clock is as stale as the save. Fix it now instead of at the next tick. + var now = timeval() + if gettimeofday(&now, nil) == 0 { + do { + try await agent.setTime(sec: Int64(now.tv_sec), usec: Int32(now.tv_usec)) + } catch { + self.logger?.warning("failed to sync time after restore: \(error)") + } + } + + // Don't close our remote context as we are providing + // it to our time sync routine. + await self.timeSyncer.start(context: agent) + } + } + public func dialAgent() async throws -> Vminitd { try await lock.withLock { _ in do { diff --git a/Sources/Containerization/VZVirtualMachineManager.swift b/Sources/Containerization/VZVirtualMachineManager.swift index 4959bee42..b451cbf8e 100644 --- a/Sources/Containerization/VZVirtualMachineManager.swift +++ b/Sources/Containerization/VZVirtualMachineManager.swift @@ -46,7 +46,7 @@ public struct VZVirtualMachineManager: VirtualMachineManager { self.logger = logger } - public func create(config: some VMCreationConfig) throws -> any VirtualMachineInstance { + public func create(config: some VMCreationConfig, from savedState: URL? = nil) throws -> any VirtualMachineInstance { let vmConfig = config.configuration // Use nested virtualization if requested in config or set as default in manager @@ -61,6 +61,7 @@ public struct VZVirtualMachineManager: VirtualMachineManager { return try VZVirtualMachineInstance( group: self.group, logger: self.logger, + checkpoint: savedState, with: { instanceConfig in instanceConfig.cpus = cpus instanceConfig.memoryInBytes = memoryInBytes diff --git a/Sources/Containerization/VirtualMachineAgent.swift b/Sources/Containerization/VirtualMachineAgent.swift index 1bbaf7839..35737f91b 100644 --- a/Sources/Containerization/VirtualMachineAgent.swift +++ b/Sources/Containerization/VirtualMachineAgent.swift @@ -71,6 +71,13 @@ public protocol VirtualMachineAgent: Sendable { func waitProcess(id: String, containerID: String?, timeoutInSeconds: Int64?) async throws -> ExitStatus func deleteProcess(id: String, containerID: String?) async throws func closeProcessStdin(id: String, containerID: String?) async throws + /// Take down the relays that carry a process's stdio to the host, without disturbing the + /// process. The host must have closed its stdin connection first and must read stdout and + /// stderr until EOF. + func detachProcessIO(id: String, containerID: String?) async throws + /// Connect a process whose stdio was detached to the host again, over new vsock ports the + /// host is listening on. + func attachProcessIO(id: String, containerID: String?, stdinPort: UInt32?, stdoutPort: UInt32?, stderrPort: UInt32?) async throws // Networking func up(name: String, mtu: UInt32?) async throws @@ -91,6 +98,14 @@ extension VirtualMachineAgent { throw ContainerizationError(.unsupported, message: "closeProcessStdin") } + public func detachProcessIO(id: String, containerID: String?) async throws { + throw ContainerizationError(.unsupported, message: "detachProcessIO") + } + + public func attachProcessIO(id: String, containerID: String?, stdinPort: UInt32?, stdoutPort: UInt32?, stderrPort: UInt32?) async throws { + throw ContainerizationError(.unsupported, message: "attachProcessIO") + } + public func configureHosts(config: Hosts, location: String) async throws { throw ContainerizationError(.unsupported, message: "configureHosts") } diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index 302e97ae7..ad2664c3c 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -65,6 +65,14 @@ public protocol VirtualMachineInstance: Sendable { /// Resume the virtual machine. func resume() async throws + /// Save the state of the paused virtual machine to a new directory at `url`. + /// The virtual machine must already be paused; its run state is not changed. + func save(to url: URL) async throws + + /// Restore the state saved by ``save(to:)`` into this freshly created (stopped) virtual + /// machine and run it. The instance must have been created from the same saved state. + func restore(from url: URL) async throws + /// Hotplug a block device, returning the attached filesystem info. /// Throws if the VMM does not support hotplug or not available /// - Parameter block: The mount configuration for the block device to hotplug @@ -103,6 +111,12 @@ extension VirtualMachineInstance { public func resume() async throws { throw ContainerizationError(.unsupported, message: "resume") } + public func save(to url: URL) async throws { + throw ContainerizationError(.unsupported, message: "save") + } + public func restore(from url: URL) async throws { + throw ContainerizationError(.unsupported, message: "restore") + } public func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem { throw ContainerizationError(.unsupported, message: "hotplug not supported") } diff --git a/Sources/Containerization/VirtualMachineManager.swift b/Sources/Containerization/VirtualMachineManager.swift index f750bd522..69512d209 100644 --- a/Sources/Containerization/VirtualMachineManager.swift +++ b/Sources/Containerization/VirtualMachineManager.swift @@ -14,7 +14,28 @@ // limitations under the License. //===----------------------------------------------------------------------===// +import Foundation + /// A protocol to implement for virtual machine isolated containers. public protocol VirtualMachineManager: Sendable { - func create(config: some VMCreationConfig) async throws -> any VirtualMachineInstance + /// Create a virtual machine. + /// + /// - Parameters: + /// - config: The configuration of the virtual machine. + /// - savedState: The directory a virtual machine's state was saved to, or `nil` to create a + /// virtual machine that boots normally. The returned instance is prepared to be restored + /// from it but is not restored yet; call ``VirtualMachineInstance/restore(from:)`` on it. + /// A manager that cannot restore MUST throw an `.unsupported` ContainerizationError for a + /// non-nil value. + func create(config: some VMCreationConfig, from savedState: URL?) async throws -> any VirtualMachineInstance +} + +extension VirtualMachineManager { + /// Create a virtual machine that boots normally. + /// + /// Swift does not allow a default argument on a protocol requirement, so this stands in for + /// `savedState: nil`. + public func create(config: some VMCreationConfig) async throws -> any VirtualMachineInstance { + try await create(config: config, from: nil) + } } diff --git a/Sources/Containerization/Vminitd.swift b/Sources/Containerization/Vminitd.swift index 630ce93b0..836009f92 100644 --- a/Sources/Containerization/Vminitd.swift +++ b/Sources/Containerization/Vminitd.swift @@ -376,6 +376,41 @@ extension Vminitd: VirtualMachineAgent { _ = try await client.closeProcessStdin(request) } + public func detachProcessIO(id: String, containerID: String?) async throws { + let request = Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest.with { + $0.id = id + if let containerID { + $0.containerID = containerID + } + } + _ = try await client.detachProcessIO(request) + } + + public func attachProcessIO( + id: String, + containerID: String?, + stdinPort: UInt32?, + stdoutPort: UInt32?, + stderrPort: UInt32? + ) async throws { + let request = Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest.with { + $0.id = id + if let containerID { + $0.containerID = containerID + } + if let stdinPort { + $0.stdin = stdinPort + } + if let stdoutPort { + $0.stdout = stdoutPort + } + if let stderrPort { + $0.stderr = stderrPort + } + } + _ = try await client.attachProcessIO(request) + } + public func up(name: String, mtu: UInt32? = nil) async throws { let request = Com_Apple_Containerization_Sandbox_V3_IpLinkSetRequest.with { $0.interface = name diff --git a/Sources/Integration/CheckpointTests.swift b/Sources/Integration/CheckpointTests.swift new file mode 100644 index 000000000..d94a88665 --- /dev/null +++ b/Sources/Integration/CheckpointTests.swift @@ -0,0 +1,306 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +#if os(macOS) +import Containerization +import ContainerizationError +import ContainerizationExtras +import ContainerizationIO +import ContainerizationOS +import Foundation +import Synchronization + +/// A stdin whose input the test controls after the container has started. It can be streamed +/// more than once, because a checkpoint reconnects the process's stdin: input sent while nothing +/// is connected waits for the next connection. +private final class LiveStdin: ReaderStream, Sendable { + private struct State { + var generation = 0 + var continuation: AsyncStream.Continuation? + var pending: [Data] = [] + } + + private let state = Mutex(State()) + + func stream() -> AsyncStream { + let (stream, continuation) = AsyncStream.makeStream() + let generation = self.state.withLock { state -> Int in + state.generation += 1 + state.continuation = continuation + for data in state.pending { + continuation.yield(data) + } + state.pending.removeAll() + return state.generation + } + continuation.onTermination = { [weak self] _ in + self?.state.withLock { state in + if state.generation == generation { + state.continuation = nil + } + } + } + return stream + } + + func send(_ line: String) { + let data = Data((line + "\n").utf8) + self.state.withLock { state in + if let continuation = state.continuation { + continuation.yield(data) + } else { + state.pending.append(data) + } + } + } +} + +extension IntegrationSuite { + /// A process with state that only memory can carry: it numbers the lines it reads. + private static let lineCounter = ["/bin/sh", "-c", "n=0; while read l; do n=$((n+1)); echo \"$n:$l\"; done"] + + private func waitForOutput( + _ buffer: BufferWriter, + toContain expected: String, + timeoutSeconds: Int = 30 + ) async throws { + for _ in 0..<(timeoutSeconds * 10) { + if String(decoding: buffer.data, as: UTF8.self).contains(expected) { + return + } + try await Task.sleep(for: .milliseconds(100)) + } + throw IntegrationError.assert( + msg: "timed out waiting for '\(expected)' in output '\(String(decoding: buffer.data, as: UTF8.self))'") + } + + private func makeCheckpointScratchDirectory() throws -> URL { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent("checkpoint-test-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + return dir + } + + /// Guest side: detaching a process's stdio for a checkpoint and attaching it again must leave + /// the process running with working stdin and stdout, every time. + func testCheckpointReattachesProcessIO() async throws { + let id = "test-checkpoint-reattach-io" + let bs = try await bootstrap(id) + let scratch = try makeCheckpointScratchDirectory() + defer { try? FileManager.default.removeItem(at: scratch) } + + let stdin = LiveStdin() + let stdout = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.process.arguments = Self.lineCounter + config.process.stdin = stdin + config.process.stdout = stdout + config.bootLog = bs.bootLog + } + + do { + try await container.create() + try await container.start() + + stdin.send("a") + try await waitForOutput(stdout, toContain: "1:a") + + // The process keeps its state and its stdio across a checkpoint, more than once. + try await container.checkpoint(to: scratch.appendingPathComponent("first")) + stdin.send("b") + try await waitForOutput(stdout, toContain: "2:b") + + try await container.checkpoint(to: scratch.appendingPathComponent("second")) + stdin.send("c") + try await waitForOutput(stdout, toContain: "3:c") + + // A checkpoint that cannot be taken must not disturb the process either. + do { + try await container.checkpoint(to: scratch.appendingPathComponent("first")) + throw IntegrationError.assert(msg: "checkpoint to an existing directory should fail") + } catch let error as ContainerizationError { + guard error.code == .exists else { + throw error + } + } + stdin.send("d") + try await waitForOutput(stdout, toContain: "4:d") + + try await container.kill(.kill) + try await container.wait() + try await container.stop() + } catch { + try? await container.stop() + throw error + } + } + + /// The checkpoint directory carries the manifest and the machine state. + func testCheckpointWritesBundle() async throws { + let id = "test-checkpoint-bundle" + let bs = try await bootstrap(id) + let scratch = try makeCheckpointScratchDirectory() + defer { try? FileManager.default.removeItem(at: scratch) } + + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.process.arguments = ["/bin/sleep", "300"] + config.bootLog = bs.bootLog + } + + do { + try await container.create() + try await container.start() + + let bundle = scratch.appendingPathComponent("bundle") + let ranWhilePaused = Mutex(false) + try await container.checkpoint(to: bundle) { + ranWhilePaused.withLock { $0 = true } + } + guard ranWhilePaused.withLock({ $0 }) else { + throw IntegrationError.assert(msg: "whilePaused was not called") + } + + let manifest = try LinuxContainer.Checkpoint.load( + from: bundle.appendingPathComponent(LinuxContainer.Checkpoint.manifestName)) + guard manifest.id == id else { + throw IntegrationError.assert(msg: "checkpoint id '\(manifest.id)' != '\(id)'") + } + guard + FileManager.default.fileExists( + atPath: bundle.appendingPathComponent(LinuxContainer.Checkpoint.stateDirectoryName).path) + else { + throw IntegrationError.assert(msg: "checkpoint has no saved state") + } + + // A failure while paused removes the checkpoint and is reported, and the container survives. + struct PausedFailure: Swift.Error {} + let failed = scratch.appendingPathComponent("failed") + do { + try await container.checkpoint(to: failed) { throw PausedFailure() } + throw IntegrationError.assert(msg: "checkpoint should have failed") + } catch is PausedFailure {} + guard !FileManager.default.fileExists(atPath: failed.path) else { + throw IntegrationError.assert(msg: "failed checkpoint was left behind") + } + + try await container.kill(.kill) + try await container.wait() + try await container.stop() + } catch { + try? await container.stop() + throw error + } + } + + /// A real save and restore: the process in the restored machine carries on from the state it + /// had when the checkpoint was taken, with its stdio connected to the new streams. + func testCheckpointAndRestore() async throws { + let id = "test-checkpoint-restore" + let bs = try await bootstrap(id) + let scratch = try makeCheckpointScratchDirectory() + defer { try? FileManager.default.removeItem(at: scratch) } + + let stdin = LiveStdin() + let stdout = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.process.arguments = Self.lineCounter + config.process.stdin = stdin + config.process.stdout = stdout + config.bootLog = bs.bootLog + } + + let bundle = scratch.appendingPathComponent("bundle") + let rootfsCopy = scratch.appendingPathComponent("rootfs.ext4") + do { + try await container.create() + try await container.start() + + stdin.send("a") + stdin.send("b") + try await waitForOutput(stdout, toContain: "2:b") + + // The disk is not part of the checkpoint. Preserve it while the guest is frozen. + let rootfsPath = URL(fileURLWithPath: bs.rootfs.source) + try await container.checkpoint(to: bundle) { + try FileManager.default.copyItem(at: rootfsPath, to: rootfsCopy) + } + + // Nothing of the original machine is needed from here on. + try await container.kill(.kill) + try await container.wait() + try await container.stop() + } catch { + try? await container.stop() + throw error + } + + let restoredBootstrap = try await bootstrap("\(id)-restored") + let restoredStdin = LiveStdin() + let restoredStdout = BufferWriter() + let restoredRootfs = Containerization.Mount.block( + format: "ext4", + source: rootfsCopy.path, + destination: "/", + options: [] + ) + let restored = try LinuxContainer(id, rootfs: restoredRootfs, vmm: restoredBootstrap.vmm) { config in + config.process.arguments = Self.lineCounter + config.process.stdin = restoredStdin + config.process.stdout = restoredStdout + config.bootLog = restoredBootstrap.bootLog + } + + do { + // The init process is already running in the restored guest: no start(). + try await restored.create(from: bundle) + + // The counter is 2 in the saved memory; a fresh process would answer "1:c". + restoredStdin.send("c") + try await waitForOutput(restoredStdout, toContain: "3:c") + + try await restored.kill(.kill) + try await restored.wait() + try await restored.stop() + } catch { + try? await restored.stop() + throw error + } + } + + /// A restore against a root filesystem or checkpoint that is not usable fails cleanly. + func testRestoreFromMissingCheckpointFails() async throws { + let id = "test-checkpoint-restore-missing" + let bs = try await bootstrap(id) + let scratch = try makeCheckpointScratchDirectory() + defer { try? FileManager.default.removeItem(at: scratch) } + + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.process.arguments = ["/bin/true"] + config.bootLog = bs.bootLog + } + + var failed = false + do { + try await container.create(from: scratch.appendingPathComponent("does-not-exist")) + } catch { + failed = true + } + try? await container.stop() + guard failed else { + throw IntegrationError.assert(msg: "restoring from a missing checkpoint should fail") + } + } +} +#endif diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 6ce16125d..16a8822f6 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -680,6 +680,12 @@ struct IntegrationSuite: AsyncParsableCommand { // Nested virtualization (VZ-only feature) Test("nested virt", testNestedVirtualizationEnabled), + // VM checkpoint / restore (VZ-only) + Test("checkpoint reattaches process IO", testCheckpointReattachesProcessIO), + Test("checkpoint writes bundle", testCheckpointWritesBundle), + Test("checkpoint and restore", testCheckpointAndRestore), + Test("restore from missing checkpoint fails", testRestoreFromMissingCheckpointFails), + // Filesystem operations (TODO: promote to cross-platform once verified on CH) Test("container frozen ext4 clone", testFrozenExt4Clone), Test("container trim ext4 clone", testTrimExt4Clone), diff --git a/Tests/ContainerizationTests/LinuxContainerCheckpointTests.swift b/Tests/ContainerizationTests/LinuxContainerCheckpointTests.swift new file mode 100644 index 000000000..e350b7695 --- /dev/null +++ b/Tests/ContainerizationTests/LinuxContainerCheckpointTests.swift @@ -0,0 +1,65 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import Containerization + +struct LinuxContainerCheckpointTests { + private func temporaryFile() -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("checkpoint-\(UUID().uuidString).json") + } + + @Test func roundTripsThroughDisk() throws { + let url = temporaryFile() + defer { try? FileManager.default.removeItem(at: url) } + + let checkpoint = LinuxContainer.Checkpoint(id: "my-container") + try checkpoint.save(to: url) + + let loaded = try LinuxContainer.Checkpoint.load(from: url) + #expect(loaded == checkpoint) + #expect(loaded.id == "my-container") + #expect(loaded.version == LinuxContainer.Checkpoint.currentVersion) + } + + @Test func loadFailsForMissingFile() { + #expect(throws: (any Error).self) { + try LinuxContainer.Checkpoint.load(from: temporaryFile()) + } + } + + @Test func loadFailsForMalformedManifest() throws { + let url = temporaryFile() + defer { try? FileManager.default.removeItem(at: url) } + try Data("not json".utf8).write(to: url) + + #expect(throws: (any Error).self) { + try LinuxContainer.Checkpoint.load(from: url) + } + } + + @Test func manifestUsesStableKeys() throws { + let url = temporaryFile() + defer { try? FileManager.default.removeItem(at: url) } + try LinuxContainer.Checkpoint(id: "abc").save(to: url) + + let object = try JSONSerialization.jsonObject(with: Data(contentsOf: url)) as? [String: Any] + #expect(object?["id"] as? String == "abc") + #expect(object?["version"] as? Int == LinuxContainer.Checkpoint.currentVersion) + } +} diff --git a/Tests/ContainerizationTests/LinuxContainerTests.swift b/Tests/ContainerizationTests/LinuxContainerTests.swift index 5156e704c..54a9133b6 100644 --- a/Tests/ContainerizationTests/LinuxContainerTests.swift +++ b/Tests/ContainerizationTests/LinuxContainerTests.swift @@ -125,7 +125,7 @@ struct LinuxContainerTests { /// A `VirtualMachineManager` that cannot create anything. `LinuxContainer.init` /// only stores the manager, so nothing below should reach `create`. private struct UnusableVMM: VirtualMachineManager { - func create(config: some VMCreationConfig) async throws -> any VirtualMachineInstance { + func create(config: some VMCreationConfig, from savedState: URL?) async throws -> any VirtualMachineInstance { throw ContainerizationError(.unsupported, message: "test stub: no VM should be created here") } } @@ -210,7 +210,7 @@ struct LinuxContainerTests { captured.withLock { $0 } } - func create(config: some VMCreationConfig) async throws -> any VirtualMachineInstance { + func create(config: some VMCreationConfig, from savedState: URL?) async throws -> any VirtualMachineInstance { captured.withLock { $0 = config.configuration } throw ContainerizationError(.unsupported, message: "stub") } diff --git a/Tests/ContainerizationTests/VZVirtualMachineConfigurationSnapshotTests.swift b/Tests/ContainerizationTests/VZVirtualMachineConfigurationSnapshotTests.swift new file mode 100644 index 000000000..fefebfd75 --- /dev/null +++ b/Tests/ContainerizationTests/VZVirtualMachineConfigurationSnapshotTests.swift @@ -0,0 +1,188 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +#if os(macOS) +import ContainerizationError +import Foundation +import Testing +@preconcurrency import Virtualization + +@testable import Containerization + +struct VZVirtualMachineConfigurationSnapshotTests { + /// A snapshot with one storage device and one network device, built the way production does. + private func makeSnapshot() throws -> VZVirtualMachineConfigurationSnapshot { + let (config, dir) = try makeConfiguration() + defer { try? FileManager.default.removeItem(at: dir) } + + let disk = dir.appendingPathComponent("a.img") + try Data(count: 4096).write(to: disk) + let block = VZVirtioBlockDeviceConfiguration(attachment: try VZDiskImageStorageDeviceAttachment(url: disk, readOnly: false)) + config.storageDevices = [block] + + let nic = VZVirtioNetworkDeviceConfiguration() + nic.attachment = VZNATNetworkDeviceAttachment() + config.networkDevices = [nic] + return try VZVirtualMachineConfigurationSnapshot(configuration: config) + } + + @Test func roundTripsThroughDisk() throws { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("snapshot-\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: url) } + + let snapshot = try makeSnapshot() + try snapshot.save(to: url) + #expect(try VZVirtualMachineConfigurationSnapshot.load(from: url) == snapshot) + } + + @Test func identicalSnapshotsHaveNoDifferences() throws { + let base = try makeSnapshot() + #expect(base.differences(from: base).isEmpty) + } + + @Test func diskImagePathsAreNotCompared() throws { + let base = try makeSnapshot() + var current = base + current.storage[0].path = "/cloned/a.img" + #expect(base.differences(from: current).isEmpty) + } + + @Test func kernelPathIsNotComparedButContentIs() throws { + let base = try makeSnapshot() + var moved = base + moved.bootLoader.kernel.path = "/elsewhere/k" + #expect(base.differences(from: moved).isEmpty) + + var changed = base + changed.bootLoader.kernel.sha256 = "bb" + let diffs = base.differences(from: changed) + #expect(diffs.count == 1) + #expect(diffs.first?.contains("kernel sha256") == true) + } + + @Test func reportsEveryMismatch() throws { + let base = try makeSnapshot() + var current = base + current.cpuCount = 4 + current.memorySize = 2 << 30 + current.platform.machineIdentifier = Data([9]) + current.network[0].macAddress = "02:00:00:00:00:02" + current.network[0].attachment = .vmnet + current.storage[0].readOnly = true + current.serialPorts = [.init(attachment: .file)] + + let diffs = base.differences(from: current) + for name in ["cpu count", "memory size", "machine identifier", "mac address", "attachment", "read-only", "serial ports"] { + #expect(diffs.contains { $0.contains(name) }, "missing difference for \(name)") + } + } + + @Test func reportsStorageCountMismatch() throws { + let base = try makeSnapshot() + var current = base + current.storage = [] + let diffs = base.differences(from: current) + #expect(diffs.contains { $0.contains("storage device count") }) + } + + // MARK: Building from a VZ configuration + + private func makeConfiguration() throws -> (VZVirtualMachineConfiguration, URL) { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent("snapshot-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + let kernel = dir.appendingPathComponent("vmlinux") + try Data("kernel".utf8).write(to: kernel) + + let config = VZVirtualMachineConfiguration() + config.cpuCount = 2 + config.memorySize = 1 << 30 + config.platform = VZGenericPlatformConfiguration() + let loader = VZLinuxBootLoader(kernelURL: kernel) + loader.commandLine = "console=hvc0" + config.bootLoader = loader + return (config, dir) + } + + @Test func buildsFromMinimalConfiguration() throws { + let (config, dir) = try makeConfiguration() + defer { try? FileManager.default.removeItem(at: dir) } + + let snapshot = try VZVirtualMachineConfigurationSnapshot(configuration: config) + #expect(snapshot.cpuCount == 2) + #expect(snapshot.memorySize == 1 << 30) + #expect(snapshot.bootLoader.commandLine == "console=hvc0") + #expect(snapshot.bootLoader.kernel.sha256.count == 64) + #expect(snapshot.storage.isEmpty) + #expect(snapshot.network.isEmpty) + #expect(snapshot.platform.nestedVirtualization == false) + + // Building twice from the same configuration is deterministic. + let again = try VZVirtualMachineConfigurationSnapshot(configuration: config) + #expect(snapshot.differences(from: again).isEmpty) + } + + @Test func kernelHashMatchesContent() throws { + let (config, dir) = try makeConfiguration() + defer { try? FileManager.default.removeItem(at: dir) } + + let snapshot = try VZVirtualMachineConfigurationSnapshot(configuration: config) + #expect(snapshot.bootLoader.kernel.sha256.count == 64) + + try Data("different".utf8).write(to: URL(fileURLWithPath: snapshot.bootLoader.kernel.path)) + let changed = try VZVirtualMachineConfigurationSnapshot(configuration: config) + #expect(snapshot.differences(from: changed).contains { $0.contains("kernel sha256") }) + } + + @Test func rejectsUnsupportedDevices() throws { + let (config, dir) = try makeConfiguration() + defer { try? FileManager.default.removeItem(at: dir) } + + config.keyboards = [VZUSBKeyboardConfiguration()] + do { + _ = try VZVirtualMachineConfigurationSnapshot(configuration: config) + Issue.record("expected an unsupported error") + } catch let error as ContainerizationError { + #expect(error.code == .unsupported) + } + } + + @Test func rejectsDirectorySharesWithContents() throws { + let (config, dir) = try makeConfiguration() + defer { try? FileManager.default.removeItem(at: dir) } + + let fs = VZVirtioFileSystemDeviceConfiguration(tag: "share") + fs.share = VZMultipleDirectoryShare(directories: ["d": VZSharedDirectory(url: dir, readOnly: true)]) + config.directorySharingDevices = [fs] + do { + _ = try VZVirtualMachineConfigurationSnapshot(configuration: config) + Issue.record("expected an unsupported error") + } catch let error as ContainerizationError { + #expect(error.code == .unsupported) + } + } + + @Test func acceptsEmptyDirectoryShare() throws { + let (config, dir) = try makeConfiguration() + defer { try? FileManager.default.removeItem(at: dir) } + + let fs = VZVirtioFileSystemDeviceConfiguration(tag: "share") + fs.share = VZMultipleDirectoryShare(directories: [:]) + config.directorySharingDevices = [fs] + let snapshot = try VZVirtualMachineConfigurationSnapshot(configuration: config) + #expect(snapshot.directorySharing == [.init(tag: "share")]) + } +} +#endif diff --git a/vminitd/Sources/VminitdCore/ContainerProcess.swift b/vminitd/Sources/VminitdCore/ContainerProcess.swift index e25ba8ddf..10b6f2890 100644 --- a/vminitd/Sources/VminitdCore/ContainerProcess.swift +++ b/vminitd/Sources/VminitdCore/ContainerProcess.swift @@ -62,6 +62,16 @@ protocol ContainerProcess: AnyObject, Sendable { /// - Throws: If stdin cannot be closed func closeStdin() throws + /// Take down the relays that carry the process's stdio to the host without disturbing the + /// process, so the virtual machine can be paused. + /// - Throws: If the relays cannot be detached, or the process does not support it. + func detachIO() throws + + /// Connect a process whose stdio was detached to the host again. + /// - Parameter stdio: The new vsock ports the host is listening on. + /// - Throws: If the relays cannot be attached, or the process does not support it. + func attachIO(stdio: HostStdio) throws + /// Delete the container process and clean up resources /// - Throws: If cleanup fails func delete() async throws diff --git a/vminitd/Sources/VminitdCore/IOPair.swift b/vminitd/Sources/VminitdCore/IOPair.swift index 648b22dce..5ced9c32c 100644 --- a/vminitd/Sources/VminitdCore/IOPair.swift +++ b/vminitd/Sources/VminitdCore/IOPair.swift @@ -16,6 +16,7 @@ #if os(Linux) +import ContainerizationError import ContainerizationOS import Foundation import Logging @@ -38,6 +39,20 @@ final class IOPair: Sendable { var readFd: Int32? var writeFd: Int32? var registeredFds: [Int32] = [] + var ignoreHup = false + + /// Bytes that were taken from the source but could not be handed to the sink yet, which + /// `drain` must write before it reads anything else. Only a failed `detach` leaves any. + var overflow = Data() + + mutating func refillFromOverflow() { + let count = min(self.overflow.count, self.buffer.count) + guard count > 0 else { return } + _ = self.overflow.prefix(count).copyBytes(to: self.buffer) + self.overflow.removeFirst(count) + self.writeOffset = 0 + self.bufferedBytes = count + } mutating func drain(logger: Logger?) { guard !self.closed else { return } @@ -49,6 +64,9 @@ final class IOPair: Sendable { let writeTo = OSFile(fd: writeFd) while true { + if self.bufferedBytes == 0 && !self.overflow.isEmpty { + self.refillFromOverflow() + } if self.bufferedBytes > 0 { let view = UnsafeMutableBufferPointer( start: self.buffer.baseAddress?.advanced(by: self.writeOffset), @@ -77,6 +95,9 @@ final class IOPair: Sendable { break } } + if !self.overflow.isEmpty { + continue + } if self.readFinished { self.close(logger: logger) return @@ -144,10 +165,15 @@ final class IOPair: Sendable { readFrom: IOCloser, writeTo: IOCloser, reason: String, + pending: Data = Data(), logger: Logger? = nil ) { let buffer = UnsafeMutableBufferPointer.allocate(capacity: Int(getpagesize())) - self.io = Mutex(IO(from: readFrom, to: writeTo, buffer: buffer)) + var io = IO(from: readFrom, to: writeTo, buffer: buffer) + // Written to the sink before anything is read from the source. These are bytes an earlier + // relay took but never delivered, so they come first. + io.overflow = pending + self.io = Mutex(io) self.reason = reason self.logger = logger } @@ -164,26 +190,15 @@ final class IOPair: Sendable { let writeFd = fcntl(io.to.fileDescriptor, F_DUPFD_CLOEXEC, 0) guard writeFd >= 0 else { throw POSIXError.fromErrno() } io.writeFd = writeFd + io.ignoreHup = ignoreHup - try ProcessSupervisor.default.registerFd(writeFd, mask: .output) { mask in - self.io.withLock { io in - if mask.isHangup && !ignoreHup { - io.close(logger: self.logger) - return - } - io.drain(logger: self.logger) - } - } - io.registeredFds.append(writeFd) - try ProcessSupervisor.default.registerFd(readFd, mask: .input) { mask in - self.io.withLock { io in - if mask.isHangup && !ignoreHup { - io.closing = true - } - io.drain(logger: self.logger) - } + try self.arm(&io) + + // Edge-triggered epoll only reports a descriptor becoming ready. Don't leave + // seeded bytes waiting for an event that may already have come and gone. + if !io.overflow.isEmpty { + io.drain(logger: self.logger) } - io.registeredFds.append(readFd) } catch { io.close(logger: self.logger) throw error @@ -191,6 +206,135 @@ final class IOPair: Sendable { } } + /// Register the active descriptors with the process supervisor's epoll. + private func arm(_ io: inout IO) throws { + guard let readFd = io.readFd, let writeFd = io.writeFd else { + throw ContainerizationError(.invalidState, message: "relay for \(reason) has no descriptors") + } + try ProcessSupervisor.default.registerFd(writeFd, mask: .output) { mask in + self.io.withLock { io in + if mask.isHangup && !io.ignoreHup { + io.close(logger: self.logger) + return + } + io.drain(logger: self.logger) + } + } + io.registeredFds.append(writeFd) + try ProcessSupervisor.default.registerFd(readFd, mask: .input) { mask in + self.io.withLock { io in + if mask.isHangup && !io.ignoreHup { + io.closing = true + } + io.drain(logger: self.logger) + } + } + io.registeredFds.append(readFd) + } + + /// Largest amount of stdin that detaching will take out of the source. + private static let maxDetachBytes = 16 * 1024 * 1024 + + /// Stop relaying so the descriptors on the process side can be handed to a new relay, and + /// return the bytes that were taken from the source but never reached the sink. + /// + /// Nothing is written to the sink. After this returns the pair owns nothing: it is + /// unregistered from epoll, its duplicated descriptors are closed, and both ends were closed. + /// The socket end really closes. An end wrapped in `UnownedIOCloser` is left open for whoever + /// holds it. The caller must write the returned bytes first on the replacement relay. + /// + /// - Parameter readUntilEOF: Whether the source is a peer that has stopped sending and is + /// expected to be at end-of-stream (a socket carrying stdin). Everything it sent is read and + /// returned. Leave it `false` when the source is a process's output: that is never at EOF + /// while the process runs, and what the process has written but not been read stays in its + /// pipe for the next relay. + /// A relay that already ended (its socket went away or its source finished) has nothing to + /// release and nothing pending, so detaching it returns no bytes. + /// + /// - Throws: If `readUntilEOF` and the source did not reach EOF, failed, or sent more than + /// `maxDetachBytes`. Nothing is lost and the pair is left relaying exactly as before, with + /// what it had read queued to be written first. + func detach(readUntilEOF: Bool) throws -> Data { + try self.io.withLock { io in + if io.closed { + return Data() + } + guard let readFd = io.readFd, io.writeFd != nil else { + throw ContainerizationError(.invalidState, message: "relay for \(self.reason) was never started") + } + + // No more epoll events means no more drain, and so no more close. + for fd in io.registeredFds { + do { + try ProcessSupervisor.default.unregisterFd(fd) + } catch { + self.logger?.error("failed to delete fd from epoll \(fd): \(error)") + } + } + io.registeredFds.removeAll() + + // Everything read but not yet written, oldest first. + var pending = Data() + if io.bufferedBytes > 0, let base = io.buffer.baseAddress { + pending.append(UnsafeBufferPointer(start: base.advanced(by: io.writeOffset), count: io.bufferedBytes)) + } + pending.append(io.overflow) + io.bufferedBytes = 0 + io.writeOffset = 0 + io.overflow = Data() + + if readUntilEOF { + let reader = OSFile(fd: readFd) + var reachedEOF = false + var failure: Error? + readLoop: while true { + let result = reader.read(io.buffer) + if result.read > 0, let base = io.buffer.baseAddress { + pending.append(UnsafeBufferPointer(start: base, count: result.read)) + } + if pending.count > Self.maxDetachBytes { + failure = ContainerizationError( + .invalidState, + message: "\(self.reason) has more than \(Self.maxDetachBytes) bytes pending while detaching" + ) + break readLoop + } + switch result.action { + case .success: + continue readLoop + case .eof: + reachedEOF = true + break readLoop + case .again, .brokenPipe: + break readLoop + case .error(let errno): + failure = POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + break readLoop + } + } + + if !reachedEOF { + // Put everything back the way it was. + io.overflow = pending + do { + try self.arm(&io) + } catch { + io.close(logger: self.logger) + throw error + } + throw failure + ?? ContainerizationError( + .invalidState, + message: "\(self.reason) did not reach end of stream while detaching" + ) + } + } + + io.close(logger: self.logger) + return pending + } + } + func close() { self.io.withLock { io in self.logger?.info("closing relay for \(reason)") diff --git a/vminitd/Sources/VminitdCore/ManagedContainer.swift b/vminitd/Sources/VminitdCore/ManagedContainer.swift index d0a126f2b..b9b44fb60 100644 --- a/vminitd/Sources/VminitdCore/ManagedContainer.swift +++ b/vminitd/Sources/VminitdCore/ManagedContainer.swift @@ -251,6 +251,16 @@ extension ManagedContainer { try proc.closeStdin() } + func detachIO(execID: String) throws { + let proc = try self.getExecOrInit(execID: execID) + try proc.detachIO() + } + + func attachIO(execID: String, stdio: HostStdio) throws { + let proc = try self.getExecOrInit(execID: execID) + try proc.attachIO(stdio: stdio) + } + func deleteExec(id: String) async throws { try ensureExecExists(id) diff --git a/vminitd/Sources/VminitdCore/ManagedProcess.swift b/vminitd/Sources/VminitdCore/ManagedProcess.swift index ba4cd2d1e..6c9dc256f 100644 --- a/vminitd/Sources/VminitdCore/ManagedProcess.swift +++ b/vminitd/Sources/VminitdCore/ManagedProcess.swift @@ -34,6 +34,8 @@ final class ManagedProcess: ContainerProcess, Sendable { func close() throws func closeStdin() throws func closeAfterExec() throws + func detach() throws + func reattach(stdio: HostStdio) throws } // swiftlint: enable type_name @@ -339,6 +341,23 @@ extension ManagedProcess { try io.closeStdin() } + func detachIO() throws { + let io = self.state.withLock { $0.io } + try io.detach() + } + + func attachIO(stdio: HostStdio) throws { + let io = self.state.withLock { $0.io } + // Whether the process has a terminal was decided when it was created. + try io.reattach( + stdio: HostStdio( + stdin: stdio.stdin, + stdout: stdio.stdout, + stderr: stdio.stderr, + terminal: self.terminal + )) + } + func delete() async throws { // vmexec doesn't require explicit cleanup - the process is cleaned up // when it exits and IO is closed via setExit() diff --git a/vminitd/Sources/VminitdCore/RuncExecProcess.swift b/vminitd/Sources/VminitdCore/RuncExecProcess.swift index 6c5a4abd6..d1f8b0af3 100644 --- a/vminitd/Sources/VminitdCore/RuncExecProcess.swift +++ b/vminitd/Sources/VminitdCore/RuncExecProcess.swift @@ -403,6 +403,14 @@ final class RuncExecProcess: ContainerProcess, Sendable { try self.io.closeStdin() } + func detachIO() throws { + throw ContainerizationError(.unsupported, message: "detaching stdio is not supported for processes run through an OCI runtime") + } + + func attachIO(stdio: HostStdio) throws { + throw ContainerizationError(.unsupported, message: "attaching stdio is not supported for processes run through an OCI runtime") + } + func delete() async throws { // No `runc delete` counterpart: runc holds no per-exec state, so the // only thing to reclaim is the console socket. The process spec and pid diff --git a/vminitd/Sources/VminitdCore/RuncProcess.swift b/vminitd/Sources/VminitdCore/RuncProcess.swift index 447fd7934..3ce870134 100644 --- a/vminitd/Sources/VminitdCore/RuncProcess.swift +++ b/vminitd/Sources/VminitdCore/RuncProcess.swift @@ -283,6 +283,14 @@ final class RuncProcess: ContainerProcess, Sendable { try self.io.closeStdin() } + func detachIO() throws { + throw ContainerizationError(.unsupported, message: "detaching stdio is not supported for processes run through an OCI runtime") + } + + func attachIO(stdio: HostStdio) throws { + throw ContainerizationError(.unsupported, message: "attaching stdio is not supported for processes run through an OCI runtime") + } + func delete() async throws { let shouldDelete = self.state.withLock { state -> Bool in switch state.state { diff --git a/vminitd/Sources/VminitdCore/Server+GRPC.swift b/vminitd/Sources/VminitdCore/Server+GRPC.swift index 7f20185ac..af3f8e86b 100644 --- a/vminitd/Sources/VminitdCore/Server+GRPC.swift +++ b/vminitd/Sources/VminitdCore/Server+GRPC.swift @@ -1404,6 +1404,110 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ } } + public func detachProcessIO( + request: Com_Apple_Containerization_Sandbox_V3_DetachProcessIORequest, context: GRPCCore.ServerContext + ) async throws -> Com_Apple_Containerization_Sandbox_V3_DetachProcessIOResponse { + log.debug( + "detachProcessIO", + metadata: [ + "id": "\(request.id)", + "containerID": "\(request.containerID)", + ]) + + do { + if !request.hasContainerID { + throw ContainerizationError( + .invalidArgument, + message: "processes in the root of the vm not implemented" + ) + } + + let ctr = try await self.state.get(container: request.containerID) + + try await ctr.detachIO(execID: request.id) + + return .init() + } catch let err as ContainerizationError { + log.error( + "detachProcessIO", + metadata: [ + "id": "\(request.id)", + "containerID": "\(request.containerID)", + "error": "\(err)", + ]) + throw err.toRPCError(operation: "detachProcessIO: failed to detach process IO") + } catch { + log.error( + "detachProcessIO", + metadata: [ + "id": "\(request.id)", + "containerID": "\(request.containerID)", + "error": "\(error)", + ]) + throw RPCError( + code: .internalError, + message: "detachProcessIO: failed to detach process IO", + cause: error + ) + } + } + + public func attachProcessIO( + request: Com_Apple_Containerization_Sandbox_V3_AttachProcessIORequest, context: GRPCCore.ServerContext + ) async throws -> Com_Apple_Containerization_Sandbox_V3_AttachProcessIOResponse { + log.debug( + "attachProcessIO", + metadata: [ + "id": "\(request.id)", + "containerID": "\(request.containerID)", + ]) + + do { + if !request.hasContainerID { + throw ContainerizationError( + .invalidArgument, + message: "processes in the root of the vm not implemented" + ) + } + + let ctr = try await self.state.get(container: request.containerID) + + // Whether the process has a terminal is decided by the process, not by the request. + let stdio = HostStdio( + stdin: request.hasStdin ? request.stdin : nil, + stdout: request.hasStdout ? request.stdout : nil, + stderr: request.hasStderr ? request.stderr : nil, + terminal: false + ) + + try await ctr.attachIO(execID: request.id, stdio: stdio) + + return .init() + } catch let err as ContainerizationError { + log.error( + "attachProcessIO", + metadata: [ + "id": "\(request.id)", + "containerID": "\(request.containerID)", + "error": "\(err)", + ]) + throw err.toRPCError(operation: "attachProcessIO: failed to attach process IO") + } catch { + log.error( + "attachProcessIO", + metadata: [ + "id": "\(request.id)", + "containerID": "\(request.containerID)", + "error": "\(error)", + ]) + throw RPCError( + code: .internalError, + message: "attachProcessIO: failed to attach process IO", + cause: error + ) + } + } + public func ipLinkSet( request: Com_Apple_Containerization_Sandbox_V3_IpLinkSetRequest, context: GRPCCore.ServerContext ) async throws -> Com_Apple_Containerization_Sandbox_V3_IpLinkSetResponse { diff --git a/vminitd/Sources/VminitdCore/StandardIO.swift b/vminitd/Sources/VminitdCore/StandardIO.swift index 49d32c406..1aca86665 100644 --- a/vminitd/Sources/VminitdCore/StandardIO.swift +++ b/vminitd/Sources/VminitdCore/StandardIO.swift @@ -31,6 +31,24 @@ final class StandardIO: ManagedProcess.IO & Sendable { var stdinPipe: Pipe? var stdoutPipe: Pipe? var stderrPipe: Pipe? + + // The read ends of the process's output pipes. The relays borrow them (see + // `UnownedIOCloser`), so a relay ending because its socket went away does not + // close the pipe under a process that is still running. `close()` does. + var stdoutReader: FileHandle? + var stderrReader: FileHandle? + + // The write end of the stdin pipe. The relay only borrows it (see `UnownedIOCloser`), + // so a relay ending because its socket went away does not give the process an EOF. + // `closeStdin()` and `close()` do. The relay keeps its own duplicate of the descriptor + // until it has flushed, so closing this one first does not cut off buffered input. + var stdinWriter: FileHandle? + + // What `detach()` took out of the relays and could not deliver, oldest first. The process + // side of each relay is untouched, so these are written first when the relays come back. + var stdinPending = Data() + var stdoutPending = Data() + var stderrPending = Data() } private let log: Logger? @@ -63,9 +81,11 @@ final class StandardIO: ManagedProcess.IO & Sendable { let stdinSocket = try Socket(type: type, closeOnDeinit: false) try stdinSocket.connect() + $0.stdinWriter = inPipe.fileHandleForWriting + let pair = IOPair( readFrom: stdinSocket, - writeTo: inPipe.fileHandleForWriting, + writeTo: UnownedIOCloser(inPipe.fileHandleForWriting), reason: "StandardIO stdin", logger: log ) @@ -86,8 +106,10 @@ final class StandardIO: ManagedProcess.IO & Sendable { let stdoutSocket = try Socket(type: type, closeOnDeinit: false) try stdoutSocket.connect() + $0.stdoutReader = outPipe.fileHandleForReading + let pair = IOPair( - readFrom: outPipe.fileHandleForReading, + readFrom: UnownedIOCloser(outPipe.fileHandleForReading), writeTo: stdoutSocket, reason: "StandardIO stdout", logger: log @@ -109,8 +131,10 @@ final class StandardIO: ManagedProcess.IO & Sendable { let stderrSocket = try Socket(type: type, closeOnDeinit: false) try stderrSocket.connect() + $0.stderrReader = errPipe.fileHandleForReading + let pair = IOPair( - readFrom: errPipe.fileHandleForReading, + readFrom: UnownedIOCloser(errPipe.fileHandleForReading), writeTo: stderrSocket, reason: "StandardIO stderr", logger: log @@ -142,15 +166,164 @@ final class StandardIO: ManagedProcess.IO & Sendable { stderr.close() $0.stderr = nil } + + // The process is gone, so there is nobody left to give EOF to. + try? $0.stdinWriter?.close() + $0.stdinWriter = nil + + // The relays only borrowed these. The process is done with its output now. + try? $0.stdoutReader?.close() + $0.stdoutReader = nil + try? $0.stderrReader?.close() + $0.stderrReader = nil } } + /// Take the relays to the host down without disturbing the process, so the virtual machine + /// can be paused. The process keeps its pipe ends: output it writes waits in the pipe and + /// input waits in `stdinPending`, and the process blocks only if it fills a pipe. + /// + /// The host must have closed (or half-closed) its stdin socket before this, so that all of + /// stdin can be read to EOF. It must keep reading stdout and stderr until it sees EOF: each + /// relay closes its own socket here, after which nothing more arrives on it. + /// + /// stdin is detached first because it is the one that can fail. If it throws, nothing has + /// changed, and the relays carry on as before. + func detach() throws { + try self.state.withLock { + if let stdin = $0.stdin { + $0.stdinPending = try stdin.detach(readUntilEOF: true) + $0.stdin = nil + } + if let stdout = $0.stdout { + $0.stdoutPending = try stdout.detach(readUntilEOF: false) + $0.stdout = nil + } + if let stderr = $0.stderr { + $0.stderrPending = try stderr.detach(readUntilEOF: false) + $0.stderr = nil + } + } + } + + /// Connect new relays to the host for a process whose relays were taken down by `detach()`. + /// + /// `stdio` names the new vsock ports the host is listening on. Each relay is built over the + /// process's own pipe end, which `detach()` left open, and writes what `detach()` could not + /// deliver before it carries anything new. The streams in `stdio` must be the ones the + /// process was started with. + /// + /// If this throws, nothing is attached and the pending bytes are kept, except when a relay + /// fails after others started: those are closed and their pending bytes are lost. + func reattach(stdio: HostStdio) throws { + try self.state.withLock { + guard $0.stdin == nil, $0.stdout == nil, $0.stderr == nil else { + throw ContainerizationError(.invalidState, message: "stdio is already attached") + } + if stdio.stdin != nil && $0.stdinWriter == nil { + throw ContainerizationError(.invalidState, message: "process has no stdin to attach") + } + if stdio.stdout != nil && $0.stdoutReader == nil { + throw ContainerizationError(.invalidState, message: "process has no stdout to attach") + } + if stdio.stderr != nil && $0.stderrReader == nil { + throw ContainerizationError(.invalidState, message: "process has no stderr to attach") + } + + var sockets: [Socket] = [] + var started: [IOPair] = [] + do { + // Everything that can fail on the host's side first, before any relay moves bytes. + var stdinSocket: Socket? + var stdoutSocket: Socket? + var stderrSocket: Socket? + if let port = stdio.stdin { + stdinSocket = try Self.connectToHost(port: port) + sockets.append(stdinSocket!) + } + if let port = stdio.stdout { + stdoutSocket = try Self.connectToHost(port: port) + sockets.append(stdoutSocket!) + } + if let port = stdio.stderr { + stderrSocket = try Self.connectToHost(port: port) + sockets.append(stderrSocket!) + } + + var stdin: IOPair? + var stdout: IOPair? + var stderr: IOPair? + if let socket = stdinSocket, let writer = $0.stdinWriter { + let pair = IOPair( + readFrom: socket, + writeTo: UnownedIOCloser(writer), + reason: "StandardIO stdin", + pending: $0.stdinPending, + logger: log + ) + started.append(pair) + try pair.relay() + stdin = pair + } + if let socket = stdoutSocket, let reader = $0.stdoutReader { + let pair = IOPair( + readFrom: UnownedIOCloser(reader), + writeTo: socket, + reason: "StandardIO stdout", + pending: $0.stdoutPending, + logger: log + ) + started.append(pair) + try pair.relay() + stdout = pair + } + if let socket = stderrSocket, let reader = $0.stderrReader { + let pair = IOPair( + readFrom: UnownedIOCloser(reader), + writeTo: socket, + reason: "StandardIO stderr", + pending: $0.stderrPending, + logger: log + ) + started.append(pair) + try pair.relay() + stderr = pair + } + + $0.stdin = stdin + $0.stdout = stdout + $0.stderr = stderr + $0.stdinPending = Data() + $0.stdoutPending = Data() + $0.stderrPending = Data() + } catch { + for pair in started { + pair.close() + } + for socket in sockets { + try? socket.close() + } + throw error + } + } + } + + private static func connectToHost(port: UInt32) throws -> Socket { + let socket = try Socket(type: VsockType(port: port, cid: VsockType.hostCID), closeOnDeinit: false) + try socket.connect() + return socket + } + func closeStdin() throws { self.state.withLock { if let stdin = $0.stdin { stdin.close() $0.stdin = nil } + // This is what gives the process its EOF. If the relay is still flushing, its + // duplicate keeps the pipe open until it is done. + try? $0.stdinWriter?.close() + $0.stdinWriter = nil } } diff --git a/vminitd/Sources/VminitdCore/TerminalIO.swift b/vminitd/Sources/VminitdCore/TerminalIO.swift index b6a2006df..566191602 100644 --- a/vminitd/Sources/VminitdCore/TerminalIO.swift +++ b/vminitd/Sources/VminitdCore/TerminalIO.swift @@ -16,6 +16,7 @@ #if os(Linux) +import ContainerizationError import ContainerizationOS import Foundation import LCShim @@ -30,6 +31,11 @@ final class TerminalIO: ManagedProcess.IO & Sendable { var stdin: IOPair? var stdout: IOPair? var parent: Terminal? + + // What `detach()` took out of the relays and could not deliver, oldest first. The pty is + // untouched, so these are written first when the relays come back. + var stdinPending = Data() + var stdoutPending = Data() } private let log: Logger? @@ -110,7 +116,7 @@ final class TerminalIO: ManagedProcess.IO & Sendable { if let stdoutSocket = $0.stdoutSocket { let pair = IOPair( - readFrom: term, + readFrom: UnownedIOCloser(term), writeTo: stdoutSocket, reason: "TerminalIO stdout", logger: log @@ -144,11 +150,126 @@ final class TerminalIO: ManagedProcess.IO & Sendable { $0.stdoutSocket = nil } + // The relays only borrowed the pty master. Closing it is what hangs up the + // process's terminal, so it happens here, with the process, and never because + // a relay's socket went away. + try? $0.parent?.close() $0.parent = nil } } // NOP + /// Take the relays to the host down without disturbing the process, so the virtual machine + /// can be paused. The pty master stays open, so the process's terminal is not hung up: + /// output it writes waits in the pty and input waits in `stdinPending`. + /// + /// The host must have closed (or half-closed) its stdin socket before this, so that all of + /// stdin can be read to EOF. It must keep reading stdout until it sees EOF: the relay closes + /// its socket here, after which nothing more arrives on it. + /// + /// stdin is detached first because it is the one that can fail. If it throws, nothing has + /// changed, and the relays carry on as before. + func detach() throws { + try self.state.withLock { + if let stdin = $0.stdin { + $0.stdinPending = try stdin.detach(readUntilEOF: true) + $0.stdin = nil + // The relay closed it. + $0.stdinSocket = nil + } + if let stdout = $0.stdout { + $0.stdoutPending = try stdout.detach(readUntilEOF: false) + $0.stdout = nil + $0.stdoutSocket = nil + } + } + } + + /// Connect new relays to the host for a process whose relays were taken down by `detach()`. + /// + /// `stdio` names the new vsock ports the host is listening on. The relays are built over the + /// pty master, which `detach()` left open, and write what `detach()` could not deliver + /// before they carry anything new. + /// + /// If this throws, nothing is attached and the pending bytes are kept, except when the + /// second relay fails after the first started: that one is closed and its bytes are lost. + func reattach(stdio: HostStdio) throws { + try self.state.withLock { + guard $0.stdin == nil, $0.stdout == nil else { + throw ContainerizationError(.invalidState, message: "stdio is already attached") + } + guard let term = $0.parent else { + throw ContainerizationError(.invalidState, message: "process has no terminal to attach") + } + guard stdio.stderr == nil else { + throw ContainerizationError(.invalidArgument, message: "stderr can't be attached to a terminal") + } + + var sockets: [Socket] = [] + var started: [IOPair] = [] + do { + var stdinSocket: Socket? + var stdoutSocket: Socket? + if let port = stdio.stdin { + stdinSocket = try Self.connectToHost(port: port) + sockets.append(stdinSocket!) + } + if let port = stdio.stdout { + stdoutSocket = try Self.connectToHost(port: port) + sockets.append(stdoutSocket!) + } + + var stdin: IOPair? + var stdout: IOPair? + if let socket = stdinSocket { + let pair = IOPair( + readFrom: socket, + writeTo: UnownedIOCloser(term), + reason: "TerminalIO stdin", + pending: $0.stdinPending, + logger: log + ) + started.append(pair) + try pair.relay(ignoreHup: true) + stdin = pair + } + if let socket = stdoutSocket { + let pair = IOPair( + readFrom: UnownedIOCloser(term), + writeTo: socket, + reason: "TerminalIO stdout", + pending: $0.stdoutPending, + logger: log + ) + started.append(pair) + try pair.relay(ignoreHup: true) + stdout = pair + } + + $0.stdinSocket = stdinSocket + $0.stdoutSocket = stdoutSocket + $0.stdin = stdin + $0.stdout = stdout + $0.stdinPending = Data() + $0.stdoutPending = Data() + } catch { + for pair in started { + pair.close() + } + for socket in sockets { + try? socket.close() + } + throw error + } + } + } + + private static func connectToHost(port: UInt32) throws -> Socket { + let socket = try Socket(type: VsockType(port: port, cid: VsockType.hostCID), closeOnDeinit: false) + try socket.connect() + return socket + } + func closeAfterExec() throws {} func closeStdin() throws {