Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions Sources/ContainerK8s/Commands/K8sCreate.swift
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,10 @@ public struct K8sCreate: AsyncParsableCommand {
@Option(help: "Node image reference (default: \(K8sHelper.nodeImage))")
var nodeImage: String = K8sHelper.nodeImage

@Option(name: .long, help: "Optional path to a CNI manifest to apply.")
@Option(
name: .long,
help: "Optional path to a CNI manifest to apply, or \"none\" (case-insensitive) to skip installing a CNI."
)
var cni: String?

public func run() async throws {
Expand All @@ -62,11 +65,7 @@ public struct K8sCreate: AsyncParsableCommand {
throw ContainerizationError(.invalidArgument, message: "cluster name \(name) is not a valid container ID")
}

if let cni {
guard FileManager.default.fileExists(atPath: cni) else {
throw ContainerizationError(.invalidArgument, message: "CNI manifest not found at \(cni)")
}
}
let cniSelection = try CNISelection.resolve(cni)

let isTTY = isatty(FileHandle.standardError.fileDescriptor) == 1
let progressConfig = try ProgressConfig(
Expand Down Expand Up @@ -112,11 +111,13 @@ public struct K8sCreate: AsyncParsableCommand {
try await K8sHelper.bootstrapControlPlane(
nodeID: name, apiServerSANs: sans, advertiseAddress: vmIP,
schedulable: provisioner.roles.contains(StandardRoles.worker),
cniManifestPath: cni,
cni: cniSelection,
client: client, log: log)

progress.set(description: "Waiting for cluster to be ready")
try await K8sHelper.waitForReady(containerId: name, client: client, log: log)
if cniSelection != .none {
progress.set(description: "Waiting for cluster to be ready")
try await K8sHelper.waitForReady(containerId: name, client: client, log: log)
}

progress.set(description: "Writing kubeconfig")
do {
Expand Down
8 changes: 7 additions & 1 deletion Sources/ContainerK8s/Commands/K8sStart.swift
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ public struct K8sStart: AsyncParsableCommand {
@Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))")
var name: String = K8sHelper.defaultName

@Flag(name: .long, help: "Wait for the node to report Ready before returning")
var wait: Bool = false

public func run() async throws {
LoggingSystem.bootstrap { _ in StderrLogHandler() }
let log = Logger(label: K8sHelper.pluginName)
Expand All @@ -57,7 +60,10 @@ public struct K8sStart: AsyncParsableCommand {
try io.closeAfterStart()

try await K8sHelper.waitForNodeBooted(containerId: name, client: client, log: log)
try await K8sHelper.waitForReady(containerId: name, client: client, log: log)

if wait {
try await K8sHelper.waitForReady(containerId: name, client: client, log: log)
}

do {
let fqdn = await K8sHelper.detectFQDN(name: name)
Expand Down
27 changes: 27 additions & 0 deletions Sources/ContainerK8s/K8sHelper.swift
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,31 @@ import ContainerizationOS
import Foundation
import Logging

// MARK: - CNISelection

/// Resolved interpretation of the `--cni` create flag: use the bundled kindnet
/// CNI, skip CNI installation entirely, or apply a manifest at a given path.
enum CNISelection: Equatable {
case kindnet
case none
case manifest(URL)

/// Parses a raw `--cni` flag value. Case-insensitively matches
/// `K8sHelper.noCNIName` ("none") as the sentinel for skipping CNI
/// installation; anything else is treated as a manifest path, which must
/// exist. `nil` resolves to `.kindnet`.
static func resolve(_ raw: String?) throws -> CNISelection {
guard let raw else { return .kindnet }
if raw.caseInsensitiveCompare(K8sHelper.noCNIName) == .orderedSame {
return .none
}
guard FileManager.default.fileExists(atPath: raw) else {
throw ContainerizationError(.invalidArgument, message: "CNI manifest not found at \(raw)")
}
return .manifest(URL(fileURLWithPath: raw))
}
}

// MARK: - K8sHelper

public struct K8sHelper {
Expand All @@ -44,6 +69,8 @@ public struct K8sHelper {
public static let ignorePreflightErrors =
"Swap,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables"
static let podSubnet = "10.244.0.0/16"
/// Sentinel value for `--cni` that skips installing a CNI entirely.
static let noCNIName = "none"
// kubeadm default service subnet; must stay in sync if ClusterConfiguration.serviceSubnet is ever set.
static let serviceSubnet = "10.96.0.0/12"

Expand Down
41 changes: 24 additions & 17 deletions Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ extension K8sHelper {

static func bootstrapControlPlane(
nodeID: String, apiServerSANs: [String], advertiseAddress: String,
schedulable: Bool, cniManifestPath: String? = nil, client: ContainerClient, log: Logger
schedulable: Bool, cni: CNISelection = .kindnet, client: ContainerClient, log: Logger
) async throws {
let configYAML = initConfigYAML(advertiseAddress: advertiseAddress, certSANs: apiServerSANs)
var r = try await execCapture(
Expand Down Expand Up @@ -73,14 +73,22 @@ extension K8sHelper {
arguments: ["taint", "nodes", "--all", "node-role.kubernetes.io/control-plane-"])
}

log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"])
let manifest = try await loadCNIManifest(path: cniManifestPath, log: log)
let apply = "\(kubeconfigEnv) kubectl apply -f - <<'EOF'\n\(manifest)\nEOF"
r = try await execCapture(
containerId: nodeID, executable: "/bin/sh",
arguments: ["-c", apply], client: client)
guard r.code == 0 else {
throw ContainerizationError(.internalError, message: "apply CNI failed on \(nodeID): \(r.output)")
switch cni {
case .none:
log.info("Skipping CNI installation", metadata: ["node": "\(nodeID)"])
case .kindnet:
log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"])
let manifest = try await loadKindnetManifest(log: log)
try await applyCNIManifest(manifest, nodeID: nodeID, client: client)
case .manifest(let url):
log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"])
let manifest: String
do {
manifest = try String(contentsOf: url, encoding: .utf8)
} catch {
throw ContainerizationError(.invalidArgument, message: "failed to read CNI manifest at \(url.path): \(error)")
}
try await applyCNIManifest(manifest, nodeID: nodeID, client: client)
}
}

Expand All @@ -101,15 +109,14 @@ extension K8sHelper {
return (token: parts[tokenIdx + 1], caCertHash: parts[hashIdx + 1])
}

static func loadCNIManifest(path: String?, log: Logger) async throws -> String {
if let path {
do {
return try String(contentsOfFile: path, encoding: .utf8)
} catch {
throw ContainerizationError(.invalidArgument, message: "failed to read CNI manifest at \(path): \(error)")
}
private static func applyCNIManifest(_ manifest: String, nodeID: String, client: ContainerClient) async throws {
let apply = "\(kubeconfigEnv) kubectl apply -f - <<'EOF'\n\(manifest)\nEOF"
let r = try await execCapture(
containerId: nodeID, executable: "/bin/sh",
arguments: ["-c", apply], client: client)
guard r.code == 0 else {
throw ContainerizationError(.internalError, message: "apply CNI failed on \(nodeID): \(r.output)")
}
return try await loadKindnetManifest(log: log)
}

private static func loadKindnetManifest(log: Logger) async throws -> String {
Expand Down
34 changes: 34 additions & 0 deletions Tests/IntegrationTests/K8s/ContainerFixture+K8sHelpers.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//

import ContainerTestSupport
import Foundation

extension ContainerFixture {
@discardableResult
func kubectl(node: String, args: [String]) throws -> (output: String, status: Int32) {
print("[k8s] kubectl \(args.joined(separator: " ")) (node: \(node))")
let result = try self.run(["exec", node, "kubectl"] + args)
print("[k8s] kubectl exit=\(result.status) output=\(result.output.prefix(120).trimmingCharacters(in: .whitespacesAndNewlines))")
let filteredStderr = result.error.components(separatedBy: "\n")
.filter { !$0.contains("Warning! Running debug build") && !$0.isEmpty }
.joined(separator: "\n")
if !filteredStderr.isEmpty {
print("[k8s] kubectl stderr: \(filteredStderr.prefix(300))")
}
return (result.output, result.status)
}
}
52 changes: 52 additions & 0 deletions Tests/IntegrationTests/K8s/TestK8sCNISerial.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//

import ContainerTestSupport
import Foundation
import Testing

@Suite(.serialized)
struct TestK8sCNISerial {

@Test func testCreateWithCNINoneSkipsCNIInstallation() async throws {
try await ContainerFixture.with { f in
let name = "k8s-\(f.testID)"
f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) }

try f.restoreWarmupImage(.kindestNodeV1_35_5)
print("[k8s-cni] k8s create --name \(name) --cni none")
let result = try f.run(["k8s", "create", "--name", name, "--cni", "none"])
print("[k8s-cni] k8s create exit=\(result.status)")
if result.status != 0 {
print("[k8s-cni] k8s create stderr: \(result.error)")
f.dumpNodeDiagnostics(node: name)
}

try result.check()
#expect(result.output.contains(name))
#expect(try f.getContainerStatus(name) == "running")

// No CNI manifest was applied, so kube-system has no CNI daemonset and the node never reaches Ready.
let (podsOutput, podsStatus) = try f.kubectl(node: name, args: ["get", "pods", "-n", "kube-system", "--no-headers"])
#expect(podsStatus == 0)
#expect(!podsOutput.lowercased().contains("kindnet"))

let (nodesOutput, nodesStatus) = try f.kubectl(node: name, args: ["get", "nodes", "--no-headers"])
#expect(nodesStatus == 0)
#expect(nodesOutput.contains("NotReady"))
}
}
}
46 changes: 16 additions & 30 deletions Tests/IntegrationTests/K8s/TestK8sNetworkingSerial.swift
Original file line number Diff line number Diff line change
Expand Up @@ -37,31 +37,17 @@ struct TestK8sNetworkingSerial {
print("=== END ENV DUMP ===")
}

@discardableResult
private func kubectl(_ f: ContainerFixture, node: String, args: [String]) throws -> (output: String, status: Int32) {
print("[k8s-net] kubectl \(args.joined(separator: " ")) (node: \(node))")
let result = try f.run(["exec", node, "kubectl"] + args)
print("[k8s-net] kubectl exit=\(result.status) output=\(result.output.prefix(120).trimmingCharacters(in: .whitespacesAndNewlines))")
let filteredStderr = result.error.components(separatedBy: "\n")
.filter { !$0.contains("Warning! Running debug build") && !$0.isEmpty }
.joined(separator: "\n")
if !filteredStderr.isEmpty {
print("[k8s-net] kubectl stderr: \(filteredStderr.prefix(300))")
}
return (result.output, result.status)
}

private func waitForPod(_ f: ContainerFixture, node: String, podName: String, timeoutSeconds: Int) throws {
print("[k8s-net] waitForPod \(podName) on \(node) (timeout=\(timeoutSeconds)s)")
let (_, status) = try kubectl(
f, node: node,
let (_, status) = try f.kubectl(
node: node,
args: [
"wait", "--for=condition=Ready", "pod/\(podName)", "--timeout=\(timeoutSeconds)s",
])
guard status == 0 else {
let (podStatus, _) = try kubectl(f, node: node, args: ["get", "pod", podName, "--no-headers"])
let (podStatus, _) = try f.kubectl(node: node, args: ["get", "pod", podName, "--no-headers"])
print("[k8s-net] pod \(podName) status: \(podStatus.trimmingCharacters(in: .whitespacesAndNewlines))")
let (podDesc, _) = try kubectl(f, node: node, args: ["describe", "pod", podName])
let (podDesc, _) = try f.kubectl(node: node, args: ["describe", "pod", podName])
print("[k8s-net] pod \(podName) describe:\n\(podDesc.prefix(1000))")
throw CommandError.executionFailed("pod \(podName) did not become ready within \(timeoutSeconds)s")
}
Expand Down Expand Up @@ -94,8 +80,8 @@ struct TestK8sNetworkingSerial {
if loadResult.status != 0 { print("[k8s-net] k8s load-image stderr: \(loadResult.error)") }
#expect(loadResult.status == 0)

let (_, createStatus) = try kubectl(
f, node: name,
let (_, createStatus) = try f.kubectl(
node: name,
args: [
"run", "test-pod",
"--image=\(Self.testImage.rawValue)",
Expand All @@ -107,8 +93,8 @@ struct TestK8sNetworkingSerial {

try waitForPod(f, node: name, podName: "test-pod", timeoutSeconds: 300)

let (output, execStatus) = try kubectl(
f, node: name,
let (output, execStatus) = try f.kubectl(
node: name,
args: [
"exec", "test-pod", "--", "echo", "hello",
])
Expand Down Expand Up @@ -143,8 +129,8 @@ struct TestK8sNetworkingSerial {
#expect(loadResult.status == 0)

// Server: alpine busybox httpd serving a static response.
let (_, serverStatus) = try kubectl(
f, node: name,
let (_, serverStatus) = try f.kubectl(
node: name,
args: [
"run", "server",
"--image=\(Self.testImage.rawValue)",
Expand All @@ -158,16 +144,16 @@ struct TestK8sNetworkingSerial {
try waitForPod(f, node: name, podName: "server", timeoutSeconds: 300)

// Expose server as a ClusterIP service.
let (_, exposeStatus) = try kubectl(
f, node: name,
let (_, exposeStatus) = try f.kubectl(
node: name,
args: [
"expose", "pod", "server", "--port=8080", "--name=server-svc",
])
#expect(exposeStatus == 0)

// Client pod that stays alive so we can exec into it.
let (_, clientStatus) = try kubectl(
f, node: name,
let (_, clientStatus) = try f.kubectl(
node: name,
args: [
"run", "client",
"--image=\(Self.testImage.rawValue)",
Expand All @@ -180,8 +166,8 @@ struct TestK8sNetworkingSerial {

// Reach server via the service DNS name — exercises CoreDNS + kube-proxy.
print("[k8s-net] wget from client to server-svc:8080")
let (response, wgetStatus) = try kubectl(
f, node: name,
let (response, wgetStatus) = try f.kubectl(
node: name,
args: [
"exec", "client", "--", "sh", "-c",
"sleep 2 && wget -qO- http://server-svc:8080",
Expand Down
Loading