Skip to content

Commit 05b13f2

Browse files
committed
cluster: migrate registry from host container to in-cluster deployment
Replace the shared host-side func-registry container with Kubernetes-native resources deployed inside each Kind cluster: - Deployment (registry:2 with hostPort 5000 + emptyDir volume) - ClusterIP Service (port 5000) - Contour Ingress at registry.localtest.me Key changes: - registryAddr is now "registry.localtest.me" (was "localhost:50000") - containerd mirrors point at http://localhost:5000 via hostPort (was http://func-registry:5000 via Docker network DNS) - Each cluster owns its own registry, destroyed with kind delete cluster - Delete flow simplified: no shared container teardown, just host trust revert on last-cluster removal - Removed: ensureRegistry, registryStatus, teardownRegistry, setupPodmanMacOSForwarding, and all host-container lifecycle code
1 parent da6327e commit 05b13f2

3 files changed

Lines changed: 102 additions & 187 deletions

File tree

‎pkg/cluster/delete.go‎

Lines changed: 18 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -8,37 +8,29 @@ import (
88
"path/filepath"
99
)
1010

11-
// Delete removes a single func-managed dev cluster. The shared registry
12-
// container and the host's insecure-registries entry are removed only when
13-
// the *last* func-managed cluster is being torn down — other surviving
14-
// clusters keep using the shared registry.
11+
// Delete removes a func-managed dev cluster. The in-cluster registry is
12+
// destroyed automatically with the Kind cluster. Host-side trust config
13+
// (insecure-registries) is only reverted when this is the last func cluster,
14+
// since other surviving clusters share the same host entry.
1515
func Delete(ctx context.Context, cfg ClusterConfig, out io.Writer) error {
16-
// Set KUBECONFIG for child processes; restore the caller's value on return.
1716
defer setKubeconfig(cfg.Kubeconfig())()
1817

19-
status(out, "Deleting Cluster")
20-
21-
if err := run(ctx, out, "",
22-
cfg.kind(), "delete", "cluster",
23-
"--name="+cfg.Name,
24-
"--kubeconfig="+cfg.Kubeconfig()); err != nil {
25-
warnf(out, "failed to delete cluster %q: %v", cfg.Name, err)
18+
if _, err := os.Stat(cfg.Kubeconfig()); err == nil {
19+
status(out, "Deleting Cluster")
20+
if err := run(ctx, out, "",
21+
cfg.kind(), "delete", "cluster",
22+
"--name="+cfg.Name,
23+
"--kubeconfig="+cfg.Kubeconfig()); err != nil {
24+
warnf(out, "failed to delete cluster %q: %v", cfg.Name, err)
25+
}
26+
_ = os.RemoveAll(filepath.Dir(cfg.Kubeconfig()))
2627
}
2728

28-
// Remove this cluster's kubeconfig dir so the "last cluster?" check
29-
// below reflects the post-delete state.
30-
_ = os.RemoveAll(filepath.Dir(cfg.Kubeconfig()))
31-
32-
remaining := List()
33-
if len(remaining) == 0 {
34-
status(out, "Last func cluster removed; tearing down shared registry")
35-
teardownRegistry(ctx, cfg, out)
36-
if !cfg.SkipRegistryConfig {
37-
revertHostRegistry(out)
38-
}
39-
} else {
40-
fmt.Fprintf(out, "Registry left running; shared with %d other func-managed cluster(s): %v\n",
41-
len(remaining), remaining)
29+
if remaining := List(); len(remaining) > 0 {
30+
fmt.Fprintf(out, "Other func-managed cluster(s) still running: %v; leaving host registry config in place.\n",
31+
remaining)
32+
} else if !cfg.SkipRegistryConfig {
33+
revertHostRegistry(out)
4234
}
4335

4436
fmt.Fprintf(out, "%s Downloaded container images are not automatically removed.\n", red("NOTE:"))

‎pkg/cluster/kubernetes.go‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ const kindConfigTemplate = `kind: Cluster
1616
apiVersion: kind.x-k8s.io/v1alpha4
1717
nodes:
1818
- role: control-plane
19-
image: kindest/node:%[1]s
19+
image: kindest/node:%s
2020
extraPortMappings:
2121
- containerPort: 80
2222
hostPort: 80
@@ -29,14 +29,14 @@ nodes:
2929
listenAddress: "127.0.0.1"
3030
containerdConfigPatches:
3131
- |-
32-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:%[2]d"]
33-
endpoint = ["http://%[3]s:%[4]d"]
34-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.default.svc.cluster.local:%[4]d"]
35-
endpoint = ["http://%[3]s:%[4]d"]
32+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.localtest.me"]
33+
endpoint = ["http://localhost:5000"]
34+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.default.svc.cluster.local:5000"]
35+
endpoint = ["http://localhost:5000"]
3636
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."ghcr.io"]
37-
endpoint = ["http://%[3]s:%[4]d"]
37+
endpoint = ["http://localhost:5000"]
3838
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"]
39-
endpoint = ["http://%[3]s:%[4]d"]
39+
endpoint = ["http://localhost:5000"]
4040
`
4141

4242
const metalLBPoolTemplate = `apiVersion: metallb.io/v1beta1
@@ -59,8 +59,7 @@ func installKubernetes(ctx context.Context, cfg ClusterConfig, out io.Writer) er
5959
start := time.Now()
6060
status(out, "Allocating")
6161

62-
kindConfig := fmt.Sprintf(kindConfigTemplate,
63-
kindNodeVersion, registryHostPort, registryContainerName, registryContainerPort)
62+
kindConfig := fmt.Sprintf(kindConfigTemplate, kindNodeVersion)
6463

6564
err := run(ctx, out, kindConfig,
6665
cfg.kind(), "create", "cluster",

‎pkg/cluster/registry.go‎

Lines changed: 76 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -15,34 +15,84 @@ import (
1515
"time"
1616
)
1717

18-
const (
19-
// registryContainerName is the fixed name of the shared local registry
20-
// container. All func-managed dev clusters on the host share this
21-
// single registry.
22-
registryContainerName = "func-registry"
23-
// registryHostPort is the TCP port the registry is published on to the
24-
// host; it also appears in the host container engine's
25-
// insecure-registries list.
26-
registryHostPort = 50000
27-
// registryContainerPort is the port the `registry:2` image listens on
28-
// inside the container.
29-
registryContainerPort = 5000
30-
)
31-
32-
// registryAddr is the host-side address used in daemon.json /
33-
// registries.conf and in the in-cluster `local-registry-hosting` ConfigMap.
34-
// Derived from registryHostPort so the two can't drift apart.
35-
var registryAddr = fmt.Sprintf("localhost:%d", registryHostPort)
18+
const registryAddr = "registry.localtest.me"
3619

37-
// installRegistry starts the shared local container registry, configures
38-
// host-side trust for it, and applies the in-cluster ConfigMap + Service
39-
// the kind cluster uses to reach it.
20+
// installRegistry deploys the container registry as in-cluster Kubernetes
21+
// resources (Deployment + ClusterIP Service + Contour Ingress), configures
22+
// host-side trust, and applies the local-registry-hosting ConfigMap.
4023
func installRegistry(ctx context.Context, cfg ClusterConfig, out io.Writer) error {
4124
start := time.Now()
4225
status(out, "Creating Registry")
4326

44-
if err := ensureRegistry(ctx, cfg, out); err != nil {
45-
return err
27+
registryManifest := `apiVersion: apps/v1
28+
kind: Deployment
29+
metadata:
30+
name: registry
31+
namespace: default
32+
spec:
33+
replicas: 1
34+
selector:
35+
matchLabels:
36+
app: registry
37+
template:
38+
metadata:
39+
labels:
40+
app: registry
41+
spec:
42+
containers:
43+
- name: registry
44+
image: registry:2
45+
ports:
46+
- containerPort: 5000
47+
hostPort: 5000
48+
volumeMounts:
49+
- name: registry-data
50+
mountPath: /var/lib/registry
51+
volumes:
52+
- name: registry-data
53+
emptyDir: {}
54+
---
55+
apiVersion: v1
56+
kind: Service
57+
metadata:
58+
name: registry
59+
namespace: default
60+
spec:
61+
selector:
62+
app: registry
63+
ports:
64+
- port: 5000
65+
targetPort: 5000
66+
---
67+
apiVersion: networking.k8s.io/v1
68+
kind: Ingress
69+
metadata:
70+
name: registry
71+
namespace: default
72+
spec:
73+
ingressClassName: contour-external
74+
rules:
75+
- host: registry.localtest.me
76+
http:
77+
paths:
78+
- backend:
79+
service:
80+
name: registry
81+
port:
82+
number: 5000
83+
pathType: Prefix
84+
path: /
85+
`
86+
87+
if err := applyManifest(ctx, out, cfg, registryManifest); err != nil {
88+
return fmt.Errorf("applying registry resources: %w", err)
89+
}
90+
91+
if err := run(ctx, out, "",
92+
cfg.kubectl(), "wait",
93+
"--for=condition=Available", "deployment/registry",
94+
"-n", "default", "--timeout=5m"); err != nil {
95+
return fmt.Errorf("waiting for registry deployment: %w", err)
4696
}
4797

4898
if !cfg.SkipRegistryConfig {
@@ -51,114 +101,25 @@ func installRegistry(ctx context.Context, cfg ClusterConfig, out io.Writer) erro
51101
}
52102
}
53103

54-
// ConfigMap for local registry hosting
55-
registryConfigMap := fmt.Sprintf(`apiVersion: v1
104+
registryConfigMap := `apiVersion: v1
56105
kind: ConfigMap
57106
metadata:
58107
name: local-registry-hosting
59108
namespace: kube-public
60109
data:
61110
localRegistryHosting.v1: |
62-
host: "localhost:%d"
111+
host: "registry.localtest.me"
63112
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
64-
`, registryHostPort)
113+
`
65114

66115
if err := applyManifest(ctx, out, cfg, registryConfigMap); err != nil {
67116
return fmt.Errorf("applying registry configmap: %w", err)
68117
}
69118

70-
// ExternalName service for in-cluster access
71-
registrySvc := fmt.Sprintf(`apiVersion: v1
72-
kind: Service
73-
metadata:
74-
name: registry
75-
namespace: default
76-
spec:
77-
type: ExternalName
78-
externalName: %s
79-
`, registryContainerName)
80-
81-
if err := applyManifest(ctx, out, cfg, registrySvc); err != nil {
82-
return fmt.Errorf("applying registry service: %w", err)
83-
}
84-
85119
success(out, "Registry", time.Since(start))
86120
return nil
87121
}
88122

89-
// ensureRegistry makes sure the shared func-registry container exists, is
90-
// running, and is attached to the `kind` docker network. Idempotent; safe
91-
// to call whether or not another func-managed cluster has already
92-
// provisioned it.
93-
//
94-
// Scope is intentionally just the container lifecycle — host-side trust
95-
// config is the orchestrator's responsibility (see installRegistry).
96-
// TODO: should we rename the kind network to "kind-func" to avoid collision
97-
// with a developer's own kind usage?
98-
func ensureRegistry(ctx context.Context, cfg ClusterConfig, out io.Writer) error {
99-
exists, running, networked, err := registryStatus(ctx, cfg)
100-
if err != nil {
101-
return err
102-
}
103-
if !exists {
104-
portMap := fmt.Sprintf("127.0.0.1:%d:%d", registryHostPort, registryContainerPort)
105-
// --net=kind attaches at creation time, so no separate network
106-
// connect is needed on this path.
107-
return run(ctx, out, "",
108-
cfg.ContainerEngine(), "run",
109-
"-d",
110-
"--restart=always",
111-
"-p", portMap,
112-
"--net=kind",
113-
"--name", registryContainerName,
114-
"registry:2")
115-
}
116-
if !running {
117-
if err := run(ctx, out, "", cfg.ContainerEngine(), "start", registryContainerName); err != nil {
118-
return fmt.Errorf("starting registry: %w", err)
119-
}
120-
}
121-
if !networked {
122-
if err := run(ctx, out, "", cfg.ContainerEngine(), "network", "connect", "kind", registryContainerName); err != nil {
123-
return fmt.Errorf("connecting registry to kind network: %w", err)
124-
}
125-
}
126-
return nil
127-
}
128-
129-
// registryStatus inspects the shared registry container. A non-nil err
130-
// means the engine itself errored in a way that isn't "no such object";
131-
// callers should surface it. A (false, false, false, nil) return means
132-
// "container is absent" or the inspect was unparseable — either way,
133-
// treated as fresh state.
134-
func registryStatus(ctx context.Context, cfg ClusterConfig) (exists, running, networked bool, err error) {
135-
output, inspectErr := runOutput(ctx, cfg.ContainerEngine(), "container", "inspect", registryContainerName)
136-
if inspectErr != nil {
137-
// `container inspect <missing>` exits non-zero; so does any real
138-
// engine failure. Treat both as "not present" — a real failure
139-
// resurfaces on the next engine command.
140-
return false, false, false, nil
141-
}
142-
var results []struct {
143-
State struct {
144-
Running bool `json:"Running"`
145-
} `json:"State"`
146-
NetworkSettings struct {
147-
Networks map[string]json.RawMessage `json:"Networks"`
148-
} `json:"NetworkSettings"`
149-
}
150-
if err := json.Unmarshal([]byte(output), &results); err != nil {
151-
return false, false, false, fmt.Errorf("parsing inspect output: %w", err)
152-
}
153-
if len(results) == 0 {
154-
return false, false, false, nil
155-
}
156-
exists = true
157-
running = results[0].State.Running
158-
_, networked = results[0].NetworkSettings.Networks["kind"]
159-
return
160-
}
161-
162123
// configureHostRegistry configures the host's container engine(s) to
163124
// trust the shared local registry. Mirror of revertHostRegistry; called
164125
// at most once per installRegistry (the caller gates on
@@ -265,7 +226,6 @@ func configurePodmanHTTP(out io.Writer) error {
265226
return fmt.Errorf("writing config: %w", err)
266227
}
267228
fmt.Fprintf(out, "Successfully created Podman registry configuration for %s\n", registryAddr)
268-
setupPodmanMacOSForwarding(out)
269229
return nil
270230
}
271231

@@ -304,34 +264,9 @@ func configurePodmanHTTP(out io.Writer) error {
304264
return err
305265
}
306266

307-
setupPodmanMacOSForwarding(out)
308267
return nil
309268
}
310269

311-
// setupPodmanMacOSForwarding sets up SSH port forwarding on macOS so the
312-
// Podman VM can access the host's local registry. Idempotent: detects an
313-
// existing backgrounded ssh forwarder and skips rather than spawning
314-
// another (which would leak or fail to bind).
315-
func setupPodmanMacOSForwarding(out io.Writer) {
316-
if runtime.GOOS != "darwin" {
317-
return
318-
}
319-
forward := fmt.Sprintf("-L %d:localhost:%d", registryHostPort, registryHostPort)
320-
if err := exec.Command("pgrep", "-f", forward).Run(); err == nil {
321-
fmt.Fprintln(out, "Podman VM port forwarding already active; skipping")
322-
return
323-
}
324-
fmt.Fprintln(out, "Setting up port forwarding for Podman VM to access registry...")
325-
port := fmt.Sprintf("%d", registryHostPort)
326-
cmd := exec.Command("podman", "machine", "ssh", "--",
327-
"-L", port+":localhost:"+port, "-N", "-f")
328-
cmd.Stdout = out
329-
cmd.Stderr = out
330-
if err := cmd.Run(); err != nil {
331-
fmt.Fprintf(out, "Warning: port forwarding setup failed: %v\n", err)
332-
}
333-
}
334-
335270
// warnNix detects Nix and emits configuration guidance.
336271
func warnNix(out io.Writer) {
337272
if !hasCommand("nix") && !hasCommand("nixos-rebuild") {
@@ -366,17 +301,6 @@ The configuration required is adding the following to registries.conf:
366301
}
367302
}
368303

369-
// Teardowns
370-
// ---------
371-
372-
// teardownRegistry stops and removes the shared registry container. Called
373-
// from Delete when the last func-managed cluster is being removed.
374-
func teardownRegistry(ctx context.Context, cfg ClusterConfig, out io.Writer) {
375-
if err := run(ctx, out, "", cfg.ContainerEngine(), "rm", "-f", registryContainerName); err != nil {
376-
fmt.Fprintf(out, "Warning: failed to remove registry container %q: %v\n", registryContainerName, err)
377-
}
378-
}
379-
380304
// revertHostRegistry removes the insecure-registries entry we added at
381305
// create time and the matching podman stanza. Best-effort: per-engine
382306
// failures warn but don't abort the delete.

0 commit comments

Comments
 (0)