From 6b077285f5d5789a0e5a07210703f2c46fa515a4 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 10 Jul 2026 12:21:54 -0500 Subject: [PATCH 1/5] test(e2e): stand up e2e on the shared test-infra federation foundation Replace compute's bespoke Kind + Karmada standup with a remote Taskfile include of the shared test-infra federation foundation, so a laptop and CI run the identical path. The foundation now owns creating the three Kind clusters, installing and tuning Karmada, and joining the POP cells; compute keeps only its service-specific layer (Karmada federation config, CRD installs, the management/cell federation credentials, image build, and the deploy overlays). Drop the absorbed plumbing: the karmadactl/helm-repo tool installs, the cluster-create and kubeconfig-export tasks, the Karmada install/tune/join tasks, and the four hack/e2e config and kubeconfig-rewrite scripts. Read the federation kubeconfigs the foundation writes under .test-infra/kubeconfigs/federation, and source the cell operators' hub address from the foundation-provided karmada-internal.yaml instead of re-deriving the hub container IP. Gitignore the foundation self-clone and Task's remote cache. --- .gitignore | 6 + Taskfile.yaml | 497 +++++---------------------- hack/e2e/kind-control-plane.yaml | 41 --- hack/e2e/kind-pop.yaml | 25 -- hack/e2e/make-internal-kubeconfig.sh | 60 ---- hack/e2e/patch-cluster-secret.sh | 90 ----- 6 files changed, 97 insertions(+), 622 deletions(-) delete mode 100644 hack/e2e/kind-control-plane.yaml delete mode 100644 hack/e2e/kind-pop.yaml delete mode 100755 hack/e2e/make-internal-kubeconfig.sh delete mode 100755 hack/e2e/patch-cluster-secret.sh diff --git a/.gitignore b/.gitignore index d5cc564d..592fed60 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,9 @@ bin/ # Local e2e environment artefacts (Kind kubeconfigs, etc.) tmp/ + +# test-infra federation foundation: its self-clone and the kubeconfigs it writes +.test-infra/ + +# Task's remote-taskfile cache (populated with TASK_X_REMOTE_TASKFILES=1) +.task/ diff --git a/Taskfile.yaml b/Taskfile.yaml index 2ae2eab1..2a2260d8 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -3,40 +3,47 @@ version: '3' # ─── Variables ────────────────────────────────────────────────────────────── vars: - # Karmada Helm chart version to install (karmada-charts/karmada) - KARMADA_VERSION: v1.16.0 - - # karmadactl CLI version for cluster registration - KARMADACTL_VERSION: v1.16.0 + # Pin for the shared test-infra federation foundation. The remote Taskfile + # include below (and the foundation's own self-clone) both resolve against + # this ref. Must be moved to a release tag before this branch merges — see the + # re-pin chain in the PR description. + TEST_INFRA_REF: feat/federation-topology # Chainsaw version for e2e testing (kyverno/chainsaw) CHAINSAW_VERSION: v0.2.15 # Local tool directory (mirrors Makefile convention) LOCALBIN: '{{.ROOT_DIR}}/bin' - KARMADACTL: '{{.ROOT_DIR}}/bin/karmadactl' CHAINSAW: '{{.ROOT_DIR}}/bin/chainsaw' - # Kind cluster names + # Kind cluster names. These are also the Karmada member/hub cluster names the + # federation foundation registers, and they name the kubeconfig files the + # foundation writes into KUBECONFIG_DIR (.yaml). KIND_CONTROL_PLANE: compute-control-plane KIND_POP_DFW: compute-pop-dfw KIND_POP_ORD: compute-pop-ord - # All cluster names (for CRD installation loops) - KIND_ALL_CLUSTERS: '{{.KIND_CONTROL_PLANE}} {{.KIND_POP_DFW}} {{.KIND_POP_ORD}}' - # Dev image tag built locally and side-loaded into every Kind cluster. This # replaces the ghcr.io/datum-cloud/compute:latest reference baked into # config/base/manager so e2e never pulls from a registry. IMAGE: compute:e2e-dev - # Working directory for e2e artefacts (gitignored) + # Working directory for compute-minted e2e artefacts (gitignored): the + # federation kubeconfigs the operators mount and per-run diagnostics. The + # cluster/Karmada kubeconfigs themselves are owned by the foundation and land + # in KUBECONFIG_DIR below. E2E_DIR: '{{.ROOT_DIR}}/tmp/e2e' - KUBECONFIG_DIR: '{{.ROOT_DIR}}/tmp/e2e/kubeconfigs' - # Fixed NodePort for the Karmada API server. - # The Kind management cluster is created with an extraPortMapping for this port - # so it is reachable at https://localhost:32443 from the developer's machine. + # Federation kubeconfig directory written by the test-infra foundation. The + # remote include self-clones test-infra into .test-infra/ and drops the hub, + # member, and Karmada kubeconfigs here; compute reads them from the same path. + KUBECONFIG_DIR: '{{.ROOT_DIR}}/.test-infra/kubeconfigs/federation' + + # Fixed NodePort for the Karmada API server, passed through to the foundation. + # The hub Kind cluster is created with an extraPortMapping for this port so it + # is reachable at https://localhost: from the developer's machine. + # Override (e.g. KARMADA_API_NODEPORT=32643) to dodge a host-port collision + # with another local cluster; the value propagates into the include. KARMADA_API_NODEPORT: "32443" # In-cluster Service address of the Karmada API server. The management @@ -48,6 +55,25 @@ vars: # Namespace the operators deploy into (matches config/overlays/* namespace:). COMPUTE_NAMESPACE: compute-system +# ─── Includes ─────────────────────────────────────────────────────────────── + +includes: + # Shared test-infra federation foundation. Standing up the Kind clusters, + # installing + tuning Karmada, and joining the cells all live here now, so the + # same path runs on a laptop and in CI. Requires TASK_X_REMOTE_TASKFILES=1 in + # the environment and `--yes` on first/non-interactive runs (Task's + # remote-taskfile trust prompt). + infra: + taskfile: https://raw.githubusercontent.com/datum-cloud/test-infra/{{.TEST_INFRA_REF}}/Taskfile.yml + vars: + # The foundation's ensure-repo self-clone must match the include ref. + REPO_REF: '{{.TEST_INFRA_REF}}' + # Compute's topology: a management/control-plane hub and two POP cells + # labelled with their city code. + FEDERATION_HUB_CLUSTER: '{{.KIND_CONTROL_PLANE}}' + FEDERATION_MEMBERS: '{{.KIND_POP_DFW}}=dfw {{.KIND_POP_ORD}}=ord' + KARMADA_API_NODEPORT: '{{.KARMADA_API_NODEPORT}}' + # ─── Tasks ────────────────────────────────────────────────────────────────── tasks: @@ -71,32 +97,31 @@ tasks: echo "╔══════════════════════════════════════════════════════════╗" echo "║ e2e environment ready — operators deployed in-cluster ║" echo "╠══════════════════════════════════════════════════════════╣" - echo "║ Control plane: {{.KUBECONFIG_DIR}}/control-plane.yaml" + echo "║ Control plane: {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml" echo "║ Karmada API: {{.KUBECONFIG_DIR}}/karmada.yaml" - echo "║ POP DFW: {{.KUBECONFIG_DIR}}/pop-dfw.yaml" - echo "║ POP ORD: {{.KUBECONFIG_DIR}}/pop-ord.yaml" + echo "║ POP DFW: {{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml" + echo "║ POP ORD: {{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml" echo "╠══════════════════════════════════════════════════════════╣" echo "║ Inspect operators: ║" - echo "║ kubectl --kubeconfig {{.KUBECONFIG_DIR}}/control-plane.yaml -n {{.COMPUTE_NAMESPACE}} get deploy" + echo "║ kubectl --kubeconfig {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml -n {{.COMPUTE_NAMESPACE}} get deploy" echo "╚══════════════════════════════════════════════════════════╝" silent: false e2e:env:up: - desc: "Create the Kind clusters, install Karmada, register cells, install CRDs (no operators)" + desc: "Stand up the federation foundation, configure Karmada, install CRDs (no operators)" cmds: - task: e2e:tools - - task: e2e:clusters:create - - task: e2e:karmada:install + # The shared foundation creates the Kind clusters, installs+tunes Karmada, + # and joins the POP cells with their city-code labels — everything the + # compute harness used to build by hand. + - task: infra:federation-up - task: e2e:karmada:configure - - task: e2e:karmada:join-clusters - task: e2e:crds:install e2e:down: desc: "Tear down the local e2e environment" cmds: - - kind delete cluster --name {{.KIND_CONTROL_PLANE}} 2>/dev/null || true - - kind delete cluster --name {{.KIND_POP_DFW}} 2>/dev/null || true - - kind delete cluster --name {{.KIND_POP_ORD}} 2>/dev/null || true + - task: infra:federation-down - rm -rf {{.E2E_DIR}} - cmd: echo "✓ e2e environment torn down" silent: false @@ -112,7 +137,7 @@ tasks: # Serial keeps the federation path uncontended on a constrained runner. # Override for a faster local run with `task e2e:test -- --parallel N`. - | - KUBECONFIG={{.KUBECONFIG_DIR}}/control-plane.yaml \ + KUBECONFIG={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ {{.CHAINSAW}} test \ --config test/e2e/chainsaw-config.yaml \ --parallel 1 \ @@ -124,7 +149,7 @@ tasks: deps: [e2e:tools:chainsaw] cmds: - | - KUBECONFIG={{.KUBECONFIG_DIR}}/control-plane.yaml \ + KUBECONFIG={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ {{.CHAINSAW}} test \ --config test/e2e/chainsaw-config.yaml \ {{.CLI_ARGS}} \ @@ -135,30 +160,11 @@ tasks: # ════════════════════════════════════════════════════════════════════════ e2e:tools: - desc: "Install e2e-specific tooling (karmadactl, chainsaw, helm repo)" + desc: "Install e2e-specific tooling (chainsaw)" cmds: - - task: e2e:tools:karmadactl + # kind/kubectl/helm/karmadactl are installed by the federation foundation's + # own ensure-tools; chainsaw is the only compute-specific e2e tool left. - task: e2e:tools:chainsaw - - task: e2e:tools:helm-repo - - e2e:tools:karmadactl: - desc: "Download karmadactl {{.KARMADACTL_VERSION}}" - cmds: - - mkdir -p {{.LOCALBIN}} - - | - if [ ! -f "{{.KARMADACTL}}" ]; then - OS=$(uname -s | tr '[:upper:]' '[:lower:]') - ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') - URL="https://github.com/karmada-io/karmada/releases/download/{{.KARMADACTL_VERSION}}/karmadactl-${OS}-${ARCH}.tgz" - echo "Downloading karmadactl {{.KARMADACTL_VERSION}} (${OS}/${ARCH}) from ${URL}..." - curl -sSfL "${URL}" | tar -xz -C {{.LOCALBIN}} karmadactl - chmod +x {{.KARMADACTL}} - echo "karmadactl installed → {{.KARMADACTL}}" - else - echo "karmadactl already present at {{.KARMADACTL}}" - fi - status: - - test -f {{.KARMADACTL}} e2e:tools:chainsaw: desc: "Download chainsaw {{.CHAINSAW_VERSION}}" @@ -179,207 +185,10 @@ tasks: status: - test -f {{.CHAINSAW}} - e2e:tools:helm-repo: - desc: "Add/update karmada-charts Helm repository" - cmds: - - | - if ! helm repo list 2>/dev/null | grep -q karmada-charts; then - helm repo add karmada-charts https://raw.githubusercontent.com/karmada-io/karmada/master/charts - echo "Added karmada-charts Helm repository" - fi - helm repo update karmada-charts - status: - - helm repo list 2>/dev/null | grep -q karmada-charts - - # ════════════════════════════════════════════════════════════════════════ - # Kind cluster management - # ════════════════════════════════════════════════════════════════════════ - - e2e:clusters:create: - desc: "Create all Kind clusters (idempotent)" - cmds: - - mkdir -p {{.E2E_DIR}} - # Render the control-plane kind config with the configured Karmada NodePort. - # The committed template pins 32443; overriding KARMADA_API_NODEPORT (for - # example to dodge a host-port collision with another local cluster) is - # substituted through here so the port has a single source of truth. - - sed 's/32443/{{.KARMADA_API_NODEPORT}}/g' hack/e2e/kind-control-plane.yaml > {{.E2E_DIR}}/kind-control-plane.yaml - # Management / control-plane cell cluster — needs extraPortMappings for - # the Karmada API server NodePort so it is accessible at localhost:. - - task: _e2e:cluster:create - vars: - CLUSTER_NAME: "{{.KIND_CONTROL_PLANE}}" - KIND_CONFIG: "{{.E2E_DIR}}/kind-control-plane.yaml" - # POP cell clusters — same busy-host control-plane tuning, no port mapping. - - task: _e2e:cluster:create - vars: - CLUSTER_NAME: "{{.KIND_POP_DFW}}" - KIND_CONFIG: hack/e2e/kind-pop.yaml - - task: _e2e:cluster:create - vars: - CLUSTER_NAME: "{{.KIND_POP_ORD}}" - KIND_CONFIG: hack/e2e/kind-pop.yaml - - mkdir -p {{.KUBECONFIG_DIR}} - - task: _e2e:kubeconfigs:export - - _e2e:cluster:create: - internal: true - cmds: - - | - if kind get clusters 2>/dev/null | grep -qx '{{.CLUSTER_NAME}}'; then - echo "Kind cluster '{{.CLUSTER_NAME}}' already exists — skipping" - else - echo "Creating Kind cluster '{{.CLUSTER_NAME}}'..." - CONFIG_FLAG="" - if [ -n "{{.KIND_CONFIG}}" ]; then - CONFIG_FLAG="--config {{.KIND_CONFIG}}" - fi - kind create cluster \ - --name {{.CLUSTER_NAME}} \ - $CONFIG_FLAG \ - --wait 90s - fi - - _e2e:kubeconfigs:export: - internal: true - desc: "Export Kind kubeconfigs and create Docker-IP variants for cross-cluster use" - cmds: - # Standard kubeconfigs (localhost-based, for developer kubectl use) - - kind export kubeconfig --name {{.KIND_CONTROL_PLANE}} --kubeconfig {{.KUBECONFIG_DIR}}/control-plane.yaml - - kind export kubeconfig --name {{.KIND_POP_DFW}} --kubeconfig {{.KUBECONFIG_DIR}}/pop-dfw.yaml - - kind export kubeconfig --name {{.KIND_POP_ORD}} --kubeconfig {{.KUBECONFIG_DIR}}/pop-ord.yaml - # Docker-IP kubeconfigs (used by Karmada controller, running inside Docker, - # to reach POP cell API servers across the kind bridge network) - - | - hack/e2e/make-internal-kubeconfig.sh \ - {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ - {{.KUBECONFIG_DIR}}/pop-dfw-internal.yaml \ - {{.KIND_POP_DFW}} - - | - hack/e2e/make-internal-kubeconfig.sh \ - {{.KUBECONFIG_DIR}}/pop-ord.yaml \ - {{.KUBECONFIG_DIR}}/pop-ord-internal.yaml \ - {{.KIND_POP_ORD}} - # ════════════════════════════════════════════════════════════════════════ - # Karmada installation + # Karmada federation configuration (compute-specific) # ════════════════════════════════════════════════════════════════════════ - e2e:karmada:install: - desc: "Install Karmada into the management cluster via Helm (idempotent)" - cmds: - # Idempotency keys on the karmada-apiserver Deployment existing rather than - # just the namespace: a Helm --wait that times out on a loaded machine - # leaves the namespace behind with the components still converging, and we - # want a re-run to resume that install (wait for it to finish) instead of - # skipping straight past a not-yet-ready control plane. The 10m timeout - # gives the Karmada components room to start when the host is busy running - # several Kind clusters at once. - - | - if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system get deploy karmada-apiserver &>/dev/null; then - echo "Karmada already present — waiting for its components to become Available..." - else - echo "Installing Karmada {{.KARMADA_VERSION}} via Helm..." - helm install karmada karmada-charts/karmada \ - --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - --namespace karmada-system \ - --create-namespace \ - --version {{.KARMADA_VERSION}} \ - --set apiServer.serviceType=NodePort \ - --set apiServer.nodePort={{.KARMADA_API_NODEPORT}} \ - --wait \ - --timeout 10m \ - || echo "Helm --wait did not settle in time; will wait on the deployments directly below" - fi - # Disable leader election on the single-replica Karmada controllers. On a - # busy host, etcd write latency spikes cause the controllers' 5s lease - # renewals to time out; each lost lease exits the process, and the - # restart re-lists every resource, which adds I/O load and provokes the - # next latency spike — a crashloop feedback loop. With one replica there is - # nothing to elect, so turning it off breaks the loop and lets the control - # plane settle. Harmless in a single-replica e2e; not for production HA. - - task: _e2e:karmada:disable-leader-election - # Disable fsync on the Karmada control plane's own etcd, for the same - # throwaway-data reason as the base clusters (see hack/e2e/kind-*.yaml). Its - # etcd is a Helm-managed StatefulSet on hostPath storage — co-located with - # the base cluster's etcd on one node — so without this its fsync contention - # keeps hub writes slow even after the base etcds are tuned. - - task: _e2e:karmada:disable-etcd-fsync - # Scale any Karmada deployment that is sitting at zero back up to one. This - # makes the target robust to a control plane that was intentionally scaled - # down (for example to keep a busy host calm between runs) or left at zero - # by a partial install: without this, the Available wait below would block - # forever on a deployment that has no desired replicas. - - | - for d in $(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system get deploy -o jsonpath='{.items[*].metadata.name}'); do - reps=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system get deploy "$d" -o jsonpath='{.spec.replicas}') - if [ "$reps" = "0" ]; then - echo "Scaling Karmada deploy $d back to 1 (was scaled to 0)..." - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system scale deploy "$d" --replicas=1 - fi - done - # Wait on the control-plane deployments regardless of how the install above - # exited, so a timed-out Helm run still converges before we build the - # kubeconfig and register cells. - - | - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system wait --for=condition=Available deploy --all --timeout=10m - echo "Karmada control plane is Available" - - task: _e2e:karmada:build-kubeconfig - - _e2e:karmada:disable-leader-election: - internal: true - cmds: - - | - for d in karmada-controller-manager karmada-kube-controller-manager karmada-scheduler; do - current=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system get deploy "$d" \ - -o jsonpath='{.spec.template.spec.containers[0].command}' 2>/dev/null) - if echo "$current" | grep -q -- "--leader-elect=false"; then - echo "Leader election already disabled on $d" - else - echo "Disabling leader election on $d..." - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system patch deploy "$d" --type=json \ - -p '[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--leader-elect=false"}]' - fi - done - - _e2e:karmada:disable-etcd-fsync: - internal: true - cmds: - # Idempotent: only patch + restart if the flag is not already set. The etcd - # data lives on a hostPath, so deleting the pod does not lose state (a - # StatefulSet does not always roll the pod on a bare command change, hence - # the explicit delete). - - | - current=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system get statefulset etcd \ - -o jsonpath='{.spec.template.spec.containers[0].command}' 2>/dev/null) - if echo "$current" | grep -q -- "--unsafe-no-fsync"; then - echo "Karmada etcd fsync already disabled" - else - echo "Disabling fsync on the Karmada etcd..." - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system patch statefulset etcd --type=json \ - -p '[{"op":"add","path":"/spec/template/spec/containers/0/command/-","value":"--unsafe-no-fsync=true"}]' - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system delete pod etcd-0 --ignore-not-found - fi - # Relax the Karmada API server's health probes. Under sustained load (e.g. - # the chainsaw suites) a transient etcd/DNS blip makes the default 5s/15s - # probes fail, and the kubelet SIGKILLs the API server — which cascades to - # a crashloop. Wider thresholds let it ride out the blip instead of being - # killed. Belt-and-suspenders with the etcd fsync + leader-election tuning. - - | - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - -n karmada-system patch deploy karmada-apiserver --type=strategic \ - -p '{"spec":{"template":{"spec":{"containers":[{"name":"karmada-apiserver","livenessProbe":{"timeoutSeconds":30,"failureThreshold":30,"periodSeconds":30,"initialDelaySeconds":60},"readinessProbe":{"timeoutSeconds":30,"failureThreshold":30,"periodSeconds":20}}]}}}}' - e2e:karmada:configure: desc: "Apply federation component config to the Karmada API server (idempotent)" cmds: @@ -399,132 +208,6 @@ tasks: done echo "ERROR: failed to apply federation component after 6 attempts"; exit 1 - _e2e:karmada:build-kubeconfig: - internal: true - desc: "Extract Karmada kubeconfig from secret and patch server to localhost:{{.KARMADA_API_NODEPORT}}" - cmds: - - | - echo "Building Karmada kubeconfig → {{.KUBECONFIG_DIR}}/karmada.yaml" - # Extract raw kubeconfig from the secret the Helm chart creates - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml \ - get secret karmada-kubeconfig \ - -n karmada-system \ - -o jsonpath='{.data.kubeconfig}' \ - | base64 -d > {{.KUBECONFIG_DIR}}/karmada-raw.yaml - # Rewrite the server address to the NodePort exposed on localhost - python3 - {{.KUBECONFIG_DIR}}/karmada-raw.yaml {{.KUBECONFIG_DIR}}/karmada.yaml 127.0.0.1 {{.KARMADA_API_NODEPORT}} << 'PYEOF' - import sys, yaml - - src, dst, host, port = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] - - with open(src) as f: - cfg = yaml.safe_load(f) - - for cluster in cfg.get('clusters', []): - old = cluster['cluster'].get('server', '') - cluster['cluster']['server'] = f'https://{host}:{port}' - # The cert is for the internal cluster IP, so skip TLS verification. - # This is a local dev-only environment. - cluster['cluster']['insecure-skip-tls-verify'] = True - cluster['cluster'].pop('certificate-authority-data', None) - print(f" karmada server: {old} → https://{host}:{port}", file=sys.stderr) - - with open(dst, 'w') as f: - yaml.dump(cfg, f, default_flow_style=False) - PYEOF - rm {{.KUBECONFIG_DIR}}/karmada-raw.yaml - # Copy karmada.yaml → downstream.yaml so Chainsaw tests that declare - # cluster: downstream can load it (chainsaw-config.yaml references this path). - - cp {{.KUBECONFIG_DIR}}/karmada.yaml {{.KUBECONFIG_DIR}}/downstream.yaml - - # ════════════════════════════════════════════════════════════════════════ - # POP cell cluster registration - # ════════════════════════════════════════════════════════════════════════ - - e2e:karmada:join-clusters: - desc: "Register POP cell clusters with Karmada and apply city-code labels" - cmds: - # cluster.karmada.io is served by the karmada-aggregated-apiserver through - # API aggregation, and can briefly return ServiceUnavailable after the pod - # reports Ready while the aggregation handshake settles. Wait for the API - # to actually answer before registering members so join does not race it. - - | - echo "Waiting for the cluster.karmada.io aggregated API to be served..." - deadline=$((SECONDS+180)) - until kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml get clusters &>/dev/null; do - if [ $SECONDS -ge $deadline ]; then - echo "ERROR: cluster.karmada.io was not served within 180s"; exit 1 - fi - sleep 3 - done - echo "cluster.karmada.io is served" - - task: _e2e:karmada:join-cluster - vars: - CLUSTER_NAME: "{{.KIND_POP_DFW}}" - CITY_CODE: dfw - EXTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-dfw.yaml" - INTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-dfw-internal.yaml" - - task: _e2e:karmada:join-cluster - vars: - CLUSTER_NAME: "{{.KIND_POP_ORD}}" - CITY_CODE: ord - EXTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-ord.yaml" - INTERNAL_KUBECONFIG: "{{.KUBECONFIG_DIR}}/pop-ord-internal.yaml" - - _e2e:karmada:join-cluster: - internal: true - cmds: - # ── Register with karmadactl join ────────────────────────────────── - # We pass the EXTERNAL kubeconfig (localhost-based) here so karmadactl - # can reach the member cluster from this macOS host to set up initial - # RBAC. The stored secret is patched below to the Docker-IP variant. - # karmadactl join creates an impersonation ServiceAccount in the member - # cluster and blocks (with a non-tunable internal timeout) on that SA's - # legacy token Secret being populated. On a busy host the member cluster's - # token controller is slow, so the first join can time out even though the - # Secret populates moments later. Retry: a subsequent attempt reuses the - # now-populated Secret and completes immediately. - - | - if kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ - get cluster {{.CLUSTER_NAME}} &>/dev/null; then - echo "Cluster '{{.CLUSTER_NAME}}' already registered in Karmada — skipping join" - else - echo "Joining '{{.CLUSTER_NAME}}' to Karmada..." - joined=false - for attempt in 1 2 3 4 5; do - if {{.KARMADACTL}} join {{.CLUSTER_NAME}} \ - --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ - --cluster-kubeconfig={{.EXTERNAL_KUBECONFIG}} \ - --cluster-context=kind-{{.CLUSTER_NAME}}; then - joined=true - echo "Cluster '{{.CLUSTER_NAME}}' registered (attempt ${attempt})" - break - fi - echo "join attempt ${attempt} failed (member token Secret may still be populating); retrying in 10s..." - sleep 10 - done - if [ "${joined}" != "true" ]; then - echo "ERROR: failed to join '{{.CLUSTER_NAME}}' after 5 attempts"; exit 1 - fi - fi - # ── Patch cluster secret → Docker-IP kubeconfig ─────────────────── - # The Karmada controller manager runs inside Docker; it cannot use - # localhost to reach POP cell API servers. We update the stored secret - # with a kubeconfig whose server address uses the Kind container IP so - # container-to-container communication works across the kind bridge. - - | - hack/e2e/patch-cluster-secret.sh \ - {{.KUBECONFIG_DIR}}/karmada.yaml \ - {{.CLUSTER_NAME}} \ - {{.INTERNAL_KUBECONFIG}} - # ── Apply city-code label ────────────────────────────────────────── - - | - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ - label cluster {{.CLUSTER_NAME}} \ - topology.datum.net/city-code={{.CITY_CODE}} \ - --overwrite - echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}}" - # ════════════════════════════════════════════════════════════════════════ # CRD installation # ════════════════════════════════════════════════════════════════════════ @@ -545,10 +228,10 @@ tasks: # WorkloadDeployment objects. - | for KC in \ - {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ {{.KUBECONFIG_DIR}}/karmada.yaml \ - {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ - {{.KUBECONFIG_DIR}}/pop-ord.yaml; do + {{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml; do echo "Installing compute CRDs → $(basename $KC .yaml)..." kubectl --kubeconfig="$KC" apply -k config/base/crd --server-side --validate=false done @@ -568,9 +251,9 @@ tasks: NSO_CRD_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/crd" echo "NSO CRDs from: ${NSO_CRD_PATH}" for KC in \ - {{.KUBECONFIG_DIR}}/control-plane.yaml \ - {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ - {{.KUBECONFIG_DIR}}/pop-ord.yaml; do + {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml; do echo "Installing NSO CRDs → $(basename $KC .yaml)..." kubectl --kubeconfig="$KC" apply -k "${NSO_CRD_PATH}" --server-side --validate=false done @@ -589,10 +272,10 @@ tasks: QUOTA_CRD_PATH="$(go env GOMODCACHE)/go.miloapis.com/milo@${MILO_VERSION}/config/crd/bases/quota" echo "Milo quota CRDs from: ${QUOTA_CRD_PATH}" for KC in \ - {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml \ {{.KUBECONFIG_DIR}}/karmada.yaml \ - {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ - {{.KUBECONFIG_DIR}}/pop-ord.yaml; do + {{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml \ + {{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml; do echo "Installing Milo quota CRDs → $(basename $KC .yaml)..." kubectl --kubeconfig="$KC" apply -k "${QUOTA_CRD_PATH}" --server-side --validate=false done @@ -611,11 +294,11 @@ tasks: e2e:image:load: desc: "Side-load {{.IMAGE}} into every Kind cluster (no registry pull)" cmds: - - | - for CLUSTER in {{.KIND_ALL_CLUSTERS}}; do - echo "Loading {{.IMAGE}} → kind cluster ${CLUSTER}..." - kind load docker-image {{.IMAGE}} --name "${CLUSTER}" - done + # The foundation owns the cluster inventory, so it loads the image into the + # hub + every member rather than compute re-deriving the cluster list. + - task: infra:federation-load-image + vars: + IMAGES: '{{.IMAGE}}' # ════════════════════════════════════════════════════════════════════════ # Federation access for the management operator @@ -659,8 +342,8 @@ tasks: # A bound token from the Karmada SA, embedded in a kubeconfig that targets # the Karmada in-cluster Service (the management pod runs alongside Karmada). # We re-mint on every run so a re-deploy always ships a fresh, unexpired - # token. insecure-skip-tls-verify mirrors the karmada.yaml the harness - # already builds — the served cert does not cover the Service DNS name. + # token. insecure-skip-tls-verify mirrors the karmada.yaml the foundation + # builds — the served cert does not cover the Service DNS name. - | mkdir -p {{.E2E_DIR}} echo "Minting Karmada token for compute-manager..." @@ -694,12 +377,12 @@ tasks: # directly rather than referencing the projected-token file, because the # token comes from a Karmada-native SA rather than a federated host token. - | - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml create namespace {{.COMPUTE_NAMESPACE}} \ - --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml apply -f - - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml -n {{.COMPUTE_NAMESPACE}} \ + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml create namespace {{.COMPUTE_NAMESPACE}} \ + --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml apply -f - + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml -n {{.COMPUTE_NAMESPACE}} \ create configmap compute-downstream-kubeconfig \ --from-file=downstream-kubeconfig.yaml={{.E2E_DIR}}/downstream-kubeconfig.yaml \ - --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml apply -f - + --dry-run=client -o yaml | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml apply -f - echo "Published compute-downstream-kubeconfig ConfigMap" # ── Cell hub credential (mirrors infra apps/compute-system/edge) ───── # Real cell deployments ALSO get a hub credential: infra's @@ -710,12 +393,14 @@ tasks: # InstanceProjector (which reads Instances only from hub write-back copies) # never sees them. We mirror it with the same compute-manager Karmada # identity — but the cell pods live in the OTHER Kind clusters, so they - # reach the hub over the control-plane node's docker-bridge IP + Karmada - # NodePort rather than the in-cluster Service the management pod uses. - - | - CP_IP=$(docker inspect {{.KIND_CONTROL_PLANE}}-control-plane \ - | python3 -c "import sys,json; print(list(json.load(sys.stdin)[0]['NetworkSettings']['Networks'].values())[0]['IPAddress'])") - echo "Cell operators reach the Karmada hub via ${CP_IP}:{{.KARMADA_API_NODEPORT}}" + # reach the hub over the docker-bridge IP + Karmada NodePort rather than the + # in-cluster Service the management pod uses. The foundation already wrote + # that docker-reachable hub address into karmada-internal.yaml, so we read + # it from there instead of re-deriving the hub container IP. + - | + HUB_SERVER=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada-internal.yaml \ + config view --minify -o jsonpath='{.clusters[0].cluster.server}') + echo "Cell operators reach the Karmada hub via ${HUB_SERVER}" TOKEN=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ create token compute-manager --duration=720h) cat > {{.E2E_DIR}}/cell-federation-kubeconfig.yaml </dev/null kubectl --kubeconfig="$KC" -n {{.COMPUTE_NAMESPACE}} \ @@ -765,10 +450,10 @@ tasks: cmds: - | echo "Deploying management-plane operator → {{.KIND_CONTROL_PLANE}}..." - kubectl --kubeconfig={{.KUBECONFIG_DIR}}/control-plane.yaml apply -k test/e2e/deploy/management --server-side --validate=false + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml apply -k test/e2e/deploy/management --server-side --validate=false - task: _e2e:deploy:wait vars: - KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/control-plane.yaml" + KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/{{.KIND_CONTROL_PLANE}}.yaml" LABEL: control-plane e2e:deploy:cells: @@ -776,11 +461,11 @@ tasks: cmds: - task: _e2e:deploy:cell vars: - KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/pop-dfw.yaml" + KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/{{.KIND_POP_DFW}}.yaml" CLUSTER_NAME: "{{.KIND_POP_DFW}}" - task: _e2e:deploy:cell vars: - KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/pop-ord.yaml" + KUBECONFIG_FILE: "{{.KUBECONFIG_DIR}}/{{.KIND_POP_ORD}}.yaml" CLUSTER_NAME: "{{.KIND_POP_ORD}}" _e2e:deploy:cell: diff --git a/hack/e2e/kind-control-plane.yaml b/hack/e2e/kind-control-plane.yaml deleted file mode 100644 index 12356ed1..00000000 --- a/hack/e2e/kind-control-plane.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Kind cluster configuration for the compute-control-plane management cluster. -# -# extraPortMappings exposes port 32443 on the macOS host so that the Karmada -# API server NodePort service (nodePort: 32443) is accessible at -# https://localhost:32443 without any additional port-forwarding. -# -# This matches KARMADA_API_NODEPORT in Taskfile.yaml. - -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -# Tune the control plane for a busy host that also runs the full Karmada control -# plane in this same cluster. kind renders its kubeadm config as v1beta3, so -# extraArgs are maps here (not the v1beta4 list form). -# - etcd unsafe-no-fsync: these are throwaway e2e clusters, and fsync -# contention between co-located etcds is what drives API server write -# latency into multi-second spikes. -# - scheduler/controllerManager leader-elect=false: single-instance static -# pods have nothing to elect, and a lease renewal that times out during a -# latency spike crash-loops them (which then re-lists everything and makes -# the spike worse). -kubeadmConfigPatches: - - | - apiVersion: kubeadm.k8s.io/v1beta3 - kind: ClusterConfiguration - etcd: - local: - extraArgs: - unsafe-no-fsync: "true" - controllerManager: - extraArgs: - leader-elect: "false" - scheduler: - extraArgs: - leader-elect: "false" -nodes: - - role: control-plane - extraPortMappings: - - containerPort: 32443 # Karmada API server NodePort - hostPort: 32443 - protocol: TCP - listenAddress: "127.0.0.1" diff --git a/hack/e2e/kind-pop.yaml b/hack/e2e/kind-pop.yaml deleted file mode 100644 index d848f43b..00000000 --- a/hack/e2e/kind-pop.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Kind cluster configuration for the compute-pop-* cell clusters. -# -# Same control-plane tuning as the management cluster (see kind-control-plane.yaml -# for the rationale): etcd runs without fsync and the single-instance scheduler -# and controller-manager run without leader election, so the cell control planes -# stay stable on a busy host instead of crash-looping on lease-renewal timeouts. -# kind renders its kubeadm config as v1beta3, so extraArgs are maps here. -# -# Unlike the management cluster, POP cells need no extraPortMappings. -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -kubeadmConfigPatches: - - | - apiVersion: kubeadm.k8s.io/v1beta3 - kind: ClusterConfiguration - etcd: - local: - extraArgs: - unsafe-no-fsync: "true" - controllerManager: - extraArgs: - leader-elect: "false" - scheduler: - extraArgs: - leader-elect: "false" diff --git a/hack/e2e/make-internal-kubeconfig.sh b/hack/e2e/make-internal-kubeconfig.sh deleted file mode 100755 index 3303a5bd..00000000 --- a/hack/e2e/make-internal-kubeconfig.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# make-internal-kubeconfig.sh -# -# Produces a kubeconfig variant that uses the Kind node's Docker container IP -# instead of localhost. This variant is stored in Karmada so the controller -# manager (running inside Docker) can reach member cluster API servers across -# the kind bridge network. -# -# Background: Kind maps each cluster's API server to a random localhost port -# on the developer machine. Inside Docker containers, "localhost" refers to the -# container's own loopback — not the host. We therefore swap the server address -# to the Kind control-plane container's Docker bridge IP (e.g. 172.18.0.x) and -# set insecure-skip-tls-verify because the node certificate does not include -# the Docker bridge IP in its SANs. -# -# Usage: -# hack/e2e/make-internal-kubeconfig.sh \ -# tmp/e2e/kubeconfigs/pop-dfw.yaml \ -# tmp/e2e/kubeconfigs/pop-dfw-internal.yaml \ -# compute-pop-dfw - -set -euo pipefail - -INPUT="${1:?usage: $0 }" -OUTPUT="${2:?usage: $0 }" -CLUSTER_NAME="${3:?usage: $0 }" - -CONTAINER_NAME="${CLUSTER_NAME}-control-plane" - -# Resolve the container's Docker bridge IP. -DOCKER_IP=$(docker inspect \ - -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ - "${CONTAINER_NAME}" 2>/dev/null || true) - -if [ -z "${DOCKER_IP}" ]; then - echo "ERROR: Could not resolve Docker IP for container '${CONTAINER_NAME}'." >&2 - echo " Is the Kind cluster '${CLUSTER_NAME}' running?" >&2 - exit 1 -fi - -echo " ${CLUSTER_NAME}: Docker IP ${DOCKER_IP} → ${OUTPUT}" - -python3 - "${INPUT}" "${OUTPUT}" "${DOCKER_IP}" <<'PYEOF' -import sys, yaml - -src, dst, docker_ip = sys.argv[1], sys.argv[2], sys.argv[3] - -with open(src) as f: - cfg = yaml.safe_load(f) - -for cluster in cfg.get('clusters', []): - # Kind API server always listens on port 6443 inside the container. - cluster['cluster']['server'] = f'https://{docker_ip}:6443' - # The node cert only covers localhost / 127.0.0.1, not the bridge IP. - cluster['cluster']['insecure-skip-tls-verify'] = True - cluster['cluster'].pop('certificate-authority-data', None) - -with open(dst, 'w') as f: - yaml.dump(cfg, f, default_flow_style=False) -PYEOF diff --git a/hack/e2e/patch-cluster-secret.sh b/hack/e2e/patch-cluster-secret.sh deleted file mode 100755 index e29ed383..00000000 --- a/hack/e2e/patch-cluster-secret.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -# patch-cluster-secret.sh -# -# After "karmadactl join", Karmada stores the member cluster's kubeconfig in a -# Secret referenced by the Cluster object's spec.secretRef, and sets -# spec.apiEndpoint to the localhost address it resolved from the external -# kubeconfig. The Karmada controller manager runs inside Docker and cannot use -# localhost to reach POP cell API servers. -# -# This script: -# 1. Replaces the kubeconfig in the Secret with the Docker-IP variant so that -# the Karmada controller can make API calls to the member cluster. -# 2. Patches spec.apiEndpoint on the Cluster object so that health checks also -# use the Docker bridge IP instead of localhost. -# -# Usage: -# hack/e2e/patch-cluster-secret.sh \ -# tmp/e2e/kubeconfigs/karmada.yaml \ -# compute-pop-dfw \ -# tmp/e2e/kubeconfigs/pop-dfw-internal.yaml - -set -euo pipefail - -KARMADA_KUBECONFIG="${1:?usage: $0 }" -CLUSTER_NAME="${2:?usage: $0 }" -INTERNAL_KUBECONFIG="${3:?usage: $0 }" - -# ------------------------------------------------------------------ -# Read the Cluster object's secretRef (name + namespace) -# ------------------------------------------------------------------ -SECRET_NAME=$(kubectl \ - --kubeconfig="${KARMADA_KUBECONFIG}" \ - get cluster "${CLUSTER_NAME}" \ - -o jsonpath='{.spec.secretRef.name}' 2>/dev/null || true) - -if [ -z "${SECRET_NAME}" ]; then - echo "ERROR: Could not find spec.secretRef.name on cluster '${CLUSTER_NAME}'." >&2 - echo " Has karmadactl join completed successfully?" >&2 - exit 1 -fi - -SECRET_NAMESPACE=$(kubectl \ - --kubeconfig="${KARMADA_KUBECONFIG}" \ - get cluster "${CLUSTER_NAME}" \ - -o jsonpath='{.spec.secretRef.namespace}' 2>/dev/null || true) - -SECRET_NAMESPACE="${SECRET_NAMESPACE:-karmada-system}" - -echo " Patching secret ${SECRET_NAMESPACE}/${SECRET_NAME} with Docker-IP kubeconfig..." - -# ------------------------------------------------------------------ -# Replace the kubeconfig data in the secret -# ------------------------------------------------------------------ -kubectl \ - --kubeconfig="${KARMADA_KUBECONFIG}" \ - create secret generic "${SECRET_NAME}" \ - --namespace="${SECRET_NAMESPACE}" \ - --from-file=kubeconfig="${INTERNAL_KUBECONFIG}" \ - --dry-run=client -o yaml \ - | kubectl \ - --kubeconfig="${KARMADA_KUBECONFIG}" \ - apply -f - - -echo " Secret ${SECRET_NAMESPACE}/${SECRET_NAME} updated — Karmada controller will use Docker bridge IP" - -# ------------------------------------------------------------------ -# Extract the Docker-IP server URL from the internal kubeconfig and -# patch spec.apiEndpoint on the Cluster object so that Karmada's -# cluster-status controller uses the same reachable address for health -# checks. Without this patch the controller continues to probe the -# localhost address stored by karmadactl join and the cluster never -# transitions to Ready. -# ------------------------------------------------------------------ -DOCKER_SERVER=$(kubectl \ - --kubeconfig="${INTERNAL_KUBECONFIG}" \ - config view --minify -o jsonpath='{.clusters[0].cluster.server}') - -if [ -z "${DOCKER_SERVER}" ]; then - echo "ERROR: Could not read server URL from ${INTERNAL_KUBECONFIG}" >&2 - exit 1 -fi - -echo " Patching spec.apiEndpoint on cluster '${CLUSTER_NAME}' → ${DOCKER_SERVER}..." -kubectl \ - --kubeconfig="${KARMADA_KUBECONFIG}" \ - patch cluster "${CLUSTER_NAME}" \ - --type=merge \ - -p "{\"spec\":{\"apiEndpoint\":\"${DOCKER_SERVER}\"}}" - -echo " Cluster '${CLUSTER_NAME}' patched — health checks will now use Docker bridge IP" From 3c56bfa9226efb76e24af0947f937fc987212b2a Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 10 Jul 2026 12:26:36 -0500 Subject: [PATCH 2/5] test(e2e): repoint suite kubeconfig paths at the federation foundation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical path substitution only — the shared foundation writes the cluster kubeconfigs under .test-infra/kubeconfigs/federation with new names, so the hardcoded --kubeconfig paths inside the Chainsaw suite steps move: tmp/e2e/kubeconfigs/control-plane.yaml -> .test-infra/kubeconfigs/federation/compute-control-plane.yaml tmp/e2e/kubeconfigs/downstream.yaml -> .test-infra/kubeconfigs/federation/karmada.yaml tmp/e2e/kubeconfigs/pop-dfw.yaml -> .test-infra/kubeconfigs/federation/compute-pop-dfw.yaml tmp/e2e/kubeconfigs/pop-ord.yaml -> .test-infra/kubeconfigs/federation/compute-pop-ord.yaml No test assertions, workload specs, or step logic change; verify with `git diff --word-diff`. The relative depth (../../../ from test/e2e//) is unchanged because .test-infra sits at the repo root just like tmp did. --- test/e2e/deletion-cascade/chainsaw-test.yaml | 4 ++-- .../full-federation-ord/chainsaw-test.yaml | 8 +++---- test/e2e/full-federation/chainsaw-test.yaml | 8 +++---- .../instance-projection/chainsaw-test.yaml | 8 +++---- .../e2e/instance-writeback/chainsaw-test.yaml | 16 +++++++------- .../chainsaw-test.yaml | 6 ++--- .../chainsaw-test.yaml | 22 +++++++++---------- .../referenced-data-mounts/chainsaw-test.yaml | 22 +++++++++---------- .../chainsaw-test.yaml | 4 ++-- 9 files changed, 49 insertions(+), 49 deletions(-) diff --git a/test/e2e/deletion-cascade/chainsaw-test.yaml b/test/e2e/deletion-cascade/chainsaw-test.yaml index 03a11ea0..e7026842 100644 --- a/test/e2e/deletion-cascade/chainsaw-test.yaml +++ b/test/e2e/deletion-cascade/chainsaw-test.yaml @@ -32,7 +32,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -63,7 +63,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: diff --git a/test/e2e/full-federation-ord/chainsaw-test.yaml b/test/e2e/full-federation-ord/chainsaw-test.yaml index b1bf1b64..388bb0bb 100644 --- a/test/e2e/full-federation-ord/chainsaw-test.yaml +++ b/test/e2e/full-federation-ord/chainsaw-test.yaml @@ -44,7 +44,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -102,7 +102,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -127,7 +127,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -154,7 +154,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: diff --git a/test/e2e/full-federation/chainsaw-test.yaml b/test/e2e/full-federation/chainsaw-test.yaml index fae94a23..ec8a57b1 100644 --- a/test/e2e/full-federation/chainsaw-test.yaml +++ b/test/e2e/full-federation/chainsaw-test.yaml @@ -46,7 +46,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -79,7 +79,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -104,7 +104,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -135,7 +135,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: diff --git a/test/e2e/instance-projection/chainsaw-test.yaml b/test/e2e/instance-projection/chainsaw-test.yaml index d871c308..89c57a49 100644 --- a/test/e2e/instance-projection/chainsaw-test.yaml +++ b/test/e2e/instance-projection/chainsaw-test.yaml @@ -39,7 +39,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -65,7 +65,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -73,7 +73,7 @@ spec: value: ($stdout) - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get workloaddeployment test-projector-wd \ --namespace "$NAMESPACE" \ -o jsonpath='{.metadata.uid}' @@ -117,7 +117,7 @@ spec: cleanup: - script: content: | - KARMADA_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + KARMADA_NS=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}') kubectl delete instance test-projected-instance \ diff --git a/test/e2e/instance-writeback/chainsaw-test.yaml b/test/e2e/instance-writeback/chainsaw-test.yaml index 50110842..4009099f 100644 --- a/test/e2e/instance-writeback/chainsaw-test.yaml +++ b/test/e2e/instance-writeback/chainsaw-test.yaml @@ -45,10 +45,10 @@ spec: - name: INSTANCE_NS value: ($instanceNS) content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-pop-dfw.yaml \ create namespace "$INSTANCE_NS" \ --dry-run=client -o yaml | \ - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml apply -f - + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-pop-dfw.yaml apply -f - - script: env: - name: INSTANCE_NS @@ -58,7 +58,7 @@ spec: # namespace) from THIS hub namespace's labels, exactly as it would from # the ns- namespace the federator stamps in the real path. Seed them # so writeBackToUpstream can resolve identity instead of erroring. - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/downstream.yaml apply -f - </dev/null) @@ -275,7 +275,7 @@ spec: echo "ERROR: gc-test-config ConfigMap was re-created on cell (Karmada recreate loop!)" exit 1 fi - SEC=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + SEC=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-pop-dfw.yaml \ get secret gc-test-secret \ --namespace "$CELL_NS" \ --ignore-not-found 2>/dev/null) diff --git a/test/e2e/referenced-data-mounts/chainsaw-test.yaml b/test/e2e/referenced-data-mounts/chainsaw-test.yaml index bcef1234..72b12b24 100644 --- a/test/e2e/referenced-data-mounts/chainsaw-test.yaml +++ b/test/e2e/referenced-data-mounts/chainsaw-test.yaml @@ -39,7 +39,7 @@ spec: - the cell operator runs with featureFlags.enableReferencedDataGate: true; the e2e cell deploy layer sets it (test/e2e/deploy/cell/config_patch.yaml), so no separate operator-start step is needed. - - tmp/e2e/kubeconfigs/downstream.yaml exists (the Taskfile writes it as a copy + - .test-infra/kubeconfigs/federation/karmada.yaml exists (the Taskfile writes it as a copy of the Karmada hub kubeconfig). template: true @@ -73,7 +73,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -99,7 +99,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -124,7 +124,7 @@ spec: - script: # Verify the annotation is present and non-empty. content: | - ANNO=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + ANNO=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get workloaddeployment test-refdata-wd \ --namespace "$NAMESPACE" \ -o jsonpath='{.metadata.annotations.compute\.datumapis\.com/expected-referenced-data}') @@ -157,7 +157,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -203,7 +203,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -248,7 +248,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -291,7 +291,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -317,7 +317,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -338,10 +338,10 @@ spec: # Verify the ReferencedData gate is absent from the Instance spec. timeout: 60s content: | - DOWNSTREAMS_NS=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + DOWNSTREAMS_NS=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}') - GATES=$(kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/pop-dfw.yaml \ + GATES=$(kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-pop-dfw.yaml \ get instance test-refdata-wd-0 \ --namespace "$DOWNSTREAMS_NS" \ -o jsonpath='{.spec.schedulingGates[*].name}') diff --git a/test/e2e/workload-deployment-federation/chainsaw-test.yaml b/test/e2e/workload-deployment-federation/chainsaw-test.yaml index 8d589310..f759072b 100644 --- a/test/e2e/workload-deployment-federation/chainsaw-test.yaml +++ b/test/e2e/workload-deployment-federation/chainsaw-test.yaml @@ -33,7 +33,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: @@ -56,7 +56,7 @@ spec: try: - script: content: | - kubectl --kubeconfig=../../../tmp/e2e/kubeconfigs/control-plane.yaml \ + kubectl --kubeconfig=../../../.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ get namespace "$NAMESPACE" \ -o template='{{printf "ns-%s" .metadata.uid}}' outputs: From 7ac55889689d8ff9aa67e80279e8cca5a9ebd056 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 10 Jul 2026 12:26:55 -0500 Subject: [PATCH 3/5] test(e2e): point the chainsaw config and docs at the federation kubeconfigs Update the chainsaw configuration's named clusters (downstream, pop-dfw, pop-ord) to the foundation-written kubeconfigs under .test-infra/kubeconfigs/federation, and refresh the referenced-data README to describe the new layout: the foundation writes the hub kubeconfig as karmada.yaml and there is no longer a downstream.yaml copy or a build-kubeconfig task. --- test/e2e/chainsaw-config.yaml | 14 +++++++------- test/e2e/referenced-data-mounts/README.md | 19 ++++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/test/e2e/chainsaw-config.yaml b/test/e2e/chainsaw-config.yaml index 9524c8e4..d7140cd1 100644 --- a/test/e2e/chainsaw-config.yaml +++ b/test/e2e/chainsaw-config.yaml @@ -3,7 +3,7 @@ # Prerequisites # ───────────── # Run `task e2e:up` to create the Kind clusters and populate kubeconfigs under -# tmp/e2e/kubeconfigs/ before running Chainsaw. +# .test-infra/kubeconfigs/federation/ before running Chainsaw. # # Running # ─────── @@ -13,7 +13,7 @@ # # Or directly: # -# KUBECONFIG=tmp/e2e/kubeconfigs/control-plane.yaml \ +# KUBECONFIG=.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ # chainsaw test --config test/e2e/chainsaw-config.yaml test/e2e/ # # The KUBECONFIG env var sets the "default" cluster (control-plane cell). @@ -40,13 +40,13 @@ spec: error: 60s exec: 30s clusters: - # Downstream control plane. WorkloadDeployments, PropagationPolicies, - # and Instance write-backs live here. + # Downstream control plane (the Karmada hub API server). WorkloadDeployments, + # PropagationPolicies, and Instance write-backs live here. downstream: - kubeconfig: tmp/e2e/kubeconfigs/downstream.yaml + kubeconfig: .test-infra/kubeconfigs/federation/karmada.yaml # POP DFW cell — downstream member cluster labelled topology.datum.net/city-code=dfw. pop-dfw: - kubeconfig: tmp/e2e/kubeconfigs/pop-dfw.yaml + kubeconfig: .test-infra/kubeconfigs/federation/compute-pop-dfw.yaml # POP ORD cell — downstream member cluster labelled topology.datum.net/city-code=ord. pop-ord: - kubeconfig: tmp/e2e/kubeconfigs/pop-ord.yaml + kubeconfig: .test-infra/kubeconfigs/federation/compute-pop-ord.yaml diff --git a/test/e2e/referenced-data-mounts/README.md b/test/e2e/referenced-data-mounts/README.md index 3536d10b..e5468208 100644 --- a/test/e2e/referenced-data-mounts/README.md +++ b/test/e2e/referenced-data-mounts/README.md @@ -29,13 +29,13 @@ against a downstream cluster. See `docs/compute/development/plans/configmap-secr **`task e2e:up` has completed successfully.** In the in-cluster harness this one target brings up the Kind clusters + Karmada AND deploys the operators from the real production overlays (plus the local e2e deviations), so there is no separate -operator-start step. It produces these kubeconfigs under `tmp/e2e/kubeconfigs/`: +operator-start step. The shared test-infra federation foundation writes these +kubeconfigs under `.test-infra/kubeconfigs/federation/`: -- `control-plane.yaml` — management cluster (also hosts the Karmada hub) -- `karmada.yaml` — the Karmada hub API server -- `downstream.yaml` — a copy of `karmada.yaml` the Taskfile writes so - `cluster: downstream` steps resolve (referenced by `chainsaw-config.yaml`) -- `pop-dfw.yaml`, `pop-ord.yaml` — the POP cell clusters +- `compute-control-plane.yaml` — management cluster (also hosts the Karmada hub) +- `karmada.yaml` — the Karmada hub API server; `chainsaw-config.yaml` maps the + `cluster: downstream` steps directly at this file +- `compute-pop-dfw.yaml`, `compute-pop-ord.yaml` — the POP cell clusters **The cell operator runs with `enableReferencedDataGate: true`.** The e2e cell deploy layer sets it in `test/e2e/deploy/cell/config_patch.yaml`. The flag's sole @@ -47,7 +47,7 @@ there because that overlay enables management controllers only. ## Running just this scenario ```sh -KUBECONFIG=tmp/e2e/kubeconfigs/control-plane.yaml \ +KUBECONFIG=.test-infra/kubeconfigs/federation/compute-control-plane.yaml \ bin/chainsaw test \ --config test/e2e/chainsaw-config.yaml \ --include-test-regex "referenced-data-mounts" \ @@ -66,8 +66,9 @@ This test targets the in-cluster harness, where `task e2e:up` builds the operato image, side-loads it into every Kind node, and deploys the management + cell operators from the real overlays. Points worth knowing: -1. `_e2e:karmada:build-kubeconfig` copies `karmada.yaml` → `downstream.yaml`, so - `cluster: downstream` steps work out of the box. +1. The federation foundation writes the hub kubeconfig as `karmada.yaml`, and + `chainsaw-config.yaml` points the `cluster: downstream` steps straight at it, + so those steps work out of the box. 2. The `enableReferencedDataGate` feature flag is delivered through the deploy layer (`test/e2e/deploy/{cell,management}/config_patch.yaml`), not a host-side `--server-config`. There is no separate operator-start task to run. From e694a75da8f0960b3849ac3b3422e316358ce429 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 10 Jul 2026 12:26:56 -0500 Subject: [PATCH 4/5] ci(e2e): run e2e against the test-infra federation foundation Drive the e2e job through the remote federation include: opt into Task's remote-taskfile support at the job level and add --yes to each task step for the one-time trust prompt. Drop the manual kind install (the foundation's ensure-tools provides it) and the PyYAML shim (no harness python parses YAML any more). Point diagnostics collection at the foundation kubeconfig directory and cluster filenames. --- .github/workflows/test-e2e.yml | 60 +++++++++++++++------------------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 89c0b007..d3a520b1 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -2,17 +2,24 @@ name: E2E Tests # In-cluster federation e2e (issue #149). # -# Stands up the full local topology the Taskfile harness builds — three Kind -# clusters (one management/control-plane hosting Karmada, two POP cells) with the -# real production kustomize overlays and hub RBAC — then runs the Chainsaw -# suites against it. This exercises the operators as deployed pods authenticating -# to Karmada as a non-admin identity, not as an in-process test binary. +# Stands up the full topology on the shared test-infra federation foundation — +# three Kind clusters (one management/control-plane hosting Karmada, two POP +# cells) with the real production kustomize overlays and hub RBAC — then runs the +# Chainsaw suites against it. This exercises the operators as deployed pods +# authenticating to Karmada as a non-admin identity, not as an in-process test +# binary. on: push: branches: [main] pull_request: +# The e2e Taskfile pulls the test-infra foundation in as a remote Taskfile +# include; opt into Task's remote-taskfile support for every step that runs a +# task (paired with `task --yes` for the non-interactive trust prompt). +env: + TASK_X_REMOTE_TASKFILES: "1" + # Cancel a superseded run on the same ref. The e2e job is expensive (three Kind # clusters + a Karmada control plane), so we don't want stale pushes burning a # runner. Unlike the cheaper test/lint workflows this only triggers on PRs and @@ -58,37 +65,24 @@ jobs: version: 3.52.0 repo-token: ${{ secrets.GITHUB_TOKEN }} - # Kind is not preinstalled on the runner. Docker, kubectl, and helm already - # are (ubuntu-24.04 image); karmadactl and chainsaw are fetched into ./bin - # by the task tooling. Pinned for reproducibility. - - name: Install kind - run: | - curl -sSfLo ./kind https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-amd64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind - kind version - - # The Karmada kubeconfig rewrite step (_e2e:karmada:build-kubeconfig) shells - # out to python3 + PyYAML. It is normally present on the runner image; guard - # the case where it is not without tripping over PEP 668. - - name: Ensure PyYAML - run: | - if ! python3 -c "import yaml" 2>/dev/null; then - sudo apt-get update - sudo apt-get install -y python3-yaml - fi + # kind, kubectl, helm, and karmadactl are installed by the test-infra + # federation foundation's own ensure-tools; chainsaw is fetched into ./bin + # by the compute task tooling. No harness-side python/PyYAML is used any + # more (the foundation rewrites kubeconfigs with kubectl; the remaining + # python3 calls parse `go list` JSON with the stdlib only). # Split into env / deploy / test so a failure lands on the phase that broke # rather than a single opaque "task e2e:up" step. e2e:env:up == e2e:up minus - # the deploy; the two together are exactly what e2e:up runs. + # the deploy; the two together are exactly what e2e:up runs. `--yes` accepts + # Task's one-time trust prompt for the remote test-infra include. - name: Provision Kind + Karmada environment - run: task e2e:env:up + run: task --yes e2e:env:up - name: Build image and deploy operators - run: task e2e:deploy + run: task --yes e2e:deploy - name: Run Chainsaw e2e suites - run: task e2e:test + run: task --yes e2e:test # Always capture what the clusters looked like when a step failed. The # runner is ephemeral so teardown is unnecessary; diagnostics are the only @@ -98,7 +92,7 @@ jobs: run: | set +e DIAG=tmp/e2e/diagnostics - KDIR=tmp/e2e/kubeconfigs + KDIR=.test-infra/kubeconfigs/federation mkdir -p "$DIAG" # Host / kind level: container state plus a full per-cluster export @@ -111,7 +105,7 @@ jobs: # Per-cluster Kubernetes state + the compute-manager operator logs # (current and previous, all containers) from every plane. - for kc in control-plane pop-dfw pop-ord karmada; do + for kc in compute-control-plane compute-pop-dfw compute-pop-ord karmada; do cfg="$KDIR/$kc.yaml" [ -f "$cfg" ] || continue out="$DIAG/$kc"; mkdir -p "$out" @@ -127,11 +121,11 @@ jobs: # Karmada control-plane pods + component logs (they live in the # management cluster) and the federation view from the Karmada API. - if [ -f "$KDIR/control-plane.yaml" ]; then - kubectl --kubeconfig="$KDIR/control-plane.yaml" -n karmada-system get pods -o wide \ + if [ -f "$KDIR/compute-control-plane.yaml" ]; then + kubectl --kubeconfig="$KDIR/compute-control-plane.yaml" -n karmada-system get pods -o wide \ > "$DIAG/karmada-pods.txt" 2>&1 for d in karmada-apiserver karmada-controller-manager karmada-scheduler; do - kubectl --kubeconfig="$KDIR/control-plane.yaml" -n karmada-system logs deploy/$d \ + kubectl --kubeconfig="$KDIR/compute-control-plane.yaml" -n karmada-system logs deploy/$d \ --tail=-1 > "$DIAG/karmada-$d.log" 2>&1 done fi From ebd3e30d790f0ed7c5ced0ef8966d22698780ac2 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 10 Jul 2026 12:58:30 -0500 Subject: [PATCH 5/5] fix(e2e): rename the tool-bin var to avoid a foundation var collision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A remote Taskfile include's root var clobbers a same-named var in the consumer Taskfile. Compute and the test-infra foundation both defined LOCALBIN, so compute's {{.ROOT_DIR}}/bin was overridden by the foundation's REPO_DIR-based definition; REPO_DIR is undefined in the compute scope, so the value rendered as a literal "%!s()/bin" — chainsaw would have installed to a bogus directory and e2e:test could not have found it. Rename the var to E2E_BIN, which does not collide. KARMADA_API_NODEPORT also shares a name with a foundation var, but it is passed through the include by design and resolves to the same value (override included), so it is unaffected. --- Taskfile.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index 2a2260d8..1a19308f 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -12,8 +12,12 @@ vars: # Chainsaw version for e2e testing (kyverno/chainsaw) CHAINSAW_VERSION: v0.2.15 - # Local tool directory (mirrors Makefile convention) - LOCALBIN: '{{.ROOT_DIR}}/bin' + # Local directory for e2e tooling. Deliberately NOT named LOCALBIN: the + # test-infra foundation defines its own LOCALBIN, and a remote include's root + # var clobbers a same-named var in this Taskfile — compute's value would be + # overwritten by the foundation's REPO_DIR-based path, which is undefined in + # this scope and renders as a nil path. + E2E_BIN: '{{.ROOT_DIR}}/bin' CHAINSAW: '{{.ROOT_DIR}}/bin/chainsaw' # Kind cluster names. These are also the Karmada member/hub cluster names the @@ -169,14 +173,14 @@ tasks: e2e:tools:chainsaw: desc: "Download chainsaw {{.CHAINSAW_VERSION}}" cmds: - - mkdir -p {{.LOCALBIN}} + - mkdir -p {{.E2E_BIN}} - | if [ ! -f "{{.CHAINSAW}}" ]; then OS=$(uname -s | tr '[:upper:]' '[:lower:]') ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') URL="https://github.com/kyverno/chainsaw/releases/download/{{.CHAINSAW_VERSION}}/chainsaw_${OS}_${ARCH}.tar.gz" echo "Downloading chainsaw {{.CHAINSAW_VERSION}} (${OS}/${ARCH}) from ${URL}..." - curl -sSfL "${URL}" | tar -xz -C {{.LOCALBIN}} chainsaw + curl -sSfL "${URL}" | tar -xz -C {{.E2E_BIN}} chainsaw chmod +x {{.CHAINSAW}} echo "chainsaw installed → {{.CHAINSAW}}" else