Skip to content
127 changes: 105 additions & 22 deletions pkg/apis/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@ package apis

import (
"context"
"errors"
"os"

v3 "github.com/tigera/api/pkg/apis/projectcalico/v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
Expand All @@ -35,58 +37,110 @@ var datastoreMigrationGVR = schema.GroupVersionResource{
Resource: "datastoremigrations",
}

const (
mutatingAdmissionPolicyGroup = "admissionregistration.k8s.io"
mutatingAdmissionPolicyKind = "MutatingAdmissionPolicy"
)

var log = ctrl.Log.WithName("apis")

// errV3RequiresMAP is returned when we've concluded v3 CRD mode but the cluster can't serve
// MutatingAdmissionPolicy. v3 mode relies on a MAP to default policy types, so we refuse to
// operate rather than run in a degraded state where defaulting silently doesn't happen.
var errV3RequiresMAP = errors.New("v3 CRD mode requires MutatingAdmissionPolicy support (Kubernetes 1.32+), which this cluster does not serve")

// UseV3CRDS detects whether we should use the crd.projectcalico.org/v1 or
// projectcalico.org/v3 API group for Calico CRDs.
func UseV3CRDS(cfg *rest.Config) (bool, error) {
if os.Getenv("CALICO_API_GROUP") != "" {
log.Info("CALICO_API_GROUP environment variable is set, using its value to determine API group", "CALICO_API_GROUP", os.Getenv("CALICO_API_GROUP"))
return os.Getenv("CALICO_API_GROUP") == "projectcalico.org/v3", nil
cs, err := kubernetes.NewForConfig(cfg)
if err != nil {
return false, err
}
dyn, err := dynamic.NewForConfig(cfg)
if err != nil {
return false, err
}
return useV3CRDs(cs.Discovery(), dyn)
}

// useV3CRDs holds the actual decision logic, taking the discovery and dynamic clients as
// interfaces so tests exercise it end-to-end with fakes rather than poking at internal helpers.
//
// - If the v1 CRDs are present, the cluster is an existing/upgraded install (or has opted out
// by pre-installing v1 CRDs), so use v1.
// - If only the v3 CRDs are present, the cluster is already on v3; never downgrade it, but v3
// mode needs MutatingAdmissionPolicy, so error out if the cluster can't serve it.
// - If neither is present, this is a brand-new install. Default to v3, but only when the cluster
// can serve MutatingAdmissionPolicy (needed to default policy types in v3 mode); otherwise v1.
func useV3CRDs(disco discovery.DiscoveryInterface, dyn dynamic.Interface) (bool, error) {
if apiGroup := os.Getenv("CALICO_API_GROUP"); apiGroup != "" {
log.Info("CALICO_API_GROUP environment variable is set, using its value to determine API group", "CALICO_API_GROUP", apiGroup)
return requireMAPForV3(apiGroup == "projectcalico.org/v3", disco)
}

// Check if a DatastoreMigration CR exists in a state that indicates v3 CRDs
// should be used. This handles operator restarts during or after migration.
// This runs before the manager cache is started, so we use a dynamic client
// directly rather than the cached datastoremigration.GetPhase().
if v3, err := checkDatastoreMigration(cfg); err != nil {
if migrated, err := checkDatastoreMigration(dyn); err != nil {
log.Info("Failed to check DatastoreMigration CR, falling through to API discovery", "error", err)
} else if v3 {
return true, nil
} else if migrated {
return requireMAPForV3(true, disco)
}

cs, err := kubernetes.NewForConfig(cfg)
if err != nil {
return false, err
}
apiGroups, err := cs.Discovery().ServerGroups()
apiGroups, err := disco.ServerGroups()
if err != nil {
return false, err
}

v3present, v1present := false, false
for _, g := range apiGroups.Groups {
if g.Name == v3.GroupName {
switch g.Name {
case v3.GroupName:
v3present = true
}
if g.Name == "crd.projectcalico.org" {
case "crd.projectcalico.org":
v1present = true
}
}

log.Info("Detected API groups from API server", "v3present", v3present, "v1present", v1present)
return v3present && !v1present, nil
// v1 CRDs present means an existing/upgraded install (or an admin who opted out by
// pre-installing v1 CRDs); stay on v1 without paying for the MutatingAdmissionPolicy lookup.
if v1present {
log.Info("Detected API groups from API server", "v3present", v3present, "v1present", v1present)
return false, nil
}

// v1 is absent, so v3 is still in play: either an existing v3 install we must not downgrade,
// or a greenfield install we default to v3. Both need MutatingAdmissionPolicy to default policy
// types, so its availability decides the outcome.
mapServed := isMutatingAdmissionPolicyServed(disco)
log.Info("Detected API groups from API server", "v3present", v3present, "v1present", v1present, "mapServed", mapServed)

if v3present {
if !mapServed {
return false, errV3RequiresMAP
}
return true, nil
}
return mapServed, nil
}

// requireMAPForV3 gates a v3 decision on MutatingAdmissionPolicy support. When v3 is chosen but
// the cluster can't serve MAP we return an error and refuse to operate; a v1 decision passes
// through untouched (and skips the discovery call). Used by the paths that assert v3 without
// cluster CRD evidence - the CALICO_API_GROUP override and a converged DatastoreMigration.
func requireMAPForV3(useV3 bool, disco discovery.DiscoveryInterface) (bool, error) {
if useV3 && !isMutatingAdmissionPolicyServed(disco) {
return false, errV3RequiresMAP
}
return useV3, nil
}

// checkDatastoreMigration uses a dynamic client to look for a DatastoreMigration CR
// and returns true if one exists in a phase that indicates v3 CRDs should be used.
// This is used at startup before the manager cache is available.
func checkDatastoreMigration(cfg *rest.Config) (bool, error) {
dc, err := dynamic.NewForConfig(cfg)
if err != nil {
return false, err
}
list, err := dc.Resource(datastoreMigrationGVR).List(context.Background(), metav1.ListOptions{})
func checkDatastoreMigration(dyn dynamic.Interface) (bool, error) {
list, err := dyn.Resource(datastoreMigrationGVR).List(context.Background(), metav1.ListOptions{})
if err != nil {
return false, err
}
Expand All @@ -103,3 +157,32 @@ func checkDatastoreMigration(cfg *rest.Config) (bool, error) {
}
return false, nil
}

// isMutatingAdmissionPolicyServed reports whether the cluster serves the MutatingAdmissionPolicy
// API (any version). v3 CRD mode relies on a MutatingAdmissionPolicy to default policy types, so
// a greenfield install only defaults to v3 when this is available (k8s 1.32+).
func isMutatingAdmissionPolicyServed(disco discovery.DiscoveryInterface) bool {
_, resourceLists, err := disco.ServerGroupsAndResources()
if err != nil {
// A partial ErrGroupDiscoveryFailed just means some aggregated APIService is unhealthy; the
// healthy groups (including the core admissionregistration.k8s.io that serves MAP) still come
// back, so continue with what we got. Any other error is a real discovery failure.
if !discovery.IsGroupDiscoveryFailedError(err) {
log.Error(err, "Failed to discover server resources while checking for MutatingAdmissionPolicy")
return false
}
log.Info("Some API groups failed discovery while checking for MutatingAdmissionPolicy; continuing with the groups that succeeded", "error", err)
}
for _, rl := range resourceLists {
gv, parseErr := schema.ParseGroupVersion(rl.GroupVersion)
if parseErr != nil || gv.Group != mutatingAdmissionPolicyGroup {
continue
}
for _, r := range rl.APIResources {
if r.Kind == mutatingAdmissionPolicyKind {
return true
}
}
}
return false
}
121 changes: 121 additions & 0 deletions pkg/apis/version_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// Copyright (c) 2026 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apis

import (
"testing"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
dynamicfake "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/fake"
)

func mapResourceList() *metav1.APIResourceList {
return &metav1.APIResourceList{
GroupVersion: "admissionregistration.k8s.io/v1beta1",
APIResources: []metav1.APIResource{{Name: "mutatingadmissionpolicies", Kind: "MutatingAdmissionPolicy"}},
}
}

// emptyDynamicClient returns a dynamic fake with no DatastoreMigration CRs, so the migration check
// finds nothing and useV3CRDs falls through to API discovery.
func emptyDynamicClient() *dynamicfake.FakeDynamicClient {
return dynamicfake.NewSimpleDynamicClientWithCustomListKinds(
runtime.NewScheme(),
map[schema.GroupVersionResource]string{datastoreMigrationGVR: "DatastoreMigrationList"},
)
}

func TestUseV3CRDs(t *testing.T) {
v1 := &metav1.APIResourceList{GroupVersion: "crd.projectcalico.org/v1"}
v3 := &metav1.APIResourceList{GroupVersion: "projectcalico.org/v3"}

cases := []struct {
name string
apiGroup string
resources []*metav1.APIResourceList
want bool
wantErr bool
}{
{"v1 present stays v1", "", []*metav1.APIResourceList{v1, mapResourceList()}, false, false},
{"both present stays v1", "", []*metav1.APIResourceList{v1, v3, mapResourceList()}, false, false},
{"v3 present with MAP stays v3", "", []*metav1.APIResourceList{v3, mapResourceList()}, true, false},
{"v3 present without MAP errors", "", []*metav1.APIResourceList{v3}, false, true},
{"greenfield capable goes v3", "", []*metav1.APIResourceList{mapResourceList()}, true, false},
{"greenfield not capable stays v1", "", []*metav1.APIResourceList{}, false, false},

{"override v3 with MAP goes v3", "projectcalico.org/v3", []*metav1.APIResourceList{mapResourceList()}, true, false},
{"override v3 without MAP errors", "projectcalico.org/v3", []*metav1.APIResourceList{}, false, true},
{"override v1 ignores MAP", "crd.projectcalico.org/v1", []*metav1.APIResourceList{}, false, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if tc.apiGroup != "" {
t.Setenv("CALICO_API_GROUP", tc.apiGroup)
}
c := fake.NewClientset()
c.Resources = tc.resources
got, err := useV3CRDs(c.Discovery(), emptyDynamicClient())
if (err != nil) != tc.wantErr {
t.Fatalf("useV3CRDs() err = %v, wantErr %t", err, tc.wantErr)
}
if got != tc.want {
t.Errorf("useV3CRDs() = %t, want %t", got, tc.want)
}
})
}
}

func TestIsMutatingAdmissionPolicyServed(t *testing.T) {
cases := []struct {
name string
resources []*metav1.APIResourceList
want bool
}{
{
name: "served when MAP resource present",
resources: []*metav1.APIResourceList{{
GroupVersion: "admissionregistration.k8s.io/v1beta1",
APIResources: []metav1.APIResource{{Name: "mutatingadmissionpolicies", Kind: "MutatingAdmissionPolicy"}},
}},
want: true,
},
{
name: "not served when group present without MAP kind",
resources: []*metav1.APIResourceList{{
GroupVersion: "admissionregistration.k8s.io/v1",
APIResources: []metav1.APIResource{{Name: "validatingwebhookconfigurations", Kind: "ValidatingWebhookConfiguration"}},
}},
want: false,
},
{
name: "not served on empty cluster",
resources: []*metav1.APIResourceList{},
want: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
c := fake.NewClientset()
c.Resources = tc.resources
got := isMutatingAdmissionPolicyServed(c.Discovery())
if got != tc.want {
t.Errorf("isMutatingAdmissionPolicyServed() = %t, want %t", got, tc.want)
}
})
}
}
11 changes: 8 additions & 3 deletions pkg/imports/admission/enterprise/protect-builtin-tiers.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
---
# ValidatingAdmissionPolicy that prevents deletion of the built-in Calico tiers
# (default, kube-admin, kube-baseline). These tiers are required for correct
# operation and should never be deleted.
# (default, adminnetworkpolicy, baselineadminnetworkpolicy). These tiers are
# required for correct operation and should never be deleted.
#
# NOTE: Calico Enterprise does not yet support ClusterNetworkPolicy, so the OSS
# kube-admin / kube-baseline tiers are not created here and are not protected.
# When ClusterNetworkPolicy lands, add kube-admin / kube-baseline to the list
# below (see EnsureInitialized in libcalico-go/lib/clientv3/client.go).
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
Expand All @@ -15,7 +20,7 @@ spec:
resources: ["tiers"]
operations: ["DELETE"]
validations:
- expression: "!(oldObject.metadata.name in ['default', 'kube-admin', 'kube-baseline'])"
- expression: "!(oldObject.metadata.name in ['default', 'adminnetworkpolicy', 'baselineadminnetworkpolicy'])"
Comment thread
caseydavenport marked this conversation as resolved.
messageExpression: "'The built-in tier ' + oldObject.metadata.name + ' cannot be deleted'"
---
apiVersion: admissionregistration.k8s.io/v1
Expand Down
33 changes: 33 additions & 0 deletions pkg/render/apiserver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,24 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() {
}))
})

It("should keep the apiserver deployment but drop the APIService for enterprise in v3 CRD mode", func() {
cfg.RequiresAggregationServer = false

component, err := render.APIServer(cfg)
Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err)
Expect(component.ResolveImages(nil)).To(BeNil())
resources, _ := component.Objects()

// The Deployment must still be present: in enterprise it hosts the queryserver.
Expect(rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment")).ToNot(BeNil())

// The v3.projectcalico.org APIService must be absent in v3 CRD mode.
for _, r := range resources {
Expect(r.GetObjectKind().GroupVersionKind().Kind).NotTo(Equal("APIService"),
"unexpected APIService registered in v3 CRD mode: %s", r.GetName())
}
})

It("should grant the calico-apiserver SA write access to globalreports/status", func() {
component, err := render.APIServer(cfg)
Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err)
Expand Down Expand Up @@ -2155,6 +2173,21 @@ var _ = Describe("API server rendering tests (Calico)", func() {
rtest.ExpectResourceInList(deleteResources, "calico-apiserver", "calico-system", "policy", "v1", "PodDisruptionBudget")
})

It("should not register the aggregation APIService when not requiring the aggregation server", func() {
cfg.RequiresAggregationServer = false

component, err := render.APIServer(cfg)
Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err)
Expect(component.ResolveImages(nil)).To(BeNil())
resources, _ := component.Objects()

// The v3.projectcalico.org APIService registration must be absent in v3 CRD mode.
for _, r := range resources {
Expect(r.GetObjectKind().GroupVersionKind().Kind).NotTo(Equal("APIService"),
"unexpected APIService registered in v3 CRD mode: %s", r.GetName())
}
})

It("should render an API server with custom configuration", func() {
expectedResources := []client.Object{
&corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ServiceAccount"}},
Expand Down
Loading