diff --git a/test/e2e/config.go b/test/e2e/config.go index f95f5613..c845ad9a 100644 --- a/test/e2e/config.go +++ b/test/e2e/config.go @@ -22,14 +22,14 @@ import ( "sigs.k8s.io/yaml" ) -// config represents the raw configuration data loaded from YAML files. +// Config represents the raw configuration data loaded from YAML files. // This struct contains file paths and basic configuration values that are used // to load and initialize the actual Kubernetes objects for testing. // -// - Loaded from YAML config files (e.g., config-quick.yaml, config-provisioning.yaml) +// - Loaded from YAML Config files (e.g., Config-quick.yaml, Config-provisioning.yaml) // - Used by applyConfig() to populate systemTestInput with actual Kubernetes objects // - Used by systemTestInput for object loading and initialization -type config struct { +type Config struct { DPUFlavorPath *string `json:"dpuFlavor,omitempty"` ProvisioningControllerPVCPath *string `json:"provisioningControllerPVC,omitempty"` BFBPath *string `json:"bfb,omitempty"` @@ -73,12 +73,12 @@ type config struct { AdditionalDPUServiceConfigurationPath *string `json:"additionalDPUServiceConfiguration,omitempty"` } -func readConfig(path string) (*config, error) { +func ReadConfig(path string) (*Config, error) { configData, err := os.ReadFile(path) if err != nil { return nil, err } - conf := &config{} + conf := &Config{} if err = yaml.UnmarshalStrict(configData, conf); err != nil { return nil, err } diff --git a/test/e2e/deprecation_warnings.go b/test/e2e/deprecation_warnings.go index b8195d53..3dc23208 100644 --- a/test/e2e/deprecation_warnings.go +++ b/test/e2e/deprecation_warnings.go @@ -52,19 +52,19 @@ func (w *warningCollector) get() []string { // warnings fire when a deprecated field is set on a DPF resource. It uses // spec.bmcIP on a DPU as one arbitrary example of a deprecated field to // trigger and assert on the warning. -func ValidateVAPDeprecationWarnings(ctx context.Context, input *systemTestInput) { +func ValidateVAPDeprecationWarnings(ctx context.Context, input *SystemTestInput) { collector := &warningCollector{} - cfg := rest.CopyConfig(input.restConfig) + cfg := rest.CopyConfig(input.RestConfig) cfg.WarningHandler = collector - warningClient, err := client.New(cfg, client.Options{Scheme: input.client.Scheme()}) + warningClient, err := client.New(cfg, client.Options{Scheme: input.Client.Scheme()}) Expect(err).NotTo(HaveOccurred()) By("Creating a DPU with deprecated spec.bmcIP set") dpu := &provisioningv1.DPU{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "e2e-vap-warning-", - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: CleanupScope.It, }, Spec: provisioningv1.DPUSpec{ @@ -90,16 +90,16 @@ func ValidateVAPDeprecationWarnings(ctx context.Context, input *systemTestInput) By("Creating a DPU without any deprecated fields set") negativeCollector := &warningCollector{} - negativeCfg := rest.CopyConfig(input.restConfig) + negativeCfg := rest.CopyConfig(input.RestConfig) negativeCfg.WarningHandler = negativeCollector - negativeClient, err := client.New(negativeCfg, client.Options{Scheme: input.client.Scheme()}) + negativeClient, err := client.New(negativeCfg, client.Options{Scheme: input.Client.Scheme()}) Expect(err).NotTo(HaveOccurred()) dpuNoDeprecated := &provisioningv1.DPU{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "e2e-vap-no-warning-", - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: CleanupScope.It, }, Spec: provisioningv1.DPUSpec{ diff --git a/test/e2e/doc/README.md b/test/e2e/doc/README.md index e52eef44..d1e1a999 100644 --- a/test/e2e/doc/README.md +++ b/test/e2e/doc/README.md @@ -18,7 +18,7 @@ General information about the E2E testing framework structure, patterns, and bes ### Infrastructure setup * `test/e2e/system_setup.go` - DPF system deployment, node setup, cluster provisioning -* `test/e2e/system_test.go` - system-level test configuration (SetInput, SystemSetupBeforeSuite) +* `test/e2e/system_bootstrap.go` - system-level test configuration (`SetInput`, `SystemSetupBeforeSuite`) ### Test suites * `test/e2e/*_test.go` @@ -83,7 +83,7 @@ Configs reference YAML manifests in `test/objects/` ### Related files * `test/e2e/config.go` - config struct definition -* `test/e2e/system_setup.go` - `systemTestInput` (holds loaded objects), `applyConfig()` (loads manifests) +* `test/e2e/system_setup.go` - `SystemTestInput` (holds loaded objects), `ApplyConfig()` (loads manifests) ### CI workflows For automated test execution workflows, see: @@ -174,7 +174,7 @@ var _ = Describe("DPF tests ...", Labels{Domain.DPFSystem}, func() { BeforeEach(func() { // If required: Check if we have DPU nodes - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { return } }) @@ -184,7 +184,7 @@ var _ = Describe("DPF tests ...", Labels{Domain.DPFSystem}, func() { // * Only It starts with a minuscule Context("Validate my fancy feature", Labels{dpfSystemLabel, requiresNodesLabel}, func() { It("create a pod consuming a DPUServiceNAD with all dependencies and check that it is created successfully", func() { - ValidateMyFeature(ctx, input) + ValidateMyFeature(Ctx, input) }) }) }) @@ -192,9 +192,9 @@ var _ = Describe("DPF tests ...", Labels{Domain.DPFSystem}, func() { ```go // myfeature.go -func ValidateMyFeature(ctx context.Context, input *systemTestInput) { +func ValidateMyFeature(ctx context.Context, input *SystemTestInput) { // If required: Check if we have DPU nodes - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { Skip("Skip test as there are not multiple nodes") } @@ -225,7 +225,7 @@ func ValidateMyFeature(ctx context.Context, input *systemTestInput) { Labels: utils.AfterAllCleanupLabels, // Define appropriate cleanup label }, } - Expect(input.client.Create(ctx, testNS)).To(Succeed()) + Expect(input.Client.Create(ctx, testNS)).To(Succeed()) By("Created test namespace: " + testNS.Name) ////////////////////////////////////////////// @@ -233,7 +233,7 @@ func ValidateMyFeature(ctx context.Context, input *systemTestInput) { By("Copy image pull secret to namespace " + testNS.Name) // Re-use generic existing helper functions if possible - CopySecretToNamespace(ctx, input.client, dpfPullSecretName, dpfOperatorSystemNamespace, testNS.Name, utils.AfterEachCleanupLabels) + CopySecretToNamespace(ctx, input.Client, DPFPullSecretName, DPFOperatorSystemNamespace, testNS.Name, utils.AfterEachCleanupLabels) ////////////////////////////////////////////// // Object creation and validation @@ -241,13 +241,13 @@ func ValidateMyFeature(ctx context.Context, input *systemTestInput) { // Easy to read as factory method abstracts away construction details and programm flow and business logic is more in focus // Separation of concerns (object construction separated from creation) dpuServiceNAD := constructDPUServiceNAD(dpuServiceNADName, testNS.Name, mtu) - Expect(input.client.Create(ctx, dpuServiceNAD)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceNAD)).To(Succeed()) // ... By("Verify DPUServiceNAD is ready") // Most of our objects have a defined status field structure and can be validated easily using helpers - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceNAD, defaultTimeout) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceNAD, defaultTimeout) By("Verify DPUService pods are created in DPU cluster") // Check with Eventually for async operations @@ -257,8 +257,8 @@ func ValidateMyFeature(ctx context.Context, input *systemTestInput) { Eventually(func(g Gomega) { const podServiceLabel string = "svc.dpu.nvidia.com/service" podList := &corev1.PodList{} - // Use `dpuClusterClient` for DPU cluster operations, `input.client` for host cluster - g.Expect(dpuClusterClient.List(ctx, podList, + // Use `DPUClusterClient` for DPU cluster operations, `input.Client` for host cluster + g.Expect(DPUClusterClient[0].List(ctx, podList, client.InNamespace(testNS.Name), client.MatchingLabels{podServiceLabel: serviceName}, )).To(Succeed()) @@ -270,7 +270,7 @@ func ValidateMyFeature(ctx context.Context, input *systemTestInput) { // Use when you need to check properties on every pod/object in the list Eventually(func(g Gomega) { podList := &corev1.PodList{} - g.Expect(dpuClusterClient.List(ctx, podList, ...)).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, podList, ...)).To(Succeed()) g.Expect(podList.Items).ToNot(BeEmpty()) // Loop through and check each pod's status for _, pod := range podList.Items { @@ -325,7 +325,7 @@ func constructDummyDPUServiceObject(serviceName, namespace, interfaceName string // values: // global: // imagePullSecretName: dpf-pull-secret - `{"imagePullSecrets": [{"name": "%s"}]}`, dpfPullSecretName, + `{"imagePullSecrets": [{"name": "%s"}]}`, DPFPullSecretName, )), } } diff --git a/test/e2e/dpfoperatorconfig.go b/test/e2e/dpfoperatorconfig.go index e78b48bc..db17cd68 100644 --- a/test/e2e/dpfoperatorconfig.go +++ b/test/e2e/dpfoperatorconfig.go @@ -57,9 +57,9 @@ const ( // ValidateDPFOperatorBaseConfiguration verifies that DPFOperatorConfiguration ContainerComponentConfiguration options work. // It changes the images for all system components to arbitrary values, checks that the changes have propagated and then // changes them back to their default versions. -func ValidateDPFOperatorBaseConfiguration(ctx context.Context, input *systemTestInput) { +func ValidateDPFOperatorBaseConfiguration(ctx context.Context, input *SystemTestInput) { modifiedConfig := &operatorv1.DPFOperatorConfig{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, modifiedConfig)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, modifiedConfig)).To(Succeed()) originalConfig := modifiedConfig.DeepCopy() dummyRegistryName := "dummy-registry.com" @@ -164,7 +164,7 @@ func ValidateDPFOperatorBaseConfiguration(ctx context.Context, input *systemTest ResourceComponentConfig: dummyResourceRequirements, }, } - if !isGinkgoLabelApplied(Domain.ZeroTrust) { + if !IsGinkgoLabelApplied(Domain.ZeroTrust) { modifiedConfig.Spec.NodeSRIOVDevicePluginController = &operatorv1.NodeSRIOVDevicePluginControllerConfiguration{ Controller: &operatorv1.DefaultOverridesConfiguration{ ImageComponentConfig: operatorv1.ImageComponentConfig{ @@ -181,7 +181,7 @@ func ValidateDPFOperatorBaseConfiguration(ctx context.Context, input *systemTest } By("Updating the DPFOperatorConfig with modified images and resources") - Expect(input.client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) + Expect(input.Client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) By("Verifying all components are updated") verifyComponentOverrides(ctx, input, dummyRegistryName, expectedDummyResources) @@ -209,27 +209,27 @@ func ValidateDPFOperatorBaseConfiguration(ctx context.Context, input *systemTest KubeFlannel: dummyRegistryName + "/kube-flannel:legacy-test", FlannelCNI: dummyRegistryName + "/flannel-cni:legacy-test", } - if !isGinkgoLabelApplied(Domain.ZeroTrust) { + if !IsGinkgoLabelApplied(Domain.ZeroTrust) { modifiedConfig.Spec.NodeSRIOVDevicePluginController.Controller.Image = ptr.To(fmt.Sprintf(imageTemplate, dummyRegistryName, operatorv1.NodeSRIOVDevicePluginControllerName)) } modifiedConfig.Spec.KataContainers.Daemon.Image = ptr.To(fmt.Sprintf(imageTemplate, dummyRegistryName, operatorv1.KataContainersName)) - Expect(input.client.Patch(ctx, modifiedConfig, client.MergeFrom(configCopy))).To(Succeed()) + Expect(input.Client.Patch(ctx, modifiedConfig, client.MergeFrom(configCopy))).To(Succeed()) By("Verifying component overrides") verifyComponentOverrides(ctx, input, dummyRegistryName, expectedDummyResources) By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) resetConfig := modifiedConfig.DeepCopy() resetConfig.Spec = originalConfig.Spec // Revert the image versions to their previous values. - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) // Ensure the changes are reverted before continuing. }).Should(Succeed()) } -func verifyComponentOverrides(ctx context.Context, input *systemTestInput, dummyRegistryName string, expectedDummyResources corev1.ResourceRequirements) { +func verifyComponentOverrides(ctx context.Context, input *SystemTestInput, dummyRegistryName string, expectedDummyResources corev1.ResourceRequirements) { // Assert the images are set for the system components. tracker := NewByTracker() Eventually(func(g Gomega) { @@ -254,7 +254,7 @@ func verifyComponentOverrides(ctx context.Context, input *systemTestInput, dummy inventory.DPUServiceControllerName: true, } - if !isGinkgoLabelApplied(Domain.ZeroTrust) { + if !IsGinkgoLabelApplied(Domain.ZeroTrust) { controller[operatorv1.NodeSRIOVDevicePluginControllerName.String()] = true } @@ -262,7 +262,7 @@ func verifyComponentOverrides(ctx context.Context, input *systemTestInput, dummy nameForCluster := fmt.Sprintf("%s-%s", clusterName, name) trackingAnnotationValuePrefix := nameForCluster if prereqsNamespace != "" { - trackingAnnotationValuePrefix = fmt.Sprintf("%s_%s", dpfOperatorSystemNamespace, nameForCluster) + trackingAnnotationValuePrefix = fmt.Sprintf("%s_%s", DPFOperatorSystemNamespace, nameForCluster) } tracker.By(nameForCluster, "verifying overrides for %s", nameForCluster) deployments := appsv1.DeploymentList{} @@ -270,7 +270,7 @@ func verifyComponentOverrides(ctx context.Context, input *systemTestInput, dummy var matchingDeployments []appsv1.Deployment for _, deploy := range deployments.Items { - if strings.HasPrefix(deploy.GetAnnotations()[argoCDTrackingIDAnnotation], trackingAnnotationValuePrefix) { + if strings.HasPrefix(deploy.GetAnnotations()[ArgoCDTrackingIDAnnotation], trackingAnnotationValuePrefix) { matchingDeployments = append(matchingDeployments, deploy) } } @@ -285,23 +285,23 @@ func verifyComponentOverrides(ctx context.Context, input *systemTestInput, dummy // Verify overrides for inCluster DPUServices for name := range inClusterDeploymentDPUServices { - n := getPerClusterDPUServiceName(name, input.dpuClusters[0].Name, input.dpuClusters[0].Namespace) - deployValidation(g, input.client, "in-cluster", n) + n := getPerClusterDPUServiceName(name, input.DPUClusters[0].Name, input.DPUClusters[0].Namespace) + deployValidation(g, input.Client, "in-cluster", n) } // Verify overrides in the DPUClusters for name := range daemonSetDPUServices { - nameForCluster := fmt.Sprintf("%s-%s", input.dpuClusters[0].Name, name) + nameForCluster := fmt.Sprintf("%s-%s", input.DPUClusters[0].Name, name) trackingAnnotationValuePrefix := nameForCluster if prereqsNamespace != "" { - trackingAnnotationValuePrefix = fmt.Sprintf("%s_%s", dpfOperatorSystemNamespace, nameForCluster) + trackingAnnotationValuePrefix = fmt.Sprintf("%s_%s", DPFOperatorSystemNamespace, nameForCluster) } tracker.By(nameForCluster, "verifying overrides for %s", nameForCluster) daemonSets := appsv1.DaemonSetList{} - g.Expect(dpuClusterClient[0].List(ctx, &daemonSets)).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, &daemonSets)).To(Succeed()) var matchingDaemonSets []appsv1.DaemonSet for _, ds := range daemonSets.Items { - if strings.HasPrefix(ds.GetAnnotations()[argoCDTrackingIDAnnotation], trackingAnnotationValuePrefix) { + if strings.HasPrefix(ds.GetAnnotations()[ArgoCDTrackingIDAnnotation], trackingAnnotationValuePrefix) { matchingDaemonSets = append(matchingDaemonSets, ds) } } @@ -319,7 +319,7 @@ func verifyComponentOverrides(ctx context.Context, input *systemTestInput, dummy for name := range controller { tracker.By(name, "verifying overrides for %s", name) deployments := appsv1.DeploymentList{} - g.Expect(input.client.List(ctx, &deployments, + g.Expect(input.Client.List(ctx, &deployments, client.MatchingLabels{operatorv1.DPFComponentLabelKey: name})).To(Succeed()) g.Expect(deployments.Items).To(HaveLen(1)) deployment := deployments.Items[0] @@ -335,17 +335,17 @@ func verifyComponentOverrides(ctx context.Context, input *systemTestInput, dummy }, 120*time.Second).Should(Succeed()) } -func ValidateDPFOperatorMTUCurrentConfiguration(ctx context.Context, input *systemTestInput) { - By("Verify flannel configmap for cluster " + input.dpuClusters[0].Name) +func ValidateDPFOperatorMTUCurrentConfiguration(ctx context.Context, input *SystemTestInput) { + By("Verify flannel configmap for cluster " + input.DPUClusters[0].Name) flannelConfigMap := &corev1.ConfigMap{} - Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: "kube-flannel-cfg"}, flannelConfigMap)).To(Succeed()) + Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: "kube-flannel-cfg"}, flannelConfigMap)).To(Succeed()) Expect(flannelConfigMap.Data["net-conf.json"]).To(ContainSubstring("MTU\": 1500,")) } -func ValidateDPFOperatorMTUConfigurationChange(ctx context.Context, input *systemTestInput) { +func ValidateDPFOperatorMTUConfigurationChange(ctx context.Context, input *SystemTestInput) { By("Get the operatorConfig") modifiedConfig := &operatorv1.DPFOperatorConfig{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, modifiedConfig)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, modifiedConfig)).To(Succeed()) By("Update the MTU in the operatorConfig") originalConfig := modifiedConfig.DeepCopy() if modifiedConfig.Spec.Networking == nil { @@ -354,13 +354,13 @@ func ValidateDPFOperatorMTUConfigurationChange(ctx context.Context, input *syste modifiedConfig.Spec.Networking.ControlPlaneMTU = ptr.To(testMTUValue) modifiedConfig.Spec.Networking.HighSpeedMTU = ptr.To(9000) Eventually(func(g Gomega) { - g.Expect(input.client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) }).Should(Succeed()) - By("Verify flannel and multus for cluster " + input.dpuClusters[0].Name) + By("Verify flannel and multus for cluster " + input.DPUClusters[0].Name) Eventually(func(g Gomega) { flannelConfigMap := &corev1.ConfigMap{} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: "kube-flannel-cfg"}, flannelConfigMap)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: "kube-flannel-cfg"}, flannelConfigMap)).To(Succeed()) g.Expect(flannelConfigMap.Data["net-conf.json"]).To(ContainSubstring(fmt.Sprintf(`MTU": %d`, testMTUValue))) netAttachDef := &unstructured.Unstructured{} @@ -370,20 +370,20 @@ func ValidateDPFOperatorMTUConfigurationChange(ctx context.Context, input *syste Kind: "NetworkAttachmentDefinition", }) - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: "mybrsfc"}, netAttachDef)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: "mybrsfc"}, netAttachDef)).To(Succeed()) netAttachConfig, exists, err := unstructured.NestedString(netAttachDef.Object, "spec", "config") g.Expect(err).ToNot(HaveOccurred()) g.Expect(exists).To(BeTrue()) g.Expect(netAttachConfig).To(ContainSubstring("mtu\": 9000,")) - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: "mybrhbn"}, netAttachDef)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: "mybrhbn"}, netAttachDef)).To(Succeed()) netAttachConfig, exists, err = unstructured.NestedString(netAttachDef.Object, "spec", "config") g.Expect(err).ToNot(HaveOccurred()) g.Expect(exists).To(BeTrue()) g.Expect(netAttachConfig).To(ContainSubstring("mtu\": 9000,")) }, time.Second*30).Should(Succeed()) - if input.hasDpuNodes() && !isGinkgoLabelApplied(Domain.ZeroTrust) { + if input.HasDpuNodes() && !IsGinkgoLabelApplied(Domain.ZeroTrust) { By("Get configured OOB bridge name from DPFOperatorConfig") bridgeName := operatorv1.DefaultDPUNodeOOBBridgeName if modifiedConfig.Spec.Networking != nil && modifiedConfig.Spec.Networking.DPUNodeOOBBridgeName != nil { @@ -393,7 +393,7 @@ func ValidateDPFOperatorMTUConfigurationChange(ctx context.Context, input *syste By(fmt.Sprintf("Verify host bridge %s MTU on DPU nodes reflects ControlPlaneMTU change", bridgeName)) Eventually(func(g Gomega) { pods := corev1.PodList{} - g.Expect(input.client.List(ctx, &pods, + g.Expect(input.Client.List(ctx, &pods, client.MatchingLabels{cutil.ProvisioningComponentLabelKey: "hostagent"})).To(Succeed()) runningPods := make([]corev1.Pod, 0, len(pods.Items)) for _, pod := range pods.Items { @@ -405,7 +405,7 @@ func ValidateDPFOperatorMTUConfigurationChange(ctx context.Context, input *syste } g.Expect(runningPods).ToNot(BeEmpty()) for _, pod := range runningPods { - stdout, err := netshoot.ExecInContainerOnce(hostClusterRESTClient, input.restConfig, + stdout, err := netshoot.ExecInContainerOnce(HostClusterRESTClient, input.RestConfig, pod.Namespace, pod.Name, "hostagent", []string{"cat", fmt.Sprintf("/sys/class/net/%s/mtu", bridgeName)}) g.Expect(err).NotTo(HaveOccurred(), "exec on pod %s/%s container hostagent: %s", pod.Namespace, pod.Name, stdout) g.Expect(strings.TrimSpace(stdout)).To(Equal(fmt.Sprintf("%d", testMTUValue))) @@ -415,23 +415,23 @@ func ValidateDPFOperatorMTUConfigurationChange(ctx context.Context, input *syste By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) resetConfig := modifiedConfig.DeepCopy() resetConfig.Spec = originalConfig.Spec // Revert the image versions to their previous values. - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) // Ensure the changes are reverted before continuing. }).Should(Succeed()) } -func ValidateDPFOperatorOOBBridgeNameChange(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPFOperatorOOBBridgeNameChange(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip OOB bridge name test as there are no DPU nodes") } By("Get the operatorConfig") modifiedConfig := &operatorv1.DPFOperatorConfig{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, modifiedConfig)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, modifiedConfig)).To(Succeed()) originalConfig := modifiedConfig.DeepCopy() By("Set a non-existent bridge name in the operatorConfig") @@ -441,13 +441,13 @@ func ValidateDPFOperatorOOBBridgeNameChange(ctx context.Context, input *systemTe } modifiedConfig.Spec.Networking.DPUNodeOOBBridgeName = ptr.To(fakeBridgeName) Eventually(func(g Gomega) { - g.Expect(input.client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) }).Should(Succeed()) By("Verify DPUNode OOBBridgeConfigured condition becomes False") Eventually(func(g Gomega) { dpuNodeList := &provisioningv1.DPUNodeList{} - g.Expect(input.client.List(ctx, dpuNodeList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuNodeList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) g.Expect(dpuNodeList.Items).ToNot(BeEmpty()) for _, dpuNode := range dpuNodeList.Items { for _, cond := range dpuNode.Status.Conditions { @@ -462,16 +462,16 @@ func ValidateDPFOperatorOOBBridgeNameChange(ctx context.Context, input *systemTe By("Revert the bridge name to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) resetConfig := modifiedConfig.DeepCopy() resetConfig.Spec = originalConfig.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) }).Should(Succeed()) By("Verify DPUNode OOBBridgeConfigured condition recovers to True") Eventually(func(g Gomega) { dpuNodeList := &provisioningv1.DPUNodeList{} - g.Expect(input.client.List(ctx, dpuNodeList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuNodeList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) g.Expect(dpuNodeList.Items).ToNot(BeEmpty()) for _, dpuNode := range dpuNodeList.Items { for _, cond := range dpuNode.Status.Conditions { @@ -484,19 +484,19 @@ func ValidateDPFOperatorOOBBridgeNameChange(ctx context.Context, input *systemTe }, 3*time.Minute).Should(Succeed()) } -func ValidateDPFOperatorOOBBridgePostProvisioning(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPFOperatorOOBBridgePostProvisioning(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip OOB bridge post-provisioning test as there are no DPU nodes") } By("Get configured OOB bridge name from DPFOperatorConfig") config := &operatorv1.DPFOperatorConfig{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, config)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, config)).To(Succeed()) bridgeName := config.Spec.Networking.GetDPUNodeOOBBridgeName() By("Get hostagent pods") pods := corev1.PodList{} - Expect(input.client.List(ctx, &pods, + Expect(input.Client.List(ctx, &pods, client.MatchingLabels{cutil.ProvisioningComponentLabelKey: "hostagent"})).To(Succeed()) runningPods := make([]corev1.Pod, 0, len(pods.Items)) for _, pod := range pods.Items { @@ -512,7 +512,7 @@ func ValidateDPFOperatorOOBBridgePostProvisioning(ctx context.Context, input *sy for _, pod := range runningPods { By(fmt.Sprintf("Verify VF is attached to bridge %s on pod %s", bridgeName, pod.Name)) Eventually(func(g Gomega) { - stdout, err := netshoot.ExecInContainerOnce(hostClusterRESTClient, input.restConfig, + stdout, err := netshoot.ExecInContainerOnce(HostClusterRESTClient, input.RestConfig, pod.Namespace, pod.Name, "hostagent", []string{"sh", "-c", fmt.Sprintf("ls /sys/class/net/%s/brif/ 2>/dev/null", bridgeName)}) g.Expect(err).NotTo(HaveOccurred(), "failed to list bridge members on pod %s: %s", pod.Name, stdout) @@ -522,7 +522,7 @@ func ValidateDPFOperatorOOBBridgePostProvisioning(ctx context.Context, input *sy By(fmt.Sprintf("Verify netplan file %s exists and references bridge %s", hostutil.BridgeMTUNetplanFile, bridgeName)) Eventually(func(g Gomega) { - stdout, err := netshoot.ExecInContainerOnce(hostClusterRESTClient, input.restConfig, + stdout, err := netshoot.ExecInContainerOnce(HostClusterRESTClient, input.RestConfig, pod.Namespace, pod.Name, "hostagent", []string{"cat", hostutil.BridgeMTUNetplanFile}) g.Expect(err).NotTo(HaveOccurred(), "netplan file not found on pod %s: %s", pod.Name, stdout) @@ -531,7 +531,7 @@ func ValidateDPFOperatorOOBBridgePostProvisioning(ctx context.Context, input *sy }, time.Minute).Should(Succeed()) By(fmt.Sprintf("Verify legacy netplan file %s is removed", hostutil.LegacyBridgeMTUNetplanFile)) - stdout, err := netshoot.ExecInContainerOnce(hostClusterRESTClient, input.restConfig, + stdout, err := netshoot.ExecInContainerOnce(HostClusterRESTClient, input.RestConfig, pod.Namespace, pod.Name, "hostagent", []string{"sh", "-c", fmt.Sprintf("test -f %s && echo EXISTS || echo GONE", hostutil.LegacyBridgeMTUNetplanFile)}) Expect(err).NotTo(HaveOccurred()) @@ -540,10 +540,10 @@ func ValidateDPFOperatorOOBBridgePostProvisioning(ctx context.Context, input *sy } } -func ValidateDPFOperatorFlannelPodCIDRChange(ctx context.Context, input *systemTestInput) { +func ValidateDPFOperatorFlannelPodCIDRChange(ctx context.Context, input *SystemTestInput) { By("Get the operatorConfig") modifiedConfig := &operatorv1.DPFOperatorConfig{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, modifiedConfig)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, modifiedConfig)).To(Succeed()) By("Update the podCIDR in the operatorConfig") originalConfig := modifiedConfig.DeepCopy() if modifiedConfig.Spec.Flannel == nil { @@ -551,37 +551,37 @@ func ValidateDPFOperatorFlannelPodCIDRChange(ctx context.Context, input *systemT } modifiedConfig.Spec.Flannel.PodCIDR = ptr.To("10.255.0.0/14") Eventually(func(g Gomega) { - g.Expect(input.client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) }).Should(Succeed()) - By("Verify flannel configmap for cluster " + input.dpuClusters[0].Name) + By("Verify flannel configmap for cluster " + input.DPUClusters[0].Name) Eventually(func(g Gomega) { flannelConfigMap := &corev1.ConfigMap{} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: "kube-flannel-cfg"}, flannelConfigMap)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: "kube-flannel-cfg"}, flannelConfigMap)).To(Succeed()) g.Expect(flannelConfigMap.Data["net-conf.json"]).To(ContainSubstring("10.255.0.0/14")) }, time.Second*30).Should(Succeed()) By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) resetConfig := modifiedConfig.DeepCopy() resetConfig.Spec = originalConfig.Spec // Revert the image versions to their previous values. - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) // Ensure the changes are reverted before continuing. }).Should(Succeed()) } -func ValidateDPFOperatorMaxDPUParallelInstallations(ctx context.Context, input *systemTestInput) { +func ValidateDPFOperatorMaxDPUParallelInstallations(ctx context.Context, input *SystemTestInput) { modifiedConfig := &operatorv1.DPFOperatorConfig{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, modifiedConfig)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, modifiedConfig)).To(Succeed()) originalConfig := modifiedConfig.DeepCopy() By("Getting the current provisioning controller pod UIDs") var originalPodUIDs []types.UID Eventually(func(g Gomega) { pods := corev1.PodList{} - g.Expect(input.client.List(ctx, &pods, + g.Expect(input.Client.List(ctx, &pods, client.MatchingLabels{operatorv1.DPFComponentLabelKey: "dpf-provisioning-controller-manager"})).To(Succeed()) g.Expect(pods.Items).ToNot(BeEmpty()) originalPodUIDs = make([]types.UID, 0, len(pods.Items)) @@ -592,13 +592,13 @@ func ValidateDPFOperatorMaxDPUParallelInstallations(ctx context.Context, input * By("Modifying the DPFOperatorConfig to set MaxDPUParallelInstallations") modifiedConfig.Spec.ProvisioningController.MaxDPUParallelInstallations = ptr.To(int32(25)) - Expect(input.client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) + Expect(input.Client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) By("Verifying that the provisioning controller pod is restarted") var restartedPodUIDs []types.UID Eventually(func(g Gomega) { pods := corev1.PodList{} - g.Expect(input.client.List(ctx, &pods, + g.Expect(input.Client.List(ctx, &pods, client.MatchingLabels{operatorv1.DPFComponentLabelKey: "dpf-provisioning-controller-manager"})).To(Succeed()) g.Expect(pods.Items).ToNot(BeEmpty()) // Verify all pods have been restarted @@ -621,16 +621,16 @@ func ValidateDPFOperatorMaxDPUParallelInstallations(ctx context.Context, input * By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) resetConfig := modifiedConfig.DeepCopy() resetConfig.Spec = originalConfig.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) }).WithTimeout(10 * time.Second).Should(Succeed()) By("Verifying that the provisioning controller pod is restarted again") Eventually(func(g Gomega) { pods := corev1.PodList{} - g.Expect(input.client.List(ctx, &pods, + g.Expect(input.Client.List(ctx, &pods, client.MatchingLabels{operatorv1.DPFComponentLabelKey: "dpf-provisioning-controller-manager"})).To(Succeed()) g.Expect(pods.Items).ToNot(BeEmpty()) // Verify all pods have been restarted again @@ -647,9 +647,9 @@ func ValidateDPFOperatorMaxDPUParallelInstallations(ctx context.Context, input * }).WithTimeout(120 * time.Second).Should(Succeed()) } -func ValidateDPFOperatorPathConfiguration(ctx context.Context, input *systemTestInput) { +func ValidateDPFOperatorPathConfiguration(ctx context.Context, input *SystemTestInput) { modifiedConfig := &operatorv1.DPFOperatorConfig{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, modifiedConfig)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, modifiedConfig)).To(Succeed()) originalConfig := modifiedConfig.DeepCopy() modifiedOVSRunPath := "/ovsrun" @@ -668,7 +668,7 @@ func ValidateDPFOperatorPathConfiguration(ctx context.Context, input *systemTest modifiedConfig.Spec.Overrides.DPUOpenvSwitchSystemSharedLibPath = ptr.To(modifiedOVSharedLibPath) modifiedConfig.Spec.Overrides.DPUOpenvSwitchSystemSharedLib64Path = ptr.To(modifiedOVSharedLib64Path) modifiedConfig.Spec.Overrides.FlannelSkipCNIConfigInstallation = ptr.To(false) - Expect(input.client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) + Expect(input.Client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) dpuServiceDaemonSetsWithPathChanges := map[operatorv1.ComponentName]bool{ operatorv1.SFCControllerName: true, @@ -681,16 +681,16 @@ func ValidateDPFOperatorPathConfiguration(ctx context.Context, input *systemTest Eventually(func(g Gomega) { for name := range dpuServiceDaemonSetsWithPathChanges { daemonSets := appsv1.DaemonSetList{} - nameForCluster := fmt.Sprintf("%s-%s", input.dpuClusters[0].Name, name) + nameForCluster := fmt.Sprintf("%s-%s", input.DPUClusters[0].Name, name) trackingAnnotationValuePrefix := nameForCluster if prereqsNamespace != "" { - trackingAnnotationValuePrefix = fmt.Sprintf("%s_%s", dpfOperatorSystemNamespace, nameForCluster) + trackingAnnotationValuePrefix = fmt.Sprintf("%s_%s", DPFOperatorSystemNamespace, nameForCluster) } - g.Expect(dpuClusterClient[0].List(ctx, &daemonSets)).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, &daemonSets)).To(Succeed()) var matchingDaemonSets []appsv1.DaemonSet for _, ds := range daemonSets.Items { - if strings.HasPrefix(ds.GetAnnotations()[argoCDTrackingIDAnnotation], trackingAnnotationValuePrefix) { + if strings.HasPrefix(ds.GetAnnotations()[ArgoCDTrackingIDAnnotation], trackingAnnotationValuePrefix) { matchingDaemonSets = append(matchingDaemonSets, ds) } } @@ -736,11 +736,11 @@ func ValidateDPFOperatorPathConfiguration(ctx context.Context, input *systemTest By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) resetConfig := modifiedConfig.DeepCopy() resetConfig.Spec = originalConfig.Spec // Revert the image versions to their previous values. - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) // Ensure the changes are reverted before continuing. }).Should(Succeed()) } @@ -758,13 +758,13 @@ func volumeNameHasPath(name string, volumes []corev1.Volume, path string) bool { // ValidateDPFOperatorKubernetesAPIServerVIPAndPort validates that the Kubernetes API Server related variables are // propagated correctly to the DMS pods. -func ValidateDPFOperatorKubernetesAPIServerVIPAndPort(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPFOperatorKubernetesAPIServerVIPAndPort(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Test requires node to trigger provisioning on, skipping") } modifiedConfig := &operatorv1.DPFOperatorConfig{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, modifiedConfig)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, modifiedConfig)).To(Succeed()) originalConfig := modifiedConfig.DeepCopy() By("Modifying the DPFOperatorConfig to set the Kubernetes API Server related variables") @@ -773,12 +773,12 @@ func ValidateDPFOperatorKubernetesAPIServerVIPAndPort(ctx context.Context, input } modifiedConfig.Spec.Overrides.KubernetesAPIServerVIP = ptr.To(testKubernetesAPIServerVIP) modifiedConfig.Spec.Overrides.KubernetesAPIServerPort = ptr.To(testKubernetesAPIServerPort) - Expect(input.client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) + Expect(input.Client.Patch(ctx, modifiedConfig, client.MergeFrom(originalConfig))).To(Succeed()) By("Validating that the provisioning controller pod has the correct argument") Eventually(func(g Gomega) { pods := corev1.PodList{} - g.Expect(input.client.List(ctx, &pods, + g.Expect(input.Client.List(ctx, &pods, // TODO: Check if we can align the operatorv1.ProvisioningControllerName with that label in the manifests // all the way client.MatchingLabels{operatorv1.DPFComponentLabelKey: "dpf-provisioning-controller-manager"})).To(Succeed()) @@ -790,12 +790,12 @@ func ValidateDPFOperatorKubernetesAPIServerVIPAndPort(ctx context.Context, input }).WithTimeout(120 * time.Second).Should(Succeed()) By("Triggering DMS Pod recreation") - triggerDMSRecreation(ctx, input.client) + triggerDMSRecreation(ctx, input.Client) By("Validating that all the DMS containers have the environment variables set correctly") Eventually(func(g Gomega) { pods := corev1.PodList{} - g.Expect(input.client.List(ctx, &pods, + g.Expect(input.Client.List(ctx, &pods, client.MatchingLabels{cutil.ProvisioningComponentLabelKey: "hostagent"})).To(Succeed()) g.Expect(pods.Items).ToNot(BeEmpty()) for _, pod := range pods.Items { @@ -816,16 +816,16 @@ func ValidateDPFOperatorKubernetesAPIServerVIPAndPort(ctx context.Context, input By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(modifiedConfig), modifiedConfig)).To(Succeed()) resetConfig := modifiedConfig.DeepCopy() resetConfig.Spec = originalConfig.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(modifiedConfig))).To(Succeed()) }).WithTimeout(10 * time.Second).Should(Succeed()) By("Validating that the provisioning controller pod has the correct argument") Eventually(func(g Gomega) { pods := corev1.PodList{} - g.Expect(input.client.List(ctx, &pods, + g.Expect(input.Client.List(ctx, &pods, // TODO: Check if we can align the operatorv1.ProvisioningControllerName with that label all the way client.MatchingLabels{operatorv1.DPFComponentLabelKey: "dpf-provisioning-controller-manager"})).To(Succeed()) g.Expect(pods.Items).ToNot(BeEmpty()) @@ -836,12 +836,12 @@ func ValidateDPFOperatorKubernetesAPIServerVIPAndPort(ctx context.Context, input }).WithTimeout(120 * time.Second).Should(Succeed()) By("Triggering DMS Pod recreation") - triggerDMSRecreation(ctx, input.client) + triggerDMSRecreation(ctx, input.Client) By("Validating that the DMS containers do not have the env variables anymore") Eventually(func(g Gomega) { pods := corev1.PodList{} - g.Expect(input.client.List(ctx, &pods, + g.Expect(input.Client.List(ctx, &pods, client.MatchingLabels{cutil.ProvisioningComponentLabelKey: "hostagent"})).To(Succeed()) g.Expect(pods.Items).ToNot(BeEmpty()) for _, pod := range pods.Items { @@ -867,7 +867,7 @@ func triggerDMSRecreation(ctx context.Context, c client.Client) { // First delete the existing DMS pods Expect(client.IgnoreNotFound(c.DeleteAllOf(ctx, &corev1.Pod{}, - client.InNamespace(dpfOperatorSystemNamespace), + client.InNamespace(DPFOperatorSystemNamespace), client.MatchingLabels{cutil.ProvisioningComponentLabelKey: "hostagent"}))).To(Succeed()) // Then trigger reconcile of DPUNode Node Controller by modifying the node objects and expect that a new dms pod is created @@ -897,12 +897,12 @@ func triggerDMSRecreation(ctx context.Context, c client.Client) { // ValidateDPFOperatorConfigCleanupPrerequisites this function ensures that the prerequisite objects exist before removing // the DPFOperatorConfig to ensure that we cover edge cases. -func ValidateDPFOperatorConfigCleanupPrerequisites(ctx context.Context, input *systemTestInput) { +func ValidateDPFOperatorConfigCleanupPrerequisites(ctx context.Context, input *SystemTestInput) { // Use case, 2 DPUServiceInterfaces, one created by DPUDeployment and a standalone. The DPF Operator should be able // to delete those gracefully without stuck finalizers due to sfc-controller missing in the DPU Cluster. By("Verify DPUServiceInterface owned by DPUDeployment exists and is not removed by previous tests") dpuServiceInterfaceList := &dpuservicev1.DPUServiceInterfaceList{} - Expect(input.client.List(ctx, dpuServiceInterfaceList, client.HasLabels{dpuservicev1.ParentDPUDeploymentNameLabel})).To(Succeed()) + Expect(input.Client.List(ctx, dpuServiceInterfaceList, client.HasLabels{dpuservicev1.ParentDPUDeploymentNameLabel})).To(Succeed()) Expect(dpuServiceInterfaceList.Items).ToNot(BeEmpty()) dpuDeploymentOwnedServiceInterfaceLabels := make([]map[string]string, 0, len(dpuServiceInterfaceList.Items)) for _, dpuServiceInterface := range dpuServiceInterfaceList.Items { @@ -916,31 +916,31 @@ func ValidateDPFOperatorConfigCleanupPrerequisites(ctx context.Context, input *s By("Create test namespace") testNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: dpuServiceInterfaceNamespace}} testNS.SetLabels(CleanupScope.Suite) - Expect(input.client.Create(ctx, testNS)).To(Succeed()) + Expect(input.Client.Create(ctx, testNS)).To(Succeed()) By("Create DPUServiceInterface") - dpuServiceInterface := input.dpuServiceInterface.DeepCopy() + dpuServiceInterface := input.DPUServiceInterface.DeepCopy() dpuServiceInterface.SetName(dpuServiceInterfaceName) dpuServiceInterface.SetNamespace(dpuServiceInterfaceNamespace) dpuServiceInterface.SetLabels(CleanupScope.Suite) dpuServiceInterface.Spec.Template.Spec.NodeSelector = nil - Expect(input.client.Create(ctx, dpuServiceInterface)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceInterface)).To(Succeed()) - if input.hasDpuNodes() { - By(fmt.Sprintf("Verify ServiceInterface is created in %d nodes", input.totalDPUs())) + if input.HasDpuNodes() { + By(fmt.Sprintf("Verify ServiceInterface is created in %d nodes", input.TotalDPUs())) Eventually(func(g Gomega) { // Expect ServiceInterface for standalone DPUServiceInterface to be created. // ServiceInterface objects are created per K8s node in the DPU cluster, and each DPU device // becomes a separate K8s node, so the count equals totalDPUs() (nodes * DPUs per node). standaloneServiceInterfaceList := &dpuservicev1.ServiceInterfaceList{} - g.Expect(dpuClusterClient[0].List(ctx, standaloneServiceInterfaceList, client.InNamespace(dpuServiceInterfaceNamespace))).To(Succeed()) - g.Expect(standaloneServiceInterfaceList.Items).To(HaveLen(input.totalDPUs())) + g.Expect(DPUClusterClient[0].List(ctx, standaloneServiceInterfaceList, client.InNamespace(dpuServiceInterfaceNamespace))).To(Succeed()) + g.Expect(standaloneServiceInterfaceList.Items).To(HaveLen(input.TotalDPUs())) // Expect ServiceInterface for DPUDeployment owned DPUServiceInterface to exist for _, serviceInterfaceLabels := range dpuDeploymentOwnedServiceInterfaceLabels { dpudeploymentOwnedServiceInterfaceList := &dpuservicev1.ServiceInterfaceList{} - g.Expect(dpuClusterClient[0].List(ctx, dpudeploymentOwnedServiceInterfaceList, client.MatchingLabels(serviceInterfaceLabels))).To(Succeed()) - g.Expect(dpudeploymentOwnedServiceInterfaceList.Items).To(HaveLen(input.totalDPUs())) + g.Expect(DPUClusterClient[0].List(ctx, dpudeploymentOwnedServiceInterfaceList, client.MatchingLabels(serviceInterfaceLabels))).To(Succeed()) + g.Expect(dpudeploymentOwnedServiceInterfaceList.Items).To(HaveLen(input.TotalDPUs())) } }).WithTimeout(2 * time.Minute).Should(Succeed()) } @@ -949,8 +949,8 @@ func ValidateDPFOperatorConfigCleanupPrerequisites(ctx context.Context, input *s func DeleteDPFOperatorConfig(ctx context.Context, testClient client.Client) { By("Delete the operatorConfig and ensure it is deleted") Eventually(func(g Gomega) { - key := client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName} - g.Expect(client.IgnoreNotFound(testClient.DeleteAllOf(ctx, &operatorv1.DPFOperatorConfig{}, client.InNamespace(dpfOperatorSystemNamespace)))).To(Succeed()) + key := client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName} + g.Expect(client.IgnoreNotFound(testClient.DeleteAllOf(ctx, &operatorv1.DPFOperatorConfig{}, client.InNamespace(DPFOperatorSystemNamespace)))).To(Succeed()) g.Expect(apierrors.IsNotFound(testClient.Get(ctx, key, &operatorv1.DPFOperatorConfig{}))).To(BeTrue()) }).WithTimeout(time.Hour).WithPolling(30 * time.Second).Should(Succeed()) // TODO: Remove once DPUSets implement foreground deletion diff --git a/test/e2e/dpudeployment.go b/test/e2e/dpudeployment.go index c5f7d644..983e20ca 100644 --- a/test/e2e/dpudeployment.go +++ b/test/e2e/dpudeployment.go @@ -51,27 +51,27 @@ const ( NodeUnschedulableTaintKey = "node.kubernetes.io/unschedulable" ) -func ValidateDPUDeploymentCreation(ctx context.Context, input *systemTestInput) { +func ValidateDPUDeploymentCreation(ctx context.Context, input *SystemTestInput) { By("Creating the dependencies") createDeploymentDependencies(ctx, input, "") By("Creating the dpudeployment") dpuDeployment := generateDPUDeployment(input, "") dpuDeployment.SetLabels(CleanupScope.It) - Expect(input.client.Create(ctx, dpuDeployment)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuDeployment)).To(Succeed()) By("Checking that the underlying objects are created") Eventually(func(g Gomega) { - g.Expect(VerifyDeploymentUnderlyingObjectsCreated(ctx, g, input.client, dpuDeployment)).To(BeTrue()) + g.Expect(VerifyDeploymentUnderlyingObjectsCreated(ctx, g, input.Client, dpuDeployment)).To(BeTrue()) }).WithTimeout(15 * time.Minute).WithPolling(time.Second).Should(Succeed()) } -func ValidateDPUDeploymentMetrics(ctx context.Context, input *systemTestInput) { +func ValidateDPUDeploymentMetrics(ctx context.Context, input *SystemTestInput) { By("Create DPUDeployment for metrics") createDeploymentDependencies(ctx, input, "metrics") dpuDeployment := generateDPUDeployment(input, "metrics") dpuDeployment.SetLabels(CleanupScope.It) - Expect(input.client.Create(ctx, dpuDeployment)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuDeployment)).To(Succeed()) By("Verify DPUDeployment and DPUServiceInterface metrics are in KSM") expectedMetricsNames := map[string][]string{ @@ -79,31 +79,31 @@ func ValidateDPUDeploymentMetrics(ctx context.Context, input *systemTestInput) { } Eventually(func(g Gomega) { - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, metricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, MetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(5 * time.Second).Should(Succeed()) } -func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.Context, input *systemTestInput) { +func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.Context, input *SystemTestInput) { // This part is needed so that we can test that the deletion logic is able to delete all the DPUServices, even // stale paused ones. By("Create DPUDeployment until deletion while disruptive upgrade is in progress") dpuServiceTemplate := generateDPUServiceTemplate(input, "disruptive") - Expect(input.client.Create(ctx, dpuServiceTemplate)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceTemplate)).To(Succeed()) dpuServiceConfiguration := generateServiceConfiguration(input, "disruptive") - Expect(input.client.Create(ctx, dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceConfiguration)).To(Succeed()) dpuDeployment := generateDPUDeployment(input, "disruptive") - Expect(input.client.Create(ctx, dpuDeployment)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuDeployment)).To(Succeed()) By("Checking that the underlying objects are created") Eventually(func(g Gomega) { - g.Expect(VerifyDeploymentUnderlyingObjectsCreated(ctx, g, input.client, dpuDeployment)).To(BeTrue()) + g.Expect(VerifyDeploymentUnderlyingObjectsCreated(ctx, g, input.Client, dpuDeployment)).To(BeTrue()) // Checking that application exists for the created DPUService. This is needed so that the HACK step is stable. gotDPUServiceList := &dpuservicev1.DPUServiceList{} - g.Expect(input.client.List(ctx, + g.Expect(input.Client.List(ctx, gotDPUServiceList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ @@ -112,7 +112,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C g.Expect(gotDPUServiceList.Items).To(HaveLen(1)) gotApplicationList := &argov1.ApplicationList{} - g.Expect(input.client.List(ctx, gotApplicationList, client.InNamespace(dpuDeployment.GetNamespace()))).To(Succeed()) + g.Expect(input.Client.List(ctx, gotApplicationList, client.InNamespace(dpuDeployment.GetNamespace()))).To(Succeed()) dpuServiceNameToApplication := getDPUServiceNameToApplication(gotDPUServiceList.Items, gotApplicationList.Items) g.Expect(dpuServiceNameToApplication).To(HaveLen(1)) }).WithTimeout(15 * time.Minute).WithPolling(time.Second).Should(Succeed()) @@ -120,16 +120,16 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C const expectedDPUServicesOnDisruptiveUpgrade = 2 By("Triggering the disruptive upgrade with bad parameters") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) originalDPUServiceConfiguration := dpuServiceConfiguration.DeepCopy() dpuServiceConfiguration.Spec.ServiceConfiguration.HelmChart.Values = &machineryruntime.RawExtension{Raw: []byte(`{"image":{"pullPolicy":"malformedPullPolicy"}}`)} - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) By(fmt.Sprintf("Checking that %d DPUServices and Applications exist and one of them is paused", expectedDPUServicesOnDisruptiveUpgrade)) Eventually(func(g Gomega) { gotDPUServiceList := &dpuservicev1.DPUServiceList{} - g.Expect(input.client.List(ctx, + g.Expect(input.Client.List(ctx, gotDPUServiceList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ @@ -139,7 +139,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C // Checking that applications exist for both the DPUServices. This is needed so that the HACK step is stable. gotApplicationList := &argov1.ApplicationList{} - g.Expect(input.client.List(ctx, gotApplicationList, client.InNamespace(dpuDeployment.GetNamespace()))).To(Succeed()) + g.Expect(input.Client.List(ctx, gotApplicationList, client.InNamespace(dpuDeployment.GetNamespace()))).To(Succeed()) dpuServiceNameToApplication := getDPUServiceNameToApplication(gotDPUServiceList.Items, gotApplicationList.Items) g.Expect(dpuServiceNameToApplication).To(HaveLen(expectedDPUServicesOnDisruptiveUpgrade)) @@ -156,7 +156,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C }).WithTimeout(30 * time.Second).MustPassRepeatedly(5).Should(Succeed()) By("Deleting the dpudeployment") - Expect(input.client.Delete(ctx, dpuDeployment)).To(Succeed()) + Expect(input.Client.Delete(ctx, dpuDeployment)).To(Succeed()) // Failed to apply ArgoCD Application deletion can take up to 5 mins based on the current configuration, // therefore we modify the application to have correct configuration so that the deletion goes faster @@ -164,7 +164,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C By("HACK: Modify the bad application to ensure that it can be deleted faster") gotDPUServiceList := &dpuservicev1.DPUServiceList{} Eventually(func(g Gomega) { - g.Expect(input.client.List(ctx, + g.Expect(input.Client.List(ctx, gotDPUServiceList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ @@ -186,7 +186,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C malformedApplications := map[client.ObjectKey]any{} Eventually(func(g Gomega) { gotApplicationList := &argov1.ApplicationList{} - g.Expect(input.client.List(ctx, gotApplicationList, client.InNamespace(dpuDeployment.GetNamespace()))).To(Succeed()) + g.Expect(input.Client.List(ctx, gotApplicationList, client.InNamespace(dpuDeployment.GetNamespace()))).To(Succeed()) dpuServiceNameToApplication := getDPUServiceNameToApplication(gotDPUServiceList.Items, gotApplicationList.Items) for _, application := range dpuServiceNameToApplication { @@ -230,10 +230,10 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C } // Use optimistic locking to ensure that we patch the latest version of the application to not forget a operation which was just triggered. - g.Expect(input.client.Patch(ctx, &application, client.MergeFromWithOptions(origApp, client.MergeFromWithOptimisticLock{}))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, &application, client.MergeFromWithOptions(origApp, client.MergeFromWithOptimisticLock{}))).To(Succeed()) // Delete the application to ensure that we haven't recreated the application in the meantime with the // patch above - g.Expect(input.client.Delete(ctx, &application)).To(Succeed()) + g.Expect(input.Client.Delete(ctx, &application)).To(Succeed()) } } }).WithTimeout(30 * time.Second).Should(Succeed()) @@ -242,7 +242,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C Eventually(func(g Gomega) { for key := range malformedApplications { application := &argov1.Application{} - err := input.client.Get(ctx, key, application) + err := input.Client.Get(ctx, key, application) if apierrors.IsNotFound(err) { continue } @@ -275,7 +275,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C Refresh: true, }, } - g.Expect(input.client.Patch(ctx, application, client.MergeFromWithOptions(origApp, client.MergeFromWithOptimisticLock{}))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, application, client.MergeFromWithOptions(origApp, client.MergeFromWithOptimisticLock{}))).To(Succeed()) } g.Expect(application.Status.OperationState.Phase).To(BeEquivalentTo("Running")) @@ -289,7 +289,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C By("Checking that the underlying objects are deleted") Eventually(func(g Gomega) { gotDPUSetList := &provisioningv1.DPUSetList{} - g.Expect(input.client.List(ctx, + g.Expect(input.Client.List(ctx, gotDPUSetList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ @@ -298,7 +298,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C g.Expect(gotDPUSetList.Items).To(BeEmpty()) gotDPUServiceList := &dpuservicev1.DPUServiceList{} - g.Expect(input.client.List(ctx, + g.Expect(input.Client.List(ctx, gotDPUServiceList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ @@ -307,7 +307,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C g.Expect(gotDPUServiceList.Items).To(BeEmpty()) gotDPUServiceChainList := &dpuservicev1.DPUServiceChainList{} - g.Expect(input.client.List(ctx, + g.Expect(input.Client.List(ctx, gotDPUServiceChainList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ @@ -316,7 +316,7 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C g.Expect(gotDPUServiceChainList.Items).To(BeEmpty()) gotDPUServiceInterfaceList := &dpuservicev1.DPUServiceInterfaceList{} - g.Expect(input.client.List(ctx, + g.Expect(input.Client.List(ctx, gotDPUServiceInterfaceList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ @@ -325,13 +325,13 @@ func ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx context.C g.Expect(gotDPUServiceInterfaceList.Items).To(BeEmpty()) // Expect the DPUDeployment to be deleted - err := input.client.Get(ctx, client.ObjectKey{Namespace: dpuDeployment.GetNamespace(), Name: dpuDeployment.GetName()}, &dpuservicev1.DPUDeployment{}) + err := input.Client.Get(ctx, client.ObjectKey{Namespace: dpuDeployment.GetNamespace(), Name: dpuDeployment.GetName()}, &dpuservicev1.DPUDeployment{}) g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) }).WithTimeout(180 * time.Second).Should(Succeed()) By("Cleanup DPUServiceConfiguration and DPUServiceTemplate") - Expect(input.client.Delete(ctx, dpuServiceTemplate)).To(Succeed()) - Expect(input.client.Delete(ctx, dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Delete(ctx, dpuServiceTemplate)).To(Succeed()) + Expect(input.Client.Delete(ctx, dpuServiceConfiguration)).To(Succeed()) } func VerifyDeploymentUnderlyingObjectsCreated(ctx context.Context, g Gomega, testClient client.Client, dpuDeployment *dpuservicev1.DPUDeployment) bool { @@ -381,67 +381,67 @@ func VerifyDeploymentUnderlyingObjectsCreated(ctx context.Context, g Gomega, tes // a little longer here. } -func ValidateDPUDeploymentFullCreation(ctx context.Context, input *systemTestInput) { +func ValidateDPUDeploymentFullCreation(ctx context.Context, input *SystemTestInput) { // TODO: Delete DPUSet not owned by DPUDeployment By("Delete DPUs and DPUSets and ensure they are deleted for a clean test condition") Eventually(func(g Gomega) { dpuSetList := &provisioningv1.DPUSetList{} - g.Expect(client.IgnoreNotFound(input.client.DeleteAllOf(ctx, &provisioningv1.DPUSet{}, client.InNamespace(dpfOperatorSystemNamespace)))).To(Succeed()) - g.Expect(input.client.List(ctx, dpuSetList)).To(Succeed()) + g.Expect(client.IgnoreNotFound(input.Client.DeleteAllOf(ctx, &provisioningv1.DPUSet{}, client.InNamespace(DPFOperatorSystemNamespace)))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuSetList)).To(Succeed()) g.Expect(dpuSetList.Items).To(BeEmpty()) // Expect all DPUs to have been deleted. dpuList := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpuList)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuList)).To(Succeed()) g.Expect(dpuList.Items).To(BeEmpty()) nodes := &corev1.NodeList{} - g.Expect(dpuClusterClient[0].List(ctx, nodes)).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, nodes)).To(Succeed()) By(fmt.Sprintf("Expected number of nodes %d to equal %d", len(nodes.Items), 0)) g.Expect(nodes.Items).To(BeEmpty()) // The timeout is so long here because for Zero Trust provisioning takes longer }).WithTimeout(45 * time.Minute).Should(Succeed()) By("Create DPUServiceIPAM to be used by dpuDeployment") - dpuServiceIPAM := input.ipPoolDPUServiceIPAM.DeepCopy() + dpuServiceIPAM := input.IPPoolDPUServiceIPAM.DeepCopy() dpuServiceIPAM.SetLabels(CleanupScope.Suite) dpuServiceIPAM.SetName("dpudeployment-ipam-pool1") - dpuServiceIPAM.SetNamespace(dpfOperatorSystemNamespace) + dpuServiceIPAM.SetNamespace(DPFOperatorSystemNamespace) // Remove selectors so it applies to all nodes dpuServiceIPAM.Spec.NodeSelector = nil - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Create a DPUDeployment with its dependencies and ensure that the underlying objects are created") dpuServiceTemplate := generateDPUServiceTemplate(input, "") useDummyDPUServiceChart(dpuServiceTemplate) - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuServiceTemplate))).To(Succeed()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuServiceTemplate))).To(Succeed()) dpuServiceConfiguration := generateServiceConfiguration(input, "") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuServiceConfiguration))).To(Succeed()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuServiceConfiguration))).To(Succeed()) dpuServiceTemplate2 := generateDPUServiceTemplate(input, "2") useDummyDPUServiceChart(dpuServiceTemplate2) - Expect(input.client.Create(ctx, dpuServiceTemplate2)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceTemplate2)).To(Succeed()) dpuServiceConfiguration2 := generateServiceConfiguration(input, "2") dpuServiceConfiguration2.Spec.Interfaces = []dpuservicev1.ServiceInterfaceTemplate{{Name: "net2", Network: "mybrsfc"}} - Expect(input.client.Create(ctx, dpuServiceConfiguration2)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceConfiguration2)).To(Succeed()) - inClusterDPUServiceTemplate := input.dpuServiceTemplate.DeepCopy() + inClusterDPUServiceTemplate := input.DPUServiceTemplate.DeepCopy() inClusterDPUServiceTemplate.SetLabels(CleanupScope.Suite) inClusterDPUServiceTemplate.SetName("dpudeployment-example-in-cluster-servicetemplate") inClusterDPUServiceTemplate.Spec.DeploymentServiceName = "example-in-cluster" - inClusterDPUServiceConfiguration := input.dpuServiceConfiguration.DeepCopy() + inClusterDPUServiceConfiguration := input.DPUServiceConfiguration.DeepCopy() inClusterDPUServiceConfiguration.SetLabels(CleanupScope.Suite) inClusterDPUServiceConfiguration.SetName("dpudeployment-example-in-cluster-serviceconfiguration") inClusterDPUServiceConfiguration.Spec.Interfaces = nil inClusterDPUServiceConfiguration.Spec.DeploymentServiceName = "example-in-cluster" inClusterDPUServiceConfiguration.Spec.ServiceConfiguration.DeployInCluster = ptr.To(true) - dpuDeployment := testutils.GenerateDPUObj("dpf-dpudeployment", input.dpuDeployment.DeepCopy().Namespace, input.dpuDeployment.DeepCopy(), CleanupScope.Suite) + dpuDeployment := testutils.GenerateDPUObj("dpf-dpudeployment", input.DPUDeployment.DeepCopy().Namespace, input.DPUDeployment.DeepCopy(), CleanupScope.Suite) // Intentionally using deprecated field, e2e tests will be updated once we have removed the deprecated field. Unit // tests cover the new field, e2e tests cover the old field since there is no more unit test coverage for the deprecated field. //nolint:staticcheck @@ -454,9 +454,9 @@ func ValidateDPUDeploymentFullCreation(ctx context.Context, input *systemTestInp ServiceConfiguration: "dpudeployment-example-serviceconfiguration-2", } - if !isGinkgoLabelApplied(Domain.ZeroTrust) { - Expect(input.client.Create(ctx, inClusterDPUServiceTemplate)).To(Succeed()) - Expect(input.client.Create(ctx, inClusterDPUServiceConfiguration)).To(Succeed()) + if !IsGinkgoLabelApplied(Domain.ZeroTrust) { + Expect(input.Client.Create(ctx, inClusterDPUServiceTemplate)).To(Succeed()) + Expect(input.Client.Create(ctx, inClusterDPUServiceConfiguration)).To(Succeed()) dpuDeployment.Spec.Services["example-in-cluster"] = dpuservicev1.DPUDeploymentServiceConfiguration{ ServiceTemplate: inClusterDPUServiceTemplate.GetName(), ServiceConfiguration: inClusterDPUServiceConfiguration.GetName(), @@ -491,10 +491,10 @@ func ValidateDPUDeploymentFullCreation(ctx context.Context, input *systemTestInp }, } - Expect(input.client.Create(ctx, dpuDeployment)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuDeployment)).To(Succeed()) Eventually(func(g Gomega) { - g.Expect(VerifyDeploymentUnderlyingObjectsCreated(ctx, g, input.client, dpuDeployment)).To(BeTrue()) + g.Expect(VerifyDeploymentUnderlyingObjectsCreated(ctx, g, input.Client, dpuDeployment)).To(BeTrue()) }).WithTimeout(180 * time.Second).Should(Succeed()) serviceInterfaceLabels := map[string]string{} @@ -502,7 +502,7 @@ func ValidateDPUDeploymentFullCreation(ctx context.Context, input *systemTestInp Eventually(func(g Gomega) { // Get the DPUServiceInterface owned by DPUDeployment dpuServiceInterfaceList := &dpuservicev1.DPUServiceInterfaceList{} - g.Expect(input.client.List(ctx, dpuServiceInterfaceList, + g.Expect(input.Client.List(ctx, dpuServiceInterfaceList, client.MatchingLabels{ "svc.dpu.nvidia.com/owned-by-dpudeployment": fmt.Sprintf("%s_%s", dpuDeployment.GetNamespace(), dpuDeployment.GetName())})). To(Succeed()) @@ -517,31 +517,31 @@ func ValidateDPUDeploymentFullCreation(ctx context.Context, input *systemTestInp return } - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { return } By("Verifying DPUs are provisioned") VerifyDPUClusterWithNodes(ctx, ProvisionDPUClustersInput{ - numberOfDPUNodes: input.numberOfDPUNodes, - numberOfDPUsPerNode: input.numberOfDPUsPerNode, - client: input.client, - NodeRebootConfigMap: input.nodeRebootConfigMap, - DPUNodeBMCs: input.dpuNodeBMCs, + NumberOfDPUNodes: input.NumberOfDPUNodes, + NumberOfDPUsPerNode: input.NumberOfDPUsPerNode, + Client: input.Client, + NodeRebootConfigMap: input.NodeRebootConfigMap, + DPUNodeBMCs: input.DPUNodeBMCs, }) - By(fmt.Sprintf("Verify ServiceInterface is created in %d nodes", input.totalDPUs())) + By(fmt.Sprintf("Verify ServiceInterface is created in %d nodes", input.TotalDPUs())) Eventually(func(g Gomega) { serviceInterfaceList := &dpuservicev1.ServiceInterfaceList{} - g.Expect(dpuClusterClient[0].List(ctx, serviceInterfaceList, client.MatchingLabels(serviceInterfaceLabels))).To(Succeed()) - g.Expect(serviceInterfaceList.Items).To(HaveLen(input.totalDPUs())) + g.Expect(DPUClusterClient[0].List(ctx, serviceInterfaceList, client.MatchingLabels(serviceInterfaceLabels))).To(Succeed()) + g.Expect(serviceInterfaceList.Items).To(HaveLen(input.TotalDPUs())) }).WithTimeout(15 * time.Minute).WithPolling(120 * time.Second).Should(Succeed()) By("Verify service pods have the service reference label") Eventually(func(g Gomega) { for serviceName, svcConfig := range dpuDeployment.Spec.Services { dpuSvcConfig := &dpuservicev1.DPUServiceConfiguration{} - g.Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpuDeployment.GetNamespace(), Name: svcConfig.ServiceConfiguration}, dpuSvcConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: dpuDeployment.GetNamespace(), Name: svcConfig.ServiceConfiguration}, dpuSvcConfig)).To(Succeed()) podList := &corev1.PodList{} // We currently don't have an in-cluster DPUService that matches the contract for e2e tests, but in theory // could do the same check as below for those services. @@ -549,16 +549,16 @@ func ValidateDPUDeploymentFullCreation(ctx context.Context, input *systemTestInp continue } - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{dpuservicev1.ServiceReferenceInDPUDeploymentLabelKey: serviceName}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(podList.Items).To(HaveLen(input.totalDPUs()), "expected %d pods for service %s", input.totalDPUs(), serviceName) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(podList.Items).To(HaveLen(input.TotalDPUs()), "expected %d pods for service %s", input.TotalDPUs(), serviceName) } }).WithTimeout(15 * time.Minute).WithPolling(120 * time.Second).Should(Succeed()) } -func VerifyDPUDeploymentIsReady(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func VerifyDPUDeploymentIsReady(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } @@ -566,58 +566,58 @@ func VerifyDPUDeploymentIsReady(ctx context.Context, input *systemTestInput) { // Get the DPUDeployment created in ValidateDPUDeploymentFullCreation dpuDeployment := &dpuservicev1.DPUDeployment{} dpuDeployment.SetName("dpf-dpudeployment") - dpuDeployment.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + dpuDeployment.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) By(fmt.Sprintf("Verifying that the dpuDeployment %s is ready", dpuDeployment.GetName())) Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) g.Expect(conditions.IsTrue(dpuDeployment, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(15 * time.Minute).WithPolling(1 * time.Second).Should(Succeed()) } // ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain validates that DPUDeployment disruptive upgrade flow for // standard DPUService works as expected with node effect drain which is the recommendation for Host Trusted -func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } By("Patching the provisioning controller to apply node effect sequentially") - dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.client) + dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.Client) By("Getting the existing DPUDeployment") // Get the DPUDeployment created in ValidateDPUDeploymentFullCreation dpuDeployment := &dpuservicev1.DPUDeployment{} dpuDeployment.SetName("dpf-dpudeployment") - dpuDeployment.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + dpuDeployment.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) By("Getting the DPUServiceConfiguration for example service") dpuServiceConfiguration := &dpuservicev1.DPUServiceConfiguration{} dpuServiceConfiguration.SetName("dpudeployment-example-serviceconfiguration") - dpuServiceConfiguration.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + dpuServiceConfiguration.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Getting initial pods for example service in DPU cluster") var initialPods []corev1.Pod Eventually(func(g Gomega) { podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceIDForExample}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(podList.Items).To(HaveLen(input.totalDPUs())) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(podList.Items).To(HaveLen(input.TotalDPUs())) initialPods = podList.Items }).WithTimeout(30 * time.Second).Should(Succeed()) By("Getting the mapping between host nodes and pods running on the DPU cluster on a DPU that is part of that node") // Get all nodes in the DPU cluster dpuClusterNodes := &corev1.NodeList{} - Expect(dpuClusterClient[0].List(ctx, dpuClusterNodes)).To(Succeed()) + Expect(DPUClusterClient[0].List(ctx, dpuClusterNodes)).To(Succeed()) // Create a map from DPU cluster node name to DPUNode name using the DPUNodeNameLabel label dpuClusterNodeToHostNodeMap := make(map[string]string) @@ -638,7 +638,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, } hostNodeToPodMap[hostNodeName] = pod } - Expect(hostNodeToPodMap).To(HaveLen(input.numberOfDPUNodes), "Expected to find a pod on the DPU for each host node") + Expect(hostNodeToPodMap).To(HaveLen(input.NumberOfDPUNodes), "Expected to find a pod on the DPU for each host node") By("Modifying the DPUServiceConfiguration by adding an extra label") originalDPUServiceConfiguration := dpuServiceConfiguration.DeepCopy() @@ -649,13 +649,13 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, dpuServiceConfiguration.Spec.ServiceConfiguration.ServiceDaemonSet.Labels = make(map[string]string) } dpuServiceConfiguration.Spec.ServiceConfiguration.ServiceDaemonSet.Labels["test-disruptive-upgrade"] = "true" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) oldServiceIDForExample := serviceIDForExample By("Waiting for the new DPUService revision and updating the ServiceID") Eventually(func(g Gomega) { updatedDPUServiceList := &dpuservicev1.DPUServiceList{} - g.Expect(input.client.List(ctx, updatedDPUServiceList, + g.Expect(input.Client.List(ctx, updatedDPUServiceList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ dpuservicev1.ParentDPUDeploymentNameLabel: fmt.Sprintf("%s_%s", dpuDeployment.GetNamespace(), dpuDeployment.GetName()), @@ -674,7 +674,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, By("Checking that one of the nodes is drained") var drainedHostNode *corev1.Node Eventually(func(g Gomega) { - drainedHostNode = verifySingleNodeDrained(g, ctx, input.client, dpuDeployment) + drainedHostNode = verifySingleNodeDrained(g, ctx, input.Client, dpuDeployment) }).WithTimeout(5 * time.Minute).Should(Succeed()) By("Checking that the pod running on the DPU which belongs to the drained node is replaced by a new one while the other DPU has its pod intact") @@ -685,12 +685,12 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, var allPods []corev1.Pod for _, sid := range []string{oldServiceIDForExample, serviceIDForExample} { podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": sid}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) allPods = append(allPods, podList.Items...) } - g.Expect(allPods).To(HaveLen(input.totalDPUs())) + g.Expect(allPods).To(HaveLen(input.TotalDPUs())) // Verify that the old pod on the DPU correlated with the drained host node is replaced with a new one foundOldPodOnDrainedNode := false @@ -735,7 +735,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, Eventually(func(g Gomega) { // Get the pod to understand if it's ready or not gotPod := &corev1.Pod{} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKeyFromObject(newPod), gotPod)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKeyFromObject(newPod), gotPod)).To(Succeed()) // Determine whether pod is ready isPodReady := false @@ -748,7 +748,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, // Get the node to understand if it's tainted or not node := &corev1.Node{} - g.Expect(input.client.Get(ctx, client.ObjectKey{Name: drainedHostNode.Name}, node)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKey{Name: drainedHostNode.Name}, node)).To(Succeed()) // Determine whether node is drained isNodeDrained := false @@ -772,17 +772,17 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, By("Verifying that the DPUDeployment becomes ready") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) g.Expect(conditions.IsTrue(dpuDeployment, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(15 * time.Minute).Should(Succeed()) By("Verifying all pods are running the new configuration") Eventually(func(g Gomega) { podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceIDForExample}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(podList.Items).To(HaveLen(input.totalDPUs())) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(podList.Items).To(HaveLen(input.TotalDPUs())) // Verify all pods have the new label for _, pod := range podList.Items { @@ -792,47 +792,47 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx context.Context, By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) resetConfig := dpfOperatorConfig.DeepCopy() resetConfig.Spec = originalDPFOperatorConfig.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) }).WithTimeout(30 * time.Second).Should(Succeed()) By("Validating that the DPFOperatorConfig is ready for the current generation") - VerifyDPFOperatorConfigReady(ctx, input.client, 2*time.Minute) + VerifyDPFOperatorConfigReady(ctx, input.Client, 2*time.Minute) } // ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold validates that DPUDeployment disruptive upgrade flow for // standard DPUService works as expected with hold node effect which is the recommendation for Zero Trust -func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 DPUNodes Skip("Skip test as there are not exactly 2 DPUNodes") } By("Patching the provisioning controller to apply node effect sequentially") - dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.client) + dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.Client) By("Getting the existing DPUDeployment") // Get the DPUDeployment created in ValidateDPUDeploymentFullCreation dpuDeployment := &dpuservicev1.DPUDeployment{} dpuDeployment.SetName("dpf-dpudeployment") - dpuDeployment.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + dpuDeployment.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) By("Getting the DPUServiceConfiguration for example service") dpuServiceConfiguration := &dpuservicev1.DPUServiceConfiguration{} dpuServiceConfiguration.SetName("dpudeployment-example-serviceconfiguration") - dpuServiceConfiguration.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + dpuServiceConfiguration.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Getting the mapping between DPUs and DPUNodes") // Get all DPUs in the system dpuList := &provisioningv1.DPUList{} - Expect(input.client.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + Expect(input.Client.List(ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Create a map from DPU name to DPUNode name dpuToDPUNodeMap := make(map[string]string) @@ -849,13 +849,13 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i dpuServiceConfiguration.Spec.ServiceConfiguration.ServiceDaemonSet.Labels = make(map[string]string) } dpuServiceConfiguration.Spec.ServiceConfiguration.ServiceDaemonSet.Labels["test-disruptive-upgrade-zt"] = "true" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) oldServiceIDForExample := serviceIDForExample By("Waiting for the new DPUService revision and updating the ServiceID") Eventually(func(g Gomega) { updatedDPUServiceList := &dpuservicev1.DPUServiceList{} - g.Expect(input.client.List(ctx, updatedDPUServiceList, + g.Expect(input.Client.List(ctx, updatedDPUServiceList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ dpuservicev1.ParentDPUDeploymentNameLabel: fmt.Sprintf("%s_%s", dpuDeployment.GetNamespace(), dpuDeployment.GetName()), @@ -877,7 +877,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i Eventually(func(g Gomega) { // Get all DPUNodeMaintenance objects dpuNodeMaintenanceList := &provisioningv1.DPUNodeMaintenanceList{} - g.Expect(input.client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Find the DPUNodeMaintenance with hold annotation set to "true" for i, dpuNodeMaintenance := range dpuNodeMaintenanceList.Items { @@ -890,7 +890,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i g.Expect(err).ToNot(HaveOccurred()) dpuNode := &provisioningv1.DPUNode{} - err = input.client.Get(ctx, client.ObjectKey{Name: dpuNodeMaintenance.Spec.DPUNodeName, Namespace: dpuNodeMaintenance.Namespace}, dpuNode) + err = input.Client.Get(ctx, client.ObjectKey{Name: dpuNodeMaintenance.Spec.DPUNodeName, Namespace: dpuNodeMaintenance.Namespace}, dpuNode) if err == nil && labelSelectorForNodes.Matches(labels.Set(dpuNode.Labels)) { dpuUnderNodeEffect = dpuNodeMaintenance.Spec.DPUNodeName inProgressDPUNodeMaintenance = &dpuNodeMaintenanceList.Items[i] @@ -904,9 +904,9 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i By("Verifying pods are NOT updated while hold annotation is true") Consistently(func(g Gomega) { podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": oldServiceIDForExample}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Verify no pods have the new label yet (update hasn't started) for _, pod := range podList.Items { @@ -916,7 +916,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i }).WithTimeout(30 * time.Second).WithPolling(5 * time.Second).Should(Succeed()) By("Simulating user action: setting hold annotation to false to allow update") - Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.client, inProgressDPUNodeMaintenance).WithTimeout(30 * time.Second).Should(Succeed()) + Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.Client, inProgressDPUNodeMaintenance).WithTimeout(30 * time.Second).Should(Succeed()) By("Checking that the pod on the DPU belonging to the DPUNode under node effect is now updated") var newPod *corev1.Pod @@ -926,12 +926,12 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i var allPods []corev1.Pod for _, sid := range []string{oldServiceIDForExample, serviceIDForExample} { podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": sid}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) allPods = append(allPods, podList.Items...) } - g.Expect(allPods).To(HaveLen(input.totalDPUs())) + g.Expect(allPods).To(HaveLen(input.TotalDPUs())) // Track pods on DPU belonging to DPUNode under node effect (should transition from old to new) foundOldPodOnDPUUnderNodeEffect := false @@ -974,7 +974,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i By("Verifying the new pod becomes ready on the DPUNode under node effect") Eventually(func(g Gomega) { gotPod := &corev1.Pod{} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKeyFromObject(newPod), gotPod)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKeyFromObject(newPod), gotPod)).To(Succeed()) // Verify pod is ready isPodReady := false @@ -990,7 +990,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i By("Verifying the DPU(s) for the updated DPUNode become ready") Eventually(func(g Gomega) { dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Find the DPU(s) belonging to the DPUNode that was updated and verify they're ready foundDPU := false @@ -1009,7 +1009,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i var secondMaintenanceWithHold *provisioningv1.DPUNodeMaintenance Eventually(func(g Gomega) { dpuNodeMaintenanceList := &provisioningv1.DPUNodeMaintenanceList{} - g.Expect(input.client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // We expect only a single DPUNodeMaintenance left g.Expect(dpuNodeMaintenanceList.Items).To(HaveLen(1)) @@ -1020,21 +1020,21 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i }).WithTimeout(5 * time.Minute).Should(Succeed()) By("Simulating user action: Setting hold annotation to false on the second DPUNode") - Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.client, secondMaintenanceWithHold).WithTimeout(30 * time.Second).Should(Succeed()) + Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.Client, secondMaintenanceWithHold).WithTimeout(30 * time.Second).Should(Succeed()) By("Verifying that the DPUDeployment becomes ready") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) g.Expect(conditions.IsTrue(dpuDeployment, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(15 * time.Minute).Should(Succeed()) By("Verifying all pods are running the new configuration") Eventually(func(g Gomega) { podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceIDForExample}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(podList.Items).To(HaveLen(input.totalDPUs())) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(podList.Items).To(HaveLen(input.TotalDPUs())) // Verify all pods have the new label for _, pod := range podList.Items { @@ -1045,7 +1045,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i By("Verifying that all DPUs are ready") Eventually(func(g Gomega) { dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) readyDPUs := 0 for _, dpu := range dpus.Items { @@ -1054,21 +1054,21 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i } } - g.Expect(readyDPUs).To(Equal(input.totalDPUs()), + g.Expect(readyDPUs).To(Equal(input.TotalDPUs()), fmt.Sprintf("expected all %d DPUs to be ready, but only %d are ready", - input.totalDPUs(), readyDPUs)) + input.TotalDPUs(), readyDPUs)) }).WithTimeout(5 * time.Minute).WithPolling(1 * time.Second).Should(Succeed()) By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) resetConfig := dpfOperatorConfig.DeepCopy() resetConfig.Spec = originalDPFOperatorConfig.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) }).WithTimeout(30 * time.Second).Should(Succeed()) By("Validating that the DPFOperatorConfig is ready for the current generation") - VerifyDPFOperatorConfigReady(ctx, input.client, 2*time.Minute) + VerifyDPFOperatorConfigReady(ctx, input.Client, 2*time.Minute) } // ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack validates that DPUDeployment disruptive upgrade flow @@ -1076,46 +1076,46 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx context.Context, i // CrashLoopBackOff). It validates that only one host node is drained, the DPU is stuck in Node Effect Removal // for 1 minute, and that reverting to the original configuration recovers the DPU. The other DPU should not be // drained at any point. -func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } By("Patching the provisioning controller to apply node effect sequentially") - dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.client) + dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.Client) By("Getting the existing DPUDeployment") // Get the DPUDeployment created in ValidateDPUDeploymentFullCreation dpuDeployment := &dpuservicev1.DPUDeployment{} dpuDeployment.SetName("dpf-dpudeployment") - dpuDeployment.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + dpuDeployment.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) By("Getting the DPUServiceConfiguration for example service") dpuServiceConfiguration := &dpuservicev1.DPUServiceConfiguration{} dpuServiceConfiguration.SetName("dpudeployment-example-serviceconfiguration") - dpuServiceConfiguration.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + dpuServiceConfiguration.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Getting initial pods for example service in DPU cluster") var initialPods []corev1.Pod Eventually(func(g Gomega) { podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceIDForExample}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(podList.Items).To(HaveLen(input.totalDPUs())) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(podList.Items).To(HaveLen(input.TotalDPUs())) initialPods = podList.Items }).WithTimeout(30 * time.Second).Should(Succeed()) By("Getting the mapping between host nodes and pods running on the DPU cluster on a DPU that is part of that node") // Get all nodes in the DPU cluster dpuClusterNodes := &corev1.NodeList{} - Expect(dpuClusterClient[0].List(ctx, dpuClusterNodes)).To(Succeed()) + Expect(DPUClusterClient[0].List(ctx, dpuClusterNodes)).To(Succeed()) // Create a map from DPU cluster node name to host node name using the DPUNodeNameLabel label dpuClusterNodeToHostNodeMap := make(map[string]string) @@ -1134,20 +1134,20 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx } hostNodeToPodMap[hostNodeName] = pod } - Expect(hostNodeToPodMap).To(HaveLen(input.numberOfDPUNodes), "Expected to find a pod on the DPU for each host node") + Expect(hostNodeToPodMap).To(HaveLen(input.NumberOfDPUNodes), "Expected to find a pod on the DPU for each host node") By("Modifying the DPUServiceConfiguration with a bad image to trigger a failing disruptive upgrade") originalDPUServiceConfiguration := dpuServiceConfiguration.DeepCopy() dpuServiceConfiguration.Spec.ServiceConfiguration.HelmChart.Values = &machineryruntime.RawExtension{ Raw: []byte(`{"image": {"repository": "invalid-image-does-not-exist", "tag": "invalid-tag-for-testing"}}`), } - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) oldServiceIDForExample := serviceIDForExample By("Waiting for the new DPUService revision with the bad image and updating the ServiceID") Eventually(func(g Gomega) { dpuServiceList := &dpuservicev1.DPUServiceList{} - g.Expect(input.client.List(ctx, dpuServiceList, + g.Expect(input.Client.List(ctx, dpuServiceList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ dpuservicev1.ParentDPUDeploymentNameLabel: fmt.Sprintf("%s_%s", dpuDeployment.GetNamespace(), dpuDeployment.GetName()), @@ -1166,7 +1166,7 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx By("Checking that one of the nodes is drained") var drainedHostNode *corev1.Node Eventually(func(g Gomega) { - drainedHostNode = verifySingleNodeDrained(g, ctx, input.client, dpuDeployment) + drainedHostNode = verifySingleNodeDrained(g, ctx, input.Client, dpuDeployment) }).WithTimeout(5 * time.Minute).Should(Succeed()) parentLabel := fmt.Sprintf("%s_%s", dpuDeployment.Namespace, dpuDeployment.Name) @@ -1175,14 +1175,14 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx // requestor is not removed from the DPUNodeMaintenance, the drained node remains drained, and no other node is drained. checkStuckState := func(g Gomega) { // Verify that the DPU for the node that is drained is in Node Effect Removal state - verifyDPUInNodeEffectRemoval(g, ctx, input.client, drainedHostNode.Name) + verifyDPUInNodeEffectRemoval(g, ctx, input.Client, drainedHostNode.Name) // Verify that the pod from the new service is deployed and not ready. // Capture the new pod's DPUService name label at the same time for the requestor check below. podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceIDForExample}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) foundNewPod := false for _, pod := range podList.Items { podHostNodeName := dpuClusterNodeToHostNodeMap[pod.Spec.NodeName] @@ -1206,8 +1206,8 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx // Identify the new DPUService by its service ID. dpuServiceList := &dpuservicev1.DPUServiceList{} - g.Expect(input.client.List(ctx, dpuServiceList, - client.InNamespace(dpfOperatorSystemNamespace), + g.Expect(input.Client.List(ctx, dpuServiceList, + client.InNamespace(DPFOperatorSystemNamespace), client.MatchingLabels{ dpuservicev1.ParentDPUDeploymentNameLabel: parentLabel, dpuservicev1.ServiceReferenceInDPUDeploymentLabelKey: "example", @@ -1226,10 +1226,10 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx expectedRequestor := fmt.Sprintf("%s_%s_%s", dpuDeployment.Namespace, dpuDeployment.Name, newDPUServiceName) // Verify that the DPUNodeMaintenance has the expected requestor - verifyDPUNodeMaintenanceHasRequestor(g, ctx, input.client, drainedHostNode.Name, expectedRequestor) + verifyDPUNodeMaintenanceHasRequestor(g, ctx, input.Client, drainedHostNode.Name, expectedRequestor) // Verify that drainedHostNode remains drained and is the only drained node. - g.Expect(verifySingleNodeDrained(g, ctx, input.client, dpuDeployment).Name).To(Equal(drainedHostNode.Name), + g.Expect(verifySingleNodeDrained(g, ctx, input.Client, dpuDeployment).Name).To(Equal(drainedHostNode.Name), "The same node should remain drained") } @@ -1243,12 +1243,12 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx var allPods []corev1.Pod for _, sid := range []string{oldServiceIDForExample, serviceIDForExample} { podList := &corev1.PodList{} - Expect(dpuClusterClient[0].List(ctx, podList, + Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": sid}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) allPods = append(allPods, podList.Items...) } - Expect(allPods).To(HaveLen(input.totalDPUs()), "Expected to find a pod in each DPU") + Expect(allPods).To(HaveLen(input.TotalDPUs()), "Expected to find a pod in each DPU") foundIntactPod := false for _, pod := range allPods { podHostNodeName := dpuClusterNodeToHostNodeMap[pod.Spec.NodeName] @@ -1265,27 +1265,27 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx By("Reverting the DPUServiceConfiguration to the original to fix the bad image") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) resetConfig := dpuServiceConfiguration.DeepCopy() resetConfig.Spec = originalDPUServiceConfiguration.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(dpuServiceConfiguration))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(dpuServiceConfiguration))).To(Succeed()) }).WithTimeout(30 * time.Second).Should(Succeed()) By("Verifying that the drained node gets its drain removed") Eventually(func(g Gomega) { - verifyNodeDrainRemoved(g, ctx, input.client, drainedHostNode) + verifyNodeDrainRemoved(g, ctx, input.Client, drainedHostNode) }).WithTimeout(15 * time.Minute).Should(Succeed()) By("Verifying that the DPUDeployment becomes ready") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) g.Expect(conditions.IsTrue(dpuDeployment, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(15 * time.Minute).Should(Succeed()) By("Getting the ServiceID for the reverted DPUService") Eventually(func(g Gomega) { dpuServiceList := &dpuservicev1.DPUServiceList{} - g.Expect(input.client.List(ctx, dpuServiceList, + g.Expect(input.Client.List(ctx, dpuServiceList, client.InNamespace(dpuDeployment.GetNamespace()), client.MatchingLabels{ dpuservicev1.ParentDPUDeploymentNameLabel: fmt.Sprintf("%s_%s", dpuDeployment.GetNamespace(), dpuDeployment.GetName()), @@ -1300,10 +1300,10 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx By("Verifying that the new pod on the DPU of the drained host has the new pod and is ready and that the other DPU has its pod remain intact") Eventually(func(g Gomega) { recoveredPodList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, recoveredPodList, + g.Expect(DPUClusterClient[0].List(ctx, recoveredPodList, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceIDForExample}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(recoveredPodList.Items).To(HaveLen(input.totalDPUs()), "Expected to find a pod in each DPU") + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(recoveredPodList.Items).To(HaveLen(input.TotalDPUs()), "Expected to find a pod in each DPU") foundNewReadyPod := false foundIntactPod := false for _, pod := range recoveredPodList.Items { @@ -1334,20 +1334,20 @@ func ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) resetConfig := dpfOperatorConfig.DeepCopy() resetConfig.Spec = originalDPFOperatorConfig.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) }).WithTimeout(30 * time.Second).Should(Succeed()) By("Validating that the DPFOperatorConfig is ready for the current generation") - VerifyDPFOperatorConfigReady(ctx, input.client, 2*time.Minute) + VerifyDPFOperatorConfigReady(ctx, input.Client, 2*time.Minute) } // ValidateDPUDeploymentInClusterDPUServiceDisruptiveUpgrade validates that DPUDeployment disruptive upgrade flow for // in-cluster DPUServices works as expected -func ValidateDPUDeploymentInClusterDPUServiceDisruptiveUpgrade(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func ValidateDPUDeploymentInClusterDPUServiceDisruptiveUpgrade(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } @@ -1355,23 +1355,23 @@ func ValidateDPUDeploymentInClusterDPUServiceDisruptiveUpgrade(ctx context.Conte By("Getting the existing DPUDeployment") dpuDeployment := &dpuservicev1.DPUDeployment{} dpuDeployment.SetName("dpf-dpudeployment") - dpuDeployment.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + dpuDeployment.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) parentLabel := fmt.Sprintf("%s_%s", dpuDeployment.Namespace, dpuDeployment.Name) By("Getting the dpuServiceConfiguration for in-cluster service") dpuServiceConfiguration := &dpuservicev1.DPUServiceConfiguration{} dpuServiceConfiguration.SetName("dpudeployment-example-in-cluster-serviceconfiguration") - dpuServiceConfiguration.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + dpuServiceConfiguration.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) dpuDeploymentKey := client.ObjectKeyFromObject(dpuDeployment) expectedVersionKey := fmt.Sprintf("%s-%s", "svc.dpu.nvidia.com/dpuservice-in-cluster-version", digest.Short(digest.FromObjects(dpuDeploymentKey, "example-in-cluster"), 10)) By("Getting the in-cluster DPUService") dpuServiceList := &dpuservicev1.DPUServiceList{} - Expect(input.client.List(ctx, dpuServiceList, + Expect(input.Client.List(ctx, dpuServiceList, client.InNamespace(dpuDeployment.Namespace), client.MatchingLabels{dpuservicev1.ParentDPUDeploymentNameLabel: parentLabel}, )).To(Succeed()) @@ -1382,12 +1382,12 @@ func ValidateDPUDeploymentInClusterDPUServiceDisruptiveUpgrade(ctx context.Conte originalInClusterService := inClusterServices[0].DeepCopy() By("Getting the target nodes") - nodesInfo := getTargetNodesAndDPUNodeNames(ctx, input.client, dpuDeployment) + nodesInfo := getTargetNodesAndDPUNodeNames(ctx, input.Client, dpuDeployment) By("Verifying that the in-cluster service is deployed") Eventually(func(g Gomega) { allNodes := &corev1.NodeList{} - g.Expect(input.client.List(ctx, allNodes)).To(Succeed()) + g.Expect(input.Client.List(ctx, allNodes)).To(Succeed()) nodesWithLabel := make(map[string]struct{}) for _, node := range allNodes.Items { @@ -1404,10 +1404,10 @@ func ValidateDPUDeploymentInClusterDPUServiceDisruptiveUpgrade(ctx context.Conte }).WithTimeout(15 * time.Minute).WithPolling(1 * time.Second).Should(Succeed()) By("Capturing old pod UIDs before update") - oldPodUIDs := captureOldPodUIDs(ctx, input.client, dpuDeployment.Namespace, originalInClusterService.Name) + oldPodUIDs := captureOldPodUIDs(ctx, input.Client, dpuDeployment.Namespace, originalInClusterService.Name) By("Capturing initial NodeEffect condition times from DPUs before update") - initialNodeEffectStates := captureInitialNodeEffectStates(ctx, input.client, nodesInfo.dpuNodeNames) + initialNodeEffectStates := captureInitialNodeEffectStates(ctx, input.Client, nodesInfo.dpuNodeNames) By("Updating the dpuServiceConfiguration by adding an extra label to trigger disruptive upgrade") originalDPUServiceConfiguration := dpuServiceConfiguration.DeepCopy() @@ -1418,60 +1418,60 @@ func ValidateDPUDeploymentInClusterDPUServiceDisruptiveUpgrade(ctx context.Conte dpuServiceConfiguration.Spec.ServiceConfiguration.ServiceDaemonSet.Labels = make(map[string]string) } dpuServiceConfiguration.Spec.ServiceConfiguration.ServiceDaemonSet.Labels["test-disruptive-upgrade"] = "true" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(originalDPUServiceConfiguration))).To(Succeed()) By("Verifying that a new DPUService is created") - newInClusterService := waitForNewInClusterDPUService(ctx, input.client, dpuDeployment.Namespace, parentLabel, originalInClusterService.Name) + newInClusterService := waitForNewInClusterDPUService(ctx, input.Client, dpuDeployment.Namespace, parentLabel, originalInClusterService.Name) By("Verifying that all target DPUs went through dpuNodeMaintenance (NodeEffectReady completed and NodeEffectRemoved)") - verifyDPUsCompletedMaintenance(ctx, input.client, nodesInfo.dpuNodeNames, initialNodeEffectStates) + verifyDPUsCompletedMaintenance(ctx, input.Client, nodesInfo.dpuNodeNames, initialNodeEffectStates) By("Verifying that pods were recreated") - verifyPodsRecreated(ctx, input.client, dpuDeployment.Namespace, newInClusterService.Name, oldPodUIDs) + verifyPodsRecreated(ctx, input.Client, dpuDeployment.Namespace, newInClusterService.Name, oldPodUIDs) By("Verifying that the DPUDeployment becomes ready") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) g.Expect(conditions.IsTrue(dpuDeployment, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(15 * time.Minute).Should(Succeed()) } // ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeDrain validates that DPUDeployment disruptive upgrade flow for // DPUServiceChain works as expected with drain node effect which is the recommendation for Host Trusted -func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeDrain(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeDrain(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } By("Patching the provisioning controller to apply node effect sequentially") - dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.client) + dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.Client) By("Getting the existing DPUDeployment") // Get the DPUDeployment created in ValidateDPUDeploymentFullCreation dpuDeployment := &dpuservicev1.DPUDeployment{} dpuDeployment.SetName("dpf-dpudeployment") - dpuDeployment.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + dpuDeployment.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) By("Getting initial ServiceChains in DPU cluster") var initialServiceChains []dpuservicev1.ServiceChain Eventually(func(g Gomega) { serviceChainList := &dpuservicev1.ServiceChainList{} - g.Expect(dpuClusterClient[0].List(ctx, serviceChainList, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(serviceChainList.Items).To(HaveLen(input.totalDPUs())) + g.Expect(DPUClusterClient[0].List(ctx, serviceChainList, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(serviceChainList.Items).To(HaveLen(input.TotalDPUs())) initialServiceChains = serviceChainList.Items }).WithTimeout(30 * time.Second).Should(Succeed()) By("Getting the mapping between host nodes and ServiceChains existing in the DPU cluster on a DPU that is part of that node") // Get all DPUs in the system dpuList := &provisioningv1.DPUList{} - Expect(input.client.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + Expect(input.Client.List(ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Get all DPUNodes in the system dpuNodeList := &provisioningv1.DPUNodeList{} - Expect(input.client.List(ctx, dpuNodeList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + Expect(input.Client.List(ctx, dpuNodeList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Create a map from DPUNode name to host node name (via KubeNodeRef) dpuNodeToHostNodeMap := make(map[string]string) @@ -1502,29 +1502,29 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeDrain(ctx context.Cont } hostNodeToServiceChainMap[hostNodeName] = serviceChain } - Expect(hostNodeToServiceChainMap).To(HaveLen(input.numberOfDPUNodes), "Expected to find a ServiceChain for each host node") + Expect(hostNodeToServiceChainMap).To(HaveLen(input.NumberOfDPUNodes), "Expected to find a ServiceChain for each host node") By("Modifying the DPUDeployment ServiceChains by changing ServiceMTU") originalDPUDeployment := dpuDeployment.DeepCopy() // Change the ServiceMTU to trigger a disruptive upgrade dpuDeployment.Spec.ServiceChains.Switches[0].ServiceMTU = ptr.To(testMTUValue) - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(originalDPUDeployment))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(originalDPUDeployment))).To(Succeed()) By("Checking that one of the nodes is drained") var drainedHostNode *corev1.Node Eventually(func(g Gomega) { - drainedHostNode = verifySingleNodeDrained(g, ctx, input.client, dpuDeployment) + drainedHostNode = verifySingleNodeDrained(g, ctx, input.Client, dpuDeployment) }).WithTimeout(5 * time.Minute).Should(Succeed()) By("Checking that the ServiceChain on the DPU correlated with the drained host node is updated while the others remain unchanged") var newServiceChain *dpuservicev1.ServiceChain Eventually(func(g Gomega) { serviceChainList := &dpuservicev1.ServiceChainList{} - g.Expect(dpuClusterClient[0].List(ctx, serviceChainList, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, serviceChainList, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Verify that we have service chains on both nodes - g.Expect(serviceChainList.Items).To(HaveLen(input.totalDPUs())) + g.Expect(serviceChainList.Items).To(HaveLen(input.TotalDPUs())) // Track which nodes have new vs old service chains foundOldServiceChainOnDrainedNode := false @@ -1573,14 +1573,14 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeDrain(ctx context.Cont Eventually(func(g Gomega) { // Get the ServiceChain to understand if it's ready or not gotServiceChain := &dpuservicev1.ServiceChain{} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKeyFromObject(newServiceChain), gotServiceChain)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKeyFromObject(newServiceChain), gotServiceChain)).To(Succeed()) // Determine whether ServiceChain is ready isServiceChainReady := conditions.IsTrue(gotServiceChain, conditions.TypeReady) // Get the node to understand if it's tainted or not node := &corev1.Node{} - g.Expect(input.client.Get(ctx, client.ObjectKey{Name: drainedHostNode.Name}, node)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKey{Name: drainedHostNode.Name}, node)).To(Succeed()) // Determine whether node is drained isNodeDrained := false @@ -1604,16 +1604,16 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeDrain(ctx context.Cont By("Verifying that the DPUDeployment becomes ready") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) g.Expect(conditions.IsTrue(dpuDeployment, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(15 * time.Minute).Should(Succeed()) By("Verifying all ServiceChains are running the new configuration") Eventually(func(g Gomega) { serviceChainList := &dpuservicev1.ServiceChainList{} - g.Expect(dpuClusterClient[0].List(ctx, serviceChainList, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(serviceChainList.Items).To(HaveLen(input.totalDPUs())) + g.Expect(DPUClusterClient[0].List(ctx, serviceChainList, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(serviceChainList.Items).To(HaveLen(input.TotalDPUs())) // Verify all ServiceChains have the new MTU for _, serviceChain := range serviceChainList.Items { @@ -1625,36 +1625,36 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeDrain(ctx context.Cont By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) resetConfig := dpfOperatorConfig.DeepCopy() resetConfig.Spec = originalDPFOperatorConfig.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) }).WithTimeout(30 * time.Second).Should(Succeed()) By("Validating that the DPFOperatorConfig is ready for the current generation") - VerifyDPFOperatorConfigReady(ctx, input.client, 2*time.Minute) + VerifyDPFOperatorConfigReady(ctx, input.Client, 2*time.Minute) } // ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold validates that DPUDeployment disruptive upgrade flow for // DPUServiceChain works as expected with hold node effect which is the default recommendation for Zero Trust -func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 DPUNodes Skip("Skip test as there are not exactly 2 DPUNodes") } By("Patching the provisioning controller to apply node effect sequentially") - dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.client) + dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.Client) By("Getting the existing DPUDeployment") dpuDeployment := &dpuservicev1.DPUDeployment{} dpuDeployment.SetName("dpf-dpudeployment") - dpuDeployment.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + dpuDeployment.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) By("Creating mapping from DPU to DPUNode") dpuList := &provisioningv1.DPUList{} - Expect(input.client.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + Expect(input.Client.List(ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Create a map from DPU name to DPUNode name dpuToDPUNodeMap := make(map[string]string) @@ -1665,14 +1665,14 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte By("Modifying the DPUDeployment ServiceChains by changing ServiceMTU") originalDPUDeployment := dpuDeployment.DeepCopy() dpuDeployment.Spec.ServiceChains.Switches[0].ServiceMTU = ptr.To(testMTUValue) - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(originalDPUDeployment))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(originalDPUDeployment))).To(Succeed()) By("Checking that DPUNodeMaintenance is created with hold annotation set to true") var dpuNodeUnderNodeEffect string var inProgressDPUNodeMaintenance *provisioningv1.DPUNodeMaintenance Eventually(func(g Gomega) { dpuNodeMaintenanceList := &provisioningv1.DPUNodeMaintenanceList{} - g.Expect(input.client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) for i, dpuNodeMaintenance := range dpuNodeMaintenanceList.Items { if isDPUNodeMaintenanceOnHold(&dpuNodeMaintenanceList.Items[i]) { @@ -1683,7 +1683,7 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte g.Expect(err).ToNot(HaveOccurred()) dpuNode := &provisioningv1.DPUNode{} - err = input.client.Get(ctx, client.ObjectKey{Name: dpuNodeMaintenance.Spec.DPUNodeName, Namespace: dpuNodeMaintenance.Namespace}, dpuNode) + err = input.Client.Get(ctx, client.ObjectKey{Name: dpuNodeMaintenance.Spec.DPUNodeName, Namespace: dpuNodeMaintenance.Namespace}, dpuNode) if err == nil && labelSelectorForNodes.Matches(labels.Set(dpuNode.Labels)) { dpuNodeUnderNodeEffect = dpuNodeMaintenance.Spec.DPUNodeName inProgressDPUNodeMaintenance = &dpuNodeMaintenanceList.Items[i] @@ -1697,8 +1697,8 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte By("Verifying ServiceChains are NOT updated while hold annotation is true") Consistently(func(g Gomega) { serviceChainList := &dpuservicev1.ServiceChainList{} - g.Expect(dpuClusterClient[0].List(ctx, serviceChainList, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, serviceChainList, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Verify no ServiceChains have the new MTU yet (update hasn't started) for _, serviceChain := range serviceChainList.Items { @@ -1709,15 +1709,15 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte }).WithTimeout(30 * time.Second).WithPolling(5 * time.Second).Should(Succeed()) By("Simulating user action: setting hold annotation to false to allow update") - Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.client, inProgressDPUNodeMaintenance).WithTimeout(30 * time.Second).Should(Succeed()) + Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.Client, inProgressDPUNodeMaintenance).WithTimeout(30 * time.Second).Should(Succeed()) By("Checking that the ServiceChain on the DPU under node effect is updated") var newServiceChain *dpuservicev1.ServiceChain Eventually(func(g Gomega) { serviceChainList := &dpuservicev1.ServiceChainList{} - g.Expect(dpuClusterClient[0].List(ctx, serviceChainList, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(serviceChainList.Items).To(HaveLen(input.totalDPUs())) + g.Expect(DPUClusterClient[0].List(ctx, serviceChainList, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(serviceChainList.Items).To(HaveLen(input.TotalDPUs())) // Track ServiceChains on DPUs under and not under node effect foundOldServiceChainOnDPUUnderNodeEffect := false @@ -1757,14 +1757,14 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte By("Verifying the new ServiceChain becomes ready") Eventually(func(g Gomega) { gotServiceChain := &dpuservicev1.ServiceChain{} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKeyFromObject(newServiceChain), gotServiceChain)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKeyFromObject(newServiceChain), gotServiceChain)).To(Succeed()) g.Expect(conditions.IsTrue(gotServiceChain, conditions.TypeReady)).To(BeTrue(), "New ServiceChain should become ready after hold annotation was set to false") }).WithTimeout(15 * time.Minute).Should(Succeed()) By("Verifying the DPU(s) for the updated DPUNode become ready") Eventually(func(g Gomega) { dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) foundDPU := false for _, dpu := range dpus.Items { @@ -1781,7 +1781,7 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte var secondMaintenanceWithHold *provisioningv1.DPUNodeMaintenance Eventually(func(g Gomega) { dpuNodeMaintenanceList := &provisioningv1.DPUNodeMaintenanceList{} - g.Expect(input.client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // We expect only a single DPUNodeMaintenance left g.Expect(dpuNodeMaintenanceList.Items).To(HaveLen(1)) @@ -1792,20 +1792,20 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte }).WithTimeout(5 * time.Minute).Should(Succeed()) By("Setting hold annotation to false on the second DPUNode") - Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.client, secondMaintenanceWithHold).WithTimeout(30 * time.Second).Should(Succeed()) + Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.Client, secondMaintenanceWithHold).WithTimeout(30 * time.Second).Should(Succeed()) By("Verifying that the DPUDeployment becomes ready") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) g.Expect(conditions.IsTrue(dpuDeployment, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(15 * time.Minute).Should(Succeed()) By("Verifying all ServiceChains are running the new configuration") Eventually(func(g Gomega) { serviceChainList := &dpuservicev1.ServiceChainList{} - g.Expect(dpuClusterClient[0].List(ctx, serviceChainList, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(serviceChainList.Items).To(HaveLen(input.totalDPUs())) + g.Expect(DPUClusterClient[0].List(ctx, serviceChainList, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(serviceChainList.Items).To(HaveLen(input.TotalDPUs())) for _, serviceChain := range serviceChainList.Items { g.Expect(serviceChain.Spec.Switches).To(HaveLen(1)) @@ -1817,7 +1817,7 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte By("Verifying that all DPUs are ready") Eventually(func(g Gomega) { dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) readyDPUs := 0 for _, dpu := range dpus.Items { @@ -1826,21 +1826,21 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte } } - g.Expect(readyDPUs).To(Equal(input.totalDPUs()), + g.Expect(readyDPUs).To(Equal(input.TotalDPUs()), fmt.Sprintf("expected all %d DPUs to be ready, but only %d are ready", - input.totalDPUs(), readyDPUs)) + input.TotalDPUs(), readyDPUs)) }).WithTimeout(5 * time.Minute).WithPolling(1 * time.Second).Should(Succeed()) By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) resetConfig := dpfOperatorConfig.DeepCopy() resetConfig.Spec = originalDPFOperatorConfig.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) }).WithTimeout(30 * time.Second).Should(Succeed()) By("Validating that the DPFOperatorConfig is ready for the current generation") - VerifyDPFOperatorConfigReady(ctx, input.client, 2*time.Minute) + VerifyDPFOperatorConfigReady(ctx, input.Client, 2*time.Minute) } // ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBack validates that the DPUDeployment @@ -1848,37 +1848,37 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx context.Conte // ServiceChain to be stuck not-ready). It validates that only one host node is drained, the DPU is stuck in Node // Effect Removal for 1 minute, and that reverting to the original configuration recovers the DPU. The other DPU // should not be drained at any point. -func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBack(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBack(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } By("Patching the provisioning controller to apply node effect sequentially") - dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.client) + dpfOperatorConfig, originalDPFOperatorConfig := setMaxUnavailableDPUNodes(ctx, input.Client) By("Getting the existing DPUDeployment") dpuDeployment := &dpuservicev1.DPUDeployment{} dpuDeployment.SetName("dpf-dpudeployment") - dpuDeployment.SetNamespace(dpfOperatorSystemNamespace) - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + dpuDeployment.SetNamespace(DPFOperatorSystemNamespace) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) By("Getting initial ServiceChains in DPU cluster") var initialServiceChains []dpuservicev1.ServiceChain Eventually(func(g Gomega) { serviceChainList := &dpuservicev1.ServiceChainList{} - g.Expect(dpuClusterClient[0].List(ctx, serviceChainList, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(serviceChainList.Items).To(HaveLen(input.totalDPUs())) + g.Expect(DPUClusterClient[0].List(ctx, serviceChainList, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(serviceChainList.Items).To(HaveLen(input.TotalDPUs())) initialServiceChains = serviceChainList.Items }).WithTimeout(30 * time.Second).Should(Succeed()) By("Getting the mapping between host nodes and ServiceChains existing in the DPU cluster on a DPU that is part of that node") dpuList := &provisioningv1.DPUList{} - Expect(input.client.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + Expect(input.Client.List(ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) dpuNodeList := &provisioningv1.DPUNodeList{} - Expect(input.client.List(ctx, dpuNodeList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + Expect(input.Client.List(ctx, dpuNodeList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) dpuNodeToHostNodeMap := make(map[string]string) for _, dpuNode := range dpuNodeList.Items { @@ -1905,17 +1905,17 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBac } hostNodeToServiceChainMap[hostNodeName] = serviceChain } - Expect(hostNodeToServiceChainMap).To(HaveLen(input.numberOfDPUNodes), "Expected to find a ServiceChain for each host node") + Expect(hostNodeToServiceChainMap).To(HaveLen(input.NumberOfDPUNodes), "Expected to find a ServiceChain for each host node") By("Modifying the DPUDeployment ServiceChains with a bad interface name to trigger a failing disruptive upgrade") originalDPUDeployment := dpuDeployment.DeepCopy() dpuDeployment.Spec.ServiceChains.Switches[0].Ports[0].Service.InterfaceName = "badnet" - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(originalDPUDeployment))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(originalDPUDeployment))).To(Succeed()) By("Checking that one of the nodes is drained") var drainedHostNode *corev1.Node Eventually(func(g Gomega) { - drainedHostNode = verifySingleNodeDrained(g, ctx, input.client, dpuDeployment) + drainedHostNode = verifySingleNodeDrained(g, ctx, input.Client, dpuDeployment) }).WithTimeout(5 * time.Minute).Should(Succeed()) parentLabel := fmt.Sprintf("%s_%s", dpuDeployment.Namespace, dpuDeployment.Name) @@ -1924,12 +1924,12 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBac // the requestor is not removed from the DPUNodeMaintenance, the drained node remains drained, and no other node is drained. checkStuckState := func(g Gomega) { // Verify that the DPU for the node that is drained is in Node Effect Removal state - verifyDPUInNodeEffectRemoval(g, ctx, input.client, drainedHostNode.Name) + verifyDPUInNodeEffectRemoval(g, ctx, input.Client, drainedHostNode.Name) // Verify that the new ServiceChain on the DPU correlated with the drained host node is not ready. serviceChainList := &dpuservicev1.ServiceChainList{} - g.Expect(dpuClusterClient[0].List(ctx, serviceChainList, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, serviceChainList, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) foundNewServiceChain := false for _, serviceChain := range serviceChainList.Items { g.Expect(serviceChain.Spec.Node).ToNot(BeNil()) @@ -1955,8 +1955,8 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBac // Identify the new DPUServiceChain by finding the one with the bad interface name in its spec. // The DPUDeployment controller converts Service.InterfaceName to MatchLabels["svc.dpu.nvidia.com/interface"]. dpuServiceChainList := &dpuservicev1.DPUServiceChainList{} - g.Expect(input.client.List(ctx, dpuServiceChainList, - client.InNamespace(dpfOperatorSystemNamespace), + g.Expect(input.Client.List(ctx, dpuServiceChainList, + client.InNamespace(DPFOperatorSystemNamespace), client.MatchingLabels{ dpuservicev1.ParentDPUDeploymentNameLabel: parentLabel, })).To(Succeed()) @@ -1977,10 +1977,10 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBac expectedRequestor := fmt.Sprintf("%s_%s_%s", dpuDeployment.Namespace, dpuDeployment.Name, newDPUServiceChainName) // Verify that the DPUNodeMaintenance has the expected requestor - verifyDPUNodeMaintenanceHasRequestor(g, ctx, input.client, drainedHostNode.Name, expectedRequestor) + verifyDPUNodeMaintenanceHasRequestor(g, ctx, input.Client, drainedHostNode.Name, expectedRequestor) // Verify that drainedHostNode remains drained and is the only drained node. - g.Expect(verifySingleNodeDrained(g, ctx, input.client, dpuDeployment).Name).To(Equal(drainedHostNode.Name), + g.Expect(verifySingleNodeDrained(g, ctx, input.Client, dpuDeployment).Name).To(Equal(drainedHostNode.Name), "The same node should remain drained") } @@ -1992,9 +1992,9 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBac By("Validating that the ServiceChain on the other node remained intact") serviceChainListCheck := &dpuservicev1.ServiceChainList{} - Expect(dpuClusterClient[0].List(ctx, serviceChainListCheck, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - Expect(serviceChainListCheck.Items).To(HaveLen(input.totalDPUs()), "Expected to find a ServiceChain in each DPU") + Expect(DPUClusterClient[0].List(ctx, serviceChainListCheck, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + Expect(serviceChainListCheck.Items).To(HaveLen(input.TotalDPUs()), "Expected to find a ServiceChain in each DPU") foundIntactServiceChain := false for _, serviceChain := range serviceChainListCheck.Items { Expect(serviceChain.Spec.Node).ToNot(BeNil()) @@ -2016,29 +2016,29 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBac By("Reverting the DPUDeployment to the original to fix the bad interface configuration") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) resetDeployment := dpuDeployment.DeepCopy() resetDeployment.Spec = originalDPUDeployment.Spec - g.Expect(input.client.Patch(ctx, resetDeployment, client.MergeFrom(dpuDeployment))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetDeployment, client.MergeFrom(dpuDeployment))).To(Succeed()) }).WithTimeout(30 * time.Second).Should(Succeed()) By("Verifying that the drained node gets its drain removed") Eventually(func(g Gomega) { - verifyNodeDrainRemoved(g, ctx, input.client, drainedHostNode) + verifyNodeDrainRemoved(g, ctx, input.Client, drainedHostNode) }).WithTimeout(15 * time.Minute).Should(Succeed()) By("Verifying that the DPUDeployment becomes ready") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) g.Expect(conditions.IsTrue(dpuDeployment, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(15 * time.Minute).Should(Succeed()) By("Verifying that the new ServiceChain on the DPU of the drained host is ready and that the other DPU has its ServiceChain remain intact") Eventually(func(g Gomega) { recoveredServiceChainList := &dpuservicev1.ServiceChainList{} - g.Expect(dpuClusterClient[0].List(ctx, recoveredServiceChainList, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(recoveredServiceChainList.Items).To(HaveLen(input.totalDPUs()), "Expected to find a ServiceChain in each DPU") + g.Expect(DPUClusterClient[0].List(ctx, recoveredServiceChainList, + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(recoveredServiceChainList.Items).To(HaveLen(input.TotalDPUs()), "Expected to find a ServiceChain in each DPU") foundNewReadyServiceChain := false foundIntactServiceChain := false for _, serviceChain := range recoveredServiceChainList.Items { @@ -2067,14 +2067,14 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBac By("Reverting the DPFOperatorConfig to its original setting") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpfOperatorConfig), dpfOperatorConfig)).To(Succeed()) resetConfig := dpfOperatorConfig.DeepCopy() resetConfig.Spec = originalDPFOperatorConfig.Spec - g.Expect(input.client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) + g.Expect(input.Client.Patch(ctx, resetConfig, client.MergeFrom(dpfOperatorConfig))).To(Succeed()) }).WithTimeout(30 * time.Second).Should(Succeed()) By("Validating that the DPFOperatorConfig is ready for the current generation") - VerifyDPFOperatorConfigReady(ctx, input.client, 2*time.Minute) + VerifyDPFOperatorConfigReady(ctx, input.Client, 2*time.Minute) } // setMaxUnavailableDPUNodes patches the DPFOperatorConfig to set MaxUnavailableDPUNodes to 1 so that only @@ -2082,7 +2082,7 @@ func ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBac // Returns the patched config and a deep copy of the original for use in revert. func setMaxUnavailableDPUNodes(ctx context.Context, c client.Client) (*operatorv1.DPFOperatorConfig, *operatorv1.DPFOperatorConfig) { dpfOperatorConfig := &operatorv1.DPFOperatorConfig{} - Expect(c.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, dpfOperatorConfig)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, dpfOperatorConfig)).To(Succeed()) originalDPFOperatorConfig := dpfOperatorConfig.DeepCopy() // Set MaxUnavailableDPUNodes to 1 to ensure only one node is upgraded at a time @@ -2117,7 +2117,7 @@ func verifyNodeDrainRemoved(g Gomega, ctx context.Context, c client.Client, node // DPUNodeEffectRemoval phase. func verifyDPUInNodeEffectRemoval(g Gomega, ctx context.Context, c client.Client, drainedHostNodeName string) { dpus := &provisioningv1.DPUList{} - g.Expect(c.List(ctx, dpus, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(c.List(ctx, dpus, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) for _, dpu := range dpus.Items { if dpu.Spec.DPUNodeName == drainedHostNodeName { @@ -2131,7 +2131,7 @@ func verifyDPUInNodeEffectRemoval(g Gomega, ctx context.Context, c client.Client // contains the expected requestor, and that it is currently active (ConditionNodeEffectApplied is True). func verifyDPUNodeMaintenanceHasRequestor(g Gomega, ctx context.Context, c client.Client, drainedHostNodeName, expectedRequestor string) { dpuNodeMaintenanceList := &provisioningv1.DPUNodeMaintenanceList{} - g.Expect(c.List(ctx, dpuNodeMaintenanceList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(c.List(ctx, dpuNodeMaintenanceList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) foundDPUNodeMaintenance := false for i := range dpuNodeMaintenanceList.Items { dpuNodeMaintenance := &dpuNodeMaintenanceList.Items[i] @@ -2177,19 +2177,19 @@ func verifySingleNodeDrained(g Gomega, ctx context.Context, c client.Client, dpu return drainedNode } -func createDeploymentDependencies(ctx context.Context, input *systemTestInput, nameDiff string) { +func createDeploymentDependencies(ctx context.Context, input *SystemTestInput, nameDiff string) { dpuServiceTemplate := generateDPUServiceTemplate(input, nameDiff) useDummyDPUServiceChart(dpuServiceTemplate) - Expect(input.client.Create(ctx, dpuServiceTemplate)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceTemplate)).To(Succeed()) dpuServiceConfiguration := generateServiceConfiguration(input, nameDiff) - Expect(input.client.Create(ctx, dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceConfiguration)).To(Succeed()) } -func generateDPUServiceTemplate(input *systemTestInput, nameDiff string) *dpuservicev1.DPUServiceTemplate { +func generateDPUServiceTemplate(input *SystemTestInput, nameDiff string) *dpuservicev1.DPUServiceTemplate { if nameDiff != "" { nameDiff = "-" + nameDiff } - dpuServiceTemplate := input.dpuServiceTemplate.DeepCopy() + dpuServiceTemplate := input.DPUServiceTemplate.DeepCopy() dpuServiceTemplate.SetLabels(CleanupScope.Suite) dpuServiceTemplate.SetName(dpuServiceTemplate.GetName() + nameDiff) dpuServiceTemplate.Spec.DeploymentServiceName += nameDiff @@ -2205,27 +2205,27 @@ func useDummyDPUServiceChart(dpuServiceTemplate *dpuservicev1.DPUServiceTemplate dpuServiceTemplate.Spec.HelmChart.Values = nil if ngcAPIKey != "" { dpuServiceTemplate.Spec.HelmChart.Values = &machineryruntime.RawExtension{ - Raw: []byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, ngcPullSecretName)), + Raw: []byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, NGCPullSecretName)), } } } -func generateServiceConfiguration(input *systemTestInput, nameDiff string) *dpuservicev1.DPUServiceConfiguration { +func generateServiceConfiguration(input *SystemTestInput, nameDiff string) *dpuservicev1.DPUServiceConfiguration { if nameDiff != "" { nameDiff = "-" + nameDiff } - dpuServiceConfiguration := input.dpuServiceConfiguration.DeepCopy() + dpuServiceConfiguration := input.DPUServiceConfiguration.DeepCopy() dpuServiceConfiguration.SetLabels(CleanupScope.Suite) dpuServiceConfiguration.SetName(dpuServiceConfiguration.GetName() + nameDiff) dpuServiceConfiguration.Spec.DeploymentServiceName += nameDiff return dpuServiceConfiguration } -func generateDPUDeployment(input *systemTestInput, nameDiff string) *dpuservicev1.DPUDeployment { +func generateDPUDeployment(input *SystemTestInput, nameDiff string) *dpuservicev1.DPUDeployment { if nameDiff != "" { nameDiff = "-" + nameDiff } - dpuDeployment := input.dpuDeployment.DeepCopy() + dpuDeployment := input.DPUDeployment.DeepCopy() dpuDeployment.SetLabels(CleanupScope.Suite) dpuDeployment.SetName(dpuDeployment.GetName() + nameDiff) currentSpecService := dpuDeployment.Spec.Services @@ -2352,7 +2352,7 @@ func captureOldPodUIDs(ctx context.Context, c client.Client, namespace string, s func captureInitialNodeEffectStates(ctx context.Context, c client.Client, dpuNodeNames []string) map[string]dpuNodeEffectState { initialNodeEffectStates := make(map[string]dpuNodeEffectState) dpuList := &provisioningv1.DPUList{} - Expect(c.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + Expect(c.List(ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) for _, dpu := range dpuList.Items { // Only track DPUs that belong to our target DPUNodes @@ -2430,7 +2430,7 @@ func verifyDPUsCompletedMaintenance(ctx context.Context, c client.Client, dpuNod dpusCompletedMaintenance := make(map[string]struct{}) dpuList := &provisioningv1.DPUList{} - g.Expect(c.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(c.List(ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) for _, dpu := range dpuList.Items { // Only check DPUs that belong to our target DPUNodes diff --git a/test/e2e/dpuservice.go b/test/e2e/dpuservice.go index d25274ea..e27f9fe8 100644 --- a/test/e2e/dpuservice.go +++ b/test/e2e/dpuservice.go @@ -49,58 +49,58 @@ var testNSImagePullSecret = &corev1.Secret{} // ValidateDPUServiceCreationAndMirroring creates the DPUService in DPU cluster and host cluster. // It verifies all triggered objects are created and ready. // Can be used as a test precondition (ex: DPUServiceDeletion test) and as a separate test. -func ValidateDPUServiceCreationAndMirroring(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceCreationAndMirroring(ctx context.Context, input *SystemTestInput) { By("Create namespace") - createTestNamespace(ctx, input.client, dpuServiceNamespace) + createTestNamespace(ctx, input.Client, dpuServiceNamespace) By("Create ImagePullSecret for DPUService in user namespace") testNSImagePullSecret = generateImagePullSecret(input, dpuServiceNamespace) - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, testNSImagePullSecret))).ToNot(HaveOccurred()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, testNSImagePullSecret))).ToNot(HaveOccurred()) By("Create a DPUServiceInterface") - dpuServiceInterface := utils.GenerateDPUObj(dpuServiceInterfaceName, dpuServiceNamespace, input.dpuServiceInterface.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceInterface)).To(Succeed()) + dpuServiceInterface := utils.GenerateDPUObj(dpuServiceInterfaceName, dpuServiceNamespace, input.DPUServiceInterface.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceInterface)).To(Succeed()) By("Create a DPUService to be deployed on the DPUCluster") - dpuService := utils.GenerateDPUObj(dpuServiceName, dpuServiceNamespace, input.dpuService.DeepCopy()) + dpuService := utils.GenerateDPUObj(dpuServiceName, dpuServiceNamespace, input.DPUService.DeepCopy()) dpuService.Spec.Interfaces = []string{dpuServiceInterfaceName} dpuService.Spec.ServiceID = ptr.To("my-service") - Expect(input.client.Create(ctx, dpuService)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuService)).To(Succeed()) By("Create a DPUService to be deployed on the host cluster") - hostDPUService := utils.GenerateDPUObj(hostDPUServiceName, dpuServiceNamespace, input.dpuService.DeepCopy()) + hostDPUService := utils.GenerateDPUObj(hostDPUServiceName, dpuServiceNamespace, input.DPUService.DeepCopy()) hostDPUService.Spec.DeployInCluster = ptr.To(true) // security.privileged must be unset when deployInCluster=true. hostDPUService.Spec.Security = nil - Expect(input.client.Create(ctx, hostDPUService)).To(Succeed()) + Expect(input.Client.Create(ctx, hostDPUService)).To(Succeed()) By("Verify DPUServices and deployments are created in DPUCluster") - verifyKubernetesDeploymentCreated(ctx, dpuClusterClient[0], dpuServiceNamespace) + verifyKubernetesDeploymentCreated(ctx, DPUClusterClient[0], dpuServiceNamespace) verifyImagePullSecretsInCluster(ctx, dpuService.Namespace, testNSImagePullSecret.Name) By("Verify DPUService is created in the host cluster") - verifyKubernetesDeploymentCreated(ctx, input.client, dpuServiceNamespace) + verifyKubernetesDeploymentCreated(ctx, input.Client, dpuServiceNamespace) } -func ValidateDPUServiceMetrics(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceMetrics(ctx context.Context, input *SystemTestInput) { By("Create namespace and DPUService") - createTestNamespace(ctx, input.client, dpuServiceNamespace) - dpuService := utils.GenerateDPUObj("dpu-01-metrics", dpuServiceNamespace, input.dpuService.DeepCopy()) - Expect(input.client.Create(ctx, dpuService)).To(Succeed()) + createTestNamespace(ctx, input.Client, dpuServiceNamespace) + dpuService := utils.GenerateDPUObj("dpu-01-metrics", dpuServiceNamespace, input.DPUService.DeepCopy()) + Expect(input.Client.Create(ctx, dpuService)).To(Succeed()) By("Verify DPUService metrics in KSM") expectedMetricsNames := map[string][]string{ "dpuservice": {"created", "info", "status_conditions", "status_condition_last_transition_time"}, } Eventually(func(g Gomega) { - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, metricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, MetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(5 * time.Second).Should(Succeed()) } -func ValidateDPUServiceDeletion(ctx context.Context, input *systemTestInput) { - if input.cleanupFlags.SkipCleanup { +func ValidateDPUServiceDeletion(ctx context.Context, input *SystemTestInput) { + if input.CleanupFlags.SkipCleanup { Skip("Skip cleanup resources") } By("Precondition") @@ -108,117 +108,117 @@ func ValidateDPUServiceDeletion(ctx context.Context, input *systemTestInput) { By("Pause dpuservice reconciliation") svc := &dpuservicev1.DPUService{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: hostDPUServiceName}, svc)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: hostDPUServiceName}, svc)).To(Succeed()) origSvc := svc.DeepCopy() svc.Spec.Paused = ptr.To(true) - Eventually(input.client.Patch).WithArguments(ctx, svc, client.MergeFrom(origSvc)).Should(Succeed()) + Eventually(input.Client.Patch).WithArguments(ctx, svc, client.MergeFrom(origSvc)).Should(Succeed()) svcHost := &dpuservicev1.DPUService{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: hostDPUServiceName}, svcHost)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: hostDPUServiceName}, svcHost)).To(Succeed()) Expect(svcHost.Spec.Paused).NotTo(BeNil()) By("Delete the DPUServices") svc = &dpuservicev1.DPUService{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: dpuServiceName}, svc)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: dpuServiceName}, svc)).To(Succeed()) // Delete the DPUCluster DPUService. - Expect(input.client.Delete(ctx, svc)).To(Succeed()) + Expect(input.Client.Delete(ctx, svc)).To(Succeed()) // Delete the host cluster DPUService. svcHost = &dpuservicev1.DPUService{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: hostDPUServiceName}, svcHost)).To(Succeed()) - Expect(input.client.Delete(ctx, svcHost)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: hostDPUServiceName}, svcHost)).To(Succeed()) + Expect(input.Client.Delete(ctx, svcHost)).To(Succeed()) // Verify that the DPUServices are deleted By("Verify DPUServices is deleted in the DPU cluster") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKey{Namespace: svc.Namespace, Name: svc.Name}, svc)).ToNot(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: svc.Namespace, Name: svc.Name}, svc)).ToNot(Succeed()) }).WithTimeout(600 * time.Second).Should(Succeed()) By("Verify DPUService is not deleted in the host cluster") Eventually(func(g Gomega) { svc = &dpuservicev1.DPUService{} - g.Expect(input.client.Get(ctx, client.ObjectKey{Namespace: svcHost.Namespace, Name: svcHost.Name}, svc)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: svcHost.Namespace, Name: svcHost.Name}, svc)).To(Succeed()) }).WithTimeout(600 * time.Second).Should(Succeed()) By("Resume dpuservice reconciliation") origSvc = svc.DeepCopy() svc.Spec.Paused = ptr.To(false) - Eventually(input.client.Patch).WithArguments(ctx, svc, client.MergeFrom(origSvc)).Should(Succeed()) + Eventually(input.Client.Patch).WithArguments(ctx, svc, client.MergeFrom(origSvc)).Should(Succeed()) // Verify that the DPUServices are deleted By("Verify DPUServices is deleted in the host cluster") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKey{Namespace: svcHost.Namespace, Name: svcHost.Name}, svc)).ToNot(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: svcHost.Namespace, Name: svcHost.Name}, svc)).ToNot(Succeed()) }).WithTimeout(600 * time.Second).Should(Succeed()) dsi := &dpuservicev1.DPUServiceInterface{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: dpuServiceInterfaceName}, dsi)).To(Succeed()) - Expect(utils.CleanupAndWait(ctx, input.client, dsi)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: dpuServiceInterfaceName}, dsi)).To(Succeed()) + Expect(utils.CleanupAndWait(ctx, input.Client, dsi)).To(Succeed()) // Check the DPUCluster DPUService is correctly deleted. Eventually(func(g Gomega) { deploymentList := appsv1.DeploymentList{} - g.Expect(dpuClusterClient[0].List(ctx, &deploymentList, client.HasLabels{"app", "release"}, client.InNamespace(dpuServiceNamespace))).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, &deploymentList, client.HasLabels{"app", "release"}, client.InNamespace(dpuServiceNamespace))).To(Succeed()) g.Expect(deploymentList.Items).To(BeEmpty()) }).WithTimeout(300 * time.Second).Should(Succeed()) // Ensure the hostDPUService deployment is deleted from the host cluster. Eventually(func(g Gomega) { deploymentList := appsv1.DeploymentList{} - g.Expect(input.client.List(ctx, &deploymentList, client.HasLabels{"app", "release"}, client.InNamespace(dpuServiceNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, &deploymentList, client.HasLabels{"app", "release"}, client.InNamespace(dpuServiceNamespace))).To(Succeed()) g.Expect(deploymentList.Items).To(BeEmpty()) }).WithTimeout(300 * time.Second).Should(Succeed()) } -func ValidateImagePullSecretsSync(ctx context.Context, input *systemTestInput) { +func ValidateImagePullSecretsSync(ctx context.Context, input *SystemTestInput) { imagePullSecretsSyncTestNamespace := "dpu-test-ns-image-pull-secrets" By("Create namespace, DPUServiceInterface and DPUService") - createTestNamespace(ctx, input.client, imagePullSecretsSyncTestNamespace) + createTestNamespace(ctx, input.Client, imagePullSecretsSyncTestNamespace) By("Create ImagePullSecret for DPUService in user namespace") testNSImagePullSecret = generateImagePullSecret(input, imagePullSecretsSyncTestNamespace) - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, testNSImagePullSecret))).ToNot(HaveOccurred()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, testNSImagePullSecret))).ToNot(HaveOccurred()) // Verify that we have the precreated secrets + the new secret in the DPU Cluster. secretCount := 2 if ngcAPIKey != "" { secretCount += 1 } - verifyImagePullSecretsCount(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, secretCount) + verifyImagePullSecretsCount(ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, secretCount) desiredConf := &operatorv1.DPFOperatorConfig{} - Eventually(input.client.Get).WithArguments(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, desiredConf).Should(Succeed()) + Eventually(input.Client.Get).WithArguments(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, desiredConf).Should(Succeed()) currentConf := desiredConf.DeepCopy() // Patch the operatorConfig to remove the second secret. This causes the label to be removed. desiredConf.Spec.ImagePullSecrets = append(desiredConf.Spec.ImagePullSecrets[:1], desiredConf.Spec.ImagePullSecrets[2:]...) - Eventually(input.client.Patch).WithArguments(ctx, desiredConf, client.MergeFrom(currentConf)).Should(Succeed()) + Eventually(input.Client.Patch).WithArguments(ctx, desiredConf, client.MergeFrom(currentConf)).Should(Succeed()) // Patch a DPUService to trigger a reconciliation. The DPUService should clean this secret up from // clusters to which it was previously mirrored. - Eventually(utils.ForceObjectReconcileWithAnnotation).WithArguments(ctx, input.client, - &dpuservicev1.DPUService{ObjectMeta: metav1.ObjectMeta{Name: operatorv1.MultusName.String(), Namespace: dpfOperatorSystemNamespace}}).Should(Succeed()) + Eventually(utils.ForceObjectReconcileWithAnnotation).WithArguments(ctx, input.Client, + &dpuservicev1.DPUService{ObjectMeta: metav1.ObjectMeta{Name: operatorv1.MultusName.String(), Namespace: DPFOperatorSystemNamespace}}).Should(Succeed()) // Verify that we have only the precreated secrets in the DPU Cluster. secretCount = 1 if ngcAPIKey != "" { secretCount += 1 } - verifyImagePullSecretsCount(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, secretCount) + verifyImagePullSecretsCount(ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, secretCount) } -func ValidateDPUServiceTemplateCreationNoAnnotations(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceTemplateCreationNoAnnotations(ctx context.Context, input *SystemTestInput) { By("Creating the DPUServiceTemplate") dpuServiceTemplate := utils.GenerateDPUObj( "dpuservice-without-annotations-metrics", - input.dpuServiceTemplate.DeepCopy().Namespace, - input.dpuServiceTemplate.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceTemplate)).To(Succeed()) + input.DPUServiceTemplate.DeepCopy().Namespace, + input.DPUServiceTemplate.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceTemplate)).To(Succeed()) By("Checking that status is ready and no versions") Eventually(func(g Gomega) { gotDPUServiceTemplate := &dpuservicev1.DPUServiceTemplate{} - g.Expect(input.client.Get(ctx, + g.Expect(input.Client.Get(ctx, types.NamespacedName{Name: dpuServiceTemplate.GetName(), Namespace: dpuServiceTemplate.GetNamespace()}, gotDPUServiceTemplate, )).To(Succeed()) @@ -227,17 +227,17 @@ func ValidateDPUServiceTemplateCreationNoAnnotations(ctx context.Context, input }).WithTimeout(180 * time.Second).Should(Succeed()) } -func VerifyDPUServiceTemplateCreationWithAnnotations(ctx context.Context, input *systemTestInput) { +func VerifyDPUServiceTemplateCreationWithAnnotations(ctx context.Context, input *SystemTestInput) { By("Creating the DPUServiceTemplate") dpuServiceTemplate := generateDPUServiceTemplate(input, "with-annotations") useDummyDPUServiceChart(dpuServiceTemplate) - Expect(input.client.Create(ctx, dpuServiceTemplate)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceTemplate)).To(Succeed()) By("Checking that status is ready and versions are set") Eventually(func(g Gomega) { gotDPUServiceTemplate := &dpuservicev1.DPUServiceTemplate{} - g.Expect(input.client.Get(ctx, + g.Expect(input.Client.Get(ctx, types.NamespacedName{Name: dpuServiceTemplate.GetName(), Namespace: dpuServiceTemplate.GetNamespace()}, gotDPUServiceTemplate, )).To(Succeed()) @@ -246,17 +246,17 @@ func VerifyDPUServiceTemplateCreationWithAnnotations(ctx context.Context, input }).WithTimeout(180 * time.Second).Should(Succeed()) } -func VerifyDPUServiceTemplateMetrics(ctx context.Context, input *systemTestInput) { +func VerifyDPUServiceTemplateMetrics(ctx context.Context, input *SystemTestInput) { By("Create namespace and DPUServiceTemplate") dpuServiceTemplate := generateDPUServiceTemplate(input, "-metrics") - Expect(input.client.Create(ctx, dpuServiceTemplate)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceTemplate)).To(Succeed()) By("Verify DPUServiceTemplate metrics in KSM") expectedMetricsNames := map[string][]string{ "dpuservicetemplate": {"created", "info", "status_conditions", "status_condition_last_transition_time"}, } Eventually(func(g Gomega) { - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, metricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, MetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(5 * time.Second).Should(Succeed()) @@ -271,7 +271,7 @@ func verifyImagePullSecretsCount(ctx context.Context, c client.Client, namespace Eventually(func(g Gomega) { // Check the imagePullSecrets has been deleted. secrets := &corev1.SecretList{} - g.Expect(dpuClusterClient[0].List(ctx, secrets, + g.Expect(DPUClusterClient[0].List(ctx, secrets, client.InNamespace(namespace), client.HasLabels{dpuservicev1.DPFImagePullSecretLabelKey}), ).To(Succeed()) @@ -279,12 +279,12 @@ func verifyImagePullSecretsCount(ctx context.Context, c client.Client, namespace }).WithTimeout(60 * time.Second).Should(Succeed()) } -func generateImagePullSecret(input *systemTestInput, dpuServiceNamespace string) *corev1.Secret { +func generateImagePullSecret(input *SystemTestInput, dpuServiceNamespace string) *corev1.Secret { labels := maps.Clone(CleanupScope.Suite) labels[dpuservicev1.DPFImagePullSecretLabelKey] = "" return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: input.pullSecretNames[0], + Name: input.PullSecretNames[0], Namespace: dpuServiceNamespace, Labels: labels, }, @@ -303,7 +303,7 @@ func verifyKubernetesDeploymentCreated(ctx context.Context, testClient client.Cl func verifyImagePullSecretsInCluster(ctx context.Context, namespace string, secretName string) { // Check an imagePullSecret was created in the same namespace in the destination cluster. Eventually(func(g Gomega) { - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{ + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{ Namespace: namespace, Name: secretName}, &corev1.Secret{})).To(Succeed()) }).WithTimeout(300 * time.Second).Should(Succeed()) diff --git a/test/e2e/dpuservice_kata_container.go b/test/e2e/dpuservice_kata_container.go index 2f6a9214..8319a6ef 100644 --- a/test/e2e/dpuservice_kata_container.go +++ b/test/e2e/dpuservice_kata_container.go @@ -43,23 +43,23 @@ const ( kataDPUServiceSFResourceID = corev1.ResourceName(kataDPUServiceSFResource) ) -func ValidateDPUServiceKataRuntimeClass(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPUServiceKataRuntimeClass(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip DPUService Kata RuntimeClass test as there are no DPU nodes") } By("Waiting for kata-containers DPUService to be ready") - dpuservice.WaitForDPUServices(ctx, input.client, dpfOperatorSystemNamespace, []string{operatorv1.KataContainersName.String()}) + dpuservice.WaitForDPUServices(ctx, input.Client, DPFOperatorSystemNamespace, []string{operatorv1.KataContainersName.String()}) By("Waiting for kata-qemu RuntimeClass to be created in the DPU cluster") Eventually(func(g Gomega) { - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{Name: kataRuntimeClassName}, &nodev1.RuntimeClass{})).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{Name: kataRuntimeClassName}, &nodev1.RuntimeClass{})).To(Succeed()) }).WithTimeout(10 * time.Minute).WithPolling(time.Second).Should(Succeed()) By("Creating a dummy DPUService that uses kata-qemu and requests an SF") - dpuService := input.dpuService.DeepCopy() + dpuService := input.DPUService.DeepCopy() dpuService.Name = kataDPUServiceName - dpuService.Namespace = dpfOperatorSystemNamespace + dpuService.Namespace = DPFOperatorSystemNamespace dpuService.SetLabels(CleanupScope.It) dpuService.Spec.HelmChart.Source = dpuservicev1.ApplicationSource{ Chart: "dummydpuservice-chart", @@ -67,17 +67,17 @@ func ValidateDPUServiceKataRuntimeClass(ctx context.Context, input *systemTestIn RepoURL: helmRegistry, } dpuService.Spec.HelmChart.Values = &machineryruntime.RawExtension{ - Raw: []byte(fmt.Sprintf(`{"runtimeClassName": %q, "imagePullSecrets": [{"name": %q}]}`, kataRuntimeClassName, dpfPullSecretName)), + Raw: []byte(fmt.Sprintf(`{"runtimeClassName": %q, "imagePullSecrets": [{"name": %q}]}`, kataRuntimeClassName, DPFPullSecretName)), } dpuService.Spec.ServiceDaemonSet = &dpuservicev1.ServiceDaemonSetValues{ Resources: corev1.ResourceList{ kataDPUServiceSFResourceID: resource.MustParse("1"), }, } - Expect(input.client.Create(ctx, dpuService)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuService)).To(Succeed()) By("Waiting for the kata dummy DPUService pods to be running") - VerifyClusterPods(ctx, dpuClusterClient[0], []string{kataDPUServiceName}) + VerifyClusterPods(ctx, DPUClusterClient[0], []string{kataDPUServiceName}) By("Verifying the running kata dummy DPUService pods use kata-qemu and an SF") Eventually(func(g Gomega) { @@ -95,7 +95,7 @@ func ValidateDPUServiceKataRuntimeClass(ctx context.Context, input *systemTestIn // kataRunningPods returns all Running pods belonging to the kata dummy DPUService DaemonSet. func kataRunningPods(ctx context.Context, g Gomega) []corev1.Pod { daemonSetList := &appsv1.DaemonSetList{} - g.Expect(dpuClusterClient[0].List(ctx, daemonSetList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, daemonSetList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) var matchLabels map[string]string for _, ds := range daemonSetList.Items { @@ -108,10 +108,10 @@ func kataRunningPods(ctx context.Context, g Gomega) []corev1.Pod { g.Expect(matchLabels).NotTo(BeEmpty(), "expected DaemonSet containing %q to exist", kataDPUServiceName) podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List( + g.Expect(DPUClusterClient[0].List( ctx, podList, - client.InNamespace(dpfOperatorSystemNamespace), + client.InNamespace(DPFOperatorSystemNamespace), client.MatchingLabels(matchLabels), client.MatchingFields{"status.phase": string(corev1.PodRunning)}, )).To(Succeed()) diff --git a/test/e2e/dpuservicechain.go b/test/e2e/dpuservicechain.go index 5379df66..798be635 100644 --- a/test/e2e/dpuservicechain.go +++ b/test/e2e/dpuservicechain.go @@ -30,55 +30,55 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func ValidateDPUServiceInterfaceCreation(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceInterfaceCreation(ctx context.Context, input *SystemTestInput) { testDPUServiceInterfaceName := "pf0-vf2" dpuServiceInterfaceNamespace := "test-service-interface" By("Create test namespace") - createTestNamespace(ctx, input.client, dpuServiceInterfaceNamespace) + createTestNamespace(ctx, input.Client, dpuServiceInterfaceNamespace) By("Create DPUServiceInterface") - dpuServiceInterface := utils.GenerateDPUObj(testDPUServiceInterfaceName, dpuServiceInterfaceNamespace, input.dpuServiceInterface.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceInterface)).To(Succeed()) + dpuServiceInterface := utils.GenerateDPUObj(testDPUServiceInterfaceName, dpuServiceInterfaceNamespace, input.DPUServiceInterface.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceInterface)).To(Succeed()) By("Verify ServiceInterfaceSet is created in DPF clusters") Eventually(func(g Gomega) { scs := &dpuservicev1.ServiceInterfaceSet{ObjectMeta: metav1.ObjectMeta{Name: testDPUServiceInterfaceName, Namespace: dpuServiceInterfaceNamespace}} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKeyFromObject(scs), scs)).NotTo(HaveOccurred()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKeyFromObject(scs), scs)).NotTo(HaveOccurred()) }, time.Second*300, time.Millisecond*250).Should(Succeed()) } -func ValidateDPUServiceChainCreation(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceChainCreation(ctx context.Context, input *SystemTestInput) { dpuServiceChainName := "svc-chain-test" dpuServiceChainNamespace := "test-2" By("Create test namespace") - createTestNamespace(ctx, input.client, dpuServiceChainNamespace) + createTestNamespace(ctx, input.Client, dpuServiceChainNamespace) By("Create DPUServiceChain") - dpuServiceChain := utils.GenerateDPUObj(dpuServiceChainName, dpuServiceChainNamespace, input.dpuServiceChain.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceChain)).To(Succeed()) + dpuServiceChain := utils.GenerateDPUObj(dpuServiceChainName, dpuServiceChainNamespace, input.DPUServiceChain.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceChain)).To(Succeed()) By("Verify ServiceChainSet is created in DPF clusters") Eventually(func(g Gomega) { scs := &dpuservicev1.ServiceChainSet{ObjectMeta: metav1.ObjectMeta{Name: dpuServiceChainName, Namespace: dpuServiceChainNamespace}} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKeyFromObject(scs), scs)).NotTo(HaveOccurred()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKeyFromObject(scs), scs)).NotTo(HaveOccurred()) }, time.Second*300, time.Millisecond*250).Should(Succeed()) } -func ValidateDPUServiceChainMetrics(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceChainMetrics(ctx context.Context, input *SystemTestInput) { dpuServiceInterfaceName := "pf0-vf2-metrics" dpuServiceInterfaceNamespace := "test-metrics" dpuServiceChainName := "svc-chain-test-metrics" By("Create test namespaces") - createTestNamespace(ctx, input.client, dpuServiceInterfaceNamespace) + createTestNamespace(ctx, input.Client, dpuServiceInterfaceNamespace) By("Create DPUServiceInterface and DPUServiceChain") - dpuServiceInterface := utils.GenerateDPUObj(dpuServiceInterfaceName, dpuServiceInterfaceNamespace, input.dpuServiceInterface.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceInterface)).To(Succeed()) - dpuServiceChain := utils.GenerateDPUObj(dpuServiceChainName, dpuServiceInterfaceNamespace, input.dpuServiceChain.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceChain)).To(Succeed()) + dpuServiceInterface := utils.GenerateDPUObj(dpuServiceInterfaceName, dpuServiceInterfaceNamespace, input.DPUServiceInterface.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceInterface)).To(Succeed()) + dpuServiceChain := utils.GenerateDPUObj(dpuServiceChainName, dpuServiceInterfaceNamespace, input.DPUServiceChain.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceChain)).To(Succeed()) By("Verify DPUServiceChain and DPUServiceInterface metrics in KSM") expectedMetricsNames := map[string][]string{ @@ -87,7 +87,7 @@ func ValidateDPUServiceChainMetrics(ctx context.Context, input *systemTestInput) } Eventually(func(g Gomega) { - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, metricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, MetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(5 * time.Second).Should(Succeed()) @@ -95,9 +95,9 @@ func ValidateDPUServiceChainMetrics(ctx context.Context, input *systemTestInput) By("Wait for ServiceChainSet and ServiceInterfaceSet to be created in DPU clusters") Eventually(func(g Gomega) { scs := &dpuservicev1.ServiceChainSet{} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{Name: dpuServiceChainName, Namespace: dpuServiceInterfaceNamespace}, scs)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{Name: dpuServiceChainName, Namespace: dpuServiceInterfaceNamespace}, scs)).To(Succeed()) sis := &dpuservicev1.ServiceInterfaceSet{} - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{Name: dpuServiceInterfaceName, Namespace: dpuServiceInterfaceNamespace}, sis)).To(Succeed()) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{Name: dpuServiceInterfaceName, Namespace: dpuServiceInterfaceNamespace}, sis)).To(Succeed()) }).WithTimeout(300 * time.Second).Should(Succeed()) // TODO: add validation for ServiceChain and ServiceInterface metrics when DPU nodes are present @@ -108,20 +108,20 @@ func ValidateDPUServiceChainMetrics(ctx context.Context, input *systemTestInput) } Eventually(func(g Gomega) { - g.Expect(input.dpuClusters).ToNot(BeEmpty(), "No DPUClusters found in test input") - dpuKSMMetricsURI, err := metrics.GetKSMMetricsURIForDPUCluster(ctx, input.client, input.dpuClusters[0], dpfOperatorSystemNamespace, kubeStateMetricsPort, "/metrics") + g.Expect(input.DPUClusters).ToNot(BeEmpty(), "No DPUClusters found in test input") + dpuKSMMetricsURI, err := metrics.GetKSMMetricsURIForDPUCluster(ctx, input.Client, input.DPUClusters[0], DPFOperatorSystemNamespace, KubeStateMetricsPort, "/metrics") g.Expect(err).NotTo(HaveOccurred(), "Failed to get KSM metrics URI for DPUCluster") g.Expect(dpuKSMMetricsURI).NotTo(BeEmpty()) // Use hostClusterRESTClient because in-cluster KSM runs on the management cluster - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, dpuKSMMetricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, dpuKSMMetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedDPUMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(10 * time.Second).Should(Succeed()) } -func ValidateDPUServiceChainDeletion(ctx context.Context, input *systemTestInput) { - if input.cleanupFlags.SkipCleanup { +func ValidateDPUServiceChainDeletion(ctx context.Context, input *SystemTestInput) { + if input.CleanupFlags.SkipCleanup { Skip("Skip cleanup resources") } dpuServiceInterfaceName := "pf0-vf2-delete" @@ -129,13 +129,13 @@ func ValidateDPUServiceChainDeletion(ctx context.Context, input *systemTestInput dpuServiceChainName := "svc-chain-test-delete" By("Create test namespaces") - createTestNamespace(ctx, input.client, dpuServiceInterfaceNamespace) + createTestNamespace(ctx, input.Client, dpuServiceInterfaceNamespace) By("Create DPUServiceInterface and DPUServiceChain") - dpuServiceInterface := utils.GenerateDPUObj(dpuServiceInterfaceName, dpuServiceInterfaceNamespace, input.dpuServiceInterface.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceInterface)).To(Succeed()) - dpuServiceChain := utils.GenerateDPUObj(dpuServiceChainName, dpuServiceInterfaceNamespace, input.dpuServiceChain.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceChain)).To(Succeed()) + dpuServiceInterface := utils.GenerateDPUObj(dpuServiceInterfaceName, dpuServiceInterfaceNamespace, input.DPUServiceInterface.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceInterface)).To(Succeed()) + dpuServiceChain := utils.GenerateDPUObj(dpuServiceChainName, dpuServiceInterfaceNamespace, input.DPUServiceChain.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceChain)).To(Succeed()) dsi := &dpuservicev1.DPUServiceInterface{} dsc := &dpuservicev1.DPUServiceChain{} @@ -144,22 +144,22 @@ func ValidateDPUServiceChainDeletion(ctx context.Context, input *systemTestInput // Delete racing with the finalizer patch can remove the object before reconcileDelete // runs and leaves the dpu-cluster object orphaned. See https://github.com/kubernetes/kubernetes/issues/77988 Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceInterface), dsi)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceInterface), dsi)).To(Succeed()) g.Expect(dsi.Finalizers).To(ContainElement(dpuservicev1.DPUServiceInterfaceFinalizer)) - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceChain), dsc)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceChain), dsc)).To(Succeed()) g.Expect(dsc.Finalizers).To(ContainElement(dpuservicev1.DPUServiceChainFinalizer)) }).WithTimeout(60 * time.Second).Should(Succeed()) - Expect(input.client.Delete(ctx, dsi)).To(Succeed()) - Expect(input.client.Delete(ctx, dsc)).To(Succeed()) + Expect(input.Client.Delete(ctx, dsi)).To(Succeed()) + Expect(input.Client.Delete(ctx, dsc)).To(Succeed()) // Get the control plane secrets. Eventually(func(g Gomega) { serviceChainSetList := dpuservicev1.ServiceChainSetList{} - g.Expect(dpuClusterClient[0].List(ctx, &serviceChainSetList, + g.Expect(DPUClusterClient[0].List(ctx, &serviceChainSetList, &client.ListOptions{Namespace: dpuServiceChain.Namespace})).To(Succeed()) g.Expect(serviceChainSetList.Items).To(BeEmpty()) serviceInterfaceSetList := dpuservicev1.ServiceInterfaceSetList{} - g.Expect(dpuClusterClient[0].List(ctx, &serviceInterfaceSetList, + g.Expect(DPUClusterClient[0].List(ctx, &serviceInterfaceSetList, &client.ListOptions{Namespace: dpuServiceInterfaceNamespace})).To(Succeed()) g.Expect(serviceInterfaceSetList.Items).To(BeEmpty()) }).WithTimeout(300 * time.Second).Should(Succeed()) diff --git a/test/e2e/dpuserviceconfigports.go b/test/e2e/dpuserviceconfigports.go index 3cf0b327..9a2ae6f1 100644 --- a/test/e2e/dpuserviceconfigports.go +++ b/test/e2e/dpuserviceconfigports.go @@ -34,8 +34,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func ValidateDPUServiceConfigPorts(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPUServiceConfigPorts(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip DPUService ConfigPorts test as there are no DPU nodes") } @@ -44,14 +44,14 @@ func ValidateDPUServiceConfigPorts(ctx context.Context, input *systemTestInput) // - Works only with single DPU per node // - Works only with single DPUCluster // See: "ConfigPorts limitations" design document for details. - if input.numberOfDPUsPerNode > 1 { + if input.NumberOfDPUsPerNode > 1 { Skip("Skip DPUService ConfigPorts test: feature does not support multiple DPUs per node") } By("Creating a DPUService with ConfigPorts") - dpuService := input.dpuService.DeepCopy() + dpuService := input.DPUService.DeepCopy() dpuService.Name = "dummydpuservice" - dpuService.Namespace = dpfOperatorSystemNamespace + dpuService.Namespace = DPFOperatorSystemNamespace dpuService.SetLabels(CleanupScope.It) dpuService.Spec.HelmChart.Source = dpuservicev1.ApplicationSource{ Chart: "dummydpuservice-chart", @@ -59,7 +59,7 @@ func ValidateDPUServiceConfigPorts(ctx context.Context, input *systemTestInput) RepoURL: helmRegistry, } dpuService.Spec.HelmChart.Values = &machineryruntime.RawExtension{ - Raw: []byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, dpfPullSecretName)), + Raw: []byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, DPFPullSecretName)), } dpuService.Spec.ConfigPorts = &dpuservicev1.ConfigPorts{ // TODO: test also ClusterIP. Currently this is not working as k3s doesn't have kube-proxy deployed. @@ -72,14 +72,14 @@ func ValidateDPUServiceConfigPorts(ctx context.Context, input *systemTestInput) }, }, } - Expect(input.client.Create(ctx, dpuService)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuService)).To(Succeed()) By("Waiting for dummydpuservice Pods to be ready") - VerifyClusterPods(ctx, dpuClusterClient[0], []string{"dummydpuservice"}) + VerifyClusterPods(ctx, DPUClusterClient[0], []string{"dummydpuservice"}) By("Verifying the ConfigPorts are exposed via the DPUService") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuService), dpuService)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuService), dpuService)).To(Succeed()) g.Expect(dpuService.Status.ConfigPorts).NotTo(BeNil()) }).WithTimeout(120 * time.Second).Should(Succeed()) @@ -94,7 +94,7 @@ func ValidateDPUServiceConfigPorts(ctx context.Context, input *systemTestInput) // Then get the name and IPs of the host nodes and check reachability. nodeIPs := make(map[string]string) nodeList := &corev1.NodeList{} - Expect(input.client.List(ctx, nodeList, client.MatchingLabels{ + Expect(input.Client.List(ctx, nodeList, client.MatchingLabels{ "feature.node.kubernetes.io/dpu-enabled": "true", })).To(Succeed()) for _, node := range nodeList.Items { @@ -108,7 +108,7 @@ func ValidateDPUServiceConfigPorts(ctx context.Context, input *systemTestInput) } // And finally check reachability by looping over all nodes. for nodeName, nodeIP := range nodeIPs { - dpuNodeIP, err := getDPUIPForHost(ctx, input.client, nodeName) + dpuNodeIP, err := getDPUIPForHost(ctx, input.Client, nodeName) Expect(err).NotTo(HaveOccurred()) Eventually(func(g Gomega) { resp, err := http.Get(fmt.Sprintf("http://%s:%d", nodeIP, *nodePort)) @@ -121,7 +121,7 @@ func ValidateDPUServiceConfigPorts(ctx context.Context, input *systemTestInput) var podInfo dummydpuservice.PodInfo g.Expect(json.NewDecoder(resp.Body).Decode(&podInfo)).To(Succeed()) g.Expect(podInfo.NodeIP).To(Equal(dpuNodeIP)) - g.Expect(podInfo.PodNamespace).To(Equal(dpfOperatorSystemNamespace)) + g.Expect(podInfo.PodNamespace).To(Equal(DPFOperatorSystemNamespace)) // This timeout needs to be big enough because in multi node nic cloud setup the image pulling may take // longer than expected }).WithTimeout(30 * time.Minute).Should(Succeed()) diff --git a/test/e2e/dpuservicecredentialrequest.go b/test/e2e/dpuservicecredentialrequest.go index c68625b8..e532d9c2 100644 --- a/test/e2e/dpuservicecredentialrequest.go +++ b/test/e2e/dpuservicecredentialrequest.go @@ -30,65 +30,65 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func ValidateDPUServiceCredentialRequestCreation(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceCredentialRequestCreation(ctx context.Context, input *SystemTestInput) { hostDPUServiceCredentialRequestName := "host-dpu-credential-request" dpuServiceCredentialRequestName := "dpu-01-credential-request" dpuServiceCredentialRequestNamespace := "dpucr-test-ns" By("Create namespace for DPUServiceCredentialRequest") - createTestNamespace(ctx, input.client, dpuServiceCredentialRequestNamespace) + createTestNamespace(ctx, input.Client, dpuServiceCredentialRequestNamespace) By("Create a DPUServiceCredentialRequest targeting the DPUCluster") - dcr := utils.GenerateDPUObj(dpuServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.dpuServiceCredentialRequest.DeepCopy()) - dcr.Spec.TargetCluster = &dpuservicev1.NamespacedName{Name: input.dpuClusters[0].Name, Namespace: ptr.To(dpfOperatorSystemNamespace)} + dcr := utils.GenerateDPUObj(dpuServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.DPUServiceCredentialRequest.DeepCopy()) + dcr.Spec.TargetCluster = &dpuservicev1.NamespacedName{Name: input.DPUClusters[0].Name, Namespace: ptr.To(DPFOperatorSystemNamespace)} dcr.Spec.ServiceAccount.Name = "dpu-sa" dcr.Spec.Secret.Name = "dpu-credential" - Expect(input.client.Create(ctx, dcr)).To(Succeed()) + Expect(input.Client.Create(ctx, dcr)).To(Succeed()) By("Create a DPUServiceCredentialRequest targeting the host cluster") - hostDcr := utils.GenerateDPUObj(hostDPUServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.dpuServiceCredentialRequest.DeepCopy()) + hostDcr := utils.GenerateDPUObj(hostDPUServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.DPUServiceCredentialRequest.DeepCopy()) hostDcr.Spec.ServiceAccount.Name = "host-dpu-sa" hostDcr.Spec.Secret.Name = "host-dpu-credential" - Expect(input.client.Create(ctx, hostDcr)).To(Succeed()) + Expect(input.Client.Create(ctx, hostDcr)).To(Succeed()) By("Verify reconciled DPUServiceCredentialRequest for DPUCluster") Eventually(func(g Gomega) { - assertDPUServiceCredentialRequest(ctx, g, input.client, dcr, false) + assertDPUServiceCredentialRequest(ctx, g, input.Client, dcr, false) }).WithTimeout(300 * time.Second).Should(Succeed()) By("Verify reconciled DPUServiceCredentialRequest for host cluster") Eventually(func(g Gomega) { - assertDPUServiceCredentialRequest(ctx, g, input.client, hostDcr, true) + assertDPUServiceCredentialRequest(ctx, g, input.Client, hostDcr, true) }).WithTimeout(600 * time.Second).Should(Succeed()) } -func ValidateDPUServiceCredentialRequestMetrics(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceCredentialRequestMetrics(ctx context.Context, input *SystemTestInput) { dpuServiceCredentialRequestName := "dpu-01-credential-request-metrics" dpuServiceCredentialRequestNamespace := "dpucr-test-ns-metrics" By("Create namespace for DPUServiceCredentialRequest") - createTestNamespace(ctx, input.client, dpuServiceCredentialRequestNamespace) + createTestNamespace(ctx, input.Client, dpuServiceCredentialRequestNamespace) By("Create a DPUServiceCredentialRequest targeting the DPUCluster") - dcr := utils.GenerateDPUObj(dpuServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.dpuServiceCredentialRequest.DeepCopy()) - dcr.Spec.TargetCluster = &dpuservicev1.NamespacedName{Name: input.dpuClusters[0].Name, Namespace: ptr.To(dpfOperatorSystemNamespace)} + dcr := utils.GenerateDPUObj(dpuServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.DPUServiceCredentialRequest.DeepCopy()) + dcr.Spec.TargetCluster = &dpuservicev1.NamespacedName{Name: input.DPUClusters[0].Name, Namespace: ptr.To(DPFOperatorSystemNamespace)} dcr.Spec.ServiceAccount.Name = "dpu-sa-metrics" dcr.Spec.Secret.Name = "dpu-credential-metrics" - Expect(input.client.Create(ctx, dcr)).To(Succeed()) + Expect(input.Client.Create(ctx, dcr)).To(Succeed()) By("Verify DPUServiceCredentialRequest metrics in KSM") expectedMetricsNames := map[string][]string{ "dpuservicecredentialrequest": {"created", "info", "expiration", "issued_at", "status_conditions", "status_condition_last_transition_time"}, } Eventually(func(g Gomega) { - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, metricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, MetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(5 * time.Second).Should(Succeed()) } -func ValidateDPUServiceCredentialRequestDeletion(ctx context.Context, input *systemTestInput) { - if input.cleanupFlags.SkipCleanup { +func ValidateDPUServiceCredentialRequestDeletion(ctx context.Context, input *SystemTestInput) { + if input.CleanupFlags.SkipCleanup { Skip("Skip cleanup resources") } @@ -97,44 +97,44 @@ func ValidateDPUServiceCredentialRequestDeletion(ctx context.Context, input *sys dpuServiceCredentialRequestNamespace := "dpucr-test-ns-delete" By("Create namespace for DPUServiceCredentialRequest") - createTestNamespace(ctx, input.client, dpuServiceCredentialRequestNamespace) + createTestNamespace(ctx, input.Client, dpuServiceCredentialRequestNamespace) By("Create a DPUServiceCredentialRequest targeting the DPUCluster") - dcr := utils.GenerateDPUObj(dpuServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.dpuServiceCredentialRequest.DeepCopy()) - dcr.Spec.TargetCluster = &dpuservicev1.NamespacedName{Name: input.dpuClusters[0].Name, Namespace: ptr.To(dpfOperatorSystemNamespace)} + dcr := utils.GenerateDPUObj(dpuServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.DPUServiceCredentialRequest.DeepCopy()) + dcr.Spec.TargetCluster = &dpuservicev1.NamespacedName{Name: input.DPUClusters[0].Name, Namespace: ptr.To(DPFOperatorSystemNamespace)} dcr.Spec.ServiceAccount.Name = "dpu-sa-delete" dcr.Spec.Secret.Name = "dpu-credential-delete" - Expect(input.client.Create(ctx, dcr)).To(Succeed()) + Expect(input.Client.Create(ctx, dcr)).To(Succeed()) By("Create a DPUServiceCredentialRequest targeting the host cluster") - hostDcr := utils.GenerateDPUObj(hostDPUServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.dpuServiceCredentialRequest.DeepCopy()) + hostDcr := utils.GenerateDPUObj(hostDPUServiceCredentialRequestName, dpuServiceCredentialRequestNamespace, input.DPUServiceCredentialRequest.DeepCopy()) hostDcr.Spec.ServiceAccount.Name = "host-dpu-sa-delete" hostDcr.Spec.Secret.Name = "host-dpu-credential-delete" - Expect(input.client.Create(ctx, hostDcr)).To(Succeed()) + Expect(input.Client.Create(ctx, hostDcr)).To(Succeed()) By("Verify reconciled DPUServiceCredentialRequest for DPUCluster") Eventually(func(g Gomega) { - assertDPUServiceCredentialRequest(ctx, g, input.client, dcr, false) + assertDPUServiceCredentialRequest(ctx, g, input.Client, dcr, false) }).WithTimeout(300 * time.Second).Should(Succeed()) By("Verify reconciled DPUServiceCredentialRequest for host cluster") Eventually(func(g Gomega) { - assertDPUServiceCredentialRequest(ctx, g, input.client, hostDcr, true) + assertDPUServiceCredentialRequest(ctx, g, input.Client, hostDcr, true) }).WithTimeout(600 * time.Second).Should(Succeed()) By("Delete the DPUServiceCredentialRequest") key := client.ObjectKey{Namespace: dpuServiceCredentialRequestNamespace, Name: dpuServiceCredentialRequestName} - Expect(input.client.Get(ctx, key, dcr)).To(Succeed()) - Expect(input.client.Delete(ctx, dcr)).To(Succeed()) + Expect(input.Client.Get(ctx, key, dcr)).To(Succeed()) + Expect(input.Client.Delete(ctx, dcr)).To(Succeed()) Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, key, dcr)).NotTo(Succeed()) + g.Expect(input.Client.Get(ctx, key, dcr)).NotTo(Succeed()) }).WithTimeout(300 * time.Second).Should(Succeed()) key = client.ObjectKey{Namespace: dpuServiceCredentialRequestNamespace, Name: hostDPUServiceCredentialRequestName} - Expect(input.client.Get(ctx, key, dcr)).To(Succeed()) - Expect(input.client.Delete(ctx, dcr)).To(Succeed()) + Expect(input.Client.Get(ctx, key, dcr)).To(Succeed()) + Expect(input.Client.Delete(ctx, dcr)).To(Succeed()) Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, key, dcr)).NotTo(Succeed()) + g.Expect(input.Client.Get(ctx, key, dcr)).NotTo(Succeed()) }).WithTimeout(300 * time.Second).Should(Succeed()) } diff --git a/test/e2e/dpuserviceipam.go b/test/e2e/dpuserviceipam.go index 0bf61d10..7f6ca327 100644 --- a/test/e2e/dpuserviceipam.go +++ b/test/e2e/dpuserviceipam.go @@ -33,11 +33,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var dpuServiceIPAMNamespace = dpfOperatorSystemNamespace +var dpuServiceIPAMNamespace = DPFOperatorSystemNamespace -func ValidateDPUServiceIPAMCreationInvalid(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMCreationInvalid(ctx context.Context, input *SystemTestInput) { By("Creating the invalid DPUServiceIPAM CR") - dpuServiceIPAMNamespace = dpfOperatorSystemNamespace + dpuServiceIPAMNamespace = DPFOperatorSystemNamespace dpuServiceIPAM := &dpuservicev1.DPUServiceIPAM{ ObjectMeta: metav1.ObjectMeta{ Name: "some-name", @@ -46,23 +46,23 @@ func ValidateDPUServiceIPAMCreationInvalid(ctx context.Context, input *systemTes } dpuServiceIPAM.SetGroupVersionKind(dpuservicev1.DPUServiceIPAMGroupVersionKind) dpuServiceIPAM.SetLabels(CleanupScope.It) - err := input.client.Create(ctx, dpuServiceIPAM) + err := input.Client.Create(ctx, dpuServiceIPAM) Expect(err).To(HaveOccurred()) fmt.Printf("Error creating the DPUServiceIPAM CR: %v\n", err) Expect(apierrors.IsBadRequest(err)).To(BeTrue()) Expect(err.Error()).To(ContainSubstring("either ipv4Subnet or ipv4Network must be specified")) } -func ValidateDPUServiceIPAMCreationSubnetSplit(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMCreationSubnetSplit(ctx context.Context, input *SystemTestInput) { dpuServiceIPAMWithIPPoolName := "switched-application" By("Creating the DPUServiceIPAM CR") - dpuServiceIPAM := testutils.GenerateDPUObj(dpuServiceIPAMWithIPPoolName, dpuServiceIPAMNamespace, input.ipPoolDPUServiceIPAM.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + dpuServiceIPAM := testutils.GenerateDPUObj(dpuServiceIPAMWithIPPoolName, dpuServiceIPAMNamespace, input.IPPoolDPUServiceIPAM.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Checking that NVIPAM IPPool CR is created in the DPU clusters") Eventually(func(g Gomega) { ipPools := &nvipamv1.IPPoolList{} - g.Expect(dpuClusterClient[0].List(ctx, ipPools, client.MatchingLabels{ + g.Expect(DPUClusterClient[0].List(ctx, ipPools, client.MatchingLabels{ "dpu.nvidia.com/dpuserviceipam-name": dpuServiceIPAM.GetName(), "dpu.nvidia.com/dpuserviceipam-namespace": dpuServiceIPAM.GetNamespace(), })).To(Succeed()) @@ -72,28 +72,28 @@ func ValidateDPUServiceIPAMCreationSubnetSplit(ctx context.Context, input *syste }).WithTimeout(180 * time.Second).Should(Succeed()) } -func ValidateDPUServiceIPAMMetrics(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMMetrics(ctx context.Context, input *SystemTestInput) { By("Creating the DPUServiceIPAM CR") - dpuServiceIPAM := testutils.GenerateDPUObj("switched-application-metrics", dpuServiceIPAMNamespace, input.ipPoolDPUServiceIPAM.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + dpuServiceIPAM := testutils.GenerateDPUObj("switched-application-metrics", dpuServiceIPAMNamespace, input.IPPoolDPUServiceIPAM.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Verify DPUServiceIPAM metrics in host cluster KSM") expectedHostMetricsNames := map[string][]string{ "dpuserviceipam": {"created", "info", "status_conditions", "status_condition_last_transition_time"}, // "network_info", "subnet_info" missed } Eventually(func(g Gomega) { - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, metricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, MetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedHostMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(5 * time.Second).Should(Succeed()) By("Waiting for DPU cluster kube-state-metrics to be ready") - VerifyClusterPods(ctx, input.client, []string{"in-cluster-kube-state-metrics"}) + VerifyClusterPods(ctx, input.Client, []string{"in-cluster-kube-state-metrics"}) By("Wait for IPPool to be created in DPU clusters") Eventually(func(g Gomega) { ipPools := &nvipamv1.IPPoolList{} - g.Expect(dpuClusterClient[0].List(ctx, ipPools, client.MatchingLabels{ + g.Expect(DPUClusterClient[0].List(ctx, ipPools, client.MatchingLabels{ "dpu.nvidia.com/dpuserviceipam-name": dpuServiceIPAM.GetName(), "dpu.nvidia.com/dpuserviceipam-namespace": dpuServiceIPAM.GetNamespace(), })).To(Succeed()) @@ -105,43 +105,43 @@ func ValidateDPUServiceIPAMMetrics(ctx context.Context, input *systemTestInput) "ippool": {"created", "info", "allocation_info"}, } Eventually(func(g Gomega) { - g.Expect(input.dpuClusters).ToNot(BeEmpty(), "No DPUClusters found in test input") - dpuKSMMetricsURI, err := metrics.GetKSMMetricsURIForDPUCluster(ctx, input.client, input.dpuClusters[0], dpfOperatorSystemNamespace, kubeStateMetricsPort, "/metrics") + g.Expect(input.DPUClusters).ToNot(BeEmpty(), "No DPUClusters found in test input") + dpuKSMMetricsURI, err := metrics.GetKSMMetricsURIForDPUCluster(ctx, input.Client, input.DPUClusters[0], DPFOperatorSystemNamespace, KubeStateMetricsPort, "/metrics") g.Expect(err).NotTo(HaveOccurred(), "Failed to get KSM metrics URI for DPUCluster") g.Expect(dpuKSMMetricsURI).NotTo(BeEmpty()) // Use hostClusterRESTClient because in-cluster KSM runs on the management cluster - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, dpuKSMMetricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, dpuKSMMetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedDPUMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(30 * time.Second).Should(Succeed()) } -func ValidateDPUServiceIPAMMetricsDeletion(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMMetricsDeletion(ctx context.Context, input *SystemTestInput) { dpuServiceIPAMWithIPPoolName := "switched-application-delete" - if input.cleanupFlags.SkipCleanup { + if input.CleanupFlags.SkipCleanup { Skip("Skip cleanup resources") } By("Creating the DPUServiceIPAM CR") - dpuServiceIPAM := testutils.GenerateDPUObj(dpuServiceIPAMWithIPPoolName, dpuServiceIPAMNamespace, input.ipPoolDPUServiceIPAM.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + dpuServiceIPAM := testutils.GenerateDPUObj(dpuServiceIPAMWithIPPoolName, dpuServiceIPAMNamespace, input.IPPoolDPUServiceIPAM.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) // Wait for the controller to set its finalizer before deleting, otherwise a // Delete racing with the finalizer patch can remove the object before reconcileDelete // runs and leaves the dpu-cluster object orphaned. See https://github.com/kubernetes/kubernetes/issues/77988 Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceIPAM), dpuServiceIPAM)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceIPAM), dpuServiceIPAM)).To(Succeed()) g.Expect(dpuServiceIPAM.Finalizers).To(ContainElement(dpuservicev1.DPUServiceIPAMFinalizer)) }).WithTimeout(60 * time.Second).Should(Succeed()) By("Deleting the DPUServiceIPAM") - Expect(input.client.Delete(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Delete(ctx, dpuServiceIPAM)).To(Succeed()) By("Checking that NVIPAM IPPool CR is deleted in each DPU cluster") Eventually(func(g Gomega) { ipPools := &nvipamv1.IPPoolList{} - g.Expect(dpuClusterClient[0].List(ctx, ipPools, client.MatchingLabels{ + g.Expect(DPUClusterClient[0].List(ctx, ipPools, client.MatchingLabels{ "dpu.nvidia.com/dpuserviceipam-name": dpuServiceIPAM.GetName(), "dpu.nvidia.com/dpuserviceipam-namespace": dpuServiceIPAM.GetNamespace(), })).To(Succeed()) @@ -149,16 +149,16 @@ func ValidateDPUServiceIPAMMetricsDeletion(ctx context.Context, input *systemTes }).WithTimeout(180 * time.Second).Should(Succeed()) } -func ValidateDPUServiceIPAMCreationCidrSplit(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMCreationCidrSplit(ctx context.Context, input *SystemTestInput) { dpuServiceIPAMWithCIDRPoolName := "routed-application" By("Creating the DPUServiceIPAM CR") - dpuServiceIPAM := testutils.GenerateDPUObj(dpuServiceIPAMWithCIDRPoolName, dpuServiceIPAMNamespace, input.cidrDPUServiceIPAM.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + dpuServiceIPAM := testutils.GenerateDPUObj(dpuServiceIPAMWithCIDRPoolName, dpuServiceIPAMNamespace, input.CIDRDPUServiceIPAM.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Checking that NVIPAM CIDRPool CR is created in the DPU clusters") Eventually(func(g Gomega) { cidrPools := &nvipamv1.CIDRPoolList{} - g.Expect(dpuClusterClient[0].List(ctx, cidrPools, client.MatchingLabels{ + g.Expect(DPUClusterClient[0].List(ctx, cidrPools, client.MatchingLabels{ "dpu.nvidia.com/dpuserviceipam-name": dpuServiceIPAM.GetName(), "dpu.nvidia.com/dpuserviceipam-namespace": dpuServiceIPAM.GetNamespace(), })).To(Succeed()) @@ -172,43 +172,43 @@ func ValidateDPUServiceIPAMCreationCidrSplit(ctx context.Context, input *systemT "cidrpool": {"created", "info", "allocation_info"}, } Eventually(func(g Gomega) { - g.Expect(input.dpuClusters).ToNot(BeEmpty(), "No DPUClusters found in test input") - dpuKSMMetricsURI, err := metrics.GetKSMMetricsURIForDPUCluster(ctx, input.client, input.dpuClusters[0], dpfOperatorSystemNamespace, kubeStateMetricsPort, "/metrics") + g.Expect(input.DPUClusters).ToNot(BeEmpty(), "No DPUClusters found in test input") + dpuKSMMetricsURI, err := metrics.GetKSMMetricsURIForDPUCluster(ctx, input.Client, input.DPUClusters[0], DPFOperatorSystemNamespace, KubeStateMetricsPort, "/metrics") g.Expect(err).NotTo(HaveOccurred(), "Failed to get KSM metrics URI for DPUCluster") g.Expect(dpuKSMMetricsURI).NotTo(BeEmpty()) // Use hostClusterRESTClient because in-cluster KSM runs on the management cluster - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, dpuKSMMetricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, dpuKSMMetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedCIDRPoolMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(10 * time.Second).Should(Succeed()) } -func ValidateDPUServiceIPAMDeletionCidrSplit(ctx context.Context, input *systemTestInput) { - if input.cleanupFlags.SkipCleanup { +func ValidateDPUServiceIPAMDeletionCidrSplit(ctx context.Context, input *SystemTestInput) { + if input.CleanupFlags.SkipCleanup { Skip("Skip cleanup resources") } dpuServiceIPAMWithCIDRPoolName := "routed-application-delete" By("Creating the DPUServiceIPAM CR") - dpuServiceIPAM := testutils.GenerateDPUObj(dpuServiceIPAMWithCIDRPoolName, dpuServiceIPAMNamespace, input.ipPoolDPUServiceIPAM.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + dpuServiceIPAM := testutils.GenerateDPUObj(dpuServiceIPAMWithCIDRPoolName, dpuServiceIPAMNamespace, input.IPPoolDPUServiceIPAM.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) // Wait for the controller to set its finalizer before deleting, otherwise a // Delete racing with the finalizer patch can remove the object before reconcileDelete // runs and leaves the dpu-cluster object orphaned. See https://github.com/kubernetes/kubernetes/issues/77988 Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceIPAM), dpuServiceIPAM)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceIPAM), dpuServiceIPAM)).To(Succeed()) g.Expect(dpuServiceIPAM.Finalizers).To(ContainElement(dpuservicev1.DPUServiceIPAMFinalizer)) }).WithTimeout(60 * time.Second).Should(Succeed()) By("Deleting the DPUServiceIPAM") - Expect(input.client.Delete(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Delete(ctx, dpuServiceIPAM)).To(Succeed()) By("Checking that NVIPAM CIDRPool CR is deleted in each DPU cluster") Eventually(func(g Gomega) { cidrPools := &nvipamv1.CIDRPoolList{} - g.Expect(dpuClusterClient[0].List(ctx, cidrPools, client.MatchingLabels{ + g.Expect(DPUClusterClient[0].List(ctx, cidrPools, client.MatchingLabels{ "dpu.nvidia.com/dpuserviceipam-name": dpuServiceIPAM.GetName(), "dpu.nvidia.com/dpuserviceipam-namespace": dpuServiceIPAM.GetNamespace(), })).To(Succeed()) diff --git a/test/e2e/dpuservicenad.go b/test/e2e/dpuservicenad.go index b92d0d2f..3f086f1d 100644 --- a/test/e2e/dpuservicenad.go +++ b/test/e2e/dpuservicenad.go @@ -37,8 +37,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func ValidateDPUServiceNADConsumedByPod(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPUServiceNADConsumedByPod(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip test as there are not multiple nodes") } @@ -58,45 +58,45 @@ func ValidateDPUServiceNADConsumedByPod(ctx context.Context, input *systemTestIn } By("Create test namespace: " + namespace) - createTestNamespace(ctx, input.client, namespace) + createTestNamespace(ctx, input.Client, namespace) By("Copy image pull secret to namespace " + namespace) - CopySecretToNamespace(ctx, input.client, dpfPullSecretName, dpfOperatorSystemNamespace, namespace, CleanupScope.It) + CopySecretToNamespace(ctx, input.Client, DPFPullSecretName, DPFOperatorSystemNamespace, namespace, CleanupScope.It) By("Create DPUServiceNAD") dpuServiceNAD := constructDPUServiceNAD(dpuServiceNADName, namespace, mtu) - Expect(input.client.Create(ctx, dpuServiceNAD)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceNAD)).To(Succeed()) By("Create DPUServiceInterface") dpuServiceInterface := constructDPUServiceInterface(dpuServiceInterfaceCustomNADName, namespace, serviceName, dpuServiceNADName, serviceInterfaceLabels) - Expect(input.client.Create(ctx, dpuServiceInterface)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceInterface)).To(Succeed()) // FIXME: There is a bug that incorrectly requires a DPUServiceChain to exist before a DPUService can be deployed successfully; remove the DPUServiceChain part if this is fixed By("Create DPUServiceChain") dpuServiceChain := constructDPUServiceChain(serviceChainName, namespace, mtu, serviceInterfaceLabels) - Expect(input.client.Create(ctx, dpuServiceChain)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceChain)).To(Succeed()) By("Deploy DummyDPUService") dpuServiceDummy := constructDummyDPUServiceObject(serviceName, namespace, dpuServiceInterfaceCustomNADName) - Expect(input.client.Create(ctx, dpuServiceDummy)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceDummy)).To(Succeed()) By("Verify DPUServiceNAD is ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceNAD, defaultTimeout) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceNAD, defaultTimeout) By("Verify DPUServiceInterface is ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceInterface, 10*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceInterface, 10*time.Minute) // Only now verify that the DPUServiceChain and DummyDPUService are ready // Reason: They depend on each other and the DPUServiceInterface and only then become ready By("Verify DPUServiceChain is ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceChain, 3*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceChain, 3*time.Minute) By("Verify DummyDPUService is ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceDummy, 3*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceDummy, 3*time.Minute) By("Verify DPUService pods are created in DPU cluster") Eventually(func(g Gomega) { const podServiceLabel string = "svc.dpu.nvidia.com/service" podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, + g.Expect(DPUClusterClient[0].List(ctx, podList, client.InNamespace(namespace), client.MatchingLabels{podServiceLabel: serviceName}, )).To(Succeed()) @@ -111,7 +111,7 @@ func ValidateDPUServiceNADMetrics(ctx context.Context) { } Eventually(func(g Gomega) { - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, metricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, MetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(5 * time.Second).Should(Succeed()) @@ -196,7 +196,7 @@ func constructDummyDPUServiceObject(serviceName, namespace, interfaceName string if ngcAPIKey != "" { dpuServiceDummy.Spec.HelmChart.Values = &machineryruntime.RawExtension{ Raw: []byte(fmt.Sprintf( - `{"imagePullSecrets": [{"name": "%s"}]}`, dpfPullSecretName, + `{"imagePullSecrets": [{"name": "%s"}]}`, DPFPullSecretName, )), } } @@ -213,8 +213,8 @@ func constructDummyDPUServiceObject(serviceName, namespace, interfaceName string } // VerifyDPUPodToPodRDMATraffic verifies that 2 Pods in the DPUCluster can run RDMA traffic between each other. -func VerifyDPUPodToPodRDMATraffic(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func VerifyDPUPodToPodRDMATraffic(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } @@ -223,7 +223,7 @@ func VerifyDPUPodToPodRDMATraffic(ctx context.Context, input *systemTestInput) { setupDPUPodToPodRDMATrafficTest(ctx, input) By("Getting the pods in the DPU cluster") - pod1, pod2 := get2DPUServicePods(ctx, input.namespace, "dummydpuservice-rdma") + pod1, pod2 := get2DPUServicePods(ctx, input.Namespace, "dummydpuservice-rdma") // Validate that IPs are available for both pods podIP1 := getPodIPForInterface(Default, pod1, "app_rdma_if") podIP2 := getPodIPForInterface(Default, pod2, "app_rdma_if") @@ -234,14 +234,14 @@ func VerifyDPUPodToPodRDMATraffic(ctx context.Context, input *systemTestInput) { // We must pass pointer of a pointer here because the dpuClusterRestClient and dpuClusterRestConfig are updated in // a goroutine in case they break and we need to ensure that the underlying function always picks up the up to date // pointer. - netshoot.RunRDMATrafficTest(&dpuClusterRestClient[0], &dpuClusterRestConfig[0], input.namespace, pod1.Name, pod2.Name, podIP2) + netshoot.RunRDMATrafficTest(&DPUClusterRestClient[0], &DPUClusterRestConfig[0], input.Namespace, pod1.Name, pod2.Name, podIP2) } -func setupDPUPodToPodRDMATrafficTest(ctx context.Context, input *systemTestInput) { +func setupDPUPodToPodRDMATrafficTest(ctx context.Context, input *SystemTestInput) { interfaceConfigs := []dpuservice.TestDPUServiceInterfaceConfig{ { Name: "p0-rdma", - Namespace: input.namespace, + Namespace: input.Namespace, Type: "physical", InterfaceName: "p0", Labels: map[string]string{ @@ -253,7 +253,7 @@ func setupDPUPodToPodRDMATrafficTest(ctx context.Context, input *systemTestInput }, { Name: "app-sf-rdma", - Namespace: input.namespace, + Namespace: input.Namespace, Type: "sf", InterfaceName: "app_rdma_if", ServiceID: "dummydpuservice-rdma", @@ -266,10 +266,10 @@ func setupDPUPodToPodRDMATrafficTest(ctx context.Context, input *systemTestInput poolLabels := map[string]string{"svc.dpu.nvidia.com/pool": "dummydpuservice-rdma"} By("Create and wait for DPU service interfaces") - createAndWaitForInterfaces(ctx, input.client, input.dpuServiceInterfaceTemplate, interfaceConfigs) + createAndWaitForInterfaces(ctx, input.Client, input.DPUServiceInterfaceTemplate, interfaceConfigs) By("Create the chain between the workload pod and p0") - fabricChain := utils.GenerateDPUObj("pod-to-fabric", input.namespace, input.dpuServiceChainTemplate.DeepCopy()) + fabricChain := utils.GenerateDPUObj("pod-to-fabric", input.Namespace, input.DPUServiceChainTemplate.DeepCopy()) fabricChain.Spec.Template.Spec.Template.Spec.Switches = []dpuservicev1.Switch{ { Ports: []dpuservicev1.Port{ @@ -289,7 +289,7 @@ func setupDPUPodToPodRDMATrafficTest(ctx context.Context, input *systemTestInput }, }, } - Expect(input.client.Create(ctx, fabricChain)).To(Succeed()) + Expect(input.Client.Create(ctx, fabricChain)).To(Succeed()) By("Create DPUServiceIPAM") dpuServiceIPAMTemplate := dpuservicev1.DPUServiceIPAM{ @@ -304,8 +304,8 @@ func setupDPUPodToPodRDMATrafficTest(ctx context.Context, input *systemTestInput }, }, } - dpuServiceIPAM := utils.GenerateDPUObj("mybrsfc-rdma", input.namespace, &dpuServiceIPAMTemplate) - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + dpuServiceIPAM := utils.GenerateDPUObj("mybrsfc-rdma", input.Namespace, &dpuServiceIPAMTemplate) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Create DPUServiceNAD") dpuServiceNADTemplate := dpuservicev1.DPUServiceNAD{ @@ -316,16 +316,16 @@ func setupDPUPodToPodRDMATrafficTest(ctx context.Context, input *systemTestInput ChainedCNIs: []dpuservicev1.CNIPlugin{{Type: ptr.To("rdma")}}, }, } - dpuServiceNAD := utils.GenerateDPUObj("mybrsfc-rdma", input.namespace, &dpuServiceNADTemplate) - Expect(input.client.Create(ctx, dpuServiceNAD)).To(Succeed()) + dpuServiceNAD := utils.GenerateDPUObj("mybrsfc-rdma", input.Namespace, &dpuServiceNADTemplate) + Expect(input.Client.Create(ctx, dpuServiceNAD)).To(Succeed()) By("Create and wait for dummydpuservice DPUService") - createDummyDPUServiceForRDMA(ctx, input.client, input.namespace, input.dpuService) - dpuservice.WaitForDPUServices(ctx, input.client, input.namespace, []string{"dummydpuservice-rdma"}) - VerifyClusterPods(ctx, dpuClusterClient[0], []string{"dummydpuservice-rdma"}) + createDummyDPUServiceForRDMA(ctx, input.Client, input.Namespace, input.DPUService) + dpuservice.WaitForDPUServices(ctx, input.Client, input.Namespace, []string{"dummydpuservice-rdma"}) + VerifyClusterPods(ctx, DPUClusterClient[0], []string{"dummydpuservice-rdma"}) By("Verify underlying ServiceChain and ServiceInterface objects are ready") - dpuservice.VerifyUnderlyingDPUObjectsReady(ctx, dpuClusterClient[0], input.namespace, interfaceConfigs, []string{"pod-to-fabric"}) + dpuservice.VerifyUnderlyingDPUObjectsReady(ctx, DPUClusterClient[0], input.Namespace, interfaceConfigs, []string{"pod-to-fabric"}) } // createDummyDPUServiceForRDMA creates a DPUService using the dummydpuservice and configures it for RDMA testing @@ -339,7 +339,7 @@ func createDummyDPUServiceForRDMA(ctx context.Context, testClient client.Client, } values := make(map[string]any) - values["imagePullSecrets"] = []map[string]string{{"name": dpfPullSecretName}} + values["imagePullSecrets"] = []map[string]string{{"name": DPFPullSecretName}} values["image"] = map[string]string{"repository": netutilsImage} values["securityContext"] = map[string]any{"capabilities": map[string]any{"add": []string{"IPC_LOCK"}}} rawValues, err := json.Marshal(values) @@ -379,7 +379,7 @@ func getPodIPForInterface(g Gomega, pod corev1.Pod, interfaceName string) string // get2DPUServicePods returns the 2 DPUService Pods associated with a service func get2DPUServicePods(ctx context.Context, namespace string, serviceID string) (corev1.Pod, corev1.Pod) { pods := &corev1.PodList{} - Expect(dpuClusterClient[0].List(ctx, pods, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceID}, client.InNamespace(namespace))).ToNot(HaveOccurred()) + Expect(DPUClusterClient[0].List(ctx, pods, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceID}, client.InNamespace(namespace))).ToNot(HaveOccurred()) Expect(pods.Items).To(HaveLen(2)) return pods.Items[0], pods.Items[1] } diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index c5aed330..ac649b04 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -58,28 +58,10 @@ var ( configPath string // testKubeconfig path to be used for this test. testKubeconfig string - // artifactsDir is the path where test artifacts will be stored. - artifactsDir string - - // collectResources indicates whether to collect logs an objects after an e2e test run. - collectResources = true - // externalTest path used to run external tests scripts - externalTest string // enableSOSReports to enable collecting SOS reports after an e2e test run failure. enableSOSReports = false ) -var ( - // cleanupFlags holds all flags to control skip cleanup behavior - cleanupFlags *cleanup.CleanupFlags - cleanupTracker *cleanup.Tracker - testClient client.Client - restConfig *rest.Config - clientset *kubernetes.Clientset - ctx = ctrl.SetupSignalHandler() - conf *config -) - func init() { testing.Init() // Initialize Go test flags (required for Go 1.24+) flag.StringVar(&testKubeconfig, "e2e.testKubeconfig", "", "path to the testKubeconfig file") @@ -87,7 +69,7 @@ func init() { flag.StringVar(&externalTest, "e2e.externalTestScript", "", "path to the external test file, script will be called in between BeforeSuite setup and AfterSuite cleanup") // Register cleanup flags and get handle for it - cleanupFlags = cleanup.NewCleanupFlagsFromCLI() + CleanupFlags = cleanup.NewCleanupFlagsFromCLI() getEnvVariables() } @@ -118,7 +100,7 @@ func getEnvVariables() { } if v, found := os.LookupEnv("DPF_E2E_COLLECT_RESOURCES"); found { var err error - collectResources, err = strconv.ParseBool(v) + CollectResources, err = strconv.ParseBool(v) if err != nil { panic(fmt.Errorf("string must be a bool: %v", err)) } @@ -166,11 +148,11 @@ func getEnvVariables() { } } if path, found := os.LookupEnv("ARTIFACTS_DIR"); found { - artifactsDir = path + ArtifactsDir = path } else { // Default to ../../artifacts relative to the current file. _, basePath, _, _ := runtime.Caller(0) - artifactsDir = filepath.Join(filepath.Dir(basePath), "../../artifacts") + ArtifactsDir = filepath.Join(filepath.Dir(basePath), "../../artifacts") } if interfaceName, found := os.LookupEnv("DPUCLUSTER_INTERFACE"); found { @@ -182,7 +164,7 @@ func getEnvVariables() { if ns, found := os.LookupEnv("PREREQS_NAMESPACE"); found { // Only set the override if it differs from the default namespace. - if ns != dpfOperatorSystemNamespace { + if ns != DPFOperatorSystemNamespace { prereqsNamespace = ns } } @@ -213,7 +195,7 @@ func TestE2E(t *testing.T) { // SchemeGroupVersion is group version used to register these objects var SchemeGroupVersion = schema.GroupVersion{Group: "", Version: "v1"} - conf, err = readConfig(configPath) + Conf, err = ReadConfig(configPath) g.Expect(err).NotTo(HaveOccurred()) // If testKubeconfig is not set default it to $HOME/.kube/config @@ -227,34 +209,34 @@ func TestE2E(t *testing.T) { _, _ = fmt.Fprintf(GinkgoWriter, "E2E Test Configuration:\n") _, _ = fmt.Fprintf(GinkgoWriter, " configPath: %s\n", configPath) _, _ = fmt.Fprintf(GinkgoWriter, " testKubeconfig: %s\n", testKubeconfig) - _, _ = fmt.Fprintf(GinkgoWriter, " numberOfDPUNodes: %d\n", conf.NumberOfDPUNodes) - _, _ = fmt.Fprintf(GinkgoWriter, " numberOfDPUsPerNode: %d\n", conf.NumberOfDPUsPerNode) - _, _ = fmt.Fprintf(GinkgoWriter, " nodeRebootConfigMap: %q\n", conf.NodeRebootConfigMap) - _, _ = fmt.Fprintf(GinkgoWriter, " nodeRebootConfigMapPath: %q\n", conf.NodeRebootConfigMapPath) + _, _ = fmt.Fprintf(GinkgoWriter, " numberOfDPUNodes: %d\n", Conf.NumberOfDPUNodes) + _, _ = fmt.Fprintf(GinkgoWriter, " numberOfDPUsPerNode: %d\n", Conf.NumberOfDPUsPerNode) + _, _ = fmt.Fprintf(GinkgoWriter, " nodeRebootConfigMap: %q\n", Conf.NodeRebootConfigMap) + _, _ = fmt.Fprintf(GinkgoWriter, " nodeRebootConfigMapPath: %q\n", Conf.NodeRebootConfigMapPath) // Create a client to use throughout the test. - restConfig, err = clientcmd.BuildConfigFromFlags("", testKubeconfig) + RestConfig, err = clientcmd.BuildConfigFromFlags("", testKubeconfig) g.Expect(err).NotTo(HaveOccurred()) - clientset, err = kubernetes.NewForConfig(restConfig) + Clientset, err = kubernetes.NewForConfig(RestConfig) g.Expect(err).NotTo(HaveOccurred()) - testClient, err = client.New(restConfig, client.Options{Scheme: s}) + TestClient, err = client.New(RestConfig, client.Options{Scheme: s}) g.Expect(err).NotTo(HaveOccurred()) // Set the path to /api for handling core resources (pods, services, etc) // for handling custom resources (deployments, etc) would need to set the API path to /apis - restConfig.APIPath = "/api" + RestConfig.APIPath = "/api" // Extend configs to restConfig for hostClusterRESTClient - restConfig.GroupVersion = &SchemeGroupVersion - restConfig.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: scheme.Codecs} - hostClusterRESTClient, err = rest.RESTClientFor(restConfig) + RestConfig.GroupVersion = &SchemeGroupVersion + RestConfig.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: scheme.Codecs} + HostClusterRESTClient, err = rest.RESTClientFor(RestConfig) g.Expect(err).NotTo(HaveOccurred()) - metricsURI = metrics.GetMetricsURI("kube-state-metrics", dpfOperatorSystemNamespace, kubeStateMetricsPort, "/metrics") - g.Expect(metricsURI).NotTo(BeEmpty()) + MetricsURI = metrics.GetMetricsURI("kube-state-metrics", DPFOperatorSystemNamespace, KubeStateMetricsPort, "/metrics") + g.Expect(MetricsURI).NotTo(BeEmpty()) // Auto-enable fail-fast when skip-cleanup-on-failure flag is set suiteConfig, _ := GinkgoConfiguration() - if cleanupFlags.SkipCleanupOnFailure { + if CleanupFlags.SkipCleanupOnFailure { suiteConfig.FailFast = true _, _ = fmt.Fprintf(GinkgoWriter, "Auto-enabled fail-fast mode (skip-cleanup-on-failure flag detected)\n") } @@ -278,23 +260,23 @@ var _ = BeforeSuite(func() { SetInput() // Initialize cleanup flags here as Ginkgo has parsed CLI arguments before BeforeSuite runs - cleanupFlags.Init() + CleanupFlags.Init() - cleanupTracker = cleanup.NewTracker(utils.CleanupWithLabelAndWait, cleanupFlags, ctx, testClient, resourcesToDelete) + CleanupTracker = cleanup.NewTracker(utils.CleanupWithLabelAndWait, CleanupFlags, Ctx, TestClient, resourcesToDelete) // Upgrade validation tests skip cleanup to preserve resources from previous test run. // isUpgradeValidationPhase matches the active label filter against every label // registered by validationPhase, so no per-phase update is needed here when a new // phase is added. - if isUpgradeValidationPhase() { + if IsUpgradeValidationPhase() { return } By("Checking for resources from previous test runs") - cleanupTracker.WarnIfStaleResources() + CleanupTracker.WarnIfStaleResources() By("Performing before suite cleanup") - cleanupTracker.HandleScopeLifecycle(nil, cleanup.GinkgoHook.BeforeSuite) + CleanupTracker.HandleScopeLifecycle(nil, cleanup.GinkgoHook.BeforeSuite) // Label filter examples supported: // (Domain.DPFSystem) -> all tests with Domain.DPFSystem running. SDN, SNAP included @@ -311,15 +293,15 @@ var _ = BeforeSuite(func() { // BeforeProvisioning(ctx, input) // CreateProvisioningDPUCluster(ctx, input) // CreateProvisioningDPUSet(ctx, input) - provInput := getProvisionDPUClustersInput() - ProvisionDPUClusters(ctx, provInput) - ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(ctx, provInput) - ProvisionDPUSet(ctx, provInput) + provInput := GetProvisionDPUClustersInput() + ProvisionDPUClusters(Ctx, provInput) + ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(Ctx, provInput) + ProvisionDPUSet(Ctx, provInput) } // Apply the ProvisioningBeforeSuite setup if directly specified Provisioning label // !skipProvisioning() branch should not be executed in provisioning-only tests - if isGinkgoLabelApplied(Domain.Provisioning) { + if IsGinkgoLabelApplied(Domain.Provisioning) { // SystemSetupBeforeSuite must run first to deploy the DPF operator and system components // Provisioning tests need the operator running but will provision DPUs from scratch (no pre-provisioning) SystemSetupBeforeSuite(false) @@ -342,7 +324,7 @@ var _ = BeforeSuite(func() { // Apply the WeaveBeforeSuite setup if !strings.Contains(GinkgoLabelFilter(), "!"+Domain.Weave) { - WeaveBeforeSuite(*conf) + WeaveBeforeSuite(*Conf) } // For Performance + OVNKHBN (physical HBN-OVN performance) scenario, deploy the full @@ -351,27 +333,27 @@ var _ = BeforeSuite(func() { // On physical environments provisioning runs above so we must also wait for DPUs to be ready. // IgnoreAlreadyExists handles objects already present (e.g. on re-runs). // Per RDG, service object creation precedes the DPU provisioning wait. - if isGinkgoLabelApplied(Domain.Performance) && isGinkgoLabelApplied(Domain.OVNKHBN) { + if IsGinkgoLabelApplied(Domain.Performance) && IsGinkgoLabelApplied(Domain.OVNKHBN) { SystemSetupBeforeSuite(false) By("Maximizing maintenance operator parallelism for performance provisioning") - restoreMaintenanceConfig := SetMaintenanceOperatorMaxParallelOperations(ctx, testClient, 50) + restoreMaintenanceConfig := SetMaintenanceOperatorMaxParallelOperations(Ctx, TestClient, 50) defer restoreMaintenanceConfig() By("Pre-provisioning DPU cluster setup") - provInput := getProvisionDPUClustersInput() - ProvisionDPUClusters(ctx, provInput) - ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(ctx, provInput) + provInput := GetProvisionDPUClustersInput() + ProvisionDPUClusters(Ctx, provInput) + ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(Ctx, provInput) By("Installing OVN-K resource injector webhook") - InstallOVNKResourceInjector(ctx, testClient) + InstallOVNKResourceInjector(Ctx, TestClient) By("Deploying HBN-OVN scenario objects") - DeployOVNKHBNScenario(ctx, input) + DeployOVNKHBNScenario(Ctx, input) By("Waiting for DPUs to be provisioned") - VerifyDPUClusterWithNodes(ctx, getProvisionDPUClustersInput()) + VerifyDPUClusterWithNodes(Ctx, GetProvisionDPUClustersInput()) } }) var _ = ReportBeforeEach(func(spec SpecReport) { // Detect entering scopes and perform "before" cleanup - cleanupTracker.HandleScopeLifecycle(&spec, cleanup.GinkgoHook.BeforeEach) + CleanupTracker.HandleScopeLifecycle(&spec, cleanup.GinkgoHook.BeforeEach) }) // reportAfterEach collects diagnostics when a test fails @@ -380,20 +362,20 @@ func reportAfterEach(spec SpecReport) { if spec.Failed() { By(fmt.Sprintf("ReportAfterEach: Test %q failed. Collecting resources and logs for the clusters", spec.FullText())) collectInput := collectResourcesInput{ - collectResources: collectResources, - testClient: testClient, - clientset: clientset, - restConfig: restConfig, - artifactsDir: artifactsDir, + collectResources: CollectResources, + testClient: TestClient, + clientset: Clientset, + restConfig: RestConfig, + artifactsDir: ArtifactsDir, } - err := collectKubernetesResources(ctx, collectInput, "failed_tests/"+spec.LeafNodeText) + err := collectKubernetesResources(Ctx, collectInput, "failed_tests/"+spec.LeafNodeText) if err != nil { GinkgoLogr.Error(err, "failed to collect resources and logs for the clusters") } // Collect SOS reports if enabled (runs at most once per suite via sync.Once). if enableSOSReports { - if err = collectSOSReports(ctx, artifactsDir); err != nil { + if err = collectSOSReports(Ctx, ArtifactsDir); err != nil { GinkgoLogr.Error(err, "SOS report collection failed") } } @@ -405,28 +387,28 @@ var _ = ReportAfterEach(func(spec SpecReport) { reportAfterEach(spec) // Handle scope lifecycle and cleanup - cleanupTracker.HandleScopeLifecycle(&spec, cleanup.GinkgoHook.AfterEach) + CleanupTracker.HandleScopeLifecycle(&spec, cleanup.GinkgoHook.AfterEach) }) var _ = AfterSuite(func() { collectInput := collectResourcesInput{ - collectResources: collectResources, - testClient: testClient, - clientset: clientset, - restConfig: restConfig, - artifactsDir: artifactsDir, + collectResources: CollectResources, + testClient: TestClient, + clientset: Clientset, + restConfig: RestConfig, + artifactsDir: ArtifactsDir, } By("Collecting resources for the clusters after suite (pre-DPF operator config cleanup)") - if err := collectKubernetesResources(ctx, collectInput, "pre-dpf-operator-config-cleanup"); err != nil { + if err := collectKubernetesResources(Ctx, collectInput, "pre-dpf-operator-config-cleanup"); err != nil { GinkgoLogr.Error(err, "failed to collect resources for the clusters (pre-DPF operator config cleanup)") } - if !cleanupFlags.SkipSuiteCleanupAfter { - DeleteDPFOperatorConfig(ctx, testClient) + if !CleanupFlags.SkipSuiteCleanupAfter { + DeleteDPFOperatorConfig(Ctx, TestClient) By("Collecting resources for the clusters after suite (post-DPF operator config cleanup)") - if err := collectKubernetesResources(ctx, collectInput, "post-dpf-operator-config-cleanup"); err != nil { + if err := collectKubernetesResources(Ctx, collectInput, "post-dpf-operator-config-cleanup"); err != nil { GinkgoLogr.Error(err, "failed to collect resources for the clusters (post-DPF operator config cleanup)") } @@ -435,30 +417,5 @@ var _ = AfterSuite(func() { } By("Performing final suite cleanup") - cleanupTracker.HandleScopeLifecycle(nil, cleanup.GinkgoHook.AfterSuite) + CleanupTracker.HandleScopeLifecycle(nil, cleanup.GinkgoHook.AfterSuite) }) - -func validateFlags() { - if !isGinkgoLabelApplied(Domain.ZeroTrust) { - return - } - - if conf.NodeRebootConfigMap == "" { - panic("ZeroTrust requires `nodeRebootConfigMap` to be set in the e2e config file") - } - if conf.NodeRebootConfigMapPath == "" { - panic("ZeroTrust requires `nodeRebootConfigMapPath` to be set in the e2e config file") - } - if bmcPassword == "" { - panic("ZeroTrust requires E2E_ZT_BMC_PASSWORD env var (BMC root password used by the in-cluster reboot script)") - } - if bmcInventoryPath == "" { - panic("ZeroTrust requires E2E_ZT_BMC_INVENTORY_PATH env var (path to the lab DPU-serial -> BMC IP inventory YAML)") - } - - if isGinkgoLabelApplied(Domain.ExternalTest) { - if len(externalTest) == 0 { - panic("This script must be provided when External label is present") - } - } -} diff --git a/test/e2e/external_test.go b/test/e2e/external_test.go index 19f2f7c8..fba70467 100644 --- a/test/e2e/external_test.go +++ b/test/e2e/external_test.go @@ -87,7 +87,7 @@ var _ = Describe("External DPF tests", Labels{Domain.ExternalTest}, func() { BeforeAll(func() { By("Wait for OVNK HBN deployment to be ready") - WaitForOVNKHBNDeploymentReady(ctx, input) + WaitForOVNKHBNDeploymentReady(Ctx, input) By("Syncing image pull secrets for Nlastic workload pods") syncNlasticImagePullSecrets() By("Setup Nlastic environment") @@ -123,8 +123,8 @@ var _ = Describe("External DPF tests", Labels{Domain.ExternalTest}, func() { func syncNlasticImagePullSecrets() { const nlasticPodNamespace = "default" - for _, secretName := range []string{dpfPullSecretName, "pull-secret-extra"} { - CopySecretToNamespace(ctx, input.client, secretName, dpfOperatorSystemNamespace, nlasticPodNamespace, CleanupScope.Suite) + for _, secretName := range []string{DPFPullSecretName, "pull-secret-extra"} { + CopySecretToNamespace(Ctx, input.Client, secretName, DPFOperatorSystemNamespace, nlasticPodNamespace, CleanupScope.Suite) } } @@ -195,7 +195,7 @@ func collectNlasticResults(sharedDir, subDir string) { if _, err := os.Stat(sharedDir); err != nil { return } - dest := filepath.Join(artifactsDir, "nlastic", subDir) + dest := filepath.Join(ArtifactsDir, "nlastic", subDir) if err := os.MkdirAll(dest, 0755); err != nil { _, _ = fmt.Fprintf(GinkgoWriter, "Failed to create nlastic artifacts dir %s: %v\n", dest, err) return diff --git a/test/e2e/globals.go b/test/e2e/globals.go new file mode 100644 index 00000000..740291a4 --- /dev/null +++ b/test/e2e/globals.go @@ -0,0 +1,56 @@ +/* +Copyright 2024 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "github.com/nvidia/doca-platform/test/e2e/cleanup" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ArtifactsDir is the path where test artifacts will be stored. +var ArtifactsDir string + +// externalTest is the path to the external test script, set via the +// -e2e.externalTestScript flag in e2e_suite_test.go's init(). +var externalTest string + +// CollectResources indicates whether to collect logs an objects after an e2e test run. +var CollectResources = true + +var ( + // CleanupFlags holds all flags to control skip cleanup behavior + CleanupFlags *cleanup.CleanupFlags + CleanupTracker *cleanup.Tracker + TestClient client.Client + RestConfig *rest.Config + Clientset *kubernetes.Clientset + Ctx = ctrl.SetupSignalHandler() + Conf *Config +) + +// input is the singleton populated by SetInput() and consumed by every DPF +// System - Core test function. External callers should use the value +// returned by SetInput() rather than this package-private variable. +var input *SystemTestInput + +// VPCOVNInput is the singleton populated by VPCOVNBeforeSuite (via +// VPCOVNTestInput.ApplyVPCOVNConfig) and consumed by the VPCOVN test funcs. +var VPCOVNInput = &VPCOVNTestInput{} diff --git a/test/e2e/leader_election.go b/test/e2e/leader_election.go index 0c74dd97..4bdeb503 100644 --- a/test/e2e/leader_election.go +++ b/test/e2e/leader_election.go @@ -64,17 +64,17 @@ const ( leaderElectionPollInterval = 1 * time.Second ) -// leaderElectionTarget names one controller's Deployment and its Lease. -type leaderElectionTarget struct { - // component is the controller's ComponentName (e.g. "provisioning-controller"). It +// LeaderElectionTarget names one controller's Deployment and its Lease. +type LeaderElectionTarget struct { + // Component is the controller's ComponentName (e.g. "provisioning-controller"). It // labels the spec and is matched against DPFOperatorConfig.ComponentConfigs() Name() // to locate the controller config when scaling replicas. - component string - // deploymentName is the Deployment's metadata.name in dpfOperatorSystemNamespace. - deploymentName string - // leaseName is the coordination.k8s.io/Lease metadata.name (matches LeaderElectionID + Component string + // DeploymentName is the Deployment's metadata.name in dpfOperatorSystemNamespace. + DeploymentName string + // LeaseName is the coordination.k8s.io/Lease metadata.name (matches LeaderElectionID // passed to ctrl.NewManager in each controller's cmd/*/main.go). - leaseName string + LeaseName string } // replicasSetter is implemented by controller component configs (those embedding @@ -88,7 +88,7 @@ type replicasSetter interface { // controller: capture the current leader -> delete that pod (simulates leader // failure) -> assert a different pod takes over the Lease and renews it at // least once -> wait for the Deployment to recover. -func ValidateLeaderElectionFailover(ctx context.Context, c client.Client, target leaderElectionTarget) { +func ValidateLeaderElectionFailover(ctx context.Context, c client.Client, target LeaderElectionTarget) { // CI deploys controllers at 1 replica; scale this target to 2 for the failover // scenario and revert to 1 afterwards. scaleControllerReplicas(ctx, c, target, 2) @@ -101,7 +101,7 @@ func ValidateLeaderElectionFailover(ctx context.Context, c client.Client, target // The lease holder identity is "_" (controller-runtime uses the // pod hostname). Match it to a live pod. pods := &corev1.PodList{} - Expect(c.List(ctx, pods, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + Expect(c.List(ctx, pods, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) var leaderPod *corev1.Pod for i := range pods.Items { if strings.HasPrefix(originalLeader, pods.Items[i].Name+"_") { @@ -109,7 +109,7 @@ func ValidateLeaderElectionFailover(ctx context.Context, c client.Client, target break } } - Expect(leaderPod).ToNot(BeNil(), "no pod found for lease holder %q in %s", originalLeader, dpfOperatorSystemNamespace) + Expect(leaderPod).ToNot(BeNil(), "no pod found for lease holder %q in %s", originalLeader, DPFOperatorSystemNamespace) By(fmt.Sprintf("Deleting leader pod %q (lease holder %q) to simulate leader failure", leaderPod.Name, originalLeader)) Expect(c.Delete(ctx, leaderPod)).To(Succeed()) @@ -120,36 +120,36 @@ func ValidateLeaderElectionFailover(ctx context.Context, c client.Client, target // scaleControllerReplicas sets the target controller's replica count via the // DPFOperatorConfig and waits for the Deployment to report that many ready replicas. -func scaleControllerReplicas(ctx context.Context, c client.Client, target leaderElectionTarget, replicas int32) { - By(fmt.Sprintf("Scaling %s to %d replica(s) via DPFOperatorConfig", target.component, replicas)) +func scaleControllerReplicas(ctx context.Context, c client.Client, target LeaderElectionTarget, replicas int32) { + By(fmt.Sprintf("Scaling %s to %d replica(s) via DPFOperatorConfig", target.Component, replicas)) Eventually(func(g Gomega) { operatorConfig := &operatorv1.DPFOperatorConfig{} g.Expect(c.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, - Name: configName, + Namespace: DPFOperatorSystemNamespace, + Name: ConfigName, }, operatorConfig)).To(Succeed()) configPatch := client.MergeFrom(operatorConfig.DeepCopy()) applied := false for _, componentConfig := range operatorConfig.ComponentConfigs() { - if componentConfig.Name() != target.component { + if componentConfig.Name() != target.Component { continue } setter, ok := componentConfig.(replicasSetter) - g.Expect(ok).To(BeTrue(), "component %q does not expose replicas", target.component) + g.Expect(ok).To(BeTrue(), "component %q does not expose replicas", target.Component) setter.SetReplicas(ptr.To(replicas)) applied = true break } - g.Expect(applied).To(BeTrue(), "component %q not found in DPFOperatorConfig", target.component) + g.Expect(applied).To(BeTrue(), "component %q not found in DPFOperatorConfig", target.Component) g.Expect(c.Patch(ctx, operatorConfig, configPatch)).To(Succeed()) }).WithTimeout(leaseReadTimeout).WithPolling(leaderElectionPollInterval).Should(Succeed()) - By(fmt.Sprintf("Waiting for the %s Deployment to report %d ready replicas", target.component, replicas)) + By(fmt.Sprintf("Waiting for the %s Deployment to report %d ready replicas", target.Component, replicas)) Eventually(func(g Gomega) { leaderDeployment := &appsv1.Deployment{} g.Expect(c.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, - Name: target.deploymentName, + Namespace: DPFOperatorSystemNamespace, + Name: target.DeploymentName, }, leaderDeployment)).To(Succeed()) g.Expect(ptr.Deref(leaderDeployment.Spec.Replicas, 0)).To(Equal(replicas)) g.Expect(leaderDeployment.Status.ReadyReplicas).To(Equal(replicas)) @@ -158,33 +158,33 @@ func scaleControllerReplicas(ctx context.Context, c client.Client, target leader // captureCurrentLeader reads the controller's Lease and returns the current // holder identity (= the active leader pod's name). -func captureCurrentLeader(ctx context.Context, c client.Client, target leaderElectionTarget) string { +func captureCurrentLeader(ctx context.Context, c client.Client, target LeaderElectionTarget) string { By("Reading the current Lease and identifying the active leader pod") lease := &coordinationv1.Lease{} Eventually(func(g Gomega) { g.Expect(c.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, - Name: target.leaseName, + Namespace: DPFOperatorSystemNamespace, + Name: target.LeaseName, }, lease)).To(Succeed()) g.Expect(lease.Spec.HolderIdentity).ToNot(BeNil()) g.Expect(*lease.Spec.HolderIdentity).ToNot(BeEmpty()) }).WithTimeout(leaseReadTimeout).WithPolling(leaderElectionPollInterval).Should(Succeed(), "expected a Lease %s/%s with a non-empty holderIdentity", - dpfOperatorSystemNamespace, target.leaseName) + DPFOperatorSystemNamespace, target.LeaseName) return *lease.Spec.HolderIdentity } // verifyLeaseHandover waits for a pod other than originalHolder to acquire the // Lease, then verifies the new leader renews the Lease at least once (proving // it is alive and healthy, not just holding a stale lease). -func verifyLeaseHandover(ctx context.Context, c client.Client, target leaderElectionTarget, originalHolder string) { +func verifyLeaseHandover(ctx context.Context, c client.Client, target LeaderElectionTarget, originalHolder string) { By("Waiting for a different pod to acquire the Lease") lease := &coordinationv1.Lease{} Eventually(func(g Gomega) { g.Expect(c.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, - Name: target.leaseName, + Namespace: DPFOperatorSystemNamespace, + Name: target.LeaseName, }, lease)).To(Succeed()) g.Expect(lease.Spec.HolderIdentity).ToNot(BeNil()) @@ -193,15 +193,15 @@ func verifyLeaseHandover(ctx context.Context, c client.Client, target leaderElec "lease is still held by the deleted pod") }).WithTimeout(leaseHandoverTimeout).WithPolling(leaderElectionPollInterval).Should(Succeed(), "expected a new pod to acquire the Lease %s/%s after the original leader was deleted", - dpfOperatorSystemNamespace, target.leaseName) + DPFOperatorSystemNamespace, target.LeaseName) By("Verifying the new leader has renewed the Lease at least once") Expect(lease.Spec.RenewTime).ToNot(BeNil()) baselineRenewTime := lease.Spec.RenewTime.Time Eventually(func(g Gomega) { g.Expect(c.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, - Name: target.leaseName, + Namespace: DPFOperatorSystemNamespace, + Name: target.LeaseName, }, lease)).To(Succeed()) g.Expect(lease.Spec.RenewTime).ToNot(BeNil()) @@ -214,20 +214,20 @@ func verifyLeaseHandover(ctx context.Context, c client.Client, target leaderElec // verifyDeploymentReady waits for the Deployment to become fully ready again // (Status.ReadyReplicas == Spec.Replicas) so the cluster is left in a healthy // state for downstream tests. -func verifyDeploymentReady(ctx context.Context, c client.Client, target leaderElectionTarget) { - By(fmt.Sprintf("Verifying the %s Deployment is fully ready again (all replicas ready)", target.component)) +func verifyDeploymentReady(ctx context.Context, c client.Client, target LeaderElectionTarget) { + By(fmt.Sprintf("Verifying the %s Deployment is fully ready again (all replicas ready)", target.Component)) leaderDeployment := &appsv1.Deployment{} Eventually(func(g Gomega) { g.Expect(c.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, - Name: target.deploymentName, + Namespace: DPFOperatorSystemNamespace, + Name: target.DeploymentName, }, leaderDeployment)).To(Succeed()) g.Expect(leaderDeployment.Status.ReadyReplicas).To( Equal(ptr.Deref(leaderDeployment.Spec.Replicas, 0)), "Deployment %s/%s did not return to fully ready (got %d/%d ready)", - dpfOperatorSystemNamespace, - target.deploymentName, + DPFOperatorSystemNamespace, + target.DeploymentName, leaderDeployment.Status.ReadyReplicas, ptr.Deref(leaderDeployment.Spec.Replicas, 0), ) diff --git a/test/e2e/leader_election_test.go b/test/e2e/leader_election_test.go index f067833f..dfd1f4bc 100644 --- a/test/e2e/leader_election_test.go +++ b/test/e2e/leader_election_test.go @@ -24,33 +24,33 @@ import ( // leaderElectionTargets enumerates the in-scope controllers for the failover // test (see leader_election.go for the inclusion criteria and exclusion list). -var leaderElectionTargets = []leaderElectionTarget{ +var leaderElectionTargets = []LeaderElectionTarget{ { - component: "provisioning-controller", - deploymentName: "dpf-provisioning-controller-manager", - leaseName: "provisioning.dpu.nvidia.com", + Component: "provisioning-controller", + DeploymentName: "dpf-provisioning-controller-manager", + LeaseName: "provisioning.dpu.nvidia.com", }, { - component: "dpuservice-controller", - deploymentName: "dpuservice-controller-manager", - leaseName: "dpuservice.dpu.nvidia.com", + Component: "dpuservice-controller", + DeploymentName: "dpuservice-controller-manager", + LeaseName: "dpuservice.dpu.nvidia.com", }, { - component: "kamaji-cluster-manager", - deploymentName: "kamaji-cm-controller-manager", - leaseName: "kamaji-cluster-manager.dpu.nvidia.com", + Component: "kamaji-cluster-manager", + DeploymentName: "kamaji-cm-controller-manager", + LeaseName: "kamaji-cluster-manager.dpu.nvidia.com", }, { - component: "static-cluster-manager", - deploymentName: "static-cm-controller-manager", - leaseName: "static-cluster-manager.dpu.nvidia.com", + Component: "static-cluster-manager", + DeploymentName: "static-cm-controller-manager", + LeaseName: "static-cluster-manager.dpu.nvidia.com", }, } var _ = Describe("DPF Leader-election failover", Labels{Domain.DPFSystem}, func() { for _, target := range leaderElectionTargets { - It(fmt.Sprintf("hands over the lease when the %s leader pod is deleted", target.component), func() { - ValidateLeaderElectionFailover(ctx, testClient, target) + It(fmt.Sprintf("hands over the lease when the %s leader pod is deleted", target.Component), func() { + ValidateLeaderElectionFailover(Ctx, TestClient, target) }) } }) diff --git a/test/e2e/logging.go b/test/e2e/logging.go index dee3a420..907bf613 100644 --- a/test/e2e/logging.go +++ b/test/e2e/logging.go @@ -38,15 +38,15 @@ const ( ) // ValidateDPUClusterOpenTelemetryConfiguration verifies DPU cluster collector configuration -func ValidateDPUClusterOpenTelemetryConfiguration(ctx context.Context, input *systemTestInput) { - for i, dpuClient := range dpuClusterClient { - clusterName := input.dpuClusters[i].Name +func ValidateDPUClusterOpenTelemetryConfiguration(ctx context.Context, input *SystemTestInput) { + for i, dpuClient := range DPUClusterClient { + clusterName := input.DPUClusters[i].Name By(fmt.Sprintf("Checking OpenTelemetry Collector ConfigMap in DPU cluster %s", clusterName)) cm := &corev1.ConfigMap{} Eventually(func(g Gomega) { err := dpuClient.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Name: clusterName + "-opentelemetry-collector-config", }, cm) g.Expect(err).NotTo(HaveOccurred()) @@ -70,16 +70,16 @@ func ValidateDPUClusterOpenTelemetryConfiguration(ctx context.Context, input *sy } // ValidateManagementClusterLogFlow verifies logs flow from management cluster to Loki -func ValidateManagementClusterLogFlow(ctx context.Context, input *systemTestInput) { - lokiClient := loki.NewClient(hostClusterRESTClient, dpfOperatorSystemNamespace) +func ValidateManagementClusterLogFlow(ctx context.Context, input *SystemTestInput) { + lokiClient := loki.NewClient(HostClusterRESTClient, DPFOperatorSystemNamespace) testNamespacePrefix := "test-logging-mgmt-" uniqueMessage := fmt.Sprintf("test-log-mgmt-%d", time.Now().Unix()) By("Creating test namespace in management cluster") - testNS := createTestNamespaceInCluster(ctx, input.client, testNamespacePrefix) + testNS := createTestNamespaceInCluster(ctx, input.Client, testNamespacePrefix) By(fmt.Sprintf("Creating log generator pod with message: %s", uniqueMessage)) - createLogGeneratorPod(ctx, input.client, testNS, "log-generator", uniqueMessage) + createLogGeneratorPod(ctx, input.Client, testNS, "log-generator", uniqueMessage) By("Waiting for logs to be collected and forwarded to Loki") Eventually(func(g Gomega) { @@ -113,19 +113,19 @@ func ValidateManagementClusterLogFlow(ctx context.Context, input *systemTestInpu } // ValidateDPUClusterLogFlow verifies logs flow from DPU cluster to Loki -func ValidateDPUClusterLogFlow(ctx context.Context, input *systemTestInput) { - lokiClient := loki.NewClient(hostClusterRESTClient, dpfOperatorSystemNamespace) +func ValidateDPUClusterLogFlow(ctx context.Context, input *SystemTestInput) { + lokiClient := loki.NewClient(HostClusterRESTClient, DPFOperatorSystemNamespace) testNamespacePrefix := "test-logging-dpu-" uniqueMessage := fmt.Sprintf("test-log-dpu-%d", time.Now().Unix()) By("Creating test namespace in DPU cluster") - testNS := createTestNamespaceInCluster(ctx, dpuClusterClient[0], testNamespacePrefix) + testNS := createTestNamespaceInCluster(ctx, DPUClusterClient[0], testNamespacePrefix) By(fmt.Sprintf("Creating log generator pod in DPU cluster with message: %s", uniqueMessage)) - createLogGeneratorPod(ctx, dpuClusterClient[0], testNS, "log-generator-dpu", uniqueMessage) + createLogGeneratorPod(ctx, DPUClusterClient[0], testNS, "log-generator-dpu", uniqueMessage) By("Waiting for logs to be collected and forwarded to Loki") - clusterName := input.dpuClusters[0].Name + clusterName := input.DPUClusters[0].Name Eventually(func(g Gomega) { labels := map[string]string{ "cluster": clusterName, diff --git a/test/e2e/metrics.go b/test/e2e/metrics.go index 3966f899..2c80cf88 100644 --- a/test/e2e/metrics.go +++ b/test/e2e/metrics.go @@ -32,51 +32,51 @@ import ( func VerifyHostKSMMetricsCollection(ctx context.Context) { By("Verify host cluster kube-state-metrics endpoint is accessible") Eventually(func(g Gomega) { - request := hostClusterRESTClient.Get().AbsPath(metricsURI) + request := HostClusterRESTClient.Get().AbsPath(MetricsURI) response, err := request.DoRaw(ctx) - g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Request %s failed with err: %v", metricsURI, err)) - g.Expect(response).NotTo(BeNil(), fmt.Sprintf("Metrics api is not accessible by url %s ", metricsURI)) + g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Request %s failed with err: %v", MetricsURI, err)) + g.Expect(response).NotTo(BeNil(), fmt.Sprintf("Metrics api is not accessible by url %s ", MetricsURI)) }).WithTimeout(30 * time.Second).Should(Succeed()) } -func VerifyDPUKSMMetricsCollection(ctx context.Context, input *systemTestInput) { +func VerifyDPUKSMMetricsCollection(ctx context.Context, input *SystemTestInput) { By("Verify DPU cluster kube-state-metrics endpoint is accessible") Eventually(func(g Gomega) { // Get the KSM metrics URI for the first DPUCluster // Note: The in-cluster kube-state-metrics service runs on the management cluster, // not on the DPU cluster. It connects remotely to collect DPU cluster metrics. - g.Expect(input.dpuClusters).ToNot(BeEmpty(), "No DPUClusters found in test input") - dpuKSMMetricsURI, err := metrics.GetKSMMetricsURIForDPUCluster(ctx, input.client, input.dpuClusters[0], dpfOperatorSystemNamespace, kubeStateMetricsPort, "/metrics") + g.Expect(input.DPUClusters).ToNot(BeEmpty(), "No DPUClusters found in test input") + dpuKSMMetricsURI, err := metrics.GetKSMMetricsURIForDPUCluster(ctx, input.Client, input.DPUClusters[0], DPFOperatorSystemNamespace, KubeStateMetricsPort, "/metrics") g.Expect(err).NotTo(HaveOccurred(), "Failed to get KSM metrics URI for DPUCluster") g.Expect(dpuKSMMetricsURI).NotTo(BeEmpty()) // Use hostClusterRESTClient because the in-cluster KSM service runs on the management cluster - request := hostClusterRESTClient.Get().AbsPath(dpuKSMMetricsURI) + request := HostClusterRESTClient.Get().AbsPath(dpuKSMMetricsURI) response, err := request.DoRaw(ctx) g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Request %s failed with err: %v", dpuKSMMetricsURI, err)) g.Expect(response).NotTo(BeNil(), fmt.Sprintf("Metrics api is not accessible by url %s ", dpuKSMMetricsURI)) }).WithTimeout(30 * time.Second).Should(Succeed()) } -func ValidateGeneralDPFMetrics(ctx context.Context, input *systemTestInput) { +func ValidateGeneralDPFMetrics(ctx context.Context, input *SystemTestInput) { By("Verify metrics are being collected") expectedMetricsNames := map[string][]string{ "dpfoperatorconfig": {"created", "info", "status_conditions", "status_condition_last_transition_time", "version"}, // "paused" missed "dpucluster": {"created", "info", "status_phase", "status_conditions", "status_condition_last_transition_time", "status_nodes_count"}, } - if input.bfb != nil { + if input.BFB != nil { expectedMetricsNames["bfb"] = []string{"created", "info", "status_phase", "version_bsp", "version_doca", "version_uefi", "version_atf", "file_name"} } - if input.hasDpuNodes() { + if input.HasDpuNodes() { By("Checking that DPUs are created") Eventually(func(g Gomega) { // A DPU object is created for each DPU device, not each DPU node. // totalDPUs() = numberOfDPUNodes * numberOfDPUsPerNode dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus)).To(Succeed()) - g.Expect(dpus.Items).To(HaveLen(input.totalDPUs())) + g.Expect(input.Client.List(ctx, dpus)).To(Succeed()) + g.Expect(dpus.Items).To(HaveLen(input.TotalDPUs())) }).WithTimeout(60 * time.Second).Should(Succeed()) expectedMetricsNames["dpu"] = []string{"created", "info", "required_reset", "status_phase", "status_conditions", "status_condition_last_transition_time", "operational_conditions", "operational_condition_last_transition_time", "agent_conditions", "agent_condition_last_transition_time", "outdated_timestamp", "outdated_reason"} @@ -86,19 +86,19 @@ func ValidateGeneralDPFMetrics(ctx context.Context, input *systemTestInput) { } Eventually(func(g Gomega) { - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, metricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, MetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(5 * time.Second).Should(Succeed()) } -func VerifyNodeProblemDetectorConditions(ctx context.Context, input *systemTestInput) { +func VerifyNodeProblemDetectorConditions(ctx context.Context, input *SystemTestInput) { Eventually(func(g Gomega) { - for i, dpuCluster := range input.dpuClusters { + for i, dpuCluster := range input.DPUClusters { By(fmt.Sprintf("Checking node conditions in DPUCluster %s", dpuCluster.Name)) nodes := &corev1.NodeList{} - g.Expect(dpuClusterClient[i].List(ctx, nodes)).To(Succeed(), + g.Expect(DPUClusterClient[i].List(ctx, nodes)).To(Succeed(), fmt.Sprintf("Failed to list nodes in DPUCluster %s", dpuCluster.Name)) g.Expect(nodes.Items).ToNot(BeEmpty(), fmt.Sprintf("No nodes found in DPUCluster %s", dpuCluster.Name)) diff --git a/test/e2e/multidpucluster.go b/test/e2e/multidpucluster.go index 785d01b5..441000a1 100644 --- a/test/e2e/multidpucluster.go +++ b/test/e2e/multidpucluster.go @@ -37,43 +37,43 @@ import ( ) // ProvisionDPUDeploymentWithEachDPUJoiningADifferentDPUCluster creates a DPUDeployment where each DPU joins a different cluster -func ProvisionDPUDeploymentWithEachDPUJoiningADifferentDPUCluster(ctx context.Context, input *systemTestInput) { - expectedTotalDPUs := input.totalDPUs() +func ProvisionDPUDeploymentWithEachDPUJoiningADifferentDPUCluster(ctx context.Context, input *SystemTestInput) { + expectedTotalDPUs := input.TotalDPUs() By("Verifying preconditions: number of clusters equals total DPUs") - Expect(input.dpuClusters).To(HaveLen(expectedTotalDPUs), + Expect(input.DPUClusters).To(HaveLen(expectedTotalDPUs), fmt.Sprintf("This test requires one DPUCluster per DPU. Expected %d clusters for %d nodes * %d DPUs/node", - expectedTotalDPUs, input.numberOfDPUNodes, input.numberOfDPUsPerNode)) + expectedTotalDPUs, input.NumberOfDPUNodes, input.NumberOfDPUsPerNode)) By("Getting DPUDevices") dpuDevices := &provisioningv1.DPUDeviceList{} - Expect(input.client.List(ctx, dpuDevices)).To(Succeed()) + Expect(input.Client.List(ctx, dpuDevices)).To(Succeed()) Expect(dpuDevices.Items).To(HaveLen(expectedTotalDPUs), fmt.Sprintf("Expected %d DPUDevices (%d nodes * %d DPUs/node)", - expectedTotalDPUs, input.numberOfDPUNodes, input.numberOfDPUsPerNode)) + expectedTotalDPUs, input.NumberOfDPUNodes, input.NumberOfDPUsPerNode)) By("Creating DPUServiceNAD") nadName := "brsfc-no-ipam" dpuServiceNAD := constructDPUServiceNAD(nadName, "dpf-operator-system", 1500) dpuServiceNAD.Labels = CleanupScope.Suite - Expect(input.client.Create(ctx, dpuServiceNAD)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceNAD)).To(Succeed()) By("Creating DPUServiceTemplate") dpuServiceTemplate := generateDPUServiceTemplate(input, "") useDummyDPUServiceChart(dpuServiceTemplate) - Expect(input.client.Create(ctx, dpuServiceTemplate)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceTemplate)).To(Succeed()) By("Creating DPUServiceConfiguration") dpuServiceConfiguration := generateServiceConfiguration(input, "") dpuServiceConfiguration.Spec.Interfaces = []dpuservicev1.ServiceInterfaceTemplate{{Name: "net1", Network: nadName}} - Expect(input.client.Create(ctx, dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceConfiguration)).To(Succeed()) By("Creating DPUDeployment with each DPU joining a different cluster") dpuDeployment := generateDPUDeployment(input, "") // Create a DPUSet for each DPU joining a different DPUCluster dpuDeployment.Spec.DPUs.DPUSets = []dpuservicev1.DPUSet{} - for i, dpuCluster := range input.dpuClusters { + for i, dpuCluster := range input.DPUClusters { dpuSet := dpuservicev1.DPUSet{ NameSuffix: fmt.Sprintf("cluster-%d", i), DPUClusterSelector: map[string]string{ @@ -103,28 +103,28 @@ func ProvisionDPUDeploymentWithEachDPUJoiningADifferentDPUCluster(ctx context.Co }, } - Expect(input.client.Create(ctx, dpuDeployment)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuDeployment)).To(Succeed()) By("Waiting for DPUDeployment underlying objects to be created") Eventually(func(g Gomega) { - g.Expect(VerifyDeploymentUnderlyingObjectsCreated(ctx, g, input.client, dpuDeployment)).To(BeTrue()) + g.Expect(VerifyDeploymentUnderlyingObjectsCreated(ctx, g, input.Client, dpuDeployment)).To(BeTrue()) }).WithTimeout(180 * time.Second).Should(Succeed()) By("Verifying that the DPUDeployment is ready") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) g.Expect(conditions.IsTrue(dpuDeployment, conditions.TypeReady)).To(BeTrue()) - }).WithTimeout(dpuDeploymentReadyTimeout).WithPolling(1 * time.Second).Should(Succeed()) + }).WithTimeout(DPUDeploymentReadyTimeout).WithPolling(1 * time.Second).Should(Succeed()) By("Verifying DPUs joined the correct clusters") - for i, dpuCluster := range input.dpuClusters { + for i, dpuCluster := range input.DPUClusters { dpuDevice := &dpuDevices.Items[i] expectedHost := dpuDevice.Labels[provisioningv1.DPUNodeNameLabel] By(fmt.Sprintf("Verifying DPU from DPUDevice %s (host: %s) joined DPUCluster %s", dpuDevice.Name, expectedHost, dpuCluster.Name)) Eventually(func(g Gomega) { nodes := &corev1.NodeList{} - g.Expect(dpuClusterClient[i].List(ctx, nodes)).To(Succeed()) + g.Expect(DPUClusterClient[i].List(ctx, nodes)).To(Succeed()) g.Expect(nodes.Items).To(HaveLen(1), fmt.Sprintf("DPUCluster %s should have exactly 1 node", dpuCluster.Name)) @@ -139,19 +139,19 @@ func ProvisionDPUDeploymentWithEachDPUJoiningADifferentDPUCluster(ctx context.Co // ValidateDPUServiceIPAMInL2ModePerDPUCluster validates per-DPUCluster DPUServiceIPAM configuration in L2 mode. // This covers the advanced use case where each DPUCluster requires its own DPUServiceIPAM object (via DPUClusterSelector), // where the user splits the CIDR on their own per DPUCluster. -func ValidateDPUServiceIPAMInL2ModePerDPUCluster(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMInL2ModePerDPUCluster(ctx context.Context, input *SystemTestInput) { By("Getting existing DPUServiceConfiguration and updating it to use br-sfc network with IPAM requirement") dpuServiceConfiguration := generateServiceConfiguration(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) dpuServiceConfigurationOriginal := dpuServiceConfiguration.DeepCopy() dpuServiceConfiguration.Spec.Interfaces[0].Network = "mybrsfc" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) poolLabel := map[string]string{ "svc.dpu.nvidia.com/pool": "l2-pool", } - dpuServiceIPAMTemplate := input.ipPoolDPUServiceIPAM.DeepCopy() - dpuServiceIPAMTemplate.SetNamespace(dpfOperatorSystemNamespace) + dpuServiceIPAMTemplate := input.IPPoolDPUServiceIPAM.DeepCopy() + dpuServiceIPAMTemplate.SetNamespace(DPFOperatorSystemNamespace) dpuServiceIPAMTemplate.Labels = CleanupScope.Suite dpuServiceIPAMTemplate.Spec.ObjectMeta.Labels = poolLabel dpuServiceIPAMTemplate.Spec.NodeSelector = nil @@ -171,13 +171,13 @@ func ValidateDPUServiceIPAMInL2ModePerDPUCluster(ctx context.Context, input *sys } dpuServiceIPAM1.Spec.DPUClusterSelector = &metav1.LabelSelector{ MatchLabels: map[string]string{ - "svc.dpu.nvidia.com/cluster": input.dpuClusters[0].Name, + "svc.dpu.nvidia.com/cluster": input.DPUClusters[0].Name, }, } - Expect(input.client.Create(ctx, dpuServiceIPAM1)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM1)).To(Succeed()) By("Waiting for DPUServiceIPAM for first cluster to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceIPAM1, 5*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceIPAM1, 5*time.Minute) By("Creating DPUServiceIPAM for second cluster") dpuServiceIPAM2 := dpuServiceIPAMTemplate.DeepCopy() @@ -195,51 +195,51 @@ func ValidateDPUServiceIPAMInL2ModePerDPUCluster(ctx context.Context, input *sys } dpuServiceIPAM2.Spec.DPUClusterSelector = &metav1.LabelSelector{ MatchLabels: map[string]string{ - "svc.dpu.nvidia.com/cluster": input.dpuClusters[1].Name, + "svc.dpu.nvidia.com/cluster": input.DPUClusters[1].Name, }, } - Expect(input.client.Create(ctx, dpuServiceIPAM2)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM2)).To(Succeed()) By("Waiting for DPUServiceIPAM for second cluster to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceIPAM2, 5*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceIPAM2, 5*time.Minute) By("Getting existing DPUDeployment and updating its ServiceChains to use DPUServiceIPAM") dpuDeployment := generateDPUDeployment(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) dpuDeploymentOriginal := dpuDeployment.DeepCopy() // Update the service port to include IPAM dpuDeployment.Spec.ServiceChains.Switches[0].Ports[0].Service.IPAM = &dpuservicev1.IPAM{MatchLabels: poolLabel} - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) By("Waiting for DPUDeployment to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuDeployment, 15*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuDeployment, 15*time.Minute) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Validating DPUService Pod in first cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[0], serviceIDForExample, "192.168.10.2", 1) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[0], serviceIDForExample, "192.168.10.2", 1) By("Validating DPUService Pod in second cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[1], serviceIDForExample, "192.168.10.7", 2) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[1], serviceIDForExample, "192.168.10.7", 2) } // ValidateDPUServiceIPAMInL3ModePerDPUCluster validates per-DPUCluster DPUServiceIPAM configuration in L3 mode. // This covers the advanced use case where each DPUCluster requires its own DPUServiceIPAM object (via DPUClusterSelector), // where the user splits the CIDR on their own per DPUCluster. -func ValidateDPUServiceIPAMInL3ModePerDPUCluster(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMInL3ModePerDPUCluster(ctx context.Context, input *SystemTestInput) { By("Getting existing DPUServiceConfiguration and updating it to use br-sfc network with IPAM requirement") dpuServiceConfiguration := generateServiceConfiguration(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) dpuServiceConfigurationOriginal := dpuServiceConfiguration.DeepCopy() dpuServiceConfiguration.Spec.Interfaces[0].Network = "mybrsfc" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) poolLabel := map[string]string{ "svc.dpu.nvidia.com/pool": "l3-pool", } - dpuServiceIPAMTemplate := input.cidrDPUServiceIPAM.DeepCopy() - dpuServiceIPAMTemplate.SetNamespace(dpfOperatorSystemNamespace) + dpuServiceIPAMTemplate := input.CIDRDPUServiceIPAM.DeepCopy() + dpuServiceIPAMTemplate.SetNamespace(DPFOperatorSystemNamespace) dpuServiceIPAMTemplate.Labels = CleanupScope.Suite dpuServiceIPAMTemplate.Spec.ObjectMeta.Labels = poolLabel dpuServiceIPAMTemplate.Spec.NodeSelector = nil @@ -260,13 +260,13 @@ func ValidateDPUServiceIPAMInL3ModePerDPUCluster(ctx context.Context, input *sys } dpuServiceIPAM1.Spec.DPUClusterSelector = &metav1.LabelSelector{ MatchLabels: map[string]string{ - "svc.dpu.nvidia.com/cluster": input.dpuClusters[0].Name, + "svc.dpu.nvidia.com/cluster": input.DPUClusters[0].Name, }, } - Expect(input.client.Create(ctx, dpuServiceIPAM1)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM1)).To(Succeed()) By("Waiting for DPUServiceIPAM for first cluster to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceIPAM1, 5*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceIPAM1, 5*time.Minute) By("Creating DPUServiceIPAM for second cluster") dpuServiceIPAM2 := dpuServiceIPAMTemplate.DeepCopy() @@ -284,54 +284,54 @@ func ValidateDPUServiceIPAMInL3ModePerDPUCluster(ctx context.Context, input *sys } dpuServiceIPAM2.Spec.DPUClusterSelector = &metav1.LabelSelector{ MatchLabels: map[string]string{ - "svc.dpu.nvidia.com/cluster": input.dpuClusters[1].Name, + "svc.dpu.nvidia.com/cluster": input.DPUClusters[1].Name, }, } - Expect(input.client.Create(ctx, dpuServiceIPAM2)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM2)).To(Succeed()) By("Waiting for DPUServiceIPAM for second cluster to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceIPAM2, 5*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceIPAM2, 5*time.Minute) By("Getting existing DPUDeployment and updating its ServiceChains to use DPUServiceIPAM") dpuDeployment := generateDPUDeployment(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) dpuDeploymentOriginal := dpuDeployment.DeepCopy() // Update the service port to include IPAM dpuDeployment.Spec.ServiceChains.Switches[0].Ports[0].Service.IPAM = &dpuservicev1.IPAM{MatchLabels: poolLabel} - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) By("Waiting for DPUDeployment to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuDeployment, 15*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuDeployment, 15*time.Minute) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Validating DPUService Pod in first cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[0], serviceIDForExample, "192.168.20.2", 1) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[0], serviceIDForExample, "192.168.20.2", 1) By("Validating DPUService Pod in second cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[1], serviceIDForExample, "192.168.20.10", 2) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[1], serviceIDForExample, "192.168.20.10", 2) } // ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClusters validates a single DPUServiceIPAM object in L2 mode that spans // all DPUClusters without a DPUClusterSelector. This is the standard multi-DPUCluster use case where the controller // distributes IP allocations from a shared pool across all clusters automatically. -func ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClusters(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClusters(ctx context.Context, input *SystemTestInput) { By("Getting existing DPUServiceConfiguration and updating it to use br-sfc network with IPAM requirement") dpuServiceConfiguration := generateServiceConfiguration(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) dpuServiceConfigurationOriginal := dpuServiceConfiguration.DeepCopy() dpuServiceConfiguration.Spec.Interfaces[0].Network = "mybrsfc" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) poolLabel := map[string]string{ "svc.dpu.nvidia.com/pool": "l2-shared-pool", } By("Creating a single DPUServiceIPAM spanning all clusters") - dpuServiceIPAM := input.ipPoolDPUServiceIPAM.DeepCopy() + dpuServiceIPAM := input.IPPoolDPUServiceIPAM.DeepCopy() dpuServiceIPAM.SetName("l2-ipam-shared") - dpuServiceIPAM.SetNamespace(dpfOperatorSystemNamespace) + dpuServiceIPAM.SetNamespace(DPFOperatorSystemNamespace) dpuServiceIPAM.Labels = CleanupScope.Suite dpuServiceIPAM.Spec.ObjectMeta.Labels = poolLabel dpuServiceIPAM.Spec.NodeSelector = nil @@ -342,51 +342,51 @@ func ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClusters(ctx context.Context, PerNodeIPCount: 6, BlocksPerDPUCluster: ptr.To[int32](2), } - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Waiting for DPUServiceIPAM to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceIPAM, 5*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceIPAM, 5*time.Minute) By("Getting existing DPUDeployment and updating its ServiceChains to use DPUServiceIPAM") dpuDeployment := generateDPUDeployment(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) dpuDeploymentOriginal := dpuDeployment.DeepCopy() dpuDeployment.Spec.ServiceChains.Switches[0].Ports[0].Service.IPAM = &dpuservicev1.IPAM{MatchLabels: poolLabel} - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) By("Waiting for DPUDeployment to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuDeployment, 15*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuDeployment, 15*time.Minute) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Validating DPUService Pod in first cluster has secondary IP from correct subnet") // .2 because we don't explicitly request the gateway - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[0], serviceIDForExample, "192.168.50.2", 1) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[0], serviceIDForExample, "192.168.50.2", 1) By("Validating DPUService Pod in second cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[1], serviceIDForExample, "192.168.50.13", 2) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[1], serviceIDForExample, "192.168.50.13", 2) } // ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClusters validates a single DPUServiceIPAM object in L3 mode that spans // all DPUClusters without a DPUClusterSelector. This is the standard multi-DPUCluster use case where the controller // distributes IP prefix allocations from a shared network across all clusters automatically. -func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClusters(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClusters(ctx context.Context, input *SystemTestInput) { By("Getting existing DPUServiceConfiguration and updating it to use br-sfc network with IPAM requirement") dpuServiceConfiguration := generateServiceConfiguration(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) dpuServiceConfigurationOriginal := dpuServiceConfiguration.DeepCopy() dpuServiceConfiguration.Spec.Interfaces[0].Network = "mybrsfc" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) poolLabel := map[string]string{ "svc.dpu.nvidia.com/pool": "l3-shared-pool", } By("Creating a single DPUServiceIPAM spanning all clusters") - dpuServiceIPAM := input.cidrDPUServiceIPAM.DeepCopy() + dpuServiceIPAM := input.CIDRDPUServiceIPAM.DeepCopy() dpuServiceIPAM.SetName("l3-ipam-shared") - dpuServiceIPAM.SetNamespace(dpfOperatorSystemNamespace) + dpuServiceIPAM.SetNamespace(DPFOperatorSystemNamespace) dpuServiceIPAM.Labels = CleanupScope.Suite dpuServiceIPAM.Spec.ObjectMeta.Labels = poolLabel dpuServiceIPAM.Spec.NodeSelector = nil @@ -397,49 +397,49 @@ func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClusters(ctx context.Context, PrefixSize: 30, SubnetsPerDPUCluster: ptr.To[int32](2), } - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Waiting for DPUServiceIPAM to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceIPAM, 5*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceIPAM, 5*time.Minute) By("Getting existing DPUDeployment and updating its ServiceChains to use DPUServiceIPAM") dpuDeployment := generateDPUDeployment(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) dpuDeploymentOriginal := dpuDeployment.DeepCopy() dpuDeployment.Spec.ServiceChains.Switches[0].Ports[0].Service.IPAM = &dpuservicev1.IPAM{MatchLabels: poolLabel} - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) By("Waiting for DPUDeployment to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuDeployment, 15*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuDeployment, 15*time.Minute) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Validating DPUService Pod in first cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[0], serviceIDForExample, "192.168.60.2", 1) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[0], serviceIDForExample, "192.168.60.2", 1) By("Validating DPUService Pod in second cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[1], serviceIDForExample, "192.168.60.10", 2) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[1], serviceIDForExample, "192.168.60.10", 2) } // ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithStaticAllocations validates a single DPUServiceIPAM in L3 mode // that uses static allocations to explicitly pin each node across all DPUClusters to a specific IP prefix. This is the // standard multi-DPUCluster use case for static allocations where one DPUServiceIPAM covers all clusters. -func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithStaticAllocations(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithStaticAllocations(ctx context.Context, input *SystemTestInput) { By("Getting existing DPUServiceConfiguration and updating it to use br-sfc network with IPAM requirement") dpuServiceConfiguration := generateServiceConfiguration(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) dpuServiceConfigurationOriginal := dpuServiceConfiguration.DeepCopy() dpuServiceConfiguration.Spec.Interfaces[0].Network = "mybrsfc" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) poolLabel := map[string]string{ "svc.dpu.nvidia.com/pool": "l3-static-shared-pool", } By("Getting node names from each DPU cluster") - nodeNames := make([]string, len(dpuClusterClient)) - for i, clusterClient := range dpuClusterClient { + nodeNames := make([]string, len(DPUClusterClient)) + for i, clusterClient := range DPUClusterClient { nodes := &corev1.NodeList{} Expect(clusterClient.List(ctx, nodes)).To(Succeed()) Expect(nodes.Items).To(HaveLen(1)) @@ -447,9 +447,9 @@ func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithStaticAllocations( } By("Creating a single DPUServiceIPAM with static allocations spanning all clusters") - dpuServiceIPAM := input.cidrDPUServiceIPAM.DeepCopy() + dpuServiceIPAM := input.CIDRDPUServiceIPAM.DeepCopy() dpuServiceIPAM.SetName("l3-ipam-static-shared") - dpuServiceIPAM.SetNamespace(dpfOperatorSystemNamespace) + dpuServiceIPAM.SetNamespace(DPFOperatorSystemNamespace) dpuServiceIPAM.Labels = CleanupScope.Suite dpuServiceIPAM.Spec.ObjectMeta.Labels = poolLabel dpuServiceIPAM.Spec.NodeSelector = nil @@ -468,49 +468,49 @@ func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithStaticAllocations( nodeNames[1]: "192.168.70.4/30", }, } - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Waiting for DPUServiceIPAM to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceIPAM, 5*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceIPAM, 5*time.Minute) By("Getting existing DPUDeployment and updating its ServiceChains to use DPUServiceIPAM") dpuDeployment := generateDPUDeployment(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) dpuDeploymentOriginal := dpuDeployment.DeepCopy() dpuDeployment.Spec.ServiceChains.Switches[0].Ports[0].Service.IPAM = &dpuservicev1.IPAM{MatchLabels: poolLabel} - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) By("Waiting for DPUDeployment to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuDeployment, 15*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuDeployment, 15*time.Minute) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Validating DPUService Pod in first cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[0], serviceIDForExample, "192.168.70.10", 1) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[0], serviceIDForExample, "192.168.70.10", 1) By("Validating DPUService Pod in second cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[1], serviceIDForExample, "192.168.70.6", 2) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[1], serviceIDForExample, "192.168.70.6", 2) } // ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClustersWithSingleIPPerNode validates a single DPUServiceIPAM in L2 mode // spanning all DPUClusters where each node receives exactly one IP (PerNodeIPCount: 1). -func ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClustersWithSingleIPPerNode(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClustersWithSingleIPPerNode(ctx context.Context, input *SystemTestInput) { By("Getting existing DPUServiceConfiguration and updating it to use br-sfc network with IPAM requirement") dpuServiceConfiguration := generateServiceConfiguration(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) dpuServiceConfigurationOriginal := dpuServiceConfiguration.DeepCopy() dpuServiceConfiguration.Spec.Interfaces[0].Network = "mybrsfc" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) poolLabel := map[string]string{ "svc.dpu.nvidia.com/pool": "l2-single-ip-pool", } By("Creating a single DPUServiceIPAM with one IP per node spanning all clusters") - dpuServiceIPAM := input.ipPoolDPUServiceIPAM.DeepCopy() + dpuServiceIPAM := input.IPPoolDPUServiceIPAM.DeepCopy() dpuServiceIPAM.SetName("l2-ipam-single-ip") - dpuServiceIPAM.SetNamespace(dpfOperatorSystemNamespace) + dpuServiceIPAM.SetNamespace(DPFOperatorSystemNamespace) dpuServiceIPAM.Labels = CleanupScope.Suite dpuServiceIPAM.Spec.ObjectMeta.Labels = poolLabel dpuServiceIPAM.Spec.NodeSelector = nil @@ -521,50 +521,50 @@ func ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClustersWithSingleIPPerNode(ct PerNodeIPCount: 1, BlocksPerDPUCluster: ptr.To[int32](2), } - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Waiting for DPUServiceIPAM to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceIPAM, 5*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceIPAM, 5*time.Minute) By("Getting existing DPUDeployment and updating its ServiceChains to use DPUServiceIPAM") dpuDeployment := generateDPUDeployment(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) dpuDeploymentOriginal := dpuDeployment.DeepCopy() dpuDeployment.Spec.ServiceChains.Switches[0].Ports[0].Service.IPAM = &dpuservicev1.IPAM{MatchLabels: poolLabel} - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) By("Waiting for DPUDeployment to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuDeployment, 15*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuDeployment, 15*time.Minute) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Validating DPUService Pod in first cluster has secondary IP from correct subnet") // .2 because we don't explicitly request the gateway - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[0], serviceIDForExample, "192.168.100.2", 1) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[0], serviceIDForExample, "192.168.100.2", 1) By("Validating DPUService Pod in second cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[1], serviceIDForExample, "192.168.100.3", 2) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[1], serviceIDForExample, "192.168.100.3", 2) } // ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithSingleIPPerNode validates a single DPUServiceIPAM in L3 mode // spanning all DPUClusters where each node receives a /32 prefix (one IP address). -func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithSingleIPPerNode(ctx context.Context, input *systemTestInput) { +func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithSingleIPPerNode(ctx context.Context, input *SystemTestInput) { By("Getting existing DPUServiceConfiguration and updating it to use br-sfc network with IPAM requirement") dpuServiceConfiguration := generateServiceConfiguration(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuServiceConfiguration), dpuServiceConfiguration)).To(Succeed()) dpuServiceConfigurationOriginal := dpuServiceConfiguration.DeepCopy() dpuServiceConfiguration.Spec.Interfaces[0].Network = "mybrsfc" - Expect(input.client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceConfiguration, client.MergeFrom(dpuServiceConfigurationOriginal))).To(Succeed()) poolLabel := map[string]string{ "svc.dpu.nvidia.com/pool": "l3-single-ip-pool", } By("Creating a single DPUServiceIPAM with /32 prefix per node spanning all clusters") - dpuServiceIPAM := input.cidrDPUServiceIPAM.DeepCopy() + dpuServiceIPAM := input.CIDRDPUServiceIPAM.DeepCopy() dpuServiceIPAM.SetName("l3-ipam-single-ip") - dpuServiceIPAM.SetNamespace(dpfOperatorSystemNamespace) + dpuServiceIPAM.SetNamespace(DPFOperatorSystemNamespace) dpuServiceIPAM.Labels = CleanupScope.Suite dpuServiceIPAM.Spec.ObjectMeta.Labels = poolLabel dpuServiceIPAM.Spec.NodeSelector = nil @@ -574,63 +574,63 @@ func ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithSingleIPPerNode(ct PrefixSize: 32, SubnetsPerDPUCluster: ptr.To[int32](2), } - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Waiting for DPUServiceIPAM to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuServiceIPAM, 5*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuServiceIPAM, 5*time.Minute) By("Getting existing DPUDeployment and updating its ServiceChains to use DPUServiceIPAM") dpuDeployment := generateDPUDeployment(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) dpuDeploymentOriginal := dpuDeployment.DeepCopy() dpuDeployment.Spec.ServiceChains.Switches[0].Ports[0].Service.IPAM = &dpuservicev1.IPAM{MatchLabels: poolLabel} - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) By("Waiting for DPUDeployment to become ready") - EventuallyCheckReadyStatusCondition(ctx, input.client, dpuDeployment, 15*time.Minute) + EventuallyCheckReadyStatusCondition(ctx, input.Client, dpuDeployment, 15*time.Minute) By("Getting the ServiceID for example service from the DPUService") - serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.client, dpuDeployment, "example") + serviceIDForExample := GetServiceIDForDPUDeploymentService(ctx, input.Client, dpuDeployment, "example") By("Validating DPUService Pod in first cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[0], serviceIDForExample, "192.168.110.0", 1) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[0], serviceIDForExample, "192.168.110.0", 1) By("Validating DPUService Pod in second cluster has secondary IP from correct subnet") - validateDPUServicePodIPInCluster(ctx, dpuClusterClient[1], serviceIDForExample, "192.168.110.2", 2) + validateDPUServicePodIPInCluster(ctx, DPUClusterClient[1], serviceIDForExample, "192.168.110.2", 2) } // ValidateDPUClusterDeletion validates the system when first DPUCluster is deleted. // It uses the existing DPUDeployment (with each DPU joining a different cluster) and verifies that after cluster 1 is // deleted the system remains healthy: DPFOperatorConfig, all DPUServices, DPUServiceChains, DPUServiceInterfaces, // DPUServiceIPAMs, and the DPUDeployment are ready. -func ValidateDPUClusterDeletion(ctx context.Context, input *systemTestInput) { - firstDPUCluster := input.dpuClusters[0] +func ValidateDPUClusterDeletion(ctx context.Context, input *SystemTestInput) { + firstDPUCluster := input.DPUClusters[0] By("Deleting first DPUCluster") - Expect(input.client.Delete(ctx, firstDPUCluster)).To(Succeed()) + Expect(input.Client.Delete(ctx, firstDPUCluster)).To(Succeed()) By("Patching DPUDeployment to remove the DPUSet referencing first DPUCluster") dpuDeployment := generateDPUDeployment(input, "") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), dpuDeployment)).To(Succeed()) dpuDeploymentOriginal := dpuDeployment.DeepCopy() dpuDeployment.Spec.DPUs.DPUSets = slices.DeleteFunc(dpuDeployment.Spec.DPUs.DPUSets, func(s dpuservicev1.DPUSet) bool { return s.DPUClusterSelector["svc.dpu.nvidia.com/cluster"] == firstDPUCluster.Name }) - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(dpuDeploymentOriginal))).To(Succeed()) By("Waiting for first DPUCluster to be completely deleted") Eventually(func(g Gomega) { - err := input.client.Get(ctx, client.ObjectKeyFromObject(firstDPUCluster), &provisioningv1.DPUCluster{}) + err := input.Client.Get(ctx, client.ObjectKeyFromObject(firstDPUCluster), &provisioningv1.DPUCluster{}) g.Expect(apierrors.IsNotFound(err)).To(BeTrue(), "DPUCluster %s should be completely deleted", client.ObjectKeyFromObject(firstDPUCluster)) }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) By("Verifying DPFOperatorConfig is ready") - VerifyDPFOperatorConfigReady(ctx, input.client, 10*time.Minute) + VerifyDPFOperatorConfigReady(ctx, input.Client, 10*time.Minute) By("Verifying all DPUServices are ready") Eventually(func(g Gomega) { dpuServiceList := &dpuservicev1.DPUServiceList{} - g.Expect(input.client.List(ctx, dpuServiceList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuServiceList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) for _, dpuService := range dpuServiceList.Items { g.Expect(conditions.IsTrue(&dpuService, conditions.TypeReady)).To(BeTrue(), fmt.Sprintf("DPUService %s should be ready", dpuService.Name)) @@ -640,7 +640,7 @@ func ValidateDPUClusterDeletion(ctx context.Context, input *systemTestInput) { By("Verifying all DPUServiceChains are ready") Eventually(func(g Gomega) { dpuServiceChainList := &dpuservicev1.DPUServiceChainList{} - g.Expect(input.client.List(ctx, dpuServiceChainList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuServiceChainList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) for _, dpuServiceChain := range dpuServiceChainList.Items { g.Expect(conditions.IsTrue(&dpuServiceChain, conditions.TypeReady)).To(BeTrue(), fmt.Sprintf("DPUServiceChain %s should be ready", dpuServiceChain.Name)) @@ -650,7 +650,7 @@ func ValidateDPUClusterDeletion(ctx context.Context, input *systemTestInput) { By("Verifying all DPUServiceInterfaces are ready") Eventually(func(g Gomega) { dpuServiceInterfaceList := &dpuservicev1.DPUServiceInterfaceList{} - g.Expect(input.client.List(ctx, dpuServiceInterfaceList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuServiceInterfaceList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) for _, dpuServiceInterface := range dpuServiceInterfaceList.Items { g.Expect(conditions.IsTrue(&dpuServiceInterface, conditions.TypeReady)).To(BeTrue(), fmt.Sprintf("DPUServiceInterface %s should be ready", dpuServiceInterface.Name)) @@ -660,7 +660,7 @@ func ValidateDPUClusterDeletion(ctx context.Context, input *systemTestInput) { By("Verifying all DPUServiceIPAMs are ready") Eventually(func(g Gomega) { dpuServiceIPAMList := &dpuservicev1.DPUServiceIPAMList{} - g.Expect(input.client.List(ctx, dpuServiceIPAMList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuServiceIPAMList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) for _, dpuServiceIPAM := range dpuServiceIPAMList.Items { g.Expect(conditions.IsTrue(&dpuServiceIPAM, conditions.TypeReady)).To(BeTrue(), fmt.Sprintf("DPUServiceIPAM %s should be ready", dpuServiceIPAM.Name)) @@ -670,7 +670,7 @@ func ValidateDPUClusterDeletion(ctx context.Context, input *systemTestInput) { By("Verifying DPUDeployment is ready") Eventually(func(g Gomega) { got := &dpuservicev1.DPUDeployment{} - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), got)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuDeployment), got)).To(Succeed()) g.Expect(conditions.IsTrue(got, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(10 * time.Minute).WithPolling(1 * time.Second).Should(Succeed()) } @@ -682,7 +682,7 @@ func validateDPUServicePodIPInCluster(ctx context.Context, clusterClient client. podList := &corev1.PodList{} g.Expect(clusterClient.List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceID}, - client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) g.Expect(podList.Items).ToNot(BeEmpty()) g.Expect(podList.Items).To(HaveLen(1)) diff --git a/test/e2e/multidpucluster_test.go b/test/e2e/multidpucluster_test.go index 6b15540e..63e4e268 100644 --- a/test/e2e/multidpucluster_test.go +++ b/test/e2e/multidpucluster_test.go @@ -25,7 +25,7 @@ import ( //nolint:dupl var _ = Describe("DPF System tests - Multi DPUCluster", Labels{Domain.MultiDPUCluster}, Ordered, func() { BeforeAll(func() { - if input.numberOfDPUNodes != 2 { + if input.NumberOfDPUNodes != 2 { Skip("Skip test as exactly 2 nodes are required for multi DPUCluster testing") } }) @@ -34,59 +34,59 @@ var _ = Describe("DPF System tests - Multi DPUCluster", Labels{Domain.MultiDPUCl SystemSetupBeforeSuite(false) }) It("create DPUClusters", func() { - ProvisionDPUClusters(ctx, getProvisionDPUClustersInput()) + ProvisionDPUClusters(Ctx, GetProvisionDPUClustersInput()) }) It("create BFB and DPUFlavor", func() { - ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(ctx, getProvisionDPUClustersInput()) + ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(Ctx, GetProvisionDPUClustersInput()) }) It("create a DPUDeployment with each of DPUs joining a different cluster", func() { - ProvisionDPUDeploymentWithEachDPUJoiningADifferentDPUCluster(ctx, input) + ProvisionDPUDeploymentWithEachDPUJoiningADifferentDPUCluster(Ctx, input) }) }) Context("Validate system behavior", Ordered, func() { BeforeAll(func() { By("Waiting for DPU cluster 0 pods to be ready") - VerifyClusterPods(ctx, dpuClusterClient[0], systemPodsToVerify) + VerifyClusterPods(Ctx, DPUClusterClient[0], systemPodsToVerify) By("Waiting for DPU cluster 1 pods to be ready") - VerifyClusterPods(ctx, dpuClusterClient[1], systemPodsToVerify) + VerifyClusterPods(Ctx, DPUClusterClient[1], systemPodsToVerify) By("Waiting for DPFOperatorConfig to be ready") - VerifyDPFOperatorConfigReady(ctx, input.client, 20*time.Minute) + VerifyDPFOperatorConfigReady(Ctx, input.Client, 20*time.Minute) }) It("create single DPUServiceIPAM in L2 mode spanning both DPUClusters and validate workload", func() { - ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClusters(ctx, input) + ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClusters(Ctx, input) }) It("create single DPUServiceIPAM in L3 mode spanning both DPUClusters and validate workload", func() { - ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClusters(ctx, input) + ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClusters(Ctx, input) }) It("create single DPUServiceIPAM in L3 mode spanning both DPUClusters with static allocations and validate workload", func() { - ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithStaticAllocations(ctx, input) + ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithStaticAllocations(Ctx, input) }) It("create single DPUServiceIPAM in L2 mode spanning both DPUClusters with single IP per node and validate workload", func() { - ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClustersWithSingleIPPerNode(ctx, input) + ValidateDPUServiceIPAMInL2ModeSharedAcrossDPUClustersWithSingleIPPerNode(Ctx, input) }) It("create single DPUServiceIPAM in L3 mode spanning both DPUClusters with single IP per node (/32) and validate workload", func() { - ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithSingleIPPerNode(ctx, input) + ValidateDPUServiceIPAMInL3ModeSharedAcrossDPUClustersWithSingleIPPerNode(Ctx, input) }) It("create per-DPUCluster DPUServiceIPAM in L2 mode and validate workload", func() { - ValidateDPUServiceIPAMInL2ModePerDPUCluster(ctx, input) + ValidateDPUServiceIPAMInL2ModePerDPUCluster(Ctx, input) }) It("create per-DPUCluster DPUServiceIPAM in L3 mode and validate workload", func() { - ValidateDPUServiceIPAMInL3ModePerDPUCluster(ctx, input) + ValidateDPUServiceIPAMInL3ModePerDPUCluster(Ctx, input) }) }) Context("Validate DPUCluster operations", Ordered, func() { BeforeAll(func() { By("Waiting for DPU cluster 0 pods to be ready") - VerifyClusterPods(ctx, dpuClusterClient[0], systemPodsToVerify) + VerifyClusterPods(Ctx, DPUClusterClient[0], systemPodsToVerify) By("Waiting for DPU cluster 1 pods to be ready") - VerifyClusterPods(ctx, dpuClusterClient[1], systemPodsToVerify) + VerifyClusterPods(Ctx, DPUClusterClient[1], systemPodsToVerify) By("Waiting for DPFOperatorConfig to be ready") - VerifyDPFOperatorConfigReady(ctx, input.client, 20*time.Minute) + VerifyDPFOperatorConfigReady(Ctx, input.Client, 20*time.Minute) }) It("Delete one of the DPUClusters and validate resource readiness", func() { - ValidateDPUClusterDeletion(ctx, input) + ValidateDPUClusterDeletion(Ctx, input) }) }) }) diff --git a/test/e2e/nodesriovdeviceplugin.go b/test/e2e/nodesriovdeviceplugin.go index e9ace531..cf08d986 100644 --- a/test/e2e/nodesriovdeviceplugin.go +++ b/test/e2e/nodesriovdeviceplugin.go @@ -43,13 +43,13 @@ import ( // verifies that the validating webhook rejects them. //nolint:dupl -func ValidateNodeSRIOVDevicePluginWebhookRejectsInvalid(ctx context.Context, input *systemTestInput) { +func ValidateNodeSRIOVDevicePluginWebhookRejectsInvalid(ctx context.Context, input *SystemTestInput) { By("Creating a NodeSRIOVDevicePluginConfig with overlapping VF ranges") Eventually(func(g Gomega) { invalidConfig := &noderesourcesv1.NodeSRIOVDevicePluginConfig{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "e2e-invalid-config-", - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: CleanupScope.It, }, Spec: noderesourcesv1.NodeSRIOVDevicePluginConfigSpec{ @@ -71,7 +71,7 @@ func ValidateNodeSRIOVDevicePluginWebhookRejectsInvalid(ctx context.Context, inp }, }, } - err := input.client.Create(ctx, invalidConfig) + err := input.Client.Create(ctx, invalidConfig) g.Expect(err).To(HaveOccurred(), "webhook should reject overlapping VF ranges") g.Expect(apierrors.IsForbidden(err) || apierrors.IsInvalid(err)).To( @@ -83,7 +83,7 @@ func ValidateNodeSRIOVDevicePluginWebhookRejectsInvalid(ctx context.Context, inp duplicateConfig := &noderesourcesv1.NodeSRIOVDevicePluginConfig{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "e2e-duplicate-config-", - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: CleanupScope.It, }, Spec: noderesourcesv1.NodeSRIOVDevicePluginConfigSpec{ @@ -105,7 +105,7 @@ func ValidateNodeSRIOVDevicePluginWebhookRejectsInvalid(ctx context.Context, inp }, }, } - err := input.client.Create(ctx, duplicateConfig) + err := input.Client.Create(ctx, duplicateConfig) g.Expect(err).To(HaveOccurred(), "webhook should reject duplicate resource names") g.Expect(apierrors.IsForbidden(err) || apierrors.IsInvalid(err)).To( @@ -115,13 +115,13 @@ func ValidateNodeSRIOVDevicePluginWebhookRejectsInvalid(ctx context.Context, inp // ValidateNodeSRIOVDevicePluginConfigValidCreate creates a valid // NodeSRIOVDevicePluginConfig and verifies it is accepted. -func ValidateNodeSRIOVDevicePluginConfigValidCreate(ctx context.Context, input *systemTestInput) { +func ValidateNodeSRIOVDevicePluginConfigValidCreate(ctx context.Context, input *SystemTestInput) { By("Creating a valid NodeSRIOVDevicePluginConfig") Eventually(func(g Gomega) { validConfig := &noderesourcesv1.NodeSRIOVDevicePluginConfig{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "e2e-valid-config-", - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: CleanupScope.It, }, Spec: noderesourcesv1.NodeSRIOVDevicePluginConfigSpec{ @@ -136,17 +136,17 @@ func ValidateNodeSRIOVDevicePluginConfigValidCreate(ctx context.Context, input * }, }, } - g.Expect(input.client.Create(ctx, validConfig)).To(Succeed()) + g.Expect(input.Client.Create(ctx, validConfig)).To(Succeed()) By("Verifying the NodeSRIOVDevicePluginConfig exists") got := &noderesourcesv1.NodeSRIOVDevicePluginConfig{} - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(validConfig), got)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(validConfig), got)).To(Succeed()) g.Expect(got.Spec.DevicePluginResources).To(HaveLen(1)) }).WithTimeout(time.Minute).Should(Succeed()) } -func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("No DPUs in test config, skipping managed pod test") } @@ -155,7 +155,7 @@ func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemT var dpuName, kubeNodeName, serialNumber string Eventually(func(g Gomega) { - dpuName, kubeNodeName, serialNumber = findTargetDPUAndNode(g, ctx, input.client) + dpuName, kubeNodeName, serialNumber = findTargetDPUAndNode(g, ctx, input.Client) g.Expect(dpuName).NotTo(BeEmpty(), "expected a DPU with HostNetworkReady=True") g.Expect(kubeNodeName).NotTo(BeEmpty()) g.Expect(serialNumber).NotTo(BeEmpty()) @@ -179,8 +179,8 @@ func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemT } By("Ensuring NodeSRIOVDevicePluginController is enabled with default settings") - removeDPUConfigAnnotation(ctx, input.client, dpuName) - patchDPFOperatorConfigAndWait(ctx, input.client, defaultControllerConfig) + removeDPUConfigAnnotation(ctx, input.Client, dpuName) + patchDPFOperatorConfigAndWait(ctx, input.Client, defaultControllerConfig) By("Creating config1 and config2") config1 := buildNodeSRIOVConfigWithResources(config1Name, []noderesourcesv1.DevicePluginResource{ @@ -200,7 +200,7 @@ func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemT }, }, }) - Expect(input.client.Create(ctx, config1)).To(Succeed()) + Expect(input.Client.Create(ctx, config1)).To(Succeed()) config2 := buildNodeSRIOVConfigWithResources(config2Name, []noderesourcesv1.DevicePluginResource{ { @@ -219,14 +219,14 @@ func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemT }, }, }) - Expect(input.client.Create(ctx, config2)).To(Succeed()) + Expect(input.Client.Create(ctx, config2)).To(Succeed()) By("Marking DPU with config1") - setDPUConfigAnnotation(ctx, input.client, dpuName, config1Name) + setDPUConfigAnnotation(ctx, input.Client, dpuName, config1Name) By("Waiting for managed pod to start and validating config + resources (config1)") Eventually(func(g Gomega) { - pod := getManagedPodForNode(ctx, g, input.client, kubeNodeName) + pod := getManagedPodForNode(ctx, g, input.Client, kubeNodeName) g.Expect(pod).NotTo(BeNil()) expectPodRunning(g, pod) raw := pod.Annotations[nodesriovctrl.PodInputAnnotationKey] @@ -234,13 +234,13 @@ func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemT g.Expect(raw).To(ContainSubstring(config1NoPrefixResource)) g.Expect(raw).To(ContainSubstring(config1ExplicitResource)) }).WithTimeout(240 * time.Second).Should(Succeed()) - waitForNodeResource(ctx, input.client, kubeNodeName, + waitForNodeResource(ctx, input.Client, kubeNodeName, fmt.Sprintf("%s/%s", nodesriovctrl.DefaultResourcePrefix, config1NoPrefixResource), 4) - waitForNodeResource(ctx, input.client, kubeNodeName, + waitForNodeResource(ctx, input.Client, kubeNodeName, fmt.Sprintf("%s/%s", explicitPrefix, config1ExplicitResource), 4) By("Updating to fake images and waiting for managed pod to update") - patchDPFOperatorConfigAndWait(ctx, input.client, &operatorv1.NodeSRIOVDevicePluginControllerConfiguration{ + patchDPFOperatorConfigAndWait(ctx, input.Client, &operatorv1.NodeSRIOVDevicePluginControllerConfiguration{ BaseComponentConfig: operatorv1.BaseComponentConfig{ Disable: ptr.To(false), }, @@ -250,22 +250,22 @@ func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemT }, }) Eventually(func(g Gomega) { - pod := getManagedPodForNode(ctx, g, input.client, kubeNodeName) + pod := getManagedPodForNode(ctx, g, input.Client, kubeNodeName) g.Expect(pod).NotTo(BeNil()) g.Expect(getContainerImageByName(pod.Spec.Containers, "sriov-device-plugin")).To(Equal(invalidDevicePluginImage)) g.Expect(getContainerImageByName(pod.Spec.InitContainers, "dpf-device-plugin-init")).To(Equal(invalidInitImage)) }).WithTimeout(240 * time.Second).Should(Succeed()) By("Reverting fake images and waiting for managed pod to recover") - patchDPFOperatorConfigAndWait(ctx, input.client, defaultControllerConfig) + patchDPFOperatorConfigAndWait(ctx, input.Client, defaultControllerConfig) Eventually(func(g Gomega) { - pod := getManagedPodForNode(ctx, g, input.client, kubeNodeName) + pod := getManagedPodForNode(ctx, g, input.Client, kubeNodeName) g.Expect(pod).NotTo(BeNil()) expectPodRunning(g, pod) }).WithTimeout(300 * time.Second).Should(Succeed()) By("Updating default resource prefix and verifying non-explicit resources are updated") - patchDPFOperatorConfigAndWait(ctx, input.client, &operatorv1.NodeSRIOVDevicePluginControllerConfiguration{ + patchDPFOperatorConfigAndWait(ctx, input.Client, &operatorv1.NodeSRIOVDevicePluginControllerConfiguration{ BaseComponentConfig: operatorv1.BaseComponentConfig{ Disable: ptr.To(false), }, @@ -275,7 +275,7 @@ func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemT }) Eventually(func(g Gomega) { node := &corev1.Node{} - g.Expect(input.client.Get(ctx, types.NamespacedName{Name: kubeNodeName}, node)).To(Succeed()) + g.Expect(input.Client.Get(ctx, types.NamespacedName{Name: kubeNodeName}, node)).To(Succeed()) newKey := fmt.Sprintf("%s/%s", alternateDefaultPrefix, config1NoPrefixResource) oldKey := fmt.Sprintf("%s/%s", nodesriovctrl.DefaultResourcePrefix, config1NoPrefixResource) @@ -293,12 +293,12 @@ func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemT }).WithTimeout(300 * time.Second).Should(Succeed()) By("Reverting default prefix and switching DPU to config2") - patchDPFOperatorConfigAndWait(ctx, input.client, defaultControllerConfig) - setDPUConfigAnnotation(ctx, input.client, dpuName, config2Name) + patchDPFOperatorConfigAndWait(ctx, input.Client, defaultControllerConfig) + setDPUConfigAnnotation(ctx, input.Client, dpuName, config2Name) By("Validating managed pod started and node exposes correct resources (config2)") Eventually(func(g Gomega) { - pod := getManagedPodForNode(ctx, g, input.client, kubeNodeName) + pod := getManagedPodForNode(ctx, g, input.Client, kubeNodeName) g.Expect(pod).NotTo(BeNil()) expectPodRunning(g, pod) raw := pod.Annotations[nodesriovctrl.PodInputAnnotationKey] @@ -306,15 +306,15 @@ func ValidateNodeSRIOVDevicePluginManagement(ctx context.Context, input *systemT g.Expect(raw).To(ContainSubstring(config2NoPrefixResource)) g.Expect(raw).To(ContainSubstring(config2ExplicitResource)) }).WithTimeout(300 * time.Second).Should(Succeed()) - waitForNodeResource(ctx, input.client, kubeNodeName, + waitForNodeResource(ctx, input.Client, kubeNodeName, fmt.Sprintf("%s/%s", nodesriovctrl.DefaultResourcePrefix, config2NoPrefixResource), 2) - waitForNodeResource(ctx, input.client, kubeNodeName, + waitForNodeResource(ctx, input.Client, kubeNodeName, fmt.Sprintf("%s/%s", explicitPrefix, config2ExplicitResource), 3) } func findTargetDPUAndNode(g Gomega, ctx context.Context, c client.Client) (dpuName, kubeNodeName, serialNumber string) { dpuList := &provisioningv1.DPUList{} - g.Expect(c.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(c.List(ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) for i := range dpuList.Items { dpu := &dpuList.Items[i] if dpu.Spec.SerialNumber == "" || !dpu.DeletionTimestamp.IsZero() { @@ -325,7 +325,7 @@ func findTargetDPUAndNode(g Gomega, ctx context.Context, c client.Client) (dpuNa } dpuNode := &provisioningv1.DPUNode{} if err := c.Get(ctx, types.NamespacedName{ - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Name: dpu.Spec.DPUNodeName, }, dpuNode); err != nil { continue @@ -353,7 +353,7 @@ func isDPUHostNetworkReady(dpu *provisioningv1.DPU) bool { func getManagedPodForNode(ctx context.Context, g Gomega, c client.Client, nodeName string) *corev1.Pod { podList := &corev1.PodList{} g.Expect(c.List(ctx, podList, - client.InNamespace(dpfOperatorSystemNamespace), + client.InNamespace(DPFOperatorSystemNamespace), client.MatchingLabels{nodesriovctrl.ManagedByLabelKey: nodesriovctrl.ManagedByLabelValue}, )).To(Succeed()) for i := range podList.Items { @@ -381,7 +381,7 @@ func setDPUConfigAnnotation(ctx context.Context, c client.Client, dpuName string Eventually(func(g Gomega) { dpu := &provisioningv1.DPU{} g.Expect(c.Get(ctx, types.NamespacedName{ - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Name: dpuName, }, dpu)).To(Succeed()) original := dpu.DeepCopy() @@ -422,8 +422,8 @@ func patchDPFOperatorConfigAndWait(ctx context.Context, c client.Client, config Eventually(func(g Gomega) { cfg := &operatorv1.DPFOperatorConfig{} g.Expect(c.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, - Name: configName, + Namespace: DPFOperatorSystemNamespace, + Name: ConfigName, }, cfg)).To(Succeed()) original := cfg.DeepCopy() cfg.Spec.NodeSRIOVDevicePluginController = config.DeepCopy() @@ -433,8 +433,8 @@ func patchDPFOperatorConfigAndWait(ctx context.Context, c client.Client, config Eventually(func(g Gomega) { cfg := &operatorv1.DPFOperatorConfig{} g.Expect(c.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, - Name: configName, + Namespace: DPFOperatorSystemNamespace, + Name: ConfigName, }, cfg)).To(Succeed()) g.Expect(cfg.Status.ObservedGeneration).To(Equal(cfg.GetGeneration())) g.Expect(conditions.IsTrue(cfg, conditions.TypeReady)).To(BeTrue()) @@ -443,7 +443,7 @@ func patchDPFOperatorConfigAndWait(ctx context.Context, c client.Client, config Eventually(func(g Gomega) { deployment := &appsv1.Deployment{} g.Expect(c.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Name: "dpf-nodesriovdeviceplugin-controller", }, deployment)).To(Succeed()) g.Expect(deployment.Spec.Replicas).NotTo(BeNil()) @@ -461,7 +461,7 @@ func buildNodeSRIOVConfigWithResources( return &noderesourcesv1.NodeSRIOVDevicePluginConfig{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: CleanupScope.It, }, Spec: noderesourcesv1.NodeSRIOVDevicePluginConfigSpec{ @@ -475,7 +475,7 @@ func removeDPUConfigAnnotation(ctx context.Context, c client.Client, dpuName str Eventually(func(g Gomega) { dpu := &provisioningv1.DPU{} g.Expect(c.Get(ctx, types.NamespacedName{ - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Name: dpuName, }, dpu)).To(Succeed()) original := dpu.DeepCopy() diff --git a/test/e2e/ovnk_test.go b/test/e2e/ovnk_test.go index af75afb0..0e2a1ddf 100644 --- a/test/e2e/ovnk_test.go +++ b/test/e2e/ovnk_test.go @@ -28,18 +28,18 @@ import ( var _ = Describe("DPF System tests - OVNK", Labels{Domain.OVNKPrimary, Domain.RequiresNodes}, func() { BeforeEach(func() { By("Wait for OVNK deployment to be ready") - dpuservice.WaitForDPUDeploymentReady(ctx, input.client, dpfOperatorSystemNamespace, []string{"ovn-kubernetes"}, 50*time.Minute) + dpuservice.WaitForDPUDeploymentReady(Ctx, input.Client, DPFOperatorSystemNamespace, []string{"ovn-kubernetes"}, 50*time.Minute) By("Waiting for multus pods to be ready") - VerifyClusterPods(ctx, input.client, []string{"kube-multus-ds"}) + VerifyClusterPods(Ctx, input.Client, []string{"kube-multus-ds"}) }) Context("OVN-Kubernetes", func() { It("verify performance of pod to pod same node", func() { - VerifyPerformancePodToPodSameNode(ctx, input, "ovnk") + VerifyPerformancePodToPodSameNode(Ctx, input, "ovnk") }) It("verify performance of pod to pod different nodes", func() { - VerifyPerformancePodToPodDifferentNode(ctx, input, "ovnk") + VerifyPerformancePodToPodDifferentNode(Ctx, input, "ovnk") }) }) }) diff --git a/test/e2e/ovnkhbn.go b/test/e2e/ovnkhbn.go index 7a5d07ee..fc879dbc 100644 --- a/test/e2e/ovnkhbn.go +++ b/test/e2e/ovnkhbn.go @@ -40,8 +40,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func WaitForOVNKHBNDeploymentReady(ctx context.Context, input *systemTestInput) { - dpuservice.WaitForDPUDeploymentReady(ctx, input.client, dpfOperatorSystemNamespace, []string{"ovn-hbn"}, 50*time.Minute) +func WaitForOVNKHBNDeploymentReady(ctx context.Context, input *SystemTestInput) { + dpuservice.WaitForDPUDeploymentReady(ctx, input.Client, DPFOperatorSystemNamespace, []string{"ovn-hbn"}, 50*time.Minute) } // DeployOVNKHBNScenario creates the application-layer objects required for the HBN-OVN scenario: @@ -49,69 +49,69 @@ func WaitForOVNKHBNDeploymentReady(ctx context.Context, input *systemTestInput) // DPUServiceConfiguration, OVN-K DPUServiceTemplate, DPUServiceConfiguration, // and the ovn-hbn DPUDeployment. // It must be called after applyConfig/applySDNConfig have run (to populate input fields). -func DeployOVNKHBNScenario(ctx context.Context, input *systemTestInput) { - for _, iface := range input.dpuServiceInterfacesHBN { +func DeployOVNKHBNScenario(ctx context.Context, input *SystemTestInput) { + for _, iface := range input.DPUServiceInterfacesHBN { By(fmt.Sprintf("Creating physical DPUServiceInterface %s for HBN uplink", iface.Name)) - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(iface.GetName(), input.namespace, iface.DeepCopy(), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(iface.GetName(), input.Namespace, iface.DeepCopy(), CleanupScope.Suite), ))).ToNot(HaveOccurred()) } - if input.dpuServiceInterfaceOVN != nil { + if input.DPUServiceInterfaceOVN != nil { By("Creating patch DPUServiceInterface for OVN-K connectivity") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(input.dpuServiceInterfaceOVN.GetName(), input.namespace, input.dpuServiceInterfaceOVN.DeepCopy(), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(input.DPUServiceInterfaceOVN.GetName(), input.Namespace, input.DPUServiceInterfaceOVN.DeepCopy(), CleanupScope.Suite), ))).ToNot(HaveOccurred()) } - if input.ovnCredentialRequest != nil { + if input.OVNCredentialRequest != nil { By("Creating DPUServiceCredentialRequest for OVN") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(input.ovnCredentialRequest.GetName(), input.namespace, input.ovnCredentialRequest.DeepCopy(), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(input.OVNCredentialRequest.GetName(), input.Namespace, input.OVNCredentialRequest.DeepCopy(), CleanupScope.Suite), ))).ToNot(HaveOccurred()) } By("Creating CIDR pool DPUServiceIPAM for HBN") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(input.cidrDPUServiceIPAM.GetName(), input.namespace, input.cidrDPUServiceIPAM.DeepCopy(), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(input.CIDRDPUServiceIPAM.GetName(), input.Namespace, input.CIDRDPUServiceIPAM.DeepCopy(), CleanupScope.Suite), ))).ToNot(HaveOccurred()) By("Creating subnet pool DPUServiceIPAM for HBN") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(input.ipPoolDPUServiceIPAM.GetName(), input.namespace, input.ipPoolDPUServiceIPAM.DeepCopy(), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(input.IPPoolDPUServiceIPAM.GetName(), input.Namespace, input.IPPoolDPUServiceIPAM.DeepCopy(), CleanupScope.Suite), ))).ToNot(HaveOccurred()) - if input.dpuServiceTemplateOVN != nil { + if input.DPUServiceTemplateOVN != nil { By("Creating DPUServiceTemplate for OVN-K") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(input.dpuServiceTemplateOVN.GetName(), input.namespace, input.dpuServiceTemplateOVN.DeepCopy(), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(input.DPUServiceTemplateOVN.GetName(), input.Namespace, input.DPUServiceTemplateOVN.DeepCopy(), CleanupScope.Suite), ))).ToNot(HaveOccurred()) } - if input.dpuServiceTemplateHBN != nil { + if input.DPUServiceTemplateHBN != nil { By("Creating DPUServiceTemplate for HBN") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(input.dpuServiceTemplateHBN.GetName(), input.namespace, input.dpuServiceTemplateHBN.DeepCopy(), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(input.DPUServiceTemplateHBN.GetName(), input.Namespace, input.DPUServiceTemplateHBN.DeepCopy(), CleanupScope.Suite), ))).ToNot(HaveOccurred()) } - if input.dpuServiceConfigurationHBN != nil { + if input.DPUServiceConfigurationHBN != nil { By("Creating DPUServiceConfiguration for HBN") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(input.dpuServiceConfigurationHBN.GetName(), input.namespace, input.dpuServiceConfigurationHBN.DeepCopy(), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(input.DPUServiceConfigurationHBN.GetName(), input.Namespace, input.DPUServiceConfigurationHBN.DeepCopy(), CleanupScope.Suite), ))).ToNot(HaveOccurred()) } - if input.dpuServiceConfigurationOVN != nil { + if input.DPUServiceConfigurationOVN != nil { By("Creating DPUServiceConfiguration for OVN-K") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(input.dpuServiceConfigurationOVN.GetName(), input.namespace, applyOVNClusterValues(ctx, input), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(input.DPUServiceConfigurationOVN.GetName(), input.Namespace, applyOVNClusterValues(ctx, input), CleanupScope.Suite), ))).ToNot(HaveOccurred()) } By("Creating ovn-hbn DPUDeployment") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, - utils.GenerateDPUObj(input.dpuDeployment.GetName(), input.namespace, input.dpuDeployment.DeepCopy(), CleanupScope.Suite), + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, + utils.GenerateDPUObj(input.DPUDeployment.GetName(), input.Namespace, input.DPUDeployment.DeepCopy(), CleanupScope.Suite), ))).ToNot(HaveOccurred()) } @@ -119,8 +119,8 @@ func DeployOVNKHBNScenario(ctx context.Context, input *systemTestInput) { // extracted from the live cluster: k8sAPIServer from restConfig, pod/service CIDRs from // kube-controller-manager, vtepCIDR and ipamPool from the cidr DPUServiceIPAM, and // hostCIDR derived from the control plane node IP. -func applyOVNClusterValues(ctx context.Context, input *systemTestInput) *dpuservicev1.DPUServiceConfiguration { - result := input.dpuServiceConfigurationOVN.DeepCopy() +func applyOVNClusterValues(ctx context.Context, input *SystemTestInput) *dpuservicev1.DPUServiceConfiguration { + result := input.DPUServiceConfigurationOVN.DeepCopy() if result.Spec.ServiceConfiguration.HelmChart.Values == nil { result.Spec.ServiceConfiguration.HelmChart.Values = &machineryruntime.RawExtension{} @@ -149,23 +149,23 @@ func applyOVNClusterValues(ctx context.Context, input *systemTestInput) *dpuserv } // k8sAPIServer from the kubeconfig used by the test - setVal(input.restConfig.Host, "k8sAPIServer") + setVal(input.RestConfig.Host, "k8sAPIServer") // pod and service CIDRs from kube-controller-manager - podCIDR, serviceCIDR := extractClusterCIDRs(ctx, input.client) + podCIDR, serviceCIDR := extractClusterCIDRs(ctx, input.Client) setVal(podCIDR, "podNetwork") setVal(serviceCIDR, "serviceNetwork") // vtepCIDR and ipamPool from the cidr DPUServiceIPAM already loaded - if input.cidrDPUServiceIPAM != nil { - setVal(input.cidrDPUServiceIPAM.Name, "dpuManifests", "ipamPool") - if input.cidrDPUServiceIPAM.Spec.IPV4Network != nil { - setVal(input.cidrDPUServiceIPAM.Spec.IPV4Network.Network, "dpuManifests", "vtepCIDR") + if input.CIDRDPUServiceIPAM != nil { + setVal(input.CIDRDPUServiceIPAM.Name, "dpuManifests", "ipamPool") + if input.CIDRDPUServiceIPAM.Spec.IPV4Network != nil { + setVal(input.CIDRDPUServiceIPAM.Spec.IPV4Network.Network, "dpuManifests", "vtepCIDR") } } // hostCIDR derived from the control plane node IP - controlPlaneIP := getClusterControlPlaneIP(ctx, input.client) + controlPlaneIP := getClusterControlPlaneIP(ctx, input.Client) setVal(deriveHostCIDR(controlPlaneIP), "dpuManifests", "hostCIDR") raw, err := json.Marshal(values) @@ -257,7 +257,7 @@ func InstallOVNKResourceInjector(ctx context.Context, c client.Client) { // parallelism to ~60% of cluster nodes. Returns a restore function that reverts to the original value. func SetMaintenanceOperatorMaxParallelOperations(ctx context.Context, c client.Client, value int32) func() { cfg := &maintenancev1alpha1.MaintenanceOperatorConfig{} - Expect(c.Get(ctx, client.ObjectKey{Name: "default", Namespace: dpfOperatorSystemNamespace}, cfg)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKey{Name: "default", Namespace: DPFOperatorSystemNamespace}, cfg)).To(Succeed()) original := cfg.Spec.MaxParallelOperations @@ -268,7 +268,7 @@ func SetMaintenanceOperatorMaxParallelOperations(ctx context.Context, c client.C return func() { cfg2 := &maintenancev1alpha1.MaintenanceOperatorConfig{} - Expect(c.Get(ctx, client.ObjectKey{Name: "default", Namespace: dpfOperatorSystemNamespace}, cfg2)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKey{Name: "default", Namespace: DPFOperatorSystemNamespace}, cfg2)).To(Succeed()) patch2 := client.MergeFrom(cfg2.DeepCopy()) cfg2.Spec.MaxParallelOperations = original Expect(c.Patch(ctx, cfg2, patch2)).To(Succeed()) diff --git a/test/e2e/ovnkhbn_test.go b/test/e2e/ovnkhbn_test.go index 465ae0bc..01a27a42 100644 --- a/test/e2e/ovnkhbn_test.go +++ b/test/e2e/ovnkhbn_test.go @@ -24,18 +24,18 @@ import ( var _ = Describe("DPF System tests - OVNK HBN", Labels{Domain.OVNKHBN, Domain.RequiresNodes}, func() { BeforeEach(func() { By("Wait for OVNK HBN deployment to be ready") - WaitForOVNKHBNDeploymentReady(ctx, input) + WaitForOVNKHBNDeploymentReady(Ctx, input) By("Waiting for multus pods to be ready") - VerifyClusterPods(ctx, input.client, []string{"kube-multus-ds"}) + VerifyClusterPods(Ctx, input.Client, []string{"kube-multus-ds"}) }) Context("OVNK HBN", func() { It("verify performance of pod to pod same node", func() { - VerifyPerformancePodToPodSameNode(ctx, input, "ovnkhbn") + VerifyPerformancePodToPodSameNode(Ctx, input, "ovnkhbn") }) It("verify performance of pod to pod different nodes", func() { - VerifyPerformancePodToPodDifferentNode(ctx, input, "ovnkhbn") + VerifyPerformancePodToPodDifferentNode(Ctx, input, "ovnkhbn") }) }) }) diff --git a/test/e2e/provisioning.go b/test/e2e/provisioning.go index a611af7c..c02a1e32 100644 --- a/test/e2e/provisioning.go +++ b/test/e2e/provisioning.go @@ -69,15 +69,15 @@ type ProvisioningExpected struct { var provisioningExpected ProvisioningExpected // initProvisioningExpected initializes the expected counts from input -func initProvisioningExpected(input *systemTestInput) { +func initProvisioningExpected(input *SystemTestInput) { provisioningExpected = ProvisioningExpected{ - DPUNodes: input.numberOfDPUNodes, - DPUsPerNode: input.numberOfDPUsPerNode, - TotalDPUs: input.totalDPUs(), + DPUNodes: input.NumberOfDPUNodes, + DPUsPerNode: input.NumberOfDPUsPerNode, + TotalDPUs: input.TotalDPUs(), DPUClusters: 1, // Provisioning tests create one DPUCluster DPUSets: 1, // Provisioning tests create one DPUSet BFBs: 1, // Provisioning tests create one BFB - Prerequisites: len(input.dpuClusterPrerequisites), + Prerequisites: len(input.DPUClusterPrerequisites), DPUServices: 6, // Multus, Flannel, SRIOV, NVIPAM, CNI installer, SFC-Controller } @@ -85,7 +85,7 @@ func initProvisioningExpected(input *systemTestInput) { } // printProvisioningConfiguration prints the expected test configuration -func printProvisioningConfiguration(input *systemTestInput) { +func printProvisioningConfiguration(input *SystemTestInput) { By("========== PROVISIONING TEST CONFIGURATION ==========") By(fmt.Sprintf(" DPU Nodes: %d", provisioningExpected.DPUNodes)) By(fmt.Sprintf(" DPUs per Node: %d", provisioningExpected.DPUsPerNode)) @@ -96,7 +96,7 @@ func printProvisioningConfiguration(input *systemTestInput) { By(fmt.Sprintf(" DPU Flavors: %d", provisioningExpected.DPUFlavors)) By(fmt.Sprintf(" Prerequisites: %d", provisioningExpected.Prerequisites)) By(fmt.Sprintf(" DPU Services: %d", provisioningExpected.DPUServices)) - By(fmt.Sprintf(" DPU Flavor Name: %s", input.dpuFlavor.Name)) + By(fmt.Sprintf(" DPU Flavor Name: %s", input.DPUFlavor.Name)) By("=====================================================") } @@ -111,9 +111,9 @@ func VerifyDPUServicesDeployed(ctx context.Context, clusterClient client.Client, g.Expect(clusterClient.List(ctx, deployments)).To(Succeed()) found := map[string]bool{} for i := range deployments.Items { - if _, hasAnnotation := deployments.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]; hasAnnotation { - g.Expect(deployments.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]).NotTo(Equal("")) - found[deployments.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]] = true + if _, hasAnnotation := deployments.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]; hasAnnotation { + g.Expect(deployments.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]).NotTo(Equal("")) + found[deployments.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]] = true } } @@ -121,9 +121,9 @@ func VerifyDPUServicesDeployed(ctx context.Context, clusterClient client.Client, daemonsets := appsv1.DaemonSetList{} g.Expect(clusterClient.List(ctx, &daemonsets, client.InNamespace(namespace))).To(Succeed()) for i := range daemonsets.Items { - if _, hasAnnotation := daemonsets.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]; hasAnnotation { - g.Expect(daemonsets.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]).NotTo(Equal("")) - found[daemonsets.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]] = true + if _, hasAnnotation := daemonsets.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]; hasAnnotation { + g.Expect(daemonsets.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]).NotTo(Equal("")) + found[daemonsets.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]] = true } } @@ -158,14 +158,14 @@ func VerifyDPUServicesDeployed(ctx context.Context, clusterClient client.Client, }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) } -func BeforeProvisioning(ctx context.Context, input *systemTestInput) { +func BeforeProvisioning(ctx context.Context, input *SystemTestInput) { // Initialize expected counts from input initProvisioningExpected(input) // Print test configuration at the start printProvisioningConfiguration(input) By("Verifying DPU nodes are available for provisioning tests") - Expect(input.hasDpuNodes()).To(BeTrue(), + Expect(input.HasDpuNodes()).To(BeTrue(), "SETUP ERROR: No DPU nodes found in cluster. "+ "Provisioning tests require DPU nodes to be configured. "+ "Please ensure DPU hardware is available and properly configured before running these tests.") @@ -180,7 +180,7 @@ func BeforeProvisioning(ctx context.Context, input *systemTestInput) { var dirty []string for name, list := range provisioningResources { - if err := input.client.List(ctx, list); err != nil { + if err := input.Client.List(ctx, list); err != nil { continue } items, err := meta.ExtractList(list) @@ -200,15 +200,15 @@ func BeforeProvisioning(ctx context.Context, input *systemTestInput) { } } -func CreateProvisioningDPUCluster(ctx context.Context, input *systemTestInput) { +func CreateProvisioningDPUCluster(ctx context.Context, input *SystemTestInput) { // Create prerequisite objects - for i, obj := range input.dpuClusterPrerequisites { + for i, obj := range input.DPUClusterPrerequisites { // Deep copy to avoid mutating the shared original object objCopy := obj.DeepCopyObject().(client.Object) objCopy.SetLabels(CleanupScope.Suite) existing := objCopy.DeepCopyObject().(client.Object) - err := input.client.Get(ctx, types.NamespacedName{ + err := input.Client.Get(ctx, types.NamespacedName{ Namespace: objCopy.GetNamespace(), Name: objCopy.GetName(), }, existing) @@ -218,7 +218,7 @@ func CreateProvisioningDPUCluster(ctx context.Context, input *systemTestInput) { i+1, provisioningExpected.Prerequisites, objCopy.GetNamespace(), objCopy.GetName())) - Expect(input.client.Create(ctx, objCopy)).To(Succeed()) + Expect(input.Client.Create(ctx, objCopy)).To(Succeed()) } else { By(fmt.Sprintf("Prerequisite [%d/%d] %s/%s already exists", i+1, provisioningExpected.Prerequisites, @@ -229,18 +229,18 @@ func CreateProvisioningDPUCluster(ctx context.Context, input *systemTestInput) { } // Deep copy to avoid mutating the shared original object - dpuCluster := input.dpuClusters[0].DeepCopy() + dpuCluster := input.DPUClusters[0].DeepCopy() dpuCluster.SetLabels(CleanupScope.Suite) By(fmt.Sprintf("Creating DPUCluster %s/%s", dpuCluster.GetNamespace(), dpuCluster.GetName())) - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuCluster))).To(Succeed()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuCluster))).To(Succeed()) By("Verifying DPUCluster exists") Eventually(func(g Gomega) { clusters := &provisioningv1.DPUClusterList{} - g.Expect(input.client.List(ctx, clusters)).To(Succeed()) + g.Expect(input.Client.List(ctx, clusters)).To(Succeed()) g.Expect(clusters.Items).To(HaveLen(provisioningExpected.DPUClusters), fmt.Sprintf("Expected %d DPU cluster(s)", provisioningExpected.DPUClusters)) }).WithTimeout(1 * time.Minute).Should(Succeed()) @@ -249,7 +249,7 @@ func CreateProvisioningDPUCluster(ctx context.Context, input *systemTestInput) { clusterTracker := NewByTracker() Eventually(func(g Gomega) { clusters := &provisioningv1.DPUClusterList{} - g.Expect(input.client.List(ctx, clusters)).To(Succeed()) + g.Expect(input.Client.List(ctx, clusters)).To(Succeed()) g.Expect(clusters.Items).To(HaveLen(provisioningExpected.DPUClusters)) cluster := clusters.Items[0] @@ -261,31 +261,31 @@ func CreateProvisioningDPUCluster(ctx context.Context, input *systemTestInput) { By("Creating DPU cluster client connection") // getDPUClusterClients requires ProvisionDPUClustersInput (defined in system_setup.go) - getDPUClusterClients(ctx, ProvisionDPUClustersInput{ - dpuClusters: input.dpuClusters, - client: input.client, - restConfig: input.restConfig, + GetDPUClusterClients(ctx, ProvisionDPUClustersInput{ + DPUClusters: input.DPUClusters, + Client: input.Client, + RestConfig: input.RestConfig, }) - bfb := input.bfb.DeepCopy() + bfb := input.BFB.DeepCopy() bfb.SetLabels(CleanupScope.Suite) // Override BFB URL if environment variable is set (on the copy, not the original) - if input.bfbImageURL != "" { - By(fmt.Sprintf("Overriding BFB URL with: %s", input.bfbImageURL)) - bfb.Spec.URL = input.bfbImageURL + if input.BFBImageURL != "" { + By(fmt.Sprintf("Overriding BFB URL with: %s", input.BFBImageURL)) + bfb.Spec.URL = input.BFBImageURL } By(fmt.Sprintf("Creating BFB %s/%s", bfb.GetNamespace(), bfb.GetName())) Eventually(func(g Gomega) { - g.Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, bfb))).To(Succeed()) + g.Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, bfb))).To(Succeed()) }).WithTimeout(10 * time.Second).Should(Succeed()) By("Verifying BFB object exists") Eventually(func(g Gomega) { bfb := &provisioningv1.BFB{} - g.Expect(input.client.Get(ctx, types.NamespacedName{ - Name: input.bfb.Name, - Namespace: input.bfb.Namespace, + g.Expect(input.Client.Get(ctx, types.NamespacedName{ + Name: input.BFB.Name, + Namespace: input.BFB.Namespace, }, bfb)).To(Succeed(), "BFB should be created") }).WithTimeout(1 * time.Minute).Should(Succeed()) @@ -293,9 +293,9 @@ func CreateProvisioningDPUCluster(ctx context.Context, input *systemTestInput) { bfbTracker := NewByTracker() Eventually(func(g Gomega) { bfb := &provisioningv1.BFB{} - g.Expect(input.client.Get(ctx, types.NamespacedName{ - Name: input.bfb.Name, - Namespace: input.bfb.Namespace, + g.Expect(input.Client.Get(ctx, types.NamespacedName{ + Name: input.BFB.Name, + Namespace: input.BFB.Namespace, }, bfb)).To(Succeed()) bfbTracker.By(bfb.Name+string(bfb.Status.Phase), "BFB %s Phase: %s", bfb.Name, bfb.Status.Phase) @@ -304,37 +304,37 @@ func CreateProvisioningDPUCluster(ctx context.Context, input *systemTestInput) { }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) } -func CreateProvisioningDPUSet(ctx context.Context, input *systemTestInput) { +func CreateProvisioningDPUSet(ctx context.Context, input *SystemTestInput) { // DPUFlavor is required for provisioning - fail fast if missing - Expect(input.dpuFlavor).NotTo(BeNil(), "dpuFlavor is required - check test configuration") + Expect(input.DPUFlavor).NotTo(BeNil(), "dpuFlavor is required - check test configuration") - dpuFlavor := input.dpuFlavor.DeepCopy() + dpuFlavor := input.DPUFlavor.DeepCopy() dpuFlavor.SetLabels(CleanupScope.Suite) By(fmt.Sprintf("Creating DPUFlavor %s/%s", dpuFlavor.GetNamespace(), dpuFlavor.GetName())) Eventually(func(g Gomega) { - g.Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuFlavor))).To(Succeed()) + g.Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuFlavor))).To(Succeed()) }).WithTimeout(60 * time.Second).Should(Succeed()) By("Verifying DPUFlavor exists") Eventually(func(g Gomega) { dpuFlavor := &provisioningv1.DPUFlavor{} - g.Expect(input.client.Get(ctx, types.NamespacedName{ - Name: input.dpuFlavor.Name, - Namespace: input.dpuFlavor.Namespace, + g.Expect(input.Client.Get(ctx, types.NamespacedName{ + Name: input.DPUFlavor.Name, + Namespace: input.DPUFlavor.Namespace, }, dpuFlavor)).To(Succeed(), "DPUFlavor should be created") }).WithTimeout(1 * time.Minute).Should(Succeed()) - dpuset := input.dpuSet.DeepCopy() + dpuset := input.DPUSet.DeepCopy() dpuset.SetLabels(CleanupScope.Suite) By(fmt.Sprintf("Creating DPUSet %s/%s", dpuset.GetNamespace(), dpuset.GetName())) Eventually(func(g Gomega) { - g.Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuset))).To(Succeed()) + g.Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuset))).To(Succeed()) }).WithTimeout(60 * time.Second).Should(Succeed()) By("Verifying DPUSet exists") Eventually(func(g Gomega) { dpusets := &provisioningv1.DPUSetList{} - g.Expect(input.client.List(ctx, dpusets)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpusets)).To(Succeed()) g.Expect(dpusets.Items).To(HaveLen(provisioningExpected.DPUSets), fmt.Sprintf("Expected %d DPUSet(s)", provisioningExpected.DPUSets)) }).WithTimeout(2 * time.Minute).Should(Succeed()) @@ -342,7 +342,7 @@ func CreateProvisioningDPUSet(ctx context.Context, input *systemTestInput) { By("Waiting for DPUSet controller to create DPU objects") Eventually(func(g Gomega) { dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus)).To(Succeed()) g.Expect(dpus.Items).To(HaveLen(provisioningExpected.TotalDPUs), fmt.Sprintf("Expected %d DPU objects, found %d", provisioningExpected.TotalDPUs, len(dpus.Items))) @@ -362,25 +362,25 @@ func CreateProvisioningDPUSet(ctx context.Context, input *systemTestInput) { Eventually(func(g Gomega) { // Track DPU phases during node joining dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus)).To(Succeed()) for _, dpu := range dpus.Items { dpuPhaseTracker.By(dpu.Name+string(dpu.Status.Phase), "DPU %s: %s", dpu.Name, dpu.Status.Phase) } nodes := &corev1.NodeList{} - g.Expect(dpuClusterClient[0].List(ctx, nodes)).To(Succeed(), "Should be able to list nodes in DPU cluster") + g.Expect(DPUClusterClient[0].List(ctx, nodes)).To(Succeed(), "Should be able to list nodes in DPU cluster") nodeKey := fmt.Sprintf("%d/%d", len(nodes.Items), provisioningExpected.TotalDPUs) nodesTracker.By(nodeKey, "K8s nodes in DPU cluster [%d/%d]", len(nodes.Items), provisioningExpected.TotalDPUs) g.Expect(nodes.Items).To(HaveLen(provisioningExpected.TotalDPUs), fmt.Sprintf("DPU cluster should have %d K8s nodes, found %d", provisioningExpected.TotalDPUs, len(nodes.Items))) - }).WithTimeout(provisioningTimeout).WithPolling(10 * time.Second).Should(Succeed()) + }).WithTimeout(ProvisioningTimeout).WithPolling(10 * time.Second).Should(Succeed()) By("Waiting for all DPU objects to reach Ready phase") dpuTracker := NewByTracker() Eventually(func(g Gomega) { dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus)).To(Succeed()) g.Expect(dpus.Items).To(HaveLen(provisioningExpected.TotalDPUs)) readyCount := 0 @@ -401,14 +401,14 @@ func CreateProvisioningDPUSet(ctx context.Context, input *systemTestInput) { }).WithTimeout(30 * time.Minute).WithPolling(30 * time.Second).Should(Succeed()) } -func VerifyProvisioning(ctx context.Context, input *systemTestInput) { - deploymentName := fmt.Sprintf("in-cluster-%s", getPerClusterDPUServiceName(operatorv1.ServiceSetControllerName, input.dpuClusters[0].Name, input.dpuClusters[0].Namespace)) +func VerifyProvisioning(ctx context.Context, input *SystemTestInput) { + deploymentName := fmt.Sprintf("in-cluster-%s", getPerClusterDPUServiceName(operatorv1.ServiceSetControllerName, input.DPUClusters[0].Name, input.DPUClusters[0].Namespace)) deploymentTracker := NewByTracker() - By(fmt.Sprintf("Verifying Deployment %s/%s", dpfOperatorSystemNamespace, deploymentName)) + By(fmt.Sprintf("Verifying Deployment %s/%s", DPFOperatorSystemNamespace, deploymentName)) Eventually(func(g Gomega) { serviceSetDeployment := &appsv1.Deployment{} - g.Expect(input.client.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, + g.Expect(input.Client.Get(ctx, client.ObjectKey{ + Namespace: DPFOperatorSystemNamespace, Name: deploymentName, }, serviceSetDeployment)).To(Succeed()) @@ -421,13 +421,13 @@ func VerifyProvisioning(ctx context.Context, input *systemTestInput) { }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) // Use shared function to verify DPUServices are deployed - VerifyDPUServicesDeployed(ctx, dpuClusterClient[0], input.dpuClusters[0].GetNamespace()) + VerifyDPUServicesDeployed(ctx, DPUClusterClient[0], input.DPUClusters[0].GetNamespace()) By("Verifying DPUSet statistics") dpuSetTracker := NewByTracker() Eventually(func(g Gomega) { dpusets := &provisioningv1.DPUSetList{} - g.Expect(input.client.List(ctx, dpusets)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpusets)).To(Succeed()) g.Expect(dpusets.Items).To(HaveLen(provisioningExpected.DPUSets), fmt.Sprintf("Expected %d DPUSet(s)", provisioningExpected.DPUSets)) @@ -453,35 +453,35 @@ func VerifyProvisioning(ctx context.Context, input *systemTestInput) { }).WithTimeout(2 * time.Minute).Should(Succeed()) By("Waiting for all system pods to be ready in DPU cluster") - VerifyClusterPods(ctx, dpuClusterClient[0], systemPodsToVerify) + VerifyClusterPods(ctx, DPUClusterClient[0], systemPodsToVerify) } -func DeleteProvisioning(ctx context.Context, input *systemTestInput) { +func DeleteProvisioning(ctx context.Context, input *SystemTestInput) { By("========== DEPROVISIONING ==========") By(fmt.Sprintf(" DPUs to remove: %d", provisioningExpected.TotalDPUs)) By(fmt.Sprintf(" Prerequisites: %d", provisioningExpected.Prerequisites)) By("=====================================") - By(fmt.Sprintf("Deleting DPUSet %s/%s", input.dpuSet.Namespace, input.dpuSet.Name)) + By(fmt.Sprintf("Deleting DPUSet %s/%s", input.DPUSet.Namespace, input.DPUSet.Name)) Eventually(func(g Gomega) { dpuset := &provisioningv1.DPUSet{} - err := input.client.Get(ctx, types.NamespacedName{ - Name: input.dpuSet.Name, - Namespace: input.dpuSet.Namespace, + err := input.Client.Get(ctx, types.NamespacedName{ + Name: input.DPUSet.Name, + Namespace: input.DPUSet.Namespace, }, dpuset) if apierrors.IsNotFound(err) { return } g.Expect(err).To(Succeed()) - g.Expect(input.client.Delete(ctx, dpuset)).To(Succeed()) + g.Expect(input.Client.Delete(ctx, dpuset)).To(Succeed()) }).WithTimeout(1 * time.Minute).Should(Succeed()) By("Waiting for DPUSet to be deleted") dpuSetDeleteTracker := NewByTracker() Eventually(func(g Gomega) { dpusets := &provisioningv1.DPUSetList{} - g.Expect(input.client.List(ctx, dpusets)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpusets)).To(Succeed()) dpuSetDeleteTracker.By(fmt.Sprintf("%d", len(dpusets.Items)), "DPUSets remaining [%d]", len(dpusets.Items)) g.Expect(dpusets.Items).To(BeEmpty(), "DPUSet should be deleted") @@ -491,7 +491,7 @@ func DeleteProvisioning(ctx context.Context, input *systemTestInput) { dpuDeleteTracker := NewByTracker() Eventually(func(g Gomega) { dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus)).To(Succeed()) dpuDeleteTracker.By(fmt.Sprintf("%d", len(dpus.Items)), "DPUs remaining [%d]", len(dpus.Items)) g.Expect(dpus.Items).To(BeEmpty(), @@ -502,84 +502,84 @@ func DeleteProvisioning(ctx context.Context, input *systemTestInput) { nodesDeleteTracker := NewByTracker() Eventually(func(g Gomega) { nodes := &corev1.NodeList{} - g.Expect(dpuClusterClient[0].List(ctx, nodes)).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, nodes)).To(Succeed()) nodesDeleteTracker.By(fmt.Sprintf("%d", len(nodes.Items)), "K8s nodes remaining [%d]", len(nodes.Items)) g.Expect(nodes.Items).To(BeEmpty(), "DPU cluster should have no nodes after deprovisioning") }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) - By(fmt.Sprintf("Deleting DPUFlavor %s/%s", input.dpuFlavor.Namespace, input.dpuFlavor.Name)) + By(fmt.Sprintf("Deleting DPUFlavor %s/%s", input.DPUFlavor.Namespace, input.DPUFlavor.Name)) Eventually(func(g Gomega) { dpuFlavor := &provisioningv1.DPUFlavor{} - err := input.client.Get(ctx, types.NamespacedName{ - Name: input.dpuFlavor.Name, - Namespace: input.dpuFlavor.Namespace, + err := input.Client.Get(ctx, types.NamespacedName{ + Name: input.DPUFlavor.Name, + Namespace: input.DPUFlavor.Namespace, }, dpuFlavor) if apierrors.IsNotFound(err) { return } g.Expect(err).To(Succeed()) - g.Expect(input.client.Delete(ctx, dpuFlavor)).To(Succeed()) + g.Expect(input.Client.Delete(ctx, dpuFlavor)).To(Succeed()) }).WithTimeout(1 * time.Minute).Should(Succeed()) By("Waiting for DPUFlavor to be deleted") Eventually(func(g Gomega) { dpuFlavor := &provisioningv1.DPUFlavor{} - err := input.client.Get(ctx, types.NamespacedName{ - Name: input.dpuFlavor.Name, - Namespace: input.dpuFlavor.Namespace, + err := input.Client.Get(ctx, types.NamespacedName{ + Name: input.DPUFlavor.Name, + Namespace: input.DPUFlavor.Namespace, }, dpuFlavor) g.Expect(apierrors.IsNotFound(err)).To(BeTrue(), "DPUFlavor should be deleted") }).WithTimeout(2 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) - By(fmt.Sprintf("Deleting BFB %s/%s", input.bfb.Namespace, input.bfb.Name)) + By(fmt.Sprintf("Deleting BFB %s/%s", input.BFB.Namespace, input.BFB.Name)) Eventually(func(g Gomega) { bfb := &provisioningv1.BFB{} - err := input.client.Get(ctx, types.NamespacedName{ - Name: input.bfb.Name, - Namespace: input.bfb.Namespace, + err := input.Client.Get(ctx, types.NamespacedName{ + Name: input.BFB.Name, + Namespace: input.BFB.Namespace, }, bfb) if apierrors.IsNotFound(err) { return } g.Expect(err).To(Succeed()) - g.Expect(input.client.Delete(ctx, bfb)).To(Succeed()) + g.Expect(input.Client.Delete(ctx, bfb)).To(Succeed()) }).WithTimeout(1 * time.Minute).Should(Succeed()) By("Waiting for BFB to be deleted") Eventually(func(g Gomega) { bfb := &provisioningv1.BFB{} - err := input.client.Get(ctx, types.NamespacedName{ - Name: input.bfb.Name, - Namespace: input.bfb.Namespace, + err := input.Client.Get(ctx, types.NamespacedName{ + Name: input.BFB.Name, + Namespace: input.BFB.Namespace, }, bfb) g.Expect(apierrors.IsNotFound(err)).To(BeTrue(), "BFB should be deleted") }).WithTimeout(5 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) if !skipDPUClusterDeletionInProvisioningTest { - By(fmt.Sprintf("Deleting DPUCluster %s/%s", input.dpuClusters[0].Namespace, input.dpuClusters[0].Name)) + By(fmt.Sprintf("Deleting DPUCluster %s/%s", input.DPUClusters[0].Namespace, input.DPUClusters[0].Name)) Eventually(func(g Gomega) { cluster := &provisioningv1.DPUCluster{} - err := input.client.Get(ctx, types.NamespacedName{ - Name: input.dpuClusters[0].Name, - Namespace: input.dpuClusters[0].Namespace, + err := input.Client.Get(ctx, types.NamespacedName{ + Name: input.DPUClusters[0].Name, + Namespace: input.DPUClusters[0].Namespace, }, cluster) if apierrors.IsNotFound(err) { return } g.Expect(err).To(Succeed()) - g.Expect(input.client.Delete(ctx, cluster)).To(Succeed()) + g.Expect(input.Client.Delete(ctx, cluster)).To(Succeed()) }).WithTimeout(1 * time.Minute).Should(Succeed()) By("Waiting for DPUCluster to be deleted") clusterDeleteTracker := NewByTracker() Eventually(func(g Gomega) { clusters := &provisioningv1.DPUClusterList{} - g.Expect(input.client.List(ctx, clusters)).To(Succeed()) + g.Expect(input.Client.List(ctx, clusters)).To(Succeed()) clusterDeleteTracker.By(fmt.Sprintf("%d", len(clusters.Items)), "DPUClusters remaining [%d]", len(clusters.Items)) g.Expect(clusters.Items).To(BeEmpty(), "DPUCluster should be deleted") @@ -588,9 +588,9 @@ func DeleteProvisioning(ctx context.Context, input *systemTestInput) { // Delete prerequisite objects (TenantControlPlane, nodeport Service) only when deleting DPUCluster. // When DPUCluster deletion is skipped (RM 4869399), leaving these in place keeps the kubeconfig secret // so DPFOperatorConfig and DPUServices can complete teardown in AfterSuite. - if len(input.dpuClusterPrerequisites) > 0 { - By(fmt.Sprintf("Deleting %d prerequisite objects", len(input.dpuClusterPrerequisites))) - Expect(testutils.CleanupAndWait(ctx, input.client, input.dpuClusterPrerequisites...)).To(Succeed()) + if len(input.DPUClusterPrerequisites) > 0 { + By(fmt.Sprintf("Deleting %d prerequisite objects", len(input.DPUClusterPrerequisites))) + Expect(testutils.CleanupAndWait(ctx, input.Client, input.DPUClusterPrerequisites...)).To(Succeed()) } } else { By("Skipping DPUCluster deletion (RM: 4869399 - DPUCluster/DPUService deletion race; cluster left for DPFOperatorConfig teardown)") @@ -599,27 +599,27 @@ func DeleteProvisioning(ctx context.Context, input *systemTestInput) { By("Verifying cleanup complete") Eventually(func(g Gomega) { dpusets := &provisioningv1.DPUSetList{} - g.Expect(input.client.List(ctx, dpusets)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpusets)).To(Succeed()) g.Expect(dpusets.Items).To(BeEmpty(), "No DPUSets should remain") dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus)).To(Succeed()) g.Expect(dpus.Items).To(BeEmpty(), "No DPU objects should remain") bfbs := &provisioningv1.BFBList{} - g.Expect(input.client.List(ctx, bfbs)).To(Succeed()) + g.Expect(input.Client.List(ctx, bfbs)).To(Succeed()) g.Expect(bfbs.Items).To(BeEmpty(), "No BFBs should remain") if !skipDPUClusterDeletionInProvisioningTest { clusters := &provisioningv1.DPUClusterList{} - g.Expect(input.client.List(ctx, clusters)).To(Succeed()) + g.Expect(input.Client.List(ctx, clusters)).To(Succeed()) g.Expect(clusters.Items).To(BeEmpty(), "No DPUClusters should remain") } flavors := &provisioningv1.DPUFlavorList{} - g.Expect(input.client.List(ctx, flavors, client.InNamespace(input.dpuFlavor.Namespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, flavors, client.InNamespace(input.DPUFlavor.Namespace))).To(Succeed()) for _, flavor := range flavors.Items { - g.Expect(flavor.Name).NotTo(Equal(input.dpuFlavor.Name), + g.Expect(flavor.Name).NotTo(Equal(input.DPUFlavor.Name), "DPUFlavor should be deleted") } }).WithTimeout(2 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) @@ -698,21 +698,21 @@ func dpuFlavorHasNodeLabelScript(dpuFlavor *provisioningv1.DPUFlavor) bool { // ValidateDPUFlavorNodeLabelScripts validates that node label scripts delivered by DPUFlavor.spec.configFiles // are executed by dpuagent and reflected as labels on tenant cluster Nodes. -func ValidateDPUFlavorNodeLabelScripts(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPUFlavorNodeLabelScripts(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip test as DPU nodes are required") } - if len(dpuClusterClient) == 0 || dpuClusterClient[0] == nil { + if len(DPUClusterClient) == 0 || DPUClusterClient[0] == nil { Fail("DPUCluster client is not initialized; expected CreateProvisioningDPUCluster to run first") } - if !dpuFlavorHasNodeLabelScript(input.dpuFlavor) { + if !dpuFlavorHasNodeLabelScript(input.DPUFlavor) { Skip("DPUFlavor has no e2e node label script; skipping DPUFlavor node label script validation") } By("Waiting for tenant Nodes to report the DPUFlavor node label script output") Eventually(func(g Gomega) { nodes := &corev1.NodeList{} - g.Expect(dpuClusterClient[0].List(ctx, nodes)).To(Succeed(), "Should be able to list nodes in DPU cluster") + g.Expect(DPUClusterClient[0].List(ctx, nodes)).To(Succeed(), "Should be able to list nodes in DPU cluster") g.Expect(nodes.Items).To(HaveLen(provisioningExpected.TotalDPUs), "DPU cluster should have one tenant Node per provisioned DPU") @@ -725,11 +725,11 @@ func ValidateDPUFlavorNodeLabelScripts(ctx context.Context, input *systemTestInp // ValidateDPUSetClusterNodeLabelsPropagation validates that changing DPUSet.spec.dpuTemplate.spec.cluster.nodeLabels/nodeAnnotations // is reflected on the tenant cluster Node for a Ready DPU. -func ValidateDPUSetClusterNodeLabelsPropagation(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPUSetClusterNodeLabelsPropagation(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip test as DPU nodes are required") } - if len(dpuClusterClient) == 0 || dpuClusterClient[0] == nil { + if len(DPUClusterClient) == 0 || DPUClusterClient[0] == nil { Fail("DPUCluster client is not initialized; expected CreateProvisioningDPUCluster to run first") } @@ -739,14 +739,14 @@ func ValidateDPUSetClusterNodeLabelsPropagation(ctx context.Context, input *syst ) By("Selecting a Ready DPU") - dpu, err := getAnyReadyDPU(ctx, input.client) + dpu, err := getAnyReadyDPU(ctx, input.Client) Expect(err).NotTo(HaveOccurred()) By("Adding a new cluster node label and annotation via DPUSet template") - dpuset, err := getProvisioningDPUSet(ctx, input.client, input.dpuSet) + dpuset, err := getProvisioningDPUSet(ctx, input.Client, input.DPUSet) Expect(err).NotTo(HaveOccurred()) dpusetCur := &provisioningv1.DPUSet{} - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuset), dpusetCur)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuset), dpusetCur)).To(Succeed()) dpusetPatch := client.MergeFrom(dpusetCur.DeepCopy()) if dpusetCur.Spec.DPUTemplate.Spec.Cluster == nil { dpusetCur.Spec.DPUTemplate.Spec.Cluster = &provisioningv1.ClusterSpec{} @@ -761,11 +761,11 @@ func ValidateDPUSetClusterNodeLabelsPropagation(ctx context.Context, input *syst const dpusetAnnKey = "e2e.provisioning.doca-platform.nvidia.com/dpuset-template-annotation" dpusetCur.Spec.DPUTemplate.Spec.Cluster.NodeLabels[dpusetLabelKey] = "tv1" dpusetCur.Spec.DPUTemplate.Spec.Cluster.NodeAnnotations[dpusetAnnKey] = "tav1" - Expect(input.client.Patch(ctx, dpusetCur, dpusetPatch)).To(Succeed()) + Expect(input.Client.Patch(ctx, dpusetCur, dpusetPatch)).To(Succeed()) By("Waiting for the tenant Node to have the DPUSet template label and annotation") Eventually(func(g Gomega) { - node, err := getTenantNode(ctx, dpuClusterClient[0], dpu.Name) + node, err := getTenantNode(ctx, DPUClusterClient[0], dpu.Name) g.Expect(err).NotTo(HaveOccurred()) g.Expect(node.Labels).To(HaveKeyWithValue(dpusetLabelKey, "tv1")) g.Expect(node.Annotations).To(HaveKeyWithValue(dpusetAnnKey, "tav1")) @@ -774,8 +774,8 @@ func ValidateDPUSetClusterNodeLabelsPropagation(ctx context.Context, input *syst // ValidateDPUSetNotReadyOnClusterMetadataConflict patches DPUSet template and a referenced DPUDevice to create a // key/value conflict and verifies the DPUSet transitions to NotReady with the expected reason. -func ValidateDPUSetNotReadyOnClusterMetadataConflict(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPUSetNotReadyOnClusterMetadataConflict(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip test as DPU nodes are required") } @@ -786,16 +786,16 @@ func ValidateDPUSetNotReadyOnClusterMetadataConflict(ctx context.Context, input ) By("Selecting a Ready DPU to locate a representative DPUDevice") - dpu, err := getAnyReadyDPU(ctx, input.client) + dpu, err := getAnyReadyDPU(ctx, input.Client) Expect(err).NotTo(HaveOccurred()) - dd, err := getDPUDeviceByName(ctx, input.client, dpu.Spec.DPUDeviceName) + dd, err := getDPUDeviceByName(ctx, input.Client, dpu.Spec.DPUDeviceName) Expect(err).NotTo(HaveOccurred()) By("Patching DPUSet template to set a conflicting label+annotation key") - dpuset, err := getProvisioningDPUSet(ctx, input.client, input.dpuSet) + dpuset, err := getProvisioningDPUSet(ctx, input.Client, input.DPUSet) Expect(err).NotTo(HaveOccurred()) dpusetCur := &provisioningv1.DPUSet{} - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuset), dpusetCur)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuset), dpusetCur)).To(Succeed()) dpusetPatch := client.MergeFrom(dpusetCur.DeepCopy()) if dpusetCur.Spec.DPUTemplate.Spec.Cluster == nil { dpusetCur.Spec.DPUTemplate.Spec.Cluster = &provisioningv1.ClusterSpec{} @@ -808,11 +808,11 @@ func ValidateDPUSetNotReadyOnClusterMetadataConflict(ctx context.Context, input } dpusetCur.Spec.DPUTemplate.Spec.Cluster.NodeLabels[conflictKey] = "from-dpuset" dpusetCur.Spec.DPUTemplate.Spec.Cluster.NodeAnnotations[conflictKey] = "from-dpuset" - Expect(input.client.Patch(ctx, dpusetCur, dpusetPatch)).To(Succeed()) + Expect(input.Client.Patch(ctx, dpusetCur, dpusetPatch)).To(Succeed()) By("Patching DPUDevice spec.cluster to set the same keys with different values") ddCur := &provisioningv1.DPUDevice{} - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dd), ddCur)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dd), ddCur)).To(Succeed()) ddPatch := client.MergeFrom(ddCur.DeepCopy()) if ddCur.Spec.Cluster == nil { ddCur.Spec.Cluster = &provisioningv1.DPUDeviceClusterSpec{} @@ -825,12 +825,12 @@ func ValidateDPUSetNotReadyOnClusterMetadataConflict(ctx context.Context, input } ddCur.Spec.Cluster.NodeLabels[conflictKey] = "from-dpudevice" ddCur.Spec.Cluster.NodeAnnotations[conflictKey] = "from-dpudevice" - Expect(input.client.Patch(ctx, ddCur, ddPatch)).To(Succeed()) + Expect(input.Client.Patch(ctx, ddCur, ddPatch)).To(Succeed()) By("Waiting for the DPUSet Ready condition to become False with reason ClusterMetadataConflict") Eventually(func(g Gomega) { cur := &provisioningv1.DPUSet{} - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpusetCur), cur)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpusetCur), cur)).To(Succeed()) cond := meta.FindStatusCondition(cur.Status.Conditions, "Ready") g.Expect(cond).NotTo(BeNil(), "DPUSet should have Ready condition") g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) @@ -840,11 +840,11 @@ func ValidateDPUSetNotReadyOnClusterMetadataConflict(ctx context.Context, input // ValidateDPUDeviceClusterNodeLabelsPropagation validates that changing DPUDevice.spec.cluster.nodeLabels/nodeAnnotations // (add/update/remove) is reflected on the tenant cluster Node for a Ready DPU. -func ValidateDPUDeviceClusterNodeLabelsPropagation(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func ValidateDPUDeviceClusterNodeLabelsPropagation(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip test as DPU nodes are required") } - if len(dpuClusterClient) == 0 || dpuClusterClient[0] == nil { + if len(DPUClusterClient) == 0 || DPUClusterClient[0] == nil { Fail("DPUCluster client is not initialized; expected CreateProvisioningDPUCluster to run first") } @@ -856,16 +856,16 @@ func ValidateDPUDeviceClusterNodeLabelsPropagation(ctx context.Context, input *s ) By("Selecting a Ready DPU") - dpu, err := getAnyReadyDPU(ctx, input.client) + dpu, err := getAnyReadyDPU(ctx, input.Client) Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Fetching DPUDevice %q referenced by DPU %q", dpu.Spec.DPUDeviceName, dpu.Name)) - dd, err := getDPUDeviceByName(ctx, input.client, dpu.Spec.DPUDeviceName) + dd, err := getDPUDeviceByName(ctx, input.Client, dpu.Spec.DPUDeviceName) Expect(err).NotTo(HaveOccurred()) By("Adding a new cluster node label and annotation via DPUDevice") ddCur := &provisioningv1.DPUDevice{} - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dd), ddCur)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dd), ddCur)).To(Succeed()) patch := client.MergeFrom(ddCur.DeepCopy()) if ddCur.Spec.Cluster == nil { ddCur.Spec.Cluster = &provisioningv1.DPUDeviceClusterSpec{} @@ -878,41 +878,41 @@ func ValidateDPUDeviceClusterNodeLabelsPropagation(ctx context.Context, input *s } ddCur.Spec.Cluster.NodeLabels[labelKey] = "v1" ddCur.Spec.Cluster.NodeAnnotations[annKey] = "av1" - Expect(input.client.Patch(ctx, ddCur, patch)).To(Succeed()) + Expect(input.Client.Patch(ctx, ddCur, patch)).To(Succeed()) By("Waiting for the tenant Node to have the added label and annotation") Eventually(func(g Gomega) { - node, err := getTenantNode(ctx, dpuClusterClient[0], dpu.Name) + node, err := getTenantNode(ctx, DPUClusterClient[0], dpu.Name) g.Expect(err).NotTo(HaveOccurred()) g.Expect(node.Labels).To(HaveKeyWithValue(labelKey, "v1")) g.Expect(node.Annotations).To(HaveKeyWithValue(annKey, "av1")) }).WithTimeout(timeout).WithPolling(pollingInterval).Should(Succeed()) By("Updating the cluster node label and annotation values via DPUDevice") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(ddCur), ddCur)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(ddCur), ddCur)).To(Succeed()) patch = client.MergeFrom(ddCur.DeepCopy()) ddCur.Spec.Cluster.NodeLabels[labelKey] = "v2" ddCur.Spec.Cluster.NodeAnnotations[annKey] = "av2" - Expect(input.client.Patch(ctx, ddCur, patch)).To(Succeed()) + Expect(input.Client.Patch(ctx, ddCur, patch)).To(Succeed()) By("Waiting for the tenant Node to have the updated label and annotation values") Eventually(func(g Gomega) { - node, err := getTenantNode(ctx, dpuClusterClient[0], dpu.Name) + node, err := getTenantNode(ctx, DPUClusterClient[0], dpu.Name) g.Expect(err).NotTo(HaveOccurred()) g.Expect(node.Labels).To(HaveKeyWithValue(labelKey, "v2")) g.Expect(node.Annotations).To(HaveKeyWithValue(annKey, "av2")) }).WithTimeout(timeout).WithPolling(pollingInterval).Should(Succeed()) By("Removing the cluster node label and annotation keys via DPUDevice") - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(ddCur), ddCur)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(ddCur), ddCur)).To(Succeed()) patch = client.MergeFrom(ddCur.DeepCopy()) delete(ddCur.Spec.Cluster.NodeLabels, labelKey) delete(ddCur.Spec.Cluster.NodeAnnotations, annKey) - Expect(input.client.Patch(ctx, ddCur, patch)).To(Succeed()) + Expect(input.Client.Patch(ctx, ddCur, patch)).To(Succeed()) By("Waiting for the tenant Node to no longer have the removed label and annotation") Eventually(func(g Gomega) { - node, err := getTenantNode(ctx, dpuClusterClient[0], dpu.Name) + node, err := getTenantNode(ctx, DPUClusterClient[0], dpu.Name) g.Expect(err).NotTo(HaveOccurred()) _, ok := node.Labels[labelKey] g.Expect(ok).To(BeFalse(), "label should be removed from tenant Node") @@ -923,3 +923,8 @@ func ValidateDPUDeviceClusterNodeLabelsPropagation(ctx context.Context, input *s By("Validating DPUSet becomes NotReady on DPUSet/DPUDevice cluster metadata conflict") ValidateDPUSetNotReadyOnClusterMetadataConflict(ctx, input) } + +func ProvisioningBeforeSuite() { + By("Setting Provisioning configs for the test") + // No additional config needed - input.ApplyConfig(*Conf) already called in SetInput() +} diff --git a/test/e2e/provisioning_test.go b/test/e2e/provisioning_test.go index 82ab24db..453cffe6 100644 --- a/test/e2e/provisioning_test.go +++ b/test/e2e/provisioning_test.go @@ -20,58 +20,53 @@ import ( . "github.com/onsi/ginkgo/v2" ) -func ProvisioningBeforeSuite() { - By("Setting Provisioning configs for the test") - // No additional config needed - input.applyConfig(*conf) already called in SetInput() -} - //nolint:dupl var _ = Describe("DPF System tests - Provisioning", Labels{Domain.Provisioning}, Ordered, func() { BeforeAll(func() { - BeforeProvisioning(ctx, input) + BeforeProvisioning(Ctx, input) }) AfterAll(func() { By("Cleaning up test suite resources") - if cleanupFlags.SkipCleanup { + if CleanupFlags.SkipCleanup { By("Skip cleanup") return } }) It("create DPU cluster and BFB", func() { - CreateProvisioningDPUCluster(ctx, input) + CreateProvisioningDPUCluster(Ctx, input) }) It("create DPUSet and provision DPUs", func() { - CreateProvisioningDPUSet(ctx, input) + CreateProvisioningDPUSet(Ctx, input) }) It("verify provisioning is complete", func() { - VerifyProvisioning(ctx, input) + VerifyProvisioning(Ctx, input) }) It("change the OOB bridge name in the operatorConfig and verify DPUNode condition updates", Labels{Domain.RequiresNodes}, func() { - ValidateDPFOperatorOOBBridgeNameChange(ctx, input) + ValidateDPFOperatorOOBBridgeNameChange(Ctx, input) }) It("verify OOB bridge VF attachment and netplan after provisioning", Labels{Domain.RequiresNodes}, func() { - ValidateDPFOperatorOOBBridgePostProvisioning(ctx, input) + ValidateDPFOperatorOOBBridgePostProvisioning(Ctx, input) }) It("verify DPUDevice and DPUSet cluster node label and annotation changes are reflected on tenant Nodes", func() { - ValidateDPUSetClusterNodeLabelsPropagation(ctx, input) - ValidateDPUDeviceClusterNodeLabelsPropagation(ctx, input) + ValidateDPUSetClusterNodeLabelsPropagation(Ctx, input) + ValidateDPUDeviceClusterNodeLabelsPropagation(Ctx, input) }) It("verify node labels are added via dpu-agent on tenant Nodes", Labels{Domain.RequiresNodes}, func() { - ValidateDPUFlavorNodeLabelScripts(ctx, input) + ValidateDPUFlavorNodeLabelScripts(Ctx, input) }) It("delete all provisioning resources", func() { - if cleanupFlags.SkipCleanup { + if CleanupFlags.SkipCleanup { Skip("Skipping deprovisioning tests because skipCleanup is enabled") } - DeleteProvisioning(ctx, input) + DeleteProvisioning(Ctx, input) }) }) diff --git a/test/e2e/scale.go b/test/e2e/scale.go new file mode 100644 index 00000000..04224c4a --- /dev/null +++ b/test/e2e/scale.go @@ -0,0 +1,88 @@ +/* +Copyright 2025 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func CreateDPUWorkerNodes(ctx context.Context, n int) { + By("Creates nodes in the target cluster") + // Get the name of the mock-dms pod + + mockDMSPod := &corev1.PodList{} + Expect(TestClient.List(ctx, mockDMSPod, client.InNamespace(DPFOperatorSystemNamespace), client.MatchingLabels{"app.kubernetes.io/instance": "mock-dms"})).To(Succeed()) + Expect(mockDMSPod.Items).To(HaveLen(1)) + mockDMSPodName := mockDMSPod.Items[0].Name + + labels := map[string]string{ + "dpf-operator-e2e-test-cleanup": "true", + "feature.node.kubernetes.io/dpu-deviceID": "0xa2d6", + "feature.node.kubernetes.io/dpu-enabled": "true", + "feature.node.kubernetes.io/dpu-oob-bridge-configured": "true", + "e2e.test.io/fake-node": "true", + } + annotations := map[string]string{ + "provisioning.dpu.nvidia.com/override-dms-pod-name": mockDMSPodName, + "kwok.x-k8s.io/node": "fake", + } + + // Get the IP address of the kind control plane node + + mockDMSIPAddress := "" + nodes := &corev1.NodeList{} + Expect(TestClient.List(ctx, nodes)).To(Succeed()) + Expect(nodes.Items).To(HaveLen(1)) + for addr := range nodes.Items[0].Status.Addresses { + if nodes.Items[0].Status.Addresses[addr].Type == corev1.NodeInternalIP { + mockDMSIPAddress = nodes.Items[0].Status.Addresses[addr].Address + break + } + } + Expect(mockDMSIPAddress).ToNot(BeEmpty()) + + for i := 0; i < n; i++ { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + // The Node should have the same name as the DPU. + Name: fmt.Sprintf("dpu-worker-%d", i), + Labels: labels, + Annotations: annotations, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + } + Expect(TestClient.Create(ctx, node)).To(Succeed()) + original := node.DeepCopy() + node.Status.Addresses = []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: mockDMSIPAddress, + }, + } + Expect(TestClient.Status().Patch(ctx, node, client.MergeFrom(original))).To(Succeed()) + } +} diff --git a/test/e2e/scale_test.go b/test/e2e/scale_test.go index 97e8bf33..8392009d 100644 --- a/test/e2e/scale_test.go +++ b/test/e2e/scale_test.go @@ -17,14 +17,7 @@ limitations under the License. package e2e import ( - "context" - "fmt" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) //nolint:dupl @@ -32,67 +25,7 @@ var _ = Describe("DPF scale tests", Labels{Domain.Scale}, func() { Context("Validate DPU Operator Cleanup", Labels{Domain.RequiresNodes}, Serial, Ordered, func() { It("should validate DPU Operator Cleanup", func() { - ValidateDPUDeploymentFullCreation(ctx, input) + ValidateDPUDeploymentFullCreation(Ctx, input) }) }) }) - -func CreateDPUWorkerNodes(ctx context.Context, n int) { - By("Creates nodes in the target cluster") - // Get the name of the mock-dms pod - - mockDMSPod := &corev1.PodList{} - Expect(testClient.List(ctx, mockDMSPod, client.InNamespace(dpfOperatorSystemNamespace), client.MatchingLabels{"app.kubernetes.io/instance": "mock-dms"})).To(Succeed()) - Expect(mockDMSPod.Items).To(HaveLen(1)) - mockDMSPodName := mockDMSPod.Items[0].Name - - labels := map[string]string{ - "dpf-operator-e2e-test-cleanup": "true", - "feature.node.kubernetes.io/dpu-deviceID": "0xa2d6", - "feature.node.kubernetes.io/dpu-enabled": "true", - "feature.node.kubernetes.io/dpu-oob-bridge-configured": "true", - "e2e.test.io/fake-node": "true", - } - annotations := map[string]string{ - "provisioning.dpu.nvidia.com/override-dms-pod-name": mockDMSPodName, - "kwok.x-k8s.io/node": "fake", - } - - // Get the IP address of the kind control plane node - - mockDMSIPAddress := "" - nodes := &corev1.NodeList{} - Expect(testClient.List(ctx, nodes)).To(Succeed()) - Expect(nodes.Items).To(HaveLen(1)) - for addr := range nodes.Items[0].Status.Addresses { - if nodes.Items[0].Status.Addresses[addr].Type == corev1.NodeInternalIP { - mockDMSIPAddress = nodes.Items[0].Status.Addresses[addr].Address - break - } - } - Expect(mockDMSIPAddress).ToNot(BeEmpty()) - - for i := 0; i < n; i++ { - node := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - // The Node should have the same name as the DPU. - Name: fmt.Sprintf("dpu-worker-%d", i), - Labels: labels, - Annotations: annotations, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Node", - APIVersion: "v1", - }, - } - Expect(testClient.Create(ctx, node)).To(Succeed()) - original := node.DeepCopy() - node.Status.Addresses = []corev1.NodeAddress{ - { - Type: corev1.NodeInternalIP, - Address: mockDMSIPAddress, - }, - } - Expect(testClient.Status().Patch(ctx, node, client.MergeFrom(original))).To(Succeed()) - } -} diff --git a/test/e2e/sdn.go b/test/e2e/sdn.go new file mode 100644 index 00000000..30ba8462 --- /dev/null +++ b/test/e2e/sdn.go @@ -0,0 +1,26 @@ +/* +Copyright 2025 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" +) + +func SDNBeforeSuite() { + By("Setting SDN configs for the test") + input.ApplySDNConfig(*Conf) +} diff --git a/test/e2e/sdn_test.go b/test/e2e/sdn_test.go index 1198c7b5..ee91275f 100644 --- a/test/e2e/sdn_test.go +++ b/test/e2e/sdn_test.go @@ -22,11 +22,6 @@ import ( . "github.com/onsi/ginkgo/v2" ) -func SDNBeforeSuite() { - By("Setting SDN configs for the test") - input.applySDNConfig(*conf) -} - //nolint:dupl var _ = Describe("DPF System tests - SDN", SpecPriority(SDNTestPriority), Labels{Domain.DPFSystem, Domain.SDN}, Ordered, func() { @@ -34,39 +29,39 @@ var _ = Describe("DPF System tests - SDN", SpecPriority(SDNTestPriority), Labels for _, label := range CurrentSpecReport().Labels() { if label == Domain.RequiresNodes { By("Waiting for provisioning") - VerifyDPUClusterWithNodes(ctx, getProvisionDPUClustersInput()) + VerifyDPUClusterWithNodes(Ctx, GetProvisionDPUClustersInput()) By("Waiting for DPU cluster pods to be ready") - VerifyClusterPods(ctx, dpuClusterClient[0], systemPodsToVerify) + VerifyClusterPods(Ctx, DPUClusterClient[0], systemPodsToVerify) By("Waiting for DPFOperatorConfig to be ready") - VerifyDPFOperatorConfigReady(ctx, input.client, 20*time.Minute) + VerifyDPFOperatorConfigReady(Ctx, input.Client, 20*time.Minute) } } }) Context("DPU Service Function Chain", Labels{Domain.RequiresNodes, Domain.L2Connectivity}, func() { It("create plain DPU chain and verify performance", func() { - VerifyPlainServiceFunctionChain(ctx, input) + VerifyPlainServiceFunctionChain(Ctx, input) }) It("create HBN only DPU chain and verify performance", func() { - VerifyHBNOnlyServiceFunctionChain(ctx, input) + VerifyHBNOnlyServiceFunctionChain(Ctx, input) }) It("create HBN only DPU chain and verify performance after killing HBN", Labels{Domain.L2Connectivity}, func() { - VerifyHBNOnlyBadFlowRecovery(ctx, input) + VerifyHBNOnlyBadFlowRecovery(Ctx, input) }) It("create simple chain and validate serviceMTU changes", func() { - VerifyServiceMTUOnDPUPods(ctx, input) + VerifyServiceMTUOnDPUPods(Ctx, input) }) It("create Pods running in the DPUCluster via DPUService and verify RDMA traffic between them", func() { - VerifyDPUPodToPodRDMATraffic(ctx, input) + VerifyDPUPodToPodRDMATraffic(Ctx, input) }) }) Context("Validate DPU Service NAD", Labels{Domain.DPFSystem, Domain.RequiresNodes}, func() { It("create a pod consuming a DPUServiceNAD with all dependencies and check that it is created successfully", func() { - ValidateDPUServiceNADConsumedByPod(ctx, input) + ValidateDPUServiceNADConsumedByPod(Ctx, input) }) It("verify DPUServiceNAD metrics", func() { - ValidateDPUServiceNADMetrics(ctx) + ValidateDPUServiceNADMetrics(Ctx) }) }) }) diff --git a/test/e2e/servicefunctionchain.go b/test/e2e/servicefunctionchain.go index 719ebe46..85ff7429 100644 --- a/test/e2e/servicefunctionchain.go +++ b/test/e2e/servicefunctionchain.go @@ -47,79 +47,79 @@ type mtuTestConfig struct { nadName string } -func VerifyPlainServiceFunctionChain(ctx context.Context, input *systemTestInput) { - if !input.hasDpuNodes() { +func VerifyPlainServiceFunctionChain(ctx context.Context, input *SystemTestInput) { + if !input.HasDpuNodes() { Skip("Skip test as there are not multiple nodes") } hostNamespace := "sfc-plain-test-ns" - createTestNamespace(ctx, input.client, hostNamespace) + createTestNamespace(ctx, input.Client, hostNamespace) vfIndex := 5 setupPlainChainTest(ctx, input, vfIndex) By("Creating test pods") pod1Config, pod2Config := getPlainChainTestPodConfigs(ctx, input, hostNamespace, vfIndex) - netshoot.CreateNadsFromConfig(ctx, input.client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) - netshoot.CreateAndWaitForPods(ctx, input.client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) + netshoot.CreateNadsFromConfig(ctx, input.Client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) + netshoot.CreateAndWaitForPods(ctx, input.Client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) By("Running traffic test between pods") - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2Config.IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2Config.IP) } -func VerifyHBNOnlyServiceFunctionChain(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func VerifyHBNOnlyServiceFunctionChain(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } hostNamespace := "sfc-hbn-test-ns" - createTestNamespace(ctx, input.client, hostNamespace) + createTestNamespace(ctx, input.Client, hostNamespace) vfIndex := 3 setupHBNOnlyTest(ctx, input, vfIndex) By("Creating test pods") pod1Config, pod2Config := getHBNOnlyTestPodConfigs(ctx, input, hostNamespace, vfIndex) - netshoot.CreateNadsFromConfig(ctx, input.client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) - netshoot.CreateAndWaitForPods(ctx, input.client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) + netshoot.CreateNadsFromConfig(ctx, input.Client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) + netshoot.CreateAndWaitForPods(ctx, input.Client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) By("Running traffic test between pods") - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2Config.IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2Config.IP) } -func VerifyHBNOnlyBadFlowRecovery(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func VerifyHBNOnlyBadFlowRecovery(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } hostNamespace := "sfc-hbn-recovery-test-ns" - createTestNamespace(ctx, input.client, hostNamespace) + createTestNamespace(ctx, input.Client, hostNamespace) vfIndex := 3 setupHBNOnlyTest(ctx, input, vfIndex) By("Creating test pods") pod1Config, pod2Config := getHBNOnlyTestPodConfigs(ctx, input, hostNamespace, vfIndex) - netshoot.CreateNadsFromConfig(ctx, input.client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) - netshoot.CreateAndWaitForPods(ctx, input.client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) + netshoot.CreateNadsFromConfig(ctx, input.Client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) + netshoot.CreateAndWaitForPods(ctx, input.Client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) By("Running initial traffic test") - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2Config.IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2Config.IP) By("Killing HBN pod to test recovery") - deleteFirstFoundPodOnDpuCluster(ctx, "doca-hbn", input.namespace) + deleteFirstFoundPodOnDpuCluster(ctx, "doca-hbn", input.Namespace) By("Waiting for HBN service to recover") - dpuservice.WaitForDPUServices(ctx, input.client, input.namespace, []string{"doca-hbn"}) + dpuservice.WaitForDPUServices(ctx, input.Client, input.Namespace, []string{"doca-hbn"}) By("Running traffic test after recovery") - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2Config.IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2Config.IP) } -func VerifyServiceMTUOnDPUPods(ctx context.Context, input *systemTestInput) { - if input.numberOfDPUNodes != 2 { +func VerifyServiceMTUOnDPUPods(ctx context.Context, input *SystemTestInput) { + if input.NumberOfDPUNodes != 2 { // Test assumes that there are exactly 2 host nodes to match the DPU cluster Skip("Skip test as there are not exactly 2 nodes") } @@ -154,7 +154,7 @@ func VerifyServiceMTUOnDPUPods(ctx context.Context, input *systemTestInput) { } for _, svc := range serviceConfigs { - pods := getActiveServicePods(ctx, g, input.namespace, svc.serviceID) + pods := getActiveServicePods(ctx, g, input.Namespace, svc.serviceID) *svc.podNames = make([]string, cfg.podCount) for i, pod := range pods { g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning), "Pod %s should be running", pod.Name) @@ -164,37 +164,37 @@ func VerifyServiceMTUOnDPUPods(ctx context.Context, input *systemTestInput) { }).WithTimeout(10 * time.Minute).Should(Succeed()) By(fmt.Sprintf("Testing inter-node (pod1->pod2) and intra-node (pod1->pod1) ping connectivity with MTU %d", mtuTestDefaultMTU)) - testPingBetweenPods(ctx, input.namespace, mtuTestDefaultMTU, cfg) + testPingBetweenPods(ctx, input.Namespace, mtuTestDefaultMTU, cfg) By("Testing serviceMTU change triggers pod recreation") - updatedPod1Names, updatedPod2Names := updateServiceMTUAndValidatePodRestart(ctx, input, input.namespace, initialPod1Names, initialPod2Names, mtuTestLowerMTU, cfg) + updatedPod1Names, updatedPod2Names := updateServiceMTUAndValidatePodRestart(ctx, input, input.Namespace, initialPod1Names, initialPod2Names, mtuTestLowerMTU, cfg) By("Waiting for tunnel connection to stabilize after pod recreation") // TODO: replace fixed delay with proper tunnel re-establishment validation time.Sleep(tunnelStabilizationWait) By(fmt.Sprintf("Testing inter-node (pod1->pod2) and intra-node (pod1->pod1) ping connectivity with MTU %d", mtuTestLowerMTU)) - testPingBetweenPods(ctx, input.namespace, mtuTestLowerMTU, cfg) + testPingBetweenPods(ctx, input.Namespace, mtuTestLowerMTU, cfg) By("Reverting the serviceMTU to the original value, and validating ping connectivity") - updateServiceMTUAndValidatePodRestart(ctx, input, input.namespace, updatedPod1Names, updatedPod2Names, mtuTestDefaultMTU, cfg) + updateServiceMTUAndValidatePodRestart(ctx, input, input.Namespace, updatedPod1Names, updatedPod2Names, mtuTestDefaultMTU, cfg) By("Waiting for tunnel connection to stabilize after pod recreation") time.Sleep(tunnelStabilizationWait) By(fmt.Sprintf("Testing inter-node (pod1->pod2) and intra-node (pod1->pod1) ping connectivity with MTU %d", mtuTestDefaultMTU)) - testPingBetweenPods(ctx, input.namespace, mtuTestDefaultMTU, cfg) + testPingBetweenPods(ctx, input.Namespace, mtuTestDefaultMTU, cfg) } // updateServiceMTUAndValidatePodRestart tests that changing the serviceMTU triggers pod recreation -func updateServiceMTUAndValidatePodRestart(ctx context.Context, input *systemTestInput, namespace string, initialPod1Names, initialPod2Names []string, newMTU int, cfg *mtuTestConfig) ([]string, []string) { +func updateServiceMTUAndValidatePodRestart(ctx context.Context, input *SystemTestInput, namespace string, initialPod1Names, initialPod2Names []string, newMTU int, cfg *mtuTestConfig) ([]string, []string) { By("Updating serviceMTU in DPUServiceChain") // Use Patch instead of Update to avoid conflicts with concurrent controller updates dpuServiceChain := &dpuservicev1.DPUServiceChain{} - Expect(input.client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: cfg.chainName}, dpuServiceChain)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: cfg.chainName}, dpuServiceChain)).To(Succeed()) originalDpuServiceChain := dpuServiceChain.DeepCopy() dpuServiceChain.Spec.Template.Spec.Template.Spec.Switches[0].ServiceMTU = ptr.To(newMTU) - Expect(input.client.Patch(ctx, dpuServiceChain, client.MergeFrom(originalDpuServiceChain))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuServiceChain, client.MergeFrom(originalDpuServiceChain))).To(Succeed()) By("Waiting for pods to be recreated with new names and be running") var newPod1Names, newPod2Names []string @@ -226,7 +226,7 @@ func updateServiceMTUAndValidatePodRestart(ctx context.Context, input *systemTes // getActiveServicePods gets and filters active (non-terminating) pods for a service func getActiveServicePods(ctx context.Context, g Gomega, namespace, serviceID string) []corev1.Pod { podList := &corev1.PodList{} - g.Expect(dpuClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceID}, client.InNamespace(namespace))).ToNot(HaveOccurred()) + g.Expect(DPUClusterClient[0].List(ctx, podList, client.MatchingLabels{"svc.dpu.nvidia.com/service": serviceID}, client.InNamespace(namespace))).ToNot(HaveOccurred()) var activePods []corev1.Pod for _, pod := range podList.Items { @@ -265,24 +265,24 @@ func testPingBetweenPods(ctx context.Context, namespace string, mtu int, cfg *mt pod2Node1IP := getPodIPForInterface(Default, *pod2Node1, cfg.ipInterface) By(fmt.Sprintf("Testing ping fails from %s (%s) to %s (%s) with MTU %d on the same node", pod1Node1.Name, pod1Node1IP, pod2Node1.Name, pod2Node1IP, mtu+1)) - netshoot.AssertPingFailureWithMTU(&dpuClusterRestClient[0], &dpuClusterRestConfig[0], namespace, pod1Node1.Name, pod2Node1IP, mtu+1, mtu) + netshoot.AssertPingFailureWithMTU(&DPUClusterRestClient[0], &DPUClusterRestConfig[0], namespace, pod1Node1.Name, pod2Node1IP, mtu+1, mtu) By(fmt.Sprintf("Testing ping fails from %s (%s) to %s (%s) with MTU %d on different nodes", pod1Node1.Name, pod1Node1IP, pod1Node2.Name, pod1Node2IP, mtu+1)) - netshoot.AssertPingFailureWithMTU(&dpuClusterRestClient[0], &dpuClusterRestConfig[0], namespace, pod1Node1.Name, pod1Node2IP, mtu+1, mtu) + netshoot.AssertPingFailureWithMTU(&DPUClusterRestClient[0], &DPUClusterRestConfig[0], namespace, pod1Node1.Name, pod1Node2IP, mtu+1, mtu) By(fmt.Sprintf("Testing ping from %s (%s) to %s (%s) with MTU %d on the same node", pod1Node1.Name, pod1Node1IP, pod2Node1.Name, pod2Node1IP, mtu)) - netshoot.AssertPingSuccessWithMTU(&dpuClusterRestClient[0], &dpuClusterRestConfig[0], namespace, pod1Node1.Name, pod2Node1IP, mtu) + netshoot.AssertPingSuccessWithMTU(&DPUClusterRestClient[0], &DPUClusterRestConfig[0], namespace, pod1Node1.Name, pod2Node1IP, mtu) By(fmt.Sprintf("Testing ping from %s (%s) to %s (%s) with MTU %d on different nodes", pod1Node1.Name, pod1Node1IP, pod1Node2.Name, pod1Node2IP, mtu)) - netshoot.AssertPingSuccessWithMTU(&dpuClusterRestClient[0], &dpuClusterRestConfig[0], namespace, pod1Node1.Name, pod1Node2IP, mtu) + netshoot.AssertPingSuccessWithMTU(&DPUClusterRestClient[0], &DPUClusterRestConfig[0], namespace, pod1Node1.Name, pod1Node2IP, mtu) By(fmt.Sprintf("Testing ping from %s (%s) to %s (%s) with MTU %d on different nodes", pod1Node2.Name, pod1Node2IP, pod1Node1.Name, pod1Node1IP, mtu)) - netshoot.AssertPingSuccessWithMTU(&dpuClusterRestClient[0], &dpuClusterRestConfig[0], namespace, pod1Node2.Name, pod1Node1IP, mtu) + netshoot.AssertPingSuccessWithMTU(&DPUClusterRestClient[0], &DPUClusterRestConfig[0], namespace, pod1Node2.Name, pod1Node1IP, mtu) } -func getPlainChainTestPodConfigs(ctx context.Context, input *systemTestInput, namespace string, vfIndex int) (netshoot.TestPodConfig, netshoot.TestPodConfig) { - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) +func getPlainChainTestPodConfigs(ctx context.Context, input *SystemTestInput, namespace string, vfIndex int) (netshoot.TestPodConfig, netshoot.TestPodConfig) { + workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.Client) pod1Config := netshoot.TestPodConfig{ Name: "pod1", @@ -304,8 +304,8 @@ func getPlainChainTestPodConfigs(ctx context.Context, input *systemTestInput, na return pod1Config, pod2Config } -func getHBNOnlyTestPodConfigs(ctx context.Context, input *systemTestInput, namespace string, vfIndex int) (netshoot.TestPodConfig, netshoot.TestPodConfig) { - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) +func getHBNOnlyTestPodConfigs(ctx context.Context, input *SystemTestInput, namespace string, vfIndex int) (netshoot.TestPodConfig, netshoot.TestPodConfig) { + workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.Client) pod1Config := netshoot.TestPodConfig{ Name: "pod1", @@ -332,12 +332,12 @@ func getHBNOnlyTestPodConfigs(ctx context.Context, input *systemTestInput, names } // setupPlainChainTest creates a test environment for a plain service function chain -func setupPlainChainTest(ctx context.Context, input *systemTestInput, vfIndex int) { +func setupPlainChainTest(ctx context.Context, input *SystemTestInput, vfIndex int) { interfaceConfigs := []dpuservice.TestDPUServiceInterfaceConfig{ { Name: "p0", Type: "physical", - Namespace: input.namespace, + Namespace: input.Namespace, InterfaceName: "p0", Labels: map[string]string{ "uplink": "p0", @@ -349,7 +349,7 @@ func setupPlainChainTest(ctx context.Context, input *systemTestInput, vfIndex in { Name: fmt.Sprintf("pf0vf%d", vfIndex), Type: "vf", - Namespace: input.namespace, + Namespace: input.Namespace, InterfaceName: fmt.Sprintf("pf0vf%d", vfIndex), PFIndex: 0, VFIndex: vfIndex, @@ -360,27 +360,27 @@ func setupPlainChainTest(ctx context.Context, input *systemTestInput, vfIndex in } By("Wait for prerequisite services") - dpuservice.WaitForDPUServices(ctx, input.client, input.namespace, []string{"sfc-controller"}) + dpuservice.WaitForDPUServices(ctx, input.Client, input.Namespace, []string{"sfc-controller"}) By("Create and wait for DPU service interfaces") - createAndWaitForInterfaces(ctx, input.client, input.dpuServiceInterfaceTemplate, interfaceConfigs) + createAndWaitForInterfaces(ctx, input.Client, input.DPUServiceInterfaceTemplate, interfaceConfigs) By("Create plain DPU service chain") - dpuServiceChain := utils.GenerateDPUObj("netshoot-to-p0", input.namespace, input.dpuServiceChainTemplate.DeepCopy()) - Expect(input.client.Create(ctx, dpuServiceChain)).To(Succeed()) + dpuServiceChain := utils.GenerateDPUObj("netshoot-to-p0", input.Namespace, input.DPUServiceChainTemplate.DeepCopy()) + Expect(input.Client.Create(ctx, dpuServiceChain)).To(Succeed()) By("Verify underlying DPU objects are ready") - dpuservice.VerifyUnderlyingDPUObjectsReady(ctx, dpuClusterClient[0], input.namespace, interfaceConfigs, []string{"netshoot-to-p0"}) + dpuservice.VerifyUnderlyingDPUObjectsReady(ctx, DPUClusterClient[0], input.Namespace, interfaceConfigs, []string{"netshoot-to-p0"}) } // setupHBNOnlyTest creates a test environment for a HBN only service function chain -func setupHBNOnlyTest(ctx context.Context, input *systemTestInput, vfIndex int) { +func setupHBNOnlyTest(ctx context.Context, input *SystemTestInput, vfIndex int) { hbnServiceID := "doca-hbn" hbnNetwork := "mybrhbn" interfaceConfigs := []dpuservice.TestDPUServiceInterfaceConfig{ { Name: "p0", - Namespace: input.namespace, + Namespace: input.Namespace, Type: "physical", InterfaceName: "p0", Labels: map[string]string{ @@ -392,7 +392,7 @@ func setupHBNOnlyTest(ctx context.Context, input *systemTestInput, vfIndex int) }, { Name: "p1", - Namespace: input.namespace, + Namespace: input.Namespace, Type: "physical", InterfaceName: "p1", Labels: map[string]string{ @@ -404,7 +404,7 @@ func setupHBNOnlyTest(ctx context.Context, input *systemTestInput, vfIndex int) }, { Name: fmt.Sprintf("pf0vf%d-rep", vfIndex), - Namespace: input.namespace, + Namespace: input.Namespace, Type: "vf", InterfaceName: fmt.Sprintf("pf0vf%d", vfIndex), PFIndex: 0, @@ -415,7 +415,7 @@ func setupHBNOnlyTest(ctx context.Context, input *systemTestInput, vfIndex int) }, { Name: fmt.Sprintf("pf1vf%d-rep", vfIndex), - Namespace: input.namespace, + Namespace: input.Namespace, Type: "vf", InterfaceName: fmt.Sprintf("pf1vf%d", vfIndex), PFIndex: 1, @@ -426,7 +426,7 @@ func setupHBNOnlyTest(ctx context.Context, input *systemTestInput, vfIndex int) }, { Name: fmt.Sprintf("pf0vf%d-sf", vfIndex), - Namespace: input.namespace, + Namespace: input.Namespace, Type: "sf", InterfaceName: fmt.Sprintf("pf0vf%d_if", vfIndex), ServiceID: hbnServiceID, @@ -438,7 +438,7 @@ func setupHBNOnlyTest(ctx context.Context, input *systemTestInput, vfIndex int) }, { Name: fmt.Sprintf("pf1vf%d-sf", vfIndex), - Namespace: input.namespace, + Namespace: input.Namespace, Type: "sf", InterfaceName: fmt.Sprintf("pf1vf%d_if", vfIndex), ServiceID: hbnServiceID, @@ -450,7 +450,7 @@ func setupHBNOnlyTest(ctx context.Context, input *systemTestInput, vfIndex int) }, { Name: "p0-sf", - Namespace: input.namespace, + Namespace: input.Namespace, Type: "sf", InterfaceName: "p0_if", ServiceID: hbnServiceID, @@ -462,7 +462,7 @@ func setupHBNOnlyTest(ctx context.Context, input *systemTestInput, vfIndex int) }, { Name: "p1-sf", - Namespace: input.namespace, + Namespace: input.Namespace, Type: "sf", InterfaceName: "p1_if", ServiceID: hbnServiceID, @@ -499,24 +499,24 @@ func setupHBNOnlyTest(ctx context.Context, input *systemTestInput, vfIndex int) } By("Wait for prerequisite services") - dpuservice.WaitForDPUServices(ctx, input.client, input.namespace, []string{"sfc-controller"}) + dpuservice.WaitForDPUServices(ctx, input.Client, input.Namespace, []string{"sfc-controller"}) By("Create and wait for DPU service interfaces") - createAndWaitForInterfaces(ctx, input.client, input.dpuServiceInterfaceTemplate, interfaceConfigs) + createAndWaitForInterfaces(ctx, input.Client, input.DPUServiceInterfaceTemplate, interfaceConfigs) By("Create HBN only service chains") - createHBNServiceChains(ctx, input.client, input.namespace, vfIndex, input.dpuServiceChainTemplate) + createHBNServiceChains(ctx, input.Client, input.Namespace, vfIndex, input.DPUServiceChainTemplate) By("Create HBN IPAMs") - createHBNIPAMs(ctx, input.client, input.namespace, input.dpuServiceIPAMTemplate, ipamConfigs) + createHBNIPAMs(ctx, input.Client, input.Namespace, input.DPUServiceIPAMTemplate, ipamConfigs) By("Create and wait for HBN service") - dpuNode1, dpuNode2 := getDPUNodesInOrder(ctx, input.client, dpuClusterClient[0]) - createHBNService(ctx, input.client, dpuNode1.Name, dpuNode2.Name, input.namespace, input.dpuServiceHBN) - dpuservice.WaitForDPUServices(ctx, input.client, input.namespace, []string{"doca-hbn"}) + dpuNode1, dpuNode2 := getDPUNodesInOrder(ctx, input.Client, DPUClusterClient[0]) + createHBNService(ctx, input.Client, dpuNode1.Name, dpuNode2.Name, input.Namespace, input.DPUServiceHBN) + dpuservice.WaitForDPUServices(ctx, input.Client, input.Namespace, []string{"doca-hbn"}) By("Verify underlying ServiceChain and ServiceInterface objects are ready") - dpuservice.VerifyUnderlyingDPUObjectsReady(ctx, dpuClusterClient[0], input.namespace, interfaceConfigs, []string{"hbn-to-fabric", "host-to-hbn"}) + dpuservice.VerifyUnderlyingDPUObjectsReady(ctx, DPUClusterClient[0], input.Namespace, interfaceConfigs, []string{"hbn-to-fabric", "host-to-hbn"}) } // createHBNService deploys the HBN service @@ -554,17 +554,17 @@ func createHBNService(ctx context.Context, testClient client.Client, node1InDPUC func deleteFirstFoundPodOnDpuCluster(ctx context.Context, podSubstrNameToDelete string, namespace string) { pods := &corev1.PodList{} deletedPodName := "" - Expect(dpuClusterClient[0].List(ctx, pods, client.InNamespace(namespace))).To(Succeed()) + Expect(DPUClusterClient[0].List(ctx, pods, client.InNamespace(namespace))).To(Succeed()) for _, pod := range pods.Items { if strings.Contains(pod.Name, podSubstrNameToDelete) { deletedPodName = pod.Name - Expect(dpuClusterClient[0].Delete(ctx, &pod)).To(Succeed()) + Expect(DPUClusterClient[0].Delete(ctx, &pod)).To(Succeed()) break } } Expect(deletedPodName).NotTo(BeEmpty()) Eventually(func(g Gomega) { - g.Expect(dpuClusterClient[0].Get(ctx, client.ObjectKey{Namespace: namespace, Name: deletedPodName}, &corev1.Pod{})).To(MatchError(ContainSubstring("not found"))) + g.Expect(DPUClusterClient[0].Get(ctx, client.ObjectKey{Namespace: namespace, Name: deletedPodName}, &corev1.Pod{})).To(MatchError(ContainSubstring("not found"))) }, 10*time.Minute).Should(Succeed()) } @@ -612,7 +612,7 @@ func createHBNServiceChains(ctx context.Context, client client.Client, namespace // createHBNIPAMs creates the IPAM configurations for HBN func createHBNIPAMs(ctx context.Context, client client.Client, namespace string, dpuServiceIPAMTemplate *dpuservicev1.DPUServiceIPAM, IPAMConfigs []dpuservice.TestIPAMConfig) { - dpuNode1, dpuNode2 := getDPUNodesInOrder(ctx, client, dpuClusterClient[0]) + dpuNode1, dpuNode2 := getDPUNodesInOrder(ctx, client, DPUClusterClient[0]) for _, config := range IPAMConfigs { DPUServiceIPAM := utils.GenerateDPUObj(config.Name, namespace, dpuServiceIPAMTemplate.DeepCopy()) dpuservice.SetDPUServiceHBNIPAM(DPUServiceIPAM, config, dpuNode1.Name, dpuNode2.Name) @@ -620,16 +620,16 @@ func createHBNIPAMs(ctx context.Context, client client.Client, namespace string, } } -func setupMTUServiceFunctionChain(ctx context.Context, input *systemTestInput, mtu int, cfg *mtuTestConfig) { +func setupMTUServiceFunctionChain(ctx context.Context, input *SystemTestInput, mtu int, cfg *mtuTestConfig) { By("Wait for prerequisite services") - dpuservice.WaitForDPUServices(ctx, input.client, input.namespace, []string{"sfc-controller"}) + dpuservice.WaitForDPUServices(ctx, input.Client, input.Namespace, []string{"sfc-controller"}) By("Create DPU service interfaces for service function") interfaceConfigs := []dpuservice.TestDPUServiceInterfaceConfig{ { Name: cfg.physicalInterfacePrefix, Type: "physical", - Namespace: input.namespace, + Namespace: input.Namespace, InterfaceName: cfg.physicalInterfacePrefix, Labels: map[string]string{ "uplink": cfg.physicalInterfacePrefix, @@ -641,7 +641,7 @@ func setupMTUServiceFunctionChain(ctx context.Context, input *systemTestInput, m interfaceConfigs = append(interfaceConfigs, dpuservice.TestDPUServiceInterfaceConfig{ Name: fmt.Sprintf("%s-%s", cfg.physicalInterfacePrefix+"-sf", serviceID), Type: "sf", - Namespace: input.namespace, + Namespace: input.Namespace, InterfaceName: cfg.ipInterface, Network: cfg.nadName, ServiceID: serviceID, @@ -653,10 +653,10 @@ func setupMTUServiceFunctionChain(ctx context.Context, input *systemTestInput, m } By("Create and wait for DPU service interfaces") - createAndWaitForInterfaces(ctx, input.client, input.dpuServiceInterfaceTemplate, interfaceConfigs) + createAndWaitForInterfaces(ctx, input.Client, input.DPUServiceInterfaceTemplate, interfaceConfigs) By("Create IPAM for service function") - dpuServiceIPAM := utils.GenerateDPUObj(cfg.subnetPoolName, input.namespace, input.dpuServiceIPAMTemplate.DeepCopy()) + dpuServiceIPAM := utils.GenerateDPUObj(cfg.subnetPoolName, input.Namespace, input.DPUServiceIPAMTemplate.DeepCopy()) dpuServiceIPAM.Spec.IPV4Subnet = &dpuservicev1.IPV4Subnet{ Subnet: "10.44.44.0/24", Gateway: "10.44.44.1", @@ -665,22 +665,22 @@ func setupMTUServiceFunctionChain(ctx context.Context, input *systemTestInput, m dpuServiceIPAM.Spec.ObjectMeta.Labels = map[string]string{ "svc.dpu.nvidia.com/pool": cfg.subnetPoolName, } - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) By("Create DPUServiceNAD for automatic resource injection") - dpuServiceNAD := utils.GenerateDPUObj(cfg.nadName, input.namespace, input.dpuServiceNAD.DeepCopy()) + dpuServiceNAD := utils.GenerateDPUObj(cfg.nadName, input.Namespace, input.DPUServiceNAD.DeepCopy()) dpuServiceNAD.Spec.ServiceMTU = mtu - Expect(input.client.Create(ctx, dpuServiceNAD)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceNAD)).To(Succeed()) By("Create netshoot DPU services") for _, serviceID := range []string{cfg.service1ID, cfg.service2ID} { - dpuService := utils.GenerateDPUObj(serviceID, input.namespace, input.dpuService.DeepCopy()) + dpuService := utils.GenerateDPUObj(serviceID, input.Namespace, input.DPUService.DeepCopy()) configureNetshootDPUService(dpuService, serviceID) - Expect(input.client.Create(ctx, dpuService)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuService)).To(Succeed()) } By("Create DPU service chain for service function") - dpuServiceChain := utils.GenerateDPUObj(cfg.chainName, input.namespace, input.dpuServiceChainTemplate.DeepCopy()) + dpuServiceChain := utils.GenerateDPUObj(cfg.chainName, input.Namespace, input.DPUServiceChainTemplate.DeepCopy()) dpuServiceChain.Spec.Template.Spec.Template.Spec.Switches = []dpuservicev1.Switch{ { Ports: []dpuservicev1.Port{ @@ -721,10 +721,10 @@ func setupMTUServiceFunctionChain(ctx context.Context, input *systemTestInput, m ServiceMTU: ptr.To(mtu), }, } - Expect(input.client.Create(ctx, dpuServiceChain)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceChain)).To(Succeed()) By("Verify underlying DPU objects are ready") - dpuservice.VerifyUnderlyingDPUObjectsReady(ctx, dpuClusterClient[0], input.namespace, interfaceConfigs, []string{cfg.chainName}) + dpuservice.VerifyUnderlyingDPUObjectsReady(ctx, DPUClusterClient[0], input.Namespace, interfaceConfigs, []string{cfg.chainName}) } // configureNetshootDPUService configures a DPUService for netshoot using the dummydpuservice chart @@ -740,7 +740,7 @@ func configureNetshootDPUService(dpuService *dpuservicev1.DPUService, serviceID } values := make(map[string]any) - values["imagePullSecrets"] = []map[string]string{{"name": dpfPullSecretName}} + values["imagePullSecrets"] = []map[string]string{{"name": DPFPullSecretName}} values["image"] = map[string]string{"repository": netutilsImage} rawValues, err := json.Marshal(values) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/snap.go b/test/e2e/snap.go new file mode 100644 index 00000000..7115fb20 --- /dev/null +++ b/test/e2e/snap.go @@ -0,0 +1,25 @@ +/* +Copyright 2025 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" +) + +func SNAPBeforeSuite() { + By("Setting SNAP configs for the test") +} diff --git a/test/e2e/snap_test.go b/test/e2e/snap_test.go index 12e99bcf..3befc829 100644 --- a/test/e2e/snap_test.go +++ b/test/e2e/snap_test.go @@ -20,10 +20,6 @@ import ( . "github.com/onsi/ginkgo/v2" ) -func SNAPBeforeSuite() { - By("Setting SNAP configs for the test") -} - //nolint:dupl var _ = Describe("DPF SNAP tests", Labels{Domain.DPFSystem, Domain.SNAP}, func() { diff --git a/test/e2e/system_bootstrap.go b/test/e2e/system_bootstrap.go new file mode 100644 index 00000000..4da3787f --- /dev/null +++ b/test/e2e/system_bootstrap.go @@ -0,0 +1,374 @@ +/* +Copyright 2025 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "maps" + "net/url" + "strconv" + "time" + + operatorv1 "github.com/nvidia/doca-platform/api/operator/v1alpha1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func validateFlags() { + if !IsGinkgoLabelApplied(Domain.ZeroTrust) { + return + } + + if Conf.NodeRebootConfigMap == "" { + panic("ZeroTrust requires `nodeRebootConfigMap` to be set in the e2e config file") + } + if Conf.NodeRebootConfigMapPath == "" { + panic("ZeroTrust requires `nodeRebootConfigMapPath` to be set in the e2e config file") + } + if bmcPassword == "" { + panic("ZeroTrust requires E2E_ZT_BMC_PASSWORD env var (BMC root password used by the in-cluster reboot script)") + } + if bmcInventoryPath == "" { + panic("ZeroTrust requires E2E_ZT_BMC_INVENTORY_PATH env var (path to the lab DPU-serial -> BMC IP inventory YAML)") + } + + if IsGinkgoLabelApplied(Domain.ExternalTest) { + if len(externalTest) == 0 { + panic("This script must be provided when External label is present") + } + } +} + +func SetInput() *SystemTestInput { + By("Validating the input") + validateFlags() + + By("Get control plane IP") + controlPlaneIP := getClusterControlPlaneIP(Ctx, TestClient) + + By("Setting operatorConfig for the test") + var bfbPVCName *string + if Conf.ProvisioningControllerPVCPath != nil { + bfbPVCName = ptr.To("bfb-pvc") + } + dpfOperatorConfig := &operatorv1.DPFOperatorConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: ConfigName, + Namespace: DPFOperatorSystemNamespace, + Labels: CleanupScope.Suite, + }, + Spec: operatorv1.DPFOperatorConfigSpec{ + DeploymentMode: operatorv1.DeploymentModeHostTrusted, + ProvisioningController: &operatorv1.ProvisioningControllerConfiguration{ + BFBPersistentVolumeClaimName: bfbPVCName, + }, + StaticClusterManager: &operatorv1.StaticClusterManagerConfiguration{ + BaseComponentConfig: operatorv1.BaseComponentConfig{ + Disable: ptr.To(false), + }, + }, + // Disable the Kamaji cluster manager so only one cluster manager is running. + // TODO: Enable Kamaji by default in the e2e tests. + KamajiClusterManager: &operatorv1.KamajiClusterManagerConfiguration{ + BaseComponentConfig: operatorv1.BaseComponentConfig{ + Disable: ptr.To(false), + }, + }, + Monitoring: &operatorv1.MonitoringConfiguration{ + Disable: ptr.To(false), + OpenTelemetryCollector: &operatorv1.OpenTelemetryCollectorConfiguration{ + Logging: &operatorv1.OpenTelemetryCollectorLoggingConfiguration{ + Endpoint: fmt.Sprintf("%s%s:%d", otelEndpointSchema, controlPlaneIP, otelNodePort), + }, + }, + }, + NodeSRIOVDevicePluginController: &operatorv1.NodeSRIOVDevicePluginControllerConfiguration{ + BaseComponentConfig: operatorv1.BaseComponentConfig{ + Disable: ptr.To(false), + }, + }, + KataContainers: &operatorv1.KataContainersConfiguration{ + BaseComponentConfig: operatorv1.BaseComponentConfig{ + Disable: ptr.To(false), + }, + }, + ImagePullSecrets: []string{DPFPullSecretName, "pull-secret-extra"}, + }, + } + if IsGinkgoLabelApplied(Domain.ZeroTrust) { + dpfOperatorConfig.Spec.DeploymentMode = operatorv1.DeploymentModeZeroTrust + dpfOperatorConfig.Spec.StaticClusterManager.BaseComponentConfig.Disable = ptr.To(true) + dpfOperatorConfig.Spec.KamajiClusterManager.BaseComponentConfig.Disable = ptr.To(false) + dpfOperatorConfig.Spec.NodeSRIOVDevicePluginController.BaseComponentConfig.Disable = ptr.To(true) + dpfOperatorConfig.Spec.ProvisioningController.InstallInterface = &operatorv1.ProvisioningInstallInterface{ + InstallViaRedfish: &operatorv1.InstallViaRedfish{ + SkipDPUNodeDiscovery: ptr.To(false), + }, + } + dpfOperatorConfig.Spec.DPUDetector = &operatorv1.DPUDetectorConfiguration{ + BaseComponentConfig: operatorv1.BaseComponentConfig{ + Disable: ptr.To(true), + }, + } + apiServerPort := 443 + if u, err := url.Parse(RestConfig.Host); err == nil { + if p := u.Port(); p != "" { + if parsed, err := strconv.Atoi(p); err == nil { + apiServerPort = parsed + } + } + } + By(fmt.Sprintf("Using API server VIP %s:%d for zero-trust kubeconfig", controlPlaneIP, apiServerPort)) + if dpfOperatorConfig.Spec.Overrides == nil { + dpfOperatorConfig.Spec.Overrides = &operatorv1.Overrides{} + } + dpfOperatorConfig.Spec.Overrides.KubernetesAPIServerVIP = ptr.To(controlPlaneIP) + dpfOperatorConfig.Spec.Overrides.KubernetesAPIServerPort = ptr.To(apiServerPort) + } + + if IsGinkgoLabelApplied(Domain.Scale) { + // For scale environments, the nodes are fake, therefore we can't have DPUDetector running + dpfOperatorConfig.Spec.DPUDetector = &operatorv1.DPUDetectorConfiguration{ + BaseComponentConfig: operatorv1.BaseComponentConfig{ + Disable: ptr.To(true), + }, + } + } + + // CI runs the host control-plane controllers at a single replica to save + // resources on control-plane nodes and keep logs easy to read. + if dpfOperatorConfig.Spec.DPUServiceController == nil { + dpfOperatorConfig.Spec.DPUServiceController = &operatorv1.DPUServiceControllerConfiguration{} + } + dpfOperatorConfig.Spec.ProvisioningController.Replicas = ptr.To[int32](1) + dpfOperatorConfig.Spec.DPUServiceController.Replicas = ptr.To[int32](1) + dpfOperatorConfig.Spec.KamajiClusterManager.Replicas = ptr.To[int32](1) + dpfOperatorConfig.Spec.StaticClusterManager.Replicas = ptr.To[int32](1) + dpfOperatorConfig.Spec.NodeSRIOVDevicePluginController.Replicas = ptr.To[int32](1) + + if prereqsNamespace != "" { + if dpfOperatorConfig.Spec.Overrides == nil { + dpfOperatorConfig.Spec.Overrides = &operatorv1.Overrides{} + } + + dpfOperatorConfig.Spec.Overrides.ArgoCDNamespace = ptr.To(prereqsNamespace) + } + + if IsGinkgoLabelApplied(Domain.Performance) { + apiServerHost := controlPlaneIP + apiServerPort := DefaultAPIServerPort + if targetClusterAPIServerHost != "" { + apiServerHost = targetClusterAPIServerHost + } else if u, err := url.Parse(RestConfig.Host); err == nil { + if h := u.Hostname(); h != "" { + apiServerHost = h + } + if p := u.Port(); p != "" { + if parsed, err := strconv.Atoi(p); err == nil { + apiServerPort = parsed + } + } + } + if dpfOperatorConfig.Spec.Overrides == nil { + dpfOperatorConfig.Spec.Overrides = &operatorv1.Overrides{} + } + dpfOperatorConfig.Spec.Overrides.KubernetesAPIServerVIP = ptr.To(apiServerHost) + dpfOperatorConfig.Spec.Overrides.KubernetesAPIServerPort = ptr.To(apiServerPort) + dpfOperatorConfig.Spec.ProvisioningController.DMSTimeout = ptr.To(15 * 60) + dpfOperatorConfig.Spec.Networking = &operatorv1.Networking{ + ControlPlaneMTU: ptr.To(PerformanceMTU), + HighSpeedMTU: ptr.To(PerformanceMTU), + } + } + + input = &SystemTestInput{ + Namespace: DPFOperatorSystemNamespace, + Config: dpfOperatorConfig, + PullSecretNames: dpfOperatorConfig.Spec.ImagePullSecrets, + Client: TestClient, + RestConfig: RestConfig, + CleanupFlags: CleanupFlags, + BFBImageURL: bfbImageURL, + BFSOsIsoURL: bfsOsIsoURL, + BFSPldmFwBundleURL: bfsPldmFwBundleURL, + } + input.ApplyConfig(*Conf) + return input +} + +// SystemSetupBeforeSuite sets up the system components for the e2e tests. +// If skipSystemComponentValidation is true, it skips the validation of system components after deployment. +func SystemSetupBeforeSuite(skipSystemComponentValidation bool) { + if Label(Domain.Scale).MatchesLabelFilter(GinkgoLabelFilter()) { + CreateDPUWorkerNodes(Ctx, input.NumberOfDPUNodes) + } + + AnnotateAndLabelNodes(Ctx, input.Client, input.UseExternalNodeReboot) + + if ngcAPIKey != "" { + createNGCImagePullSecret(Ctx, input.Client) + } + + By("Deploy DPF System components") + DeployDPFSystemComponents(Ctx, DeployDPFSystemComponentsInput{ + SystemNamespace: input.Namespace, + OperatorConfig: input.Config, + ImagePullSecrets: input.PullSecretNames, + ProvisioningControllerPVC: input.PVC, + DPUDiscovery: input.DPUDiscovery, + Client: input.Client, + NumberOfDPUNodes: input.NumberOfDPUNodes, + SkipSystemComponentValidation: skipSystemComponentValidation, + }) + + if IsGinkgoLabelApplied(Domain.ZeroTrust) { + // In ZeroTrust mode, build a DPUNode-to-host BMC IP map from the lab inventory file + // for the script-based reboot path (nodeRebootMethod.script). + input.DPUNodeBMCs = GetDPUNodeToBMCIPs( + Ctx, input.Client, input.NumberOfDPUNodes) + + // Ensure ConfigMap and DPUNode BMC IP labels are set ahead of any DPU reaching the reboot state, + // so the controller can drive in-cluster Redfish reboots through the named ConfigMap. + ApplyNodeRebootConfigMap(Ctx, input.Client, input.NodeRebootConfigMapPath) + PatchDPUNodesForScriptReboot(Ctx, input.Client, input.NumberOfDPUNodes, + input.NodeRebootConfigMap, input.DPUNodeBMCs) + } + + if IsGinkgoLabelApplied(Domain.Performance) { + vip := *input.Config.Spec.Overrides.KubernetesAPIServerVIP + port := *input.Config.Spec.Overrides.KubernetesAPIServerPort + PatchNFDWorkerForVIP(Ctx, input.Client, input.Namespace, vip, port) + } +} + +// createNGCImagePullSecret creates a secret to be able to pull images from NGC, this secret can be used by DPUservices and should not be used for core components. +func createNGCImagePullSecret(ctx context.Context, testClient client.Client) { + // Docker registry credentials + registry := "nvcr.io" + username := "$oauthtoken" + password := ngcAPIKey + + // Create the auth string + auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password))) + + // Build the config.json structure + dockerConfig := map[string]interface{}{ + "auths": map[string]interface{}{ + registry: map[string]string{ + "auth": auth, + }, + }, + } + + dockerConfigJSON, err := json.Marshal(dockerConfig) + Expect(err).ToNot(HaveOccurred()) + + labels := maps.Clone(CleanupScope.Suite) + labels["dpu.nvidia.com/image-pull-secret"] = "" + + // Create the Secret object + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: NGCPullSecretName, + Namespace: DPFOperatorSystemNamespace, + Labels: labels, + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + ".dockerconfigjson": dockerConfigJSON, + }, + } + + // Create the secret + Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, secret))).NotTo(HaveOccurred()) +} + +// AnnotateAndLabelNodes stamps host-cluster Nodes with reboot-related labels +// consumed by the host agent. When useExternalNodeReboot is true (NIC cloud +// e2e tests), the labels make the host agent delegate host reboots to lab +// infrastructure (e.g. NIC cloud's `nic-cloud-reset`). Independent from +// ZeroTrust's in-cluster script reboot path (`nodeRebootMethod.script` set +// per-DPUNode by the e2e suite). +func AnnotateAndLabelNodes(ctx context.Context, c client.Client, useExternalNodeReboot bool) { + nodeAnnotations := make(map[string]string) + nodeLabels := make(map[string]string) + + if useExternalNodeReboot { + nodeLabels["provisioning.dpu.nvidia.com/reboot-method"] = "external" + nodeLabels["provisioning.dpu.nvidia.com/dpu-reboot-after-install"] = "" + } + + if len(nodeAnnotations) == 0 && len(nodeLabels) == 0 { + return + } + + By("Annotate and Label nodes in the main cluster") + Eventually(func(g Gomega) { + nodes := &corev1.NodeList{} + g.Expect(c.List(ctx, nodes)).To(Succeed()) + for _, node := range nodes.Items { + original := node.DeepCopy() + annotations := node.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + for k, v := range nodeAnnotations { + annotations[k] = v + } + node.SetAnnotations(annotations) + + labels := node.GetLabels() + if labels == nil { + labels = map[string]string{} + } + for k, v := range nodeLabels { + labels[k] = v + } + node.SetLabels(labels) + + g.Expect(c.Patch(ctx, &node, client.MergeFrom(original))).To(Succeed()) + } + }).WithTimeout(10 * time.Second).Should(Succeed()) +} + +func GetProvisionDPUClustersInput() ProvisionDPUClustersInput { + return ProvisionDPUClustersInput{ + NumberOfDPUNodes: input.NumberOfDPUNodes, + NumberOfDPUsPerNode: input.NumberOfDPUsPerNode, + DPUClusterPrerequisites: input.DPUClusterPrerequisites, + DPUClusters: input.DPUClusters, + DPUSet: input.DPUSet, + BFB: input.BFB, + BlueFieldSoftware: input.BlueFieldSoftware, + DPUFlavor: input.DPUFlavor, + Client: input.Client, + BFBImageURL: input.BFBImageURL, + BFSOsIsoURL: input.BFSOsIsoURL, + BFSPldmFwBundleURL: input.BFSPldmFwBundleURL, + RestConfig: RestConfig, + NodeRebootConfigMap: input.NodeRebootConfigMap, + DPUNodeBMCs: input.DPUNodeBMCs, + } +} diff --git a/test/e2e/system_setup.go b/test/e2e/system_setup.go index fad01040..dfd3c189 100644 --- a/test/e2e/system_setup.go +++ b/test/e2e/system_setup.go @@ -60,22 +60,22 @@ import ( ) type ProvisionDPUClustersInput struct { - numberOfDPUNodes int - numberOfDPUsPerNode int - dpuClusterPrerequisites []client.Object - dpuClusters []*provisioningv1.DPUCluster - dpuFlavor *provisioningv1.DPUFlavor - bfb *provisioningv1.BFB - blueFieldSoftware *provisioningv1.BlueFieldSoftware - dpuSet *provisioningv1.DPUSet - client client.Client - bfbImageURL string - bfsOsIsoURL string - bfsPldmFwBundleURL string - restConfig *rest.Config + NumberOfDPUNodes int + NumberOfDPUsPerNode int + DPUClusterPrerequisites []client.Object + DPUClusters []*provisioningv1.DPUCluster + DPUFlavor *provisioningv1.DPUFlavor + BFB *provisioningv1.BFB + BlueFieldSoftware *provisioningv1.BlueFieldSoftware + DPUSet *provisioningv1.DPUSet + Client client.Client + BFBImageURL string + BFSOsIsoURL string + BFSPldmFwBundleURL string + RestConfig *rest.Config NodeRebootConfigMap string DPUNodeBMCs map[string]string - expectedKubernetesVersion string + ExpectedKubernetesVersion string } func isPreUpgradeFromLastReleasedGA(ctx context.Context, kclient client.Client, objectKey client.ObjectKey) (bool, error) { @@ -89,7 +89,7 @@ func isPreUpgradeFromLastReleasedGA(ctx context.Context, kclient client.Client, return operatorutils.IsUpgradeFromLastReleasedGA(*dpfOperatorConfig.Status.Version), nil } -// systemTestInput represents the fully loaded and processed test environment. +// SystemTestInput represents the fully loaded and processed test environment. // This struct contains actual Kubernetes API objects and runtime configuration // that are ready for use in end-to-end tests. // @@ -99,63 +99,63 @@ func isPreUpgradeFromLastReleasedGA(ctx context.Context, kclient client.Client, // - Passed to individual test functions as the primary test context // - Provides all necessary objects and configuration for test execution // - Depends on `config` struct for file paths and basic configuration -type systemTestInput struct { - namespace string - config *operatorv1.DPFOperatorConfig - pvc *corev1.PersistentVolumeClaim - dpuClusterPrerequisites []client.Object - dpuClusters []*provisioningv1.DPUCluster - dpuFlavor *provisioningv1.DPUFlavor - dpuDiscovery *provisioningv1.DPUDiscovery - dpuService *dpuservicev1.DPUService - dpuServiceHBN *dpuservicev1.DPUService - dpuServiceInterface *dpuservicev1.DPUServiceInterface - dpuServiceInterfaceTemplate *dpuservicev1.DPUServiceInterface - dpuServiceChain *dpuservicev1.DPUServiceChain - dpuServiceChainTemplate *dpuservicev1.DPUServiceChain - bfb *provisioningv1.BFB - blueFieldSoftware *provisioningv1.BlueFieldSoftware - dpuSet *provisioningv1.DPUSet - bfsOsIsoURL string - bfsPldmFwBundleURL string - dpuDeployment *dpuservicev1.DPUDeployment - dpuServiceConfiguration *dpuservicev1.DPUServiceConfiguration - dpuServiceInterfacesHBN []*dpuservicev1.DPUServiceInterface - dpuServiceInterfaceOVN *dpuservicev1.DPUServiceInterface - dpuServiceTemplate *dpuservicev1.DPUServiceTemplate - additionalDPUServiceTemplate *dpuservicev1.DPUServiceTemplate - dpuServiceTemplateOVN *dpuservicev1.DPUServiceTemplate - dpuServiceTemplateHBN *dpuservicev1.DPUServiceTemplate - dpuServiceConfigurationOVN *dpuservicev1.DPUServiceConfiguration - dpuServiceConfigurationHBN *dpuservicev1.DPUServiceConfiguration - additionalDPUServiceConfiguration *dpuservicev1.DPUServiceConfiguration - dpuServiceIPAMTemplate *dpuservicev1.DPUServiceIPAM - dpuServiceNAD *dpuservicev1.DPUServiceNAD - cidrDPUServiceIPAM *dpuservicev1.DPUServiceIPAM - ipPoolDPUServiceIPAM *dpuservicev1.DPUServiceIPAM - dpuServiceCredentialRequest *dpuservicev1.DPUServiceCredentialRequest - ovnCredentialRequest *dpuservicev1.DPUServiceCredentialRequest - numberOfDPUNodes int - numberOfDPUsPerNode int - pullSecretNames []string - client client.Client - cleanupFlags *cleanup.CleanupFlags - bfbImageURL string - restConfig *rest.Config - nodeRebootConfigMap string - nodeRebootConfigMapPath string - useExternalNodeReboot bool - dpuNodeBMCs map[string]string +type SystemTestInput struct { + Namespace string + Config *operatorv1.DPFOperatorConfig + PVC *corev1.PersistentVolumeClaim + DPUClusterPrerequisites []client.Object + DPUClusters []*provisioningv1.DPUCluster + DPUFlavor *provisioningv1.DPUFlavor + DPUDiscovery *provisioningv1.DPUDiscovery + DPUService *dpuservicev1.DPUService + DPUServiceHBN *dpuservicev1.DPUService + DPUServiceInterface *dpuservicev1.DPUServiceInterface + DPUServiceInterfaceTemplate *dpuservicev1.DPUServiceInterface + DPUServiceChain *dpuservicev1.DPUServiceChain + DPUServiceChainTemplate *dpuservicev1.DPUServiceChain + BFB *provisioningv1.BFB + BlueFieldSoftware *provisioningv1.BlueFieldSoftware + DPUSet *provisioningv1.DPUSet + BFSOsIsoURL string + BFSPldmFwBundleURL string + DPUDeployment *dpuservicev1.DPUDeployment + DPUServiceConfiguration *dpuservicev1.DPUServiceConfiguration + DPUServiceInterfacesHBN []*dpuservicev1.DPUServiceInterface + DPUServiceInterfaceOVN *dpuservicev1.DPUServiceInterface + DPUServiceTemplate *dpuservicev1.DPUServiceTemplate + AdditionalDPUServiceTemplate *dpuservicev1.DPUServiceTemplate + DPUServiceTemplateOVN *dpuservicev1.DPUServiceTemplate + DPUServiceTemplateHBN *dpuservicev1.DPUServiceTemplate + DPUServiceConfigurationOVN *dpuservicev1.DPUServiceConfiguration + DPUServiceConfigurationHBN *dpuservicev1.DPUServiceConfiguration + AdditionalDPUServiceConfiguration *dpuservicev1.DPUServiceConfiguration + DPUServiceIPAMTemplate *dpuservicev1.DPUServiceIPAM + DPUServiceNAD *dpuservicev1.DPUServiceNAD + CIDRDPUServiceIPAM *dpuservicev1.DPUServiceIPAM + IPPoolDPUServiceIPAM *dpuservicev1.DPUServiceIPAM + DPUServiceCredentialRequest *dpuservicev1.DPUServiceCredentialRequest + OVNCredentialRequest *dpuservicev1.DPUServiceCredentialRequest + NumberOfDPUNodes int + NumberOfDPUsPerNode int + PullSecretNames []string + Client client.Client + CleanupFlags *cleanup.CleanupFlags + BFBImageURL string + RestConfig *rest.Config + NodeRebootConfigMap string + NodeRebootConfigMapPath string + UseExternalNodeReboot bool + DPUNodeBMCs map[string]string } -func (t *systemTestInput) applySDNConfig(conf config) { +func (t *SystemTestInput) ApplySDNConfig(conf Config) { dpuServiceInterfaceTemplate := &dpuservicev1.DPUServiceInterface{} - dsiTemplate := unstructuredFromFile(conf.DPUServiceInterfaceTemplatePath) + dsiTemplate := UnstructuredFromFile(conf.DPUServiceInterfaceTemplatePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(dsiTemplate.Object, dpuServiceInterfaceTemplate)).To(Succeed()) - t.dpuServiceInterfaceTemplate = dpuServiceInterfaceTemplate + t.DPUServiceInterfaceTemplate = dpuServiceInterfaceTemplate dpuServiceHBN := &dpuservicev1.DPUService{} - svcHBN := unstructuredFromFile(conf.DPUServiceHBNPath) + svcHBN := UnstructuredFromFile(conf.DPUServiceHBNPath) // Override HBN image if HBN_IMAGE_URL is set if hbnImageURL != "" { @@ -172,26 +172,26 @@ func (t *systemTestInput) applySDNConfig(conf config) { } if ngcAPIKey != "" { - updateImagePullSecret(svcHBN, ngcPullSecretName) + updateImagePullSecret(svcHBN, NGCPullSecretName) } Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(svcHBN.Object, dpuServiceHBN)).To(Succeed()) - t.dpuServiceHBN = dpuServiceHBN + t.DPUServiceHBN = dpuServiceHBN dpuServiceIPAMTemplate := &dpuservicev1.DPUServiceIPAM{} - ipam := unstructuredFromFile(conf.DPUServiceIPAMTemplatePath) + ipam := UnstructuredFromFile(conf.DPUServiceIPAMTemplatePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(ipam.Object, dpuServiceIPAMTemplate)).To(Succeed()) - t.dpuServiceIPAMTemplate = dpuServiceIPAMTemplate + t.DPUServiceIPAMTemplate = dpuServiceIPAMTemplate dpuServiceNAD := &dpuservicev1.DPUServiceNAD{} - nad := unstructuredFromFile(conf.DPUServiceNADPath) + nad := UnstructuredFromFile(conf.DPUServiceNADPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(nad.Object, dpuServiceNAD)).To(Succeed()) - t.dpuServiceNAD = dpuServiceNAD + t.DPUServiceNAD = dpuServiceNAD dpuServiceChainTemplate := &dpuservicev1.DPUServiceChain{} - chainTemplate := unstructuredFromFile(conf.DPUServiceChainTemplatePath) + chainTemplate := UnstructuredFromFile(conf.DPUServiceChainTemplatePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(chainTemplate.Object, dpuServiceChainTemplate)).To(Succeed()) - t.dpuServiceChainTemplate = dpuServiceChainTemplate + t.DPUServiceChainTemplate = dpuServiceChainTemplate } func updateHBNImage(svcHBN *unstructured.Unstructured, repository, tag string) { @@ -229,110 +229,110 @@ func updateImagePullSecret(svc *unstructured.Unstructured, secretName string) { Expect(err).ToNot(HaveOccurred()) } -func (t *systemTestInput) applyConfig(conf config) { +func (t *SystemTestInput) ApplyConfig(conf Config) { if conf.BFBPath != nil { bfb := &provisioningv1.BFB{} - bfbUnstructured := unstructuredFromFile(*conf.BFBPath) + bfbUnstructured := UnstructuredFromFile(*conf.BFBPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(bfbUnstructured.Object, bfb)).To(Succeed()) - t.bfb = bfb + t.BFB = bfb } if conf.BlueFieldSoftwarePath != nil { blueFieldSoftware := &provisioningv1.BlueFieldSoftware{} - bfsUnstructured := unstructuredFromFile(*conf.BlueFieldSoftwarePath) + bfsUnstructured := UnstructuredFromFile(*conf.BlueFieldSoftwarePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(bfsUnstructured.Object, blueFieldSoftware)).To(Succeed()) - t.blueFieldSoftware = blueFieldSoftware + t.BlueFieldSoftware = blueFieldSoftware } dpuSet := &provisioningv1.DPUSet{} - dpuSetUnstructured := unstructuredFromFile(conf.DPUSetPath) + dpuSetUnstructured := UnstructuredFromFile(conf.DPUSetPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(dpuSetUnstructured.Object, dpuSet)).To(Succeed()) - t.dpuSet = dpuSet + t.DPUSet = dpuSet pvc := &corev1.PersistentVolumeClaim{} if conf.ProvisioningControllerPVCPath != nil { - pvcUnstructured := unstructuredFromFile(*conf.ProvisioningControllerPVCPath) + pvcUnstructured := UnstructuredFromFile(*conf.ProvisioningControllerPVCPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(pvcUnstructured.Object, pvc)).To(Succeed()) - t.pvc = pvc + t.PVC = pvc } // Load all DPU clusters - t.dpuClusters = make([]*provisioningv1.DPUCluster, 0, len(conf.DPUClusterPaths)) + t.DPUClusters = make([]*provisioningv1.DPUCluster, 0, len(conf.DPUClusterPaths)) for _, dpuClusterPath := range conf.DPUClusterPaths { dpuCluster := &provisioningv1.DPUCluster{} - dpuClusterUnstructured := unstructuredFromFile(dpuClusterPath) + dpuClusterUnstructured := UnstructuredFromFile(dpuClusterPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(dpuClusterUnstructured.Object, dpuCluster)).To(Succeed()) // Override interface if DPUCLUSTER_INTERFACE environment variable is set if dpuClusterInterface != "" && dpuCluster.Spec.ClusterEndpoint != nil && dpuCluster.Spec.ClusterEndpoint.Keepalived != nil { By(fmt.Sprintf("Overriding DPUCluster interface with DPUCLUSTER_INTERFACE=%s", dpuClusterInterface)) dpuCluster.Spec.ClusterEndpoint.Keepalived.Interface = dpuClusterInterface } - t.dpuClusters = append(t.dpuClusters, dpuCluster) + t.DPUClusters = append(t.DPUClusters, dpuCluster) } if conf.DPUDiscoveryPath != nil { dpuDiscovery := &provisioningv1.DPUDiscovery{} - dpuDiscoveryUnstructured := unstructuredFromFile(*conf.DPUDiscoveryPath) + dpuDiscoveryUnstructured := UnstructuredFromFile(*conf.DPUDiscoveryPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(dpuDiscoveryUnstructured.Object, dpuDiscovery)).To(Succeed()) - t.dpuDiscovery = dpuDiscovery + t.DPUDiscovery = dpuDiscovery } dpuServiceInterface := &dpuservicev1.DPUServiceInterface{} - dsi := unstructuredFromFile(conf.DPUServiceInterfacePath) + dsi := UnstructuredFromFile(conf.DPUServiceInterfacePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(dsi.Object, dpuServiceInterface)).To(Succeed()) - t.dpuServiceInterface = dpuServiceInterface + t.DPUServiceInterface = dpuServiceInterface dpuService := &dpuservicev1.DPUService{} - svc := unstructuredFromFile(conf.DPUServicePath) + svc := UnstructuredFromFile(conf.DPUServicePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(svc.Object, dpuService)).To(Succeed()) - t.dpuService = dpuService + t.DPUService = dpuService dpuClusterPrerequisiteObjects := []client.Object{} for _, path := range conf.DPUClusterPrerequisiteObjectPaths { - dpuClusterPrerequisiteObjects = append(dpuClusterPrerequisiteObjects, unstructuredFromFile(path)) + dpuClusterPrerequisiteObjects = append(dpuClusterPrerequisiteObjects, UnstructuredFromFile(path)) } - t.dpuClusterPrerequisites = dpuClusterPrerequisiteObjects + t.DPUClusterPrerequisites = dpuClusterPrerequisiteObjects dpuServiceTemplate := &dpuservicev1.DPUServiceTemplate{} - tmp := unstructuredFromFile(conf.DPUServiceTemplatePath) + tmp := UnstructuredFromFile(conf.DPUServiceTemplatePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(tmp.Object, dpuServiceTemplate)).To(Succeed()) - t.dpuServiceTemplate = dpuServiceTemplate + t.DPUServiceTemplate = dpuServiceTemplate dpuServiceConfiguration := &dpuservicev1.DPUServiceConfiguration{} - svcConfig := unstructuredFromFile(conf.DPUServiceConfiguration) + svcConfig := UnstructuredFromFile(conf.DPUServiceConfiguration) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(svcConfig.Object, dpuServiceConfiguration)).To(Succeed()) - t.dpuServiceConfiguration = dpuServiceConfiguration + t.DPUServiceConfiguration = dpuServiceConfiguration if conf.AdditionalDPUServiceTemplatePath != nil { additionalTemplate := &dpuservicev1.DPUServiceTemplate{} - tmp := unstructuredFromFile(*conf.AdditionalDPUServiceTemplatePath) + tmp := UnstructuredFromFile(*conf.AdditionalDPUServiceTemplatePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(tmp.Object, additionalTemplate)).To(Succeed()) - t.additionalDPUServiceTemplate = additionalTemplate + t.AdditionalDPUServiceTemplate = additionalTemplate } if conf.AdditionalDPUServiceConfigurationPath != nil { additionalConfiguration := &dpuservicev1.DPUServiceConfiguration{} - svcConfig := unstructuredFromFile(*conf.AdditionalDPUServiceConfigurationPath) + svcConfig := UnstructuredFromFile(*conf.AdditionalDPUServiceConfigurationPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(svcConfig.Object, additionalConfiguration)).To(Succeed()) - t.additionalDPUServiceConfiguration = additionalConfiguration + t.AdditionalDPUServiceConfiguration = additionalConfiguration } - t.dpuServiceInterfacesHBN = make([]*dpuservicev1.DPUServiceInterface, 0, len(conf.DPUServiceInterfacesHBNPaths)) + t.DPUServiceInterfacesHBN = make([]*dpuservicev1.DPUServiceInterface, 0, len(conf.DPUServiceInterfacesHBNPaths)) for _, path := range conf.DPUServiceInterfacesHBNPaths { iface := &dpuservicev1.DPUServiceInterface{} - Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(unstructuredFromFile(path).Object, iface)).To(Succeed()) - t.dpuServiceInterfacesHBN = append(t.dpuServiceInterfacesHBN, iface) + Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(UnstructuredFromFile(path).Object, iface)).To(Succeed()) + t.DPUServiceInterfacesHBN = append(t.DPUServiceInterfacesHBN, iface) } if conf.DPUServiceInterfaceOVNPath != nil { ovnIface := &dpuservicev1.DPUServiceInterface{} - Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(unstructuredFromFile(*conf.DPUServiceInterfaceOVNPath).Object, ovnIface)).To(Succeed()) - t.dpuServiceInterfaceOVN = ovnIface + Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(UnstructuredFromFile(*conf.DPUServiceInterfaceOVNPath).Object, ovnIface)).To(Succeed()) + t.DPUServiceInterfaceOVN = ovnIface } if conf.DPUServiceTemplateOVNPath != nil { dpuServiceTemplateOVN := &dpuservicev1.DPUServiceTemplate{} - ovnTmp := unstructuredFromFile(*conf.DPUServiceTemplateOVNPath) + ovnTmp := UnstructuredFromFile(*conf.DPUServiceTemplateOVNPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(ovnTmp.Object, dpuServiceTemplateOVN)).To(Succeed()) if repoURL, found := os.LookupEnv("OVN_KUBERNETES_REPO_URL"); found { dpuServiceTemplateOVN.Spec.HelmChart.Source.RepoURL = repoURL @@ -340,12 +340,12 @@ func (t *systemTestInput) applyConfig(conf config) { if chartTag, found := os.LookupEnv("OVN_KUBERNETES_CHART_TAG"); found { dpuServiceTemplateOVN.Spec.HelmChart.Source.Version = chartTag } - t.dpuServiceTemplateOVN = dpuServiceTemplateOVN + t.DPUServiceTemplateOVN = dpuServiceTemplateOVN } if conf.DPUServiceTemplateHBNPath != nil { dpuServiceTemplateHBN := &dpuservicev1.DPUServiceTemplate{} - hbnTmp := unstructuredFromFile(*conf.DPUServiceTemplateHBNPath) + hbnTmp := UnstructuredFromFile(*conf.DPUServiceTemplateHBNPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(hbnTmp.Object, dpuServiceTemplateHBN)).To(Succeed()) if repoURL, found := os.LookupEnv("HBN_CHART_REPO"); found { dpuServiceTemplateHBN.Spec.HelmChart.Source.RepoURL = repoURL @@ -353,94 +353,94 @@ func (t *systemTestInput) applyConfig(conf config) { if chartVersion, found := os.LookupEnv("HBN_CHART_VERSION"); found { dpuServiceTemplateHBN.Spec.HelmChart.Source.Version = chartVersion } - t.dpuServiceTemplateHBN = dpuServiceTemplateHBN + t.DPUServiceTemplateHBN = dpuServiceTemplateHBN } if conf.DPUServiceConfigurationOVNPath != nil { dpuServiceConfigurationOVN := &dpuservicev1.DPUServiceConfiguration{} - ovnCfg := unstructuredFromFile(*conf.DPUServiceConfigurationOVNPath) + ovnCfg := UnstructuredFromFile(*conf.DPUServiceConfigurationOVNPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(ovnCfg.Object, dpuServiceConfigurationOVN)).To(Succeed()) - t.dpuServiceConfigurationOVN = dpuServiceConfigurationOVN + t.DPUServiceConfigurationOVN = dpuServiceConfigurationOVN } if conf.DPUServiceConfigurationHBNPath != nil { dpuServiceConfigurationHBN := &dpuservicev1.DPUServiceConfiguration{} - hbnCfg := unstructuredFromFile(*conf.DPUServiceConfigurationHBNPath) + hbnCfg := UnstructuredFromFile(*conf.DPUServiceConfigurationHBNPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(hbnCfg.Object, dpuServiceConfigurationHBN)).To(Succeed()) - t.dpuServiceConfigurationHBN = dpuServiceConfigurationHBN + t.DPUServiceConfigurationHBN = dpuServiceConfigurationHBN } dpuDeployment := &dpuservicev1.DPUDeployment{} - deployment := unstructuredFromFile(conf.DPUDeploymentPath) + deployment := UnstructuredFromFile(conf.DPUDeploymentPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(deployment.Object, dpuDeployment)).To(Succeed()) - t.dpuDeployment = dpuDeployment + t.DPUDeployment = dpuDeployment dpuFlavor := &provisioningv1.DPUFlavor{} if conf.DPUFlavorPath != nil { - dpuFlavorUnstructured := unstructuredFromFile(*conf.DPUFlavorPath) + dpuFlavorUnstructured := UnstructuredFromFile(*conf.DPUFlavorPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(dpuFlavorUnstructured.Object, dpuFlavor)).To(Succeed()) - t.dpuFlavor = dpuFlavor - t.dpuDeployment.Spec.DPUs.Flavor = dpuFlavor.Name - t.dpuSet.Spec.DPUTemplate.Spec.DPUFlavor = dpuFlavor.Name + t.DPUFlavor = dpuFlavor + t.DPUDeployment.Spec.DPUs.Flavor = dpuFlavor.Name + t.DPUSet.Spec.DPUTemplate.Spec.DPUFlavor = dpuFlavor.Name } ipPoolDPUServiceIPAM := &dpuservicev1.DPUServiceIPAM{} - subnetIPAM := unstructuredFromFile(conf.IPPoolDPUServiceIPAMPath) + subnetIPAM := UnstructuredFromFile(conf.IPPoolDPUServiceIPAMPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(subnetIPAM.Object, ipPoolDPUServiceIPAM)).To(Succeed()) - t.ipPoolDPUServiceIPAM = ipPoolDPUServiceIPAM + t.IPPoolDPUServiceIPAM = ipPoolDPUServiceIPAM cidrDPUServiceIPAM := &dpuservicev1.DPUServiceIPAM{} - cidrIPAM := unstructuredFromFile(conf.CIDRPoolDPUServiceIPAMPath) + cidrIPAM := UnstructuredFromFile(conf.CIDRPoolDPUServiceIPAMPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(cidrIPAM.Object, cidrDPUServiceIPAM)).To(Succeed()) - t.cidrDPUServiceIPAM = cidrDPUServiceIPAM + t.CIDRDPUServiceIPAM = cidrDPUServiceIPAM dpuServiceChain := &dpuservicev1.DPUServiceChain{} - chain := unstructuredFromFile(conf.DPUServiceChainPath) + chain := UnstructuredFromFile(conf.DPUServiceChainPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(chain.Object, dpuServiceChain)).To(Succeed()) - t.dpuServiceChain = dpuServiceChain + t.DPUServiceChain = dpuServiceChain dpuServiceCredentialRequest := &dpuservicev1.DPUServiceCredentialRequest{} - request := unstructuredFromFile(conf.DPUServiceCredentialRequestPath) + request := UnstructuredFromFile(conf.DPUServiceCredentialRequestPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(request.Object, dpuServiceCredentialRequest)).To(Succeed()) - t.dpuServiceCredentialRequest = dpuServiceCredentialRequest + t.DPUServiceCredentialRequest = dpuServiceCredentialRequest if conf.OVNCredentialRequestPath != nil { ovnCredentialRequest := &dpuservicev1.DPUServiceCredentialRequest{} - ovnCR := unstructuredFromFile(*conf.OVNCredentialRequestPath) + ovnCR := UnstructuredFromFile(*conf.OVNCredentialRequestPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(ovnCR.Object, ovnCredentialRequest)).To(Succeed()) - t.ovnCredentialRequest = ovnCredentialRequest + t.OVNCredentialRequest = ovnCredentialRequest } - t.numberOfDPUNodes = conf.NumberOfDPUNodes - t.numberOfDPUsPerNode = conf.NumberOfDPUsPerNode - t.nodeRebootConfigMap = conf.NodeRebootConfigMap - t.nodeRebootConfigMapPath = conf.NodeRebootConfigMapPath - t.useExternalNodeReboot = conf.UseExternalNodeReboot + t.NumberOfDPUNodes = conf.NumberOfDPUNodes + t.NumberOfDPUsPerNode = conf.NumberOfDPUsPerNode + t.NodeRebootConfigMap = conf.NodeRebootConfigMap + t.NodeRebootConfigMapPath = conf.NodeRebootConfigMapPath + t.UseExternalNodeReboot = conf.UseExternalNodeReboot } -func (t *systemTestInput) hasDpuNodes() bool { - return t.numberOfDPUNodes > 0 +func (t *SystemTestInput) HasDpuNodes() bool { + return t.NumberOfDPUNodes > 0 } -// totalDPUs returns the total number of DPUs (nodes * DPUs per node) -func (t *systemTestInput) totalDPUs() int { - return t.numberOfDPUNodes * t.numberOfDPUsPerNode +// TotalDPUs returns the total number of DPUs (nodes * DPUs per node) +func (t *SystemTestInput) TotalDPUs() int { + return t.NumberOfDPUNodes * t.NumberOfDPUsPerNode } type DeployDPFSystemComponentsInput struct { - operatorConfig *operatorv1.DPFOperatorConfig - systemNamespace string + OperatorConfig *operatorv1.DPFOperatorConfig + SystemNamespace string ProvisioningControllerPVC *corev1.PersistentVolumeClaim ImagePullSecrets []string - dpuDiscovery *provisioningv1.DPUDiscovery - client client.Client - numberOfDPUNodes int - // skipSystemComponentValidation skips the post-deploy system-component checks + DPUDiscovery *provisioningv1.DPUDiscovery + Client client.Client + NumberOfDPUNodes int + // SkipSystemComponentValidation skips the post-deploy system-component checks // (the current-shape DPUService assertion and the DPFOperatorConfig ready // wait). Set for previous-release installs (e.g. BFB LTS v25.10) whose // component shape differs and whose servicechainset-controller stays // not-ready under the current CRD schema. - skipSystemComponentValidation bool + SkipSystemComponentValidation bool } // DeployDPFSystemComponents creates the operatorConfig and some dependencies and checks that the system components @@ -451,12 +451,12 @@ type DeployDPFSystemComponentsInput struct { // 4) Creates the operatorConfig for the test // 5) Ensures the DPF System components - including DPUServices - have been deployed. func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemComponentsInput) { - testClient := input.client + testClient := input.Client By("Ensure the DPF Operator is running and ready") Eventually(func(g Gomega) { deployment := &appsv1.Deployment{} g.Expect(testClient.Get(ctx, client.ObjectKey{ - Namespace: input.systemNamespace, + Namespace: input.SystemNamespace, Name: "dpf-operator-controller-manager"}, deployment)).To(Succeed()) g.Expect(deployment.Status.ReadyReplicas).To(Equal(*deployment.Spec.Replicas)) @@ -467,10 +467,10 @@ func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemCompone By("No PVC provided for the provisioning controller, skipping PVC creation") } else { pvc := input.ProvisioningControllerPVC.DeepCopy() - if n := input.operatorConfig.Spec.ProvisioningController.BFBPersistentVolumeClaimName; n != nil { + if n := input.OperatorConfig.Spec.ProvisioningController.BFBPersistentVolumeClaimName; n != nil { pvc.SetName(*n) } - pvc.SetNamespace(input.systemNamespace) + pvc.SetNamespace(input.SystemNamespace) pvc.SetLabels(CleanupScope.Suite) Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, pvc))).NotTo(HaveOccurred()) } @@ -480,7 +480,7 @@ func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemCompone secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, - Namespace: input.systemNamespace, + Namespace: input.SystemNamespace, Labels: CleanupScope.Suite, }, } @@ -488,9 +488,9 @@ func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemCompone } By("Create the DPFOperatorConfig for the system") - Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, input.operatorConfig))).NotTo(HaveOccurred()) + Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, input.OperatorConfig))).NotTo(HaveOccurred()) - if isGinkgoLabelApplied(Domain.ZeroTrust) { + if IsGinkgoLabelApplied(Domain.ZeroTrust) { By("Deploy DPUDiscovery for ZeroTrust") CreateDPUDiscovery(ctx, input) } @@ -500,7 +500,7 @@ func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemCompone // Check the DPUService controller manager is up and ready. dpuServiceDeployment := &appsv1.Deployment{} g.Expect(testClient.Get(ctx, client.ObjectKey{ - Namespace: input.systemNamespace, + Namespace: input.SystemNamespace, Name: "dpuservice-controller-manager"}, dpuServiceDeployment)).To(Succeed()) g.Expect(dpuServiceDeployment.Status.ReadyReplicas).To(Equal(*dpuServiceDeployment.Spec.Replicas)) @@ -508,18 +508,18 @@ func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemCompone // Check the DPF provisioning controller manager is up and ready. dpfProvisioningDeployment := &appsv1.Deployment{} g.Expect(testClient.Get(ctx, client.ObjectKey{ - Namespace: input.systemNamespace, + Namespace: input.SystemNamespace, Name: "dpf-provisioning-controller-manager"}, dpfProvisioningDeployment)).To(Succeed()) g.Expect(dpfProvisioningDeployment.Status.ReadyReplicas).To(Equal(*dpfProvisioningDeployment.Spec.Replicas)) // Check the NodeSRIOV Device Plugin controller deployment only when it is explicitly enabled. - if input.operatorConfig.Spec.NodeSRIOVDevicePluginController != nil && - input.operatorConfig.Spec.NodeSRIOVDevicePluginController.Disable != nil && - !*input.operatorConfig.Spec.NodeSRIOVDevicePluginController.Disable { + if input.OperatorConfig.Spec.NodeSRIOVDevicePluginController != nil && + input.OperatorConfig.Spec.NodeSRIOVDevicePluginController.Disable != nil && + !*input.OperatorConfig.Spec.NodeSRIOVDevicePluginController.Disable { nodesriovDevicePluginDeployment := &appsv1.Deployment{} g.Expect(testClient.Get(ctx, client.ObjectKey{ - Namespace: input.systemNamespace, + Namespace: input.SystemNamespace, Name: "dpf-nodesriovdeviceplugin-controller"}, nodesriovDevicePluginDeployment)).To(Succeed()) g.Expect(nodesriovDevicePluginDeployment.Status.ReadyReplicas).To(Equal(*nodesriovDevicePluginDeployment.Spec.Replicas)) @@ -527,18 +527,18 @@ func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemCompone }).WithTimeout(300 * time.Second).Should(Succeed()) - if isGinkgoLabelApplied(Domain.ZeroTrust) { + if IsGinkgoLabelApplied(Domain.ZeroTrust) { By("Verify bfb-registry Service and pods (created by provisioning controller leader)") Eventually(func(g Gomega) { svc := &corev1.Service{} g.Expect(testClient.Get(ctx, client.ObjectKey{ - Namespace: input.systemNamespace, + Namespace: input.SystemNamespace, Name: "bfb-registry", }, svc)).To(Succeed(), "bfb-registry Service should be created by provisioning controller leader") g.Expect(svc.Spec.Ports).ToNot(BeEmpty()) pods := &corev1.PodList{} g.Expect(testClient.List(ctx, pods, - client.InNamespace(input.systemNamespace), + client.InNamespace(input.SystemNamespace), client.MatchingLabels(map[string]string{ "app.kubernetes.io/part-of": "bfb-registry", "dpu.nvidia.com/component": "bfb-registry", @@ -559,7 +559,7 @@ func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemCompone }).WithTimeout(5 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) } - if input.skipSystemComponentValidation { + if input.SkipSystemComponentValidation { By("Skipping system component validation") return } @@ -569,7 +569,7 @@ func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemCompone Eventually(func(g Gomega) { // TODO: Remove as soon as we have version aware upgrade logic for the pre-upgrade validation var err error - isCurrentVersionLastReleasedGA, err = isPreUpgradeFromLastReleasedGA(ctx, testClient, client.ObjectKeyFromObject(input.operatorConfig)) + isCurrentVersionLastReleasedGA, err = isPreUpgradeFromLastReleasedGA(ctx, testClient, client.ObjectKeyFromObject(input.OperatorConfig)) g.Expect(err).NotTo(HaveOccurred()) dpuServices := &dpuservicev1.DPUServiceList{} @@ -614,17 +614,17 @@ func DeployDPFSystemComponents(ctx context.Context, input DeployDPFSystemCompone // ProvisionDPUClusters provisions DPUClusters. func ProvisionDPUClusters(ctx context.Context, input ProvisionDPUClustersInput) { By("Create prerequisites objects for DPUClusters") - for _, obj := range input.dpuClusterPrerequisites { + for _, obj := range input.DPUClusterPrerequisites { obj.SetLabels(CleanupScope.Suite) // We need to check if object already exists before creating. client.IgnoreAlreadyExists does not work in this case as the error will be "port is already allocated" existing := obj.DeepCopyObject().(client.Object) - err := input.client.Get(ctx, types.NamespacedName{ + err := input.Client.Get(ctx, types.NamespacedName{ Namespace: obj.GetNamespace(), Name: obj.GetName(), }, existing) if apierrors.IsNotFound(err) { By(fmt.Sprintf("Creating prerequisite object %s %s/%s", obj.GetObjectKind().GroupVersionKind().String(), obj.GetNamespace(), obj.GetName())) - Expect(input.client.Create(ctx, obj)).To(Succeed()) + Expect(input.Client.Create(ctx, obj)).To(Succeed()) } else { By(fmt.Sprintf("Skipping creation of existing object %s %s/%s", obj.GetObjectKind().GroupVersionKind().String(), @@ -633,26 +633,26 @@ func ProvisionDPUClusters(ctx context.Context, input ProvisionDPUClustersInput) } By("Create DPUClusters") - for _, dpuCluster := range input.dpuClusters { + for _, dpuCluster := range input.DPUClusters { dpuClusterLabels := map[string]string{ "svc.dpu.nvidia.com/cluster": dpuCluster.Name, } maps.Copy(dpuClusterLabels, CleanupScope.Suite) dpuCluster.SetLabels(dpuClusterLabels) By(fmt.Sprintf("Creating DPU Cluster %s/%s", dpuCluster.GetNamespace(), dpuCluster.GetName())) - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuCluster))).NotTo(HaveOccurred()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuCluster))).NotTo(HaveOccurred()) } - By(fmt.Sprintf("Waiting for %d DPUCluster(s) to be ready", len(input.dpuClusters))) + By(fmt.Sprintf("Waiting for %d DPUCluster(s) to be ready", len(input.DPUClusters))) Eventually(func(g Gomega) { clusters := &provisioningv1.DPUClusterList{} - g.Expect(input.client.List(ctx, clusters)).To(Succeed()) - g.Expect(clusters.Items).To(HaveLen(len(input.dpuClusters))) - for _, dpuCluster := range input.dpuClusters { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(dpuCluster), dpuCluster)).To(Succeed()) + g.Expect(input.Client.List(ctx, clusters)).To(Succeed()) + g.Expect(clusters.Items).To(HaveLen(len(input.DPUClusters))) + for _, dpuCluster := range input.DPUClusters { + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(dpuCluster), dpuCluster)).To(Succeed()) g.Expect(dpuCluster.Status.Phase).Should(Equal(provisioningv1.PhaseReady)) - if input.expectedKubernetesVersion != "" { - g.Expect(dpuCluster.Status.Version).Should(Equal(input.expectedKubernetesVersion)) + if input.ExpectedKubernetesVersion != "" { + g.Expect(dpuCluster.Status.Version).Should(Equal(input.ExpectedKubernetesVersion)) } else { g.Expect(dpuCluster.Status.Version).Should(Equal(util.KubernetesVersion)) } @@ -660,22 +660,22 @@ func ProvisionDPUClusters(ctx context.Context, input ProvisionDPUClustersInput) }).WithTimeout(300 * time.Second).Should(Succeed()) By("Creating a client for the DPUCluster") - getDPUClusterClients(ctx, input) + GetDPUClusterClients(ctx, input) } // ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor creates the BFB or BlueFieldSoftware and optionally the DPUFlavor resources. func ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(ctx context.Context, input ProvisionDPUClustersInput) { - Expect(input.bfb == nil && input.blueFieldSoftware == nil).To(BeFalse(), + Expect(input.BFB == nil && input.BlueFieldSoftware == nil).To(BeFalse(), "one of bfb or blueFieldSoftware must be set") - Expect(input.bfb != nil && input.blueFieldSoftware != nil).To(BeFalse(), + Expect(input.BFB != nil && input.BlueFieldSoftware != nil).To(BeFalse(), "bfb and blueFieldSoftware cannot both be set") - if input.bfb != nil { + if input.BFB != nil { ProvisionBFB(ctx, input) } - if input.blueFieldSoftware != nil { + if input.BlueFieldSoftware != nil { ProvisionBlueFieldSoftware(ctx, input) } - if input.dpuFlavor != nil { + if input.DPUFlavor != nil { ProvisionDPUFlavor(ctx, input) } } @@ -684,40 +684,40 @@ func ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(ctx context.Context, input Prov // the BFB file is reachable via the bfb-registry service (ZeroTrust only). func ProvisionBFB(ctx context.Context, input ProvisionDPUClustersInput) { // TODO: Pass this in as config instead of as a global. - if input.bfbImageURL != "" { - By(fmt.Sprintf("Override BFB URL with env variable BFB_IMAGE_URL=%s", input.bfbImageURL)) - input.bfb.Spec.URL = input.bfbImageURL + if input.BFBImageURL != "" { + By(fmt.Sprintf("Override BFB URL with env variable BFB_IMAGE_URL=%s", input.BFBImageURL)) + input.BFB.Spec.URL = input.BFBImageURL } By("Create the BFB") Eventually(func(g Gomega) { - bfb := input.bfb.DeepCopy() + bfb := input.BFB.DeepCopy() bfb.SetLabels(CleanupScope.Suite) - g.Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, bfb))).NotTo(HaveOccurred()) + g.Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, bfb))).NotTo(HaveOccurred()) }).WithTimeout(10 * time.Second).Should(Succeed()) By("Checking that BFB is ready") Eventually(func(g Gomega) { bfb := &provisioningv1.BFB{} - g.Expect(input.client.Get(ctx, client.ObjectKey{ - Name: input.bfb.Name, - Namespace: input.bfb.Namespace, + g.Expect(input.Client.Get(ctx, client.ObjectKey{ + Name: input.BFB.Name, + Namespace: input.BFB.Namespace, }, bfb)).To(Succeed()) g.Expect(bfb.Status.Phase).To(Equal(provisioningv1.BFBReady)) }).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) - if isGinkgoLabelApplied(Domain.ZeroTrust) { + if IsGinkgoLabelApplied(Domain.ZeroTrust) { By("Verifying BFB file is reachable") bfb := &provisioningv1.BFB{} - Expect(input.client.Get(ctx, client.ObjectKey{ - Name: input.bfb.Name, - Namespace: input.bfb.Namespace, + Expect(input.Client.Get(ctx, client.ObjectKey{ + Name: input.BFB.Name, + Namespace: input.BFB.Namespace, }, bfb)).To(Succeed()) Expect(bfb.Status.FileName).ToNot(BeEmpty(), "BFB status should have a FileName after reaching Ready") - controlPlaneIP := getClusterControlPlaneIP(ctx, input.client) + controlPlaneIP := getClusterControlPlaneIP(ctx, input.Client) svc := &corev1.Service{} - Expect(input.client.Get(ctx, client.ObjectKey{ - Namespace: input.bfb.Namespace, + Expect(input.Client.Get(ctx, client.ObjectKey{ + Namespace: input.BFB.Namespace, Name: "bfb-registry", }, svc)).To(Succeed()) Expect(svc.Spec.Ports).ToNot(BeEmpty(), "bfb-registry Service should have ports") @@ -739,27 +739,27 @@ func ProvisionBFB(ctx context.Context, input ProvisionDPUClustersInput) { // ProvisionBlueFieldSoftware creates the BlueFieldSoftware resource and waits for it to reach Ready phase. func ProvisionBlueFieldSoftware(ctx context.Context, input ProvisionDPUClustersInput) { - if input.bfsOsIsoURL != "" { - By(fmt.Sprintf("Override BlueFieldSoftware OS ISO URL with env variable BFS_OS_ISO_URL=%s", input.bfsOsIsoURL)) - input.blueFieldSoftware.Spec.OsIso = input.bfsOsIsoURL + if input.BFSOsIsoURL != "" { + By(fmt.Sprintf("Override BlueFieldSoftware OS ISO URL with env variable BFS_OS_ISO_URL=%s", input.BFSOsIsoURL)) + input.BlueFieldSoftware.Spec.OsIso = input.BFSOsIsoURL } - if input.bfsPldmFwBundleURL != "" { - By(fmt.Sprintf("Override BlueFieldSoftware PLDM FW bundle URL with env variable BFS_PLDM_FW_BUNDLE_URL=%s", input.bfsPldmFwBundleURL)) - input.blueFieldSoftware.Spec.PldmFwBundle = &input.bfsPldmFwBundleURL + if input.BFSPldmFwBundleURL != "" { + By(fmt.Sprintf("Override BlueFieldSoftware PLDM FW bundle URL with env variable BFS_PLDM_FW_BUNDLE_URL=%s", input.BFSPldmFwBundleURL)) + input.BlueFieldSoftware.Spec.PldmFwBundle = &input.BFSPldmFwBundleURL } By("Create the BlueFieldSoftware") Eventually(func(g Gomega) { - bfs := input.blueFieldSoftware.DeepCopy() + bfs := input.BlueFieldSoftware.DeepCopy() bfs.SetLabels(CleanupScope.Suite) - g.Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, bfs))).NotTo(HaveOccurred()) + g.Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, bfs))).NotTo(HaveOccurred()) }).WithTimeout(10 * time.Second).Should(Succeed()) By("Checking that BlueFieldSoftware is ready") Eventually(func(g Gomega) { bfs := &provisioningv1.BlueFieldSoftware{} - g.Expect(input.client.Get(ctx, client.ObjectKey{ - Name: input.blueFieldSoftware.Name, - Namespace: input.blueFieldSoftware.Namespace, + g.Expect(input.Client.Get(ctx, client.ObjectKey{ + Name: input.BlueFieldSoftware.Name, + Namespace: input.BlueFieldSoftware.Namespace, }, bfs)).To(Succeed()) g.Expect(bfs.Status.Phase).To(Equal(provisioningv1.BlueFieldSoftwareReady)) }).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) @@ -769,9 +769,9 @@ func ProvisionBlueFieldSoftware(ctx context.Context, input ProvisionDPUClustersI func ProvisionDPUFlavor(ctx context.Context, input ProvisionDPUClustersInput) { By("Creating the DPUFlavor") Eventually(func(g Gomega) { - dpuFlavor := input.dpuFlavor.DeepCopy() + dpuFlavor := input.DPUFlavor.DeepCopy() dpuFlavor.SetLabels(CleanupScope.Suite) - g.Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuFlavor))).NotTo(HaveOccurred()) + g.Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuFlavor))).NotTo(HaveOccurred()) }).WithTimeout(60 * time.Second).Should(Succeed()) } @@ -780,10 +780,10 @@ func ProvisionDPUFlavor(ctx context.Context, input ProvisionDPUClustersInput) { func ProvisionDPUSet(ctx context.Context, input ProvisionDPUClustersInput) { Eventually(func(g Gomega) { By("Creating the DPUSet") - dpuset := input.dpuSet.DeepCopy() + dpuset := input.DPUSet.DeepCopy() // TODO: Test the cleanup of the node related to the DPU. dpuset.SetLabels(CleanupScope.Suite) - g.Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuset))).NotTo(HaveOccurred()) + g.Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuset))).NotTo(HaveOccurred()) }).WithTimeout(60 * time.Second).Should(Succeed()) By("Checking the DPUServices have been mirrored to the target cluster") @@ -791,11 +791,11 @@ func ProvisionDPUSet(ctx context.Context, input ProvisionDPUClustersInput) { operatorv1.ServiceSetControllerName, operatorv1.NVIPAMControllerName, } { - deploymentName := fmt.Sprintf("in-cluster-%s", getPerClusterDPUServiceName(componentName, input.dpuClusters[0].Name, input.dpuClusters[0].Namespace)) + deploymentName := fmt.Sprintf("in-cluster-%s", getPerClusterDPUServiceName(componentName, input.DPUClusters[0].Name, input.DPUClusters[0].Namespace)) Eventually(func(g Gomega) { deployment := &appsv1.Deployment{} - g.Expect(input.client.Get(ctx, client.ObjectKey{ - Namespace: dpfOperatorSystemNamespace, + g.Expect(input.Client.Get(ctx, client.ObjectKey{ + Namespace: DPFOperatorSystemNamespace, Name: deploymentName}, deployment)).To(Succeed()) g.Expect(deployment.Status.ReadyReplicas).To(Equal(*deployment.Spec.Replicas)) @@ -805,20 +805,20 @@ func ProvisionDPUSet(ctx context.Context, input ProvisionDPUClustersInput) { By("Checking that DPUService objects have been mirrored to the DPUClusters") Eventually(func(g Gomega) { deployments := &appsv1.DeploymentList{} - g.Expect(dpuClusterClient[0].List(ctx, deployments)).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, deployments)).To(Succeed()) found := map[string]bool{} for i := range deployments.Items { - if _, hasAnnotation := deployments.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]; hasAnnotation { - g.Expect(deployments.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]).NotTo(Equal("")) - found[deployments.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]] = true + if _, hasAnnotation := deployments.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]; hasAnnotation { + g.Expect(deployments.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]).NotTo(Equal("")) + found[deployments.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]] = true } } daemonsets := appsv1.DaemonSetList{} - g.Expect(dpuClusterClient[0].List(ctx, &daemonsets, client.InNamespace(input.dpuClusters[0].GetNamespace()))).To(Succeed()) + g.Expect(DPUClusterClient[0].List(ctx, &daemonsets, client.InNamespace(input.DPUClusters[0].GetNamespace()))).To(Succeed()) for i := range daemonsets.Items { - if _, hasAnnotation := daemonsets.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]; hasAnnotation { - g.Expect(daemonsets.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]).NotTo(Equal("")) - found[daemonsets.Items[i].GetAnnotations()[argoCDTrackingIDAnnotation]] = true + if _, hasAnnotation := daemonsets.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]; hasAnnotation { + g.Expect(daemonsets.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]).NotTo(Equal("")) + found[daemonsets.Items[i].GetAnnotations()[ArgoCDTrackingIDAnnotation]] = true } } @@ -838,7 +838,7 @@ func ProvisionDPUSet(ctx context.Context, input ProvisionDPUClustersInput) { // addition verifies that the DPUs become ready. // Note: Each DPU joins the DPU cluster as a separate K8s node, so the number of nodes in the DPU cluster equals totalDPUs. func VerifyDPUClusterWithNodes(ctx context.Context, input ProvisionDPUClustersInput) { - expectedDPUs := input.numberOfDPUNodes * input.numberOfDPUsPerNode + expectedDPUs := input.NumberOfDPUNodes * input.NumberOfDPUsPerNode tracker := NewByTracker() if err := verifyExpectedDPUsToBeReady(ctx, nil, input, expectedDPUs); err == nil { @@ -846,7 +846,7 @@ func VerifyDPUClusterWithNodes(ctx context.Context, input ProvisionDPUClustersIn return } - if isGinkgoLabelApplied(Domain.ZeroTrust) { + if IsGinkgoLabelApplied(Domain.ZeroTrust) { ProcessDPUNodeMaintenanceHold(ctx, input) WaitForDPUReboot(ctx, input) } @@ -854,11 +854,11 @@ func VerifyDPUClusterWithNodes(ctx context.Context, input ProvisionDPUClustersIn // Verify nodes are present in DPUCluster, Eventually(func(g Gomega) { nodes := &corev1.NodeList{} - g.Expect(dpuClusterClient[0].List(ctx, nodes)).ToNot(HaveOccurred()) + g.Expect(DPUClusterClient[0].List(ctx, nodes)).ToNot(HaveOccurred()) nodeKey := fmt.Sprintf("%d/%d", len(nodes.Items), expectedDPUs) tracker.By(nodeKey, "Checking that the number of nodes %d is equal to %d", len(nodes.Items), expectedDPUs) g.Expect(nodes.Items).To(HaveLen(expectedDPUs)) - }).WithTimeout(provisioningTimeout).WithPolling(1 * time.Second).Should(Succeed()) + }).WithTimeout(ProvisioningTimeout).WithPolling(1 * time.Second).Should(Succeed()) // Verify DPUs are ready Eventually(func(g Gomega) { @@ -869,7 +869,7 @@ func VerifyDPUClusterWithNodes(ctx context.Context, input ProvisionDPUClustersIn func verifyExpectedDPUsToBeReady(ctx context.Context, tracker *ByTracker, input ProvisionDPUClustersInput, expectedDPUs int) error { dpus := &provisioningv1.DPUList{} - if err := input.client.List(ctx, dpus); err != nil { + if err := input.Client.List(ctx, dpus); err != nil { return err } if len(dpus.Items) != expectedDPUs { @@ -918,13 +918,13 @@ func ProcessDPUNodeMaintenanceHold(ctx context.Context, input ProvisionDPUCluste By("Processing DPUNodeMaintenance with Node Effect Hold") tracker := NewByTracker() - expectedDPUs := input.numberOfDPUNodes * input.numberOfDPUsPerNode + expectedDPUs := input.NumberOfDPUNodes * input.NumberOfDPUsPerNode // Wait for DPUNodeMaintenance CRs to exist with hold annotation set to "true" var dpuNodeMaintenanceList *provisioningv1.DPUNodeMaintenanceList Eventually(func(g Gomega) { dpuNodeMaintenanceList = &provisioningv1.DPUNodeMaintenanceList{} - g.Expect(input.client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, dpuNodeMaintenanceList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) // Count how many have the hold annotation set to "true" holdCount := 0 @@ -943,7 +943,7 @@ func ProcessDPUNodeMaintenanceHold(ctx context.Context, input ProvisionDPUCluste By("Setting hold annotation to false on all DPUNodeMaintenance CRs to allow provisioning to continue") for i := range dpuNodeMaintenanceList.Items { if isDPUNodeMaintenanceOnHold(&dpuNodeMaintenanceList.Items[i]) { - Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.client, &dpuNodeMaintenanceList.Items[i]).WithTimeout(30 * time.Second).Should(Succeed()) + Eventually(releaseDPUNodeMaintenanceHold).WithArguments(ctx, input.Client, &dpuNodeMaintenanceList.Items[i]).WithTimeout(30 * time.Second).Should(Succeed()) By(fmt.Sprintf("Released hold on DPUNodeMaintenance %s", dpuNodeMaintenanceList.Items[i].Name)) } } @@ -964,8 +964,8 @@ func WaitForDPUReboot(ctx context.Context, input ProvisionDPUClustersInput) { By("Wait for DPUs to reach DPURebooting state in ZeroTrust") Eventually(func(g Gomega) { - g.Expect(input.client.List(ctx, dpus)).ToNot(HaveOccurred()) - g.Expect(dpus.Items).To(HaveLen(input.numberOfDPUNodes * input.numberOfDPUsPerNode)) + g.Expect(input.Client.List(ctx, dpus)).ToNot(HaveOccurred()) + g.Expect(dpus.Items).To(HaveLen(input.NumberOfDPUNodes * input.NumberOfDPUsPerNode)) for _, dpu := range dpus.Items { dpuStatusKey := fmt.Sprintf("%s/%v", dpu.Name, dpu.Status.Phase) @@ -974,18 +974,18 @@ func WaitForDPUReboot(ctx context.Context, input ProvisionDPUClustersInput) { if dpu.Status.Phase != provisioningv1.DPUReady { dpuKey := client.ObjectKey{Name: dpu.Name, Namespace: dpu.Namespace} current := &provisioningv1.DPU{} - g.Expect(input.client.Get(ctx, dpuKey, current)).To(Succeed()) + g.Expect(input.Client.Get(ctx, dpuKey, current)).To(Succeed()) // TODO: update this behavior when retry during provisioning is introduced // Failing test instantly when facing Error during provisioning Expect(current.Status.Phase).NotTo(Equal(provisioningv1.DPUError)) g.Expect(current.Status.Phase).To(Equal(provisioningv1.DPURebooting)) } } - }).WithTimeout(provisioningTimeout).Should(Succeed()) + }).WithTimeout(ProvisioningTimeout).Should(Succeed()) By("Reboot driven by in-cluster script Job (nodeRebootMethod.script); waiting for completion") - waitForScriptRebootCompletion(ctx, input.client, - input.numberOfDPUNodes*input.numberOfDPUsPerNode) + waitForScriptRebootCompletion(ctx, input.Client, + input.NumberOfDPUNodes*input.NumberOfDPUsPerNode) } // Waits for all DPU host reboots to finish in script-reboot mode by checking DPU.Status.RebootStatus, @@ -1081,7 +1081,7 @@ func VerifyClusterPods(ctx context.Context, client client.Client, podSubstrToVer func VerifyDPFOperatorConfigReady(ctx context.Context, kclient client.Client, timeout time.Duration) { Eventually(func(g Gomega) { dpfOperatorConfig := &operatorv1.DPFOperatorConfig{} - g.Expect(kclient.Get(ctx, client.ObjectKey{Namespace: dpfOperatorSystemNamespace, Name: configName}, dpfOperatorConfig)).To(Succeed()) + g.Expect(kclient.Get(ctx, client.ObjectKey{Namespace: DPFOperatorSystemNamespace, Name: ConfigName}, dpfOperatorConfig)).To(Succeed()) g.Expect(conditions.IsTrue(dpfOperatorConfig, conditions.TypeReady)).To(BeTrue()) }).WithTimeout(timeout).WithPolling(1 * time.Second).Should(Succeed()) } @@ -1092,7 +1092,7 @@ func VerifyProvisioningControllerPodsArg(ctx context.Context, kclient client.Cli Eventually(func(g Gomega) { pods := &corev1.PodList{} g.Expect(kclient.List(ctx, pods, - client.InNamespace(dpfOperatorSystemNamespace), + client.InNamespace(DPFOperatorSystemNamespace), client.MatchingLabels{operatorv1.DPFComponentLabelKey: "dpf-provisioning-controller-manager"}, )).To(Succeed()) g.Expect(pods.Items).ToNot(BeEmpty()) @@ -1115,7 +1115,7 @@ func CreateDPUDiscovery(ctx context.Context, input DeployDPFSystemComponentsInpu By("Verify worker nodes are not present") workerNodes := &corev1.NodeList{} Eventually(func(g Gomega) int { - err := input.client.List(ctx, workerNodes, client.InNamespace(dpfOperatorSystemNamespace), client.MatchingLabels(map[string]string{"node-role.kubernetes.io/worker": ""})) + err := input.Client.List(ctx, workerNodes, client.InNamespace(DPFOperatorSystemNamespace), client.MatchingLabels(map[string]string{"node-role.kubernetes.io/worker": ""})) g.Expect(err).NotTo(HaveOccurred()) return len(workerNodes.Items) }, time.Second*30, time.Millisecond*250).Should(Equal(0)) @@ -1123,41 +1123,41 @@ func CreateDPUDiscovery(ctx context.Context, input DeployDPFSystemComponentsInpu By("Verify DPU devices are not present") dpuDeviceList := &provisioningv1.DPUDeviceList{} Eventually(func(g Gomega) int { - err := input.client.List(ctx, dpuDeviceList, client.InNamespace(input.systemNamespace)) + err := input.Client.List(ctx, dpuDeviceList, client.InNamespace(input.SystemNamespace)) g.Expect(err).NotTo(HaveOccurred()) return len(dpuDeviceList.Items) }, time.Second*30, time.Millisecond*250).Should(Equal(0)) By("Creating DpuDiscovery") - Expect(input.dpuDiscovery).NotTo(BeNil(), "dpuDiscovery config is required for ZeroTrust") - discovery := input.dpuDiscovery.DeepCopy() - discovery.SetNamespace(input.systemNamespace) + Expect(input.DPUDiscovery).NotTo(BeNil(), "dpuDiscovery config is required for ZeroTrust") + discovery := input.DPUDiscovery.DeepCopy() + discovery.SetNamespace(input.SystemNamespace) discovery.SetLabels(CleanupScope.Suite) - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, discovery))).NotTo(HaveOccurred()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, discovery))).NotTo(HaveOccurred()) By("Waiting for DPU discovery to complete and create DPU devices") dpuDeviceList = &provisioningv1.DPUDeviceList{} Eventually(func(g Gomega) int { - err := input.client.List(ctx, dpuDeviceList, client.InNamespace(input.systemNamespace)) + err := input.Client.List(ctx, dpuDeviceList, client.InNamespace(input.SystemNamespace)) g.Expect(err).NotTo(HaveOccurred()) return len(dpuDeviceList.Items) - }, time.Minute*5, time.Millisecond*250).Should(Equal(input.numberOfDPUNodes)) + }, time.Minute*5, time.Millisecond*250).Should(Equal(input.NumberOfDPUNodes)) } // ValidateDPUAgentStatus verifies that the DPU agent has reported its status correctly // on every ready DPU. Each DPU is validated against the supplied expected AgentStatus. -func ValidateDPUAgentStatus(ctx context.Context, input *systemTestInput, expected provisioningv1.AgentStatus) { - if !input.hasDpuNodes() { +func ValidateDPUAgentStatus(ctx context.Context, input *SystemTestInput, expected provisioningv1.AgentStatus) { + if !input.HasDpuNodes() { Skip("Skip DPU Agent validation as there are no DPU nodes") } - expectedDPUs := input.totalDPUs() + expectedDPUs := input.TotalDPUs() By("Listing all DPUs and validating agent status") Eventually(func(g Gomega) { dpus := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpus)).To(Succeed()) + g.Expect(input.Client.List(ctx, dpus)).To(Succeed()) g.Expect(dpus.Items).To(HaveLen(expectedDPUs), "expected %d DPUs", expectedDPUs) for i := range dpus.Items { @@ -1251,13 +1251,13 @@ func validateSingleDPUAgentStatus(g Gomega, dpu *provisioningv1.DPU, expectedAge } // verifyDPUServicesReady checks that the DPUService is ready. -func verifyDPUServicesReady(ctx context.Context, input *systemTestInput, dpuServiceNamespace string, dpuServiceName []string) { +func verifyDPUServicesReady(ctx context.Context, input *SystemTestInput, dpuServiceNamespace string, dpuServiceName []string) { tracker := NewByTracker() Eventually(func(g Gomega) { for _, name := range dpuServiceName { tracker.By(name, "verify DPUService %s is ready", name) dpuService := &dpuservicev1.DPUService{} - g.Expect(input.client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: name}, dpuService)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKey{Namespace: dpuServiceNamespace, Name: name}, dpuService)).To(Succeed()) g.Expect(conditions.IsTrue(dpuService, conditions.TypeReady)).To(BeTrue()) } // A timeout of 20 minutes is necessary here. We have alot of trouble pulling our images for all @@ -1265,21 +1265,21 @@ func verifyDPUServicesReady(ctx context.Context, input *systemTestInput, dpuServ }).WithTimeout(20 * time.Minute).Should(Succeed()) } -// getDPUClusterClient retrieves the DPUCluster client for the cluster at the given index. This function is internal and should not be called directly. +// GetDPUClusterClient retrieves the DPUCluster client for the cluster at the given index. This function is internal and should not be called directly. // Instead, use getDPUClusterClients to retrieve all clients for all clusters. -func getDPUClusterClient(ctx context.Context, input ProvisionDPUClustersInput, clusterIndex int) { +func GetDPUClusterClient(ctx context.Context, input ProvisionDPUClustersInput, clusterIndex int) { var tun *tunnel.Tunnel Eventually(func(g Gomega) { - refreshable, ok := dpuClusterClient[clusterIndex].(*refreshableclient.Client) + refreshable, ok := DPUClusterClient[clusterIndex].(*refreshableclient.Client) g.Expect(ok).To(BeTrue(), "DPUCluster client %d should be a refreshable client", clusterIndex) - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(input.dpuClusters[clusterIndex]), input.dpuClusters[clusterIndex])).To(Succeed()) - g.Expect(input.dpuClusters[clusterIndex].Spec.Kubeconfig).ToNot(BeEmpty(), "DPUCluster kubeconfig should be populated") + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(input.DPUClusters[clusterIndex]), input.DPUClusters[clusterIndex])).To(Succeed()) + g.Expect(input.DPUClusters[clusterIndex].Spec.Kubeconfig).ToNot(BeEmpty(), "DPUCluster kubeconfig should be populated") var err error var restCfg *rest.Config - restCfg, tun, err = tunnel.NewTunneledRestConfig(ctx, input.client, input.restConfig, input.dpuClusters[clusterIndex]) + restCfg, tun, err = tunnel.NewTunneledRestConfig(ctx, input.Client, input.RestConfig, input.DPUClusters[clusterIndex]) g.Expect(err).NotTo(HaveOccurred(), "Should create tunneled REST config") dpuClient, err := client.New(restCfg, client.Options{}) @@ -1290,8 +1290,8 @@ func getDPUClusterClient(ctx context.Context, input ProvisionDPUClustersInput, c restCfg.APIPath = "/api" restCfg.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"} restCfg.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: scheme.Codecs} - dpuClusterRestConfig[clusterIndex] = restCfg - dpuClusterRestClient[clusterIndex], err = rest.RESTClientFor(restCfg) + DPUClusterRestConfig[clusterIndex] = restCfg + DPUClusterRestClient[clusterIndex], err = rest.RESTClientFor(restCfg) g.Expect(err).ToNot(HaveOccurred()) }).WithTimeout(3 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) @@ -1313,7 +1313,7 @@ func getDPUClusterClient(ctx context.Context, input ProvisionDPUClustersInput, c if !tun.IsHealthy() { By("Tunnel health check failed, recreating client and rest config") tun.Close() - getDPUClusterClient(ctx, input, clusterIndex) + GetDPUClusterClient(ctx, input, clusterIndex) return } } @@ -1321,9 +1321,9 @@ func getDPUClusterClient(ctx context.Context, input ProvisionDPUClustersInput, c }() } -// getDPUClusterClients retrieves the DPUCluster clients for all clusters in the input. +// GetDPUClusterClients retrieves the DPUCluster clients for all clusters in the input. // This function must only be called once per test suite as it initializes stable global client wrappers. -func getDPUClusterClients(ctx context.Context, input ProvisionDPUClustersInput) { +func GetDPUClusterClients(ctx context.Context, input ProvisionDPUClustersInput) { if dpuClusterClientsInitialized { warningMsg := "WARNING: getDPUClusterClients called multiple times - " + "skipping reinitialization (this may indicate a test structure issue)" @@ -1334,16 +1334,16 @@ func getDPUClusterClients(ctx context.Context, input ProvisionDPUClustersInput) dpuClusterClientsInitialized = true // Pre-initialize the global slices with the correct size - numClusters := len(input.dpuClusters) - dpuClusterClient = make([]client.Client, numClusters) - dpuClusterRestConfig = make([]*rest.Config, numClusters) - dpuClusterRestClient = make([]*rest.RESTClient, numClusters) - for i := range input.dpuClusters { - dpuClusterClient[i] = refreshableclient.New() + numClusters := len(input.DPUClusters) + DPUClusterClient = make([]client.Client, numClusters) + DPUClusterRestConfig = make([]*rest.Config, numClusters) + DPUClusterRestClient = make([]*rest.RESTClient, numClusters) + for i := range input.DPUClusters { + DPUClusterClient[i] = refreshableclient.New() } - for i := range input.dpuClusters { - getDPUClusterClient(ctx, input, i) + for i := range input.DPUClusters { + GetDPUClusterClient(ctx, input, i) } } @@ -1368,7 +1368,7 @@ func GetDPUNodeToBMCIPs(ctx context.Context, c client.Client, var observed []provisioningv1.DPUNode Eventually(func(g Gomega) { nodes := &provisioningv1.DPUNodeList{} - g.Expect(c.List(ctx, nodes, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(c.List(ctx, nodes, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) g.Expect(nodes.Items).To(HaveLen(expectedDPUNodes)) observed = nodes.Items }).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) @@ -1405,7 +1405,7 @@ func ApplyNodeRebootConfigMap(ctx context.Context, c client.Client, configMapPat obj := &unstructured.Unstructured{} Expect(yaml.Unmarshal(data, obj)).To(Succeed()) if obj.GetNamespace() == "" { - obj.SetNamespace(dpfOperatorSystemNamespace) + obj.SetNamespace(DPFOperatorSystemNamespace) } labels := obj.GetLabels() if labels == nil { @@ -1420,11 +1420,11 @@ func ApplyNodeRebootConfigMap(ctx context.Context, c client.Client, configMapPat // bmcPassword is sourced from $E2E_ZT_BMC_PASSWORD by getEnvVariables() and required-ness is enforced by validateFlags() for ZT runs. func applyBMCCredentialsSecret(ctx context.Context, c client.Client) { By(fmt.Sprintf("Creating BMC credentials Secret %s/%s", - dpfOperatorSystemNamespace, bmcCredentialsSecretName)) + DPFOperatorSystemNamespace, bmcCredentialsSecretName)) secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: bmcCredentialsSecretName, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: maps.Clone(CleanupScope.Suite), }, Type: corev1.SecretTypeOpaque, @@ -1453,7 +1453,7 @@ func PatchDPUNodesForScriptReboot(ctx context.Context, c client.Client, var observed []provisioningv1.DPUNode Eventually(func(g Gomega) { nodes := &provisioningv1.DPUNodeList{} - g.Expect(c.List(ctx, nodes, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(c.List(ctx, nodes, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) g.Expect(nodes.Items).To(HaveLen(expectedDPUNodes)) observed = nodes.Items }).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) @@ -1482,7 +1482,7 @@ func PatchDPUNodesForScriptReboot(ctx context.Context, c client.Client, } } -func unstructuredFromFile(path string) *unstructured.Unstructured { +func UnstructuredFromFile(path string) *unstructured.Unstructured { data, err := os.ReadFile(path) Expect(err).ToNot(HaveOccurred()) obj := &unstructured.Unstructured{} diff --git a/test/e2e/system_test.go b/test/e2e/system_test.go index f05d6d63..f273ed73 100644 --- a/test/e2e/system_test.go +++ b/test/e2e/system_test.go @@ -17,325 +17,20 @@ limitations under the License. package e2e import ( - "context" - "encoding/base64" - "encoding/json" - "fmt" - "maps" - "net/url" - "strconv" "time" - operatorv1 "github.com/nvidia/doca-platform/api/operator/v1alpha1" provisioningv1 "github.com/nvidia/doca-platform/api/provisioning/v1alpha1" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" ) -var input *systemTestInput -var vpcOvnInput = &vpcOvnTestInput{} - -func SetInput() { - By("Validating the input") - validateFlags() - - By("Get control plane IP") - controlPlaneIP := getClusterControlPlaneIP(ctx, testClient) - - By("Setting operatorConfig for the test") - var bfbPVCName *string - if conf.ProvisioningControllerPVCPath != nil { - bfbPVCName = ptr.To("bfb-pvc") - } - dpfOperatorConfig := &operatorv1.DPFOperatorConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: configName, - Namespace: dpfOperatorSystemNamespace, - Labels: CleanupScope.Suite, - }, - Spec: operatorv1.DPFOperatorConfigSpec{ - DeploymentMode: operatorv1.DeploymentModeHostTrusted, - ProvisioningController: &operatorv1.ProvisioningControllerConfiguration{ - BFBPersistentVolumeClaimName: bfbPVCName, - }, - StaticClusterManager: &operatorv1.StaticClusterManagerConfiguration{ - BaseComponentConfig: operatorv1.BaseComponentConfig{ - Disable: ptr.To(false), - }, - }, - // Disable the Kamaji cluster manager so only one cluster manager is running. - // TODO: Enable Kamaji by default in the e2e tests. - KamajiClusterManager: &operatorv1.KamajiClusterManagerConfiguration{ - BaseComponentConfig: operatorv1.BaseComponentConfig{ - Disable: ptr.To(false), - }, - }, - Monitoring: &operatorv1.MonitoringConfiguration{ - Disable: ptr.To(false), - OpenTelemetryCollector: &operatorv1.OpenTelemetryCollectorConfiguration{ - Logging: &operatorv1.OpenTelemetryCollectorLoggingConfiguration{ - Endpoint: fmt.Sprintf("%s%s:%d", otelEndpointSchema, controlPlaneIP, otelNodePort), - }, - }, - }, - NodeSRIOVDevicePluginController: &operatorv1.NodeSRIOVDevicePluginControllerConfiguration{ - BaseComponentConfig: operatorv1.BaseComponentConfig{ - Disable: ptr.To(false), - }, - }, - KataContainers: &operatorv1.KataContainersConfiguration{ - BaseComponentConfig: operatorv1.BaseComponentConfig{ - Disable: ptr.To(false), - }, - }, - ImagePullSecrets: []string{dpfPullSecretName, "pull-secret-extra"}, - }, - } - if isGinkgoLabelApplied(Domain.ZeroTrust) { - dpfOperatorConfig.Spec.DeploymentMode = operatorv1.DeploymentModeZeroTrust - dpfOperatorConfig.Spec.StaticClusterManager.BaseComponentConfig.Disable = ptr.To(true) - dpfOperatorConfig.Spec.KamajiClusterManager.BaseComponentConfig.Disable = ptr.To(false) - dpfOperatorConfig.Spec.NodeSRIOVDevicePluginController.BaseComponentConfig.Disable = ptr.To(true) - dpfOperatorConfig.Spec.ProvisioningController.InstallInterface = &operatorv1.ProvisioningInstallInterface{ - InstallViaRedfish: &operatorv1.InstallViaRedfish{ - SkipDPUNodeDiscovery: ptr.To(false), - }, - } - dpfOperatorConfig.Spec.DPUDetector = &operatorv1.DPUDetectorConfiguration{ - BaseComponentConfig: operatorv1.BaseComponentConfig{ - Disable: ptr.To(true), - }, - } - apiServerPort := 443 - if u, err := url.Parse(restConfig.Host); err == nil { - if p := u.Port(); p != "" { - if parsed, err := strconv.Atoi(p); err == nil { - apiServerPort = parsed - } - } - } - By(fmt.Sprintf("Using API server VIP %s:%d for zero-trust kubeconfig", controlPlaneIP, apiServerPort)) - if dpfOperatorConfig.Spec.Overrides == nil { - dpfOperatorConfig.Spec.Overrides = &operatorv1.Overrides{} - } - dpfOperatorConfig.Spec.Overrides.KubernetesAPIServerVIP = ptr.To(controlPlaneIP) - dpfOperatorConfig.Spec.Overrides.KubernetesAPIServerPort = ptr.To(apiServerPort) - } - - if isGinkgoLabelApplied(Domain.Scale) { - // For scale environments, the nodes are fake, therefore we can't have DPUDetector running - dpfOperatorConfig.Spec.DPUDetector = &operatorv1.DPUDetectorConfiguration{ - BaseComponentConfig: operatorv1.BaseComponentConfig{ - Disable: ptr.To(true), - }, - } - } - - // CI runs the host control-plane controllers at a single replica to save - // resources on control-plane nodes and keep logs easy to read. - if dpfOperatorConfig.Spec.DPUServiceController == nil { - dpfOperatorConfig.Spec.DPUServiceController = &operatorv1.DPUServiceControllerConfiguration{} - } - dpfOperatorConfig.Spec.ProvisioningController.Replicas = ptr.To[int32](1) - dpfOperatorConfig.Spec.DPUServiceController.Replicas = ptr.To[int32](1) - dpfOperatorConfig.Spec.KamajiClusterManager.Replicas = ptr.To[int32](1) - dpfOperatorConfig.Spec.StaticClusterManager.Replicas = ptr.To[int32](1) - dpfOperatorConfig.Spec.NodeSRIOVDevicePluginController.Replicas = ptr.To[int32](1) - - if prereqsNamespace != "" { - if dpfOperatorConfig.Spec.Overrides == nil { - dpfOperatorConfig.Spec.Overrides = &operatorv1.Overrides{} - } - - dpfOperatorConfig.Spec.Overrides.ArgoCDNamespace = ptr.To(prereqsNamespace) - } - - if isGinkgoLabelApplied(Domain.Performance) { - apiServerHost := controlPlaneIP - apiServerPort := defaultAPIServerPort - if targetClusterAPIServerHost != "" { - apiServerHost = targetClusterAPIServerHost - } else if u, err := url.Parse(restConfig.Host); err == nil { - if h := u.Hostname(); h != "" { - apiServerHost = h - } - if p := u.Port(); p != "" { - if parsed, err := strconv.Atoi(p); err == nil { - apiServerPort = parsed - } - } - } - if dpfOperatorConfig.Spec.Overrides == nil { - dpfOperatorConfig.Spec.Overrides = &operatorv1.Overrides{} - } - dpfOperatorConfig.Spec.Overrides.KubernetesAPIServerVIP = ptr.To(apiServerHost) - dpfOperatorConfig.Spec.Overrides.KubernetesAPIServerPort = ptr.To(apiServerPort) - dpfOperatorConfig.Spec.ProvisioningController.DMSTimeout = ptr.To(15 * 60) - dpfOperatorConfig.Spec.Networking = &operatorv1.Networking{ - ControlPlaneMTU: ptr.To(performanceMTU), - HighSpeedMTU: ptr.To(performanceMTU), - } - } - - input = &systemTestInput{ - namespace: dpfOperatorSystemNamespace, - config: dpfOperatorConfig, - pullSecretNames: dpfOperatorConfig.Spec.ImagePullSecrets, - client: testClient, - restConfig: restConfig, - cleanupFlags: cleanupFlags, - bfbImageURL: bfbImageURL, - bfsOsIsoURL: bfsOsIsoURL, - bfsPldmFwBundleURL: bfsPldmFwBundleURL, - } - input.applyConfig(*conf) -} - -// SystemSetupBeforeSuite sets up the system components for the e2e tests. -// If skipSystemComponentValidation is true, it skips the validation of system components after deployment. -func SystemSetupBeforeSuite(skipSystemComponentValidation bool) { - if Label(Domain.Scale).MatchesLabelFilter(GinkgoLabelFilter()) { - CreateDPUWorkerNodes(ctx, input.numberOfDPUNodes) - } - - AnnotateAndLabelNodes(ctx, input.client, input.useExternalNodeReboot) - - if ngcAPIKey != "" { - createNGCImagePullSecret(ctx, input.client) - } - - By("Deploy DPF System components") - DeployDPFSystemComponents(ctx, DeployDPFSystemComponentsInput{ - systemNamespace: input.namespace, - operatorConfig: input.config, - ImagePullSecrets: input.pullSecretNames, - ProvisioningControllerPVC: input.pvc, - dpuDiscovery: input.dpuDiscovery, - client: input.client, - numberOfDPUNodes: input.numberOfDPUNodes, - skipSystemComponentValidation: skipSystemComponentValidation, - }) - - if isGinkgoLabelApplied(Domain.ZeroTrust) { - // In ZeroTrust mode, build a DPUNode-to-host BMC IP map from the lab inventory file - // for the script-based reboot path (nodeRebootMethod.script). - input.dpuNodeBMCs = GetDPUNodeToBMCIPs( - ctx, input.client, input.numberOfDPUNodes) - - // Ensure ConfigMap and DPUNode BMC IP labels are set ahead of any DPU reaching the reboot state, - // so the controller can drive in-cluster Redfish reboots through the named ConfigMap. - ApplyNodeRebootConfigMap(ctx, input.client, input.nodeRebootConfigMapPath) - PatchDPUNodesForScriptReboot(ctx, input.client, input.numberOfDPUNodes, - input.nodeRebootConfigMap, input.dpuNodeBMCs) - } - - if isGinkgoLabelApplied(Domain.Performance) { - vip := *input.config.Spec.Overrides.KubernetesAPIServerVIP - port := *input.config.Spec.Overrides.KubernetesAPIServerPort - PatchNFDWorkerForVIP(ctx, input.client, input.namespace, vip, port) - } -} - -// createNGCImagePullSecret creates a secret to be able to pull images from NGC, this secret can be used by DPUservices and should not be used for core components. -func createNGCImagePullSecret(ctx context.Context, testClient client.Client) { - // Docker registry credentials - registry := "nvcr.io" - username := "$oauthtoken" - password := ngcAPIKey - - // Create the auth string - auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password))) - - // Build the config.json structure - dockerConfig := map[string]interface{}{ - "auths": map[string]interface{}{ - registry: map[string]string{ - "auth": auth, - }, - }, - } - - dockerConfigJSON, err := json.Marshal(dockerConfig) - Expect(err).ToNot(HaveOccurred()) - - labels := maps.Clone(CleanupScope.Suite) - labels["dpu.nvidia.com/image-pull-secret"] = "" - - // Create the Secret object - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: ngcPullSecretName, - Namespace: dpfOperatorSystemNamespace, - Labels: labels, - }, - Type: corev1.SecretTypeDockerConfigJson, - Data: map[string][]byte{ - ".dockerconfigjson": dockerConfigJSON, - }, - } - - // Create the secret - Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, secret))).NotTo(HaveOccurred()) -} - -// AnnotateAndLabelNodes stamps host-cluster Nodes with reboot-related labels -// consumed by the host agent. When useExternalNodeReboot is true (NIC cloud -// e2e tests), the labels make the host agent delegate host reboots to lab -// infrastructure (e.g. NIC cloud's `nic-cloud-reset`). Independent from -// ZeroTrust's in-cluster script reboot path (`nodeRebootMethod.script` set -// per-DPUNode by the e2e suite). -func AnnotateAndLabelNodes(ctx context.Context, c client.Client, useExternalNodeReboot bool) { - nodeAnnotations := make(map[string]string) - nodeLabels := make(map[string]string) - - if useExternalNodeReboot { - nodeLabels["provisioning.dpu.nvidia.com/reboot-method"] = "external" - nodeLabels["provisioning.dpu.nvidia.com/dpu-reboot-after-install"] = "" - } - - if len(nodeAnnotations) == 0 && len(nodeLabels) == 0 { - return - } - - By("Annotate and Label nodes in the main cluster") - Eventually(func(g Gomega) { - nodes := &corev1.NodeList{} - g.Expect(c.List(ctx, nodes)).To(Succeed()) - for _, node := range nodes.Items { - original := node.DeepCopy() - annotations := node.GetAnnotations() - if annotations == nil { - annotations = map[string]string{} - } - for k, v := range nodeAnnotations { - annotations[k] = v - } - node.SetAnnotations(annotations) - - labels := node.GetLabels() - if labels == nil { - labels = map[string]string{} - } - for k, v := range nodeLabels { - labels[k] = v - } - node.SetLabels(labels) - - g.Expect(c.Patch(ctx, &node, client.MergeFrom(original))).To(Succeed()) - } - }).WithTimeout(10 * time.Second).Should(Succeed()) -} - //nolint:dupl var _ = Describe("DPF System tests - Core", SpecPriority(CoreTestPriority), Labels{Domain.DPFSystem}, func() { BeforeEach(func() { - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { return } for _, label := range CurrentSpecReport().Labels() { @@ -344,109 +39,109 @@ var _ = Describe("DPF System tests - Core", SpecPriority(CoreTestPriority), Labe } By("Waiting for provisioning") - VerifyDPUClusterWithNodes(ctx, getProvisionDPUClustersInput()) + VerifyDPUClusterWithNodes(Ctx, GetProvisionDPUClustersInput()) By("Waiting for DPU cluster pods to be ready") - VerifyClusterPods(ctx, dpuClusterClient[0], systemPodsToVerify) + VerifyClusterPods(Ctx, DPUClusterClient[0], systemPodsToVerify) By("Waiting for DPFOperatorConfig to be ready") - VerifyDPFOperatorConfigReady(ctx, input.client, 20*time.Minute) + VerifyDPFOperatorConfigReady(Ctx, input.Client, 20*time.Minute) } }) Context("DPU Deployment", Labels{Domain.ZeroTrust}, func() { It("create a DPUDeployment with its dependencies and ensure that the underlying objects are created", func() { - ValidateDPUDeploymentCreation(ctx, input) + ValidateDPUDeploymentCreation(Ctx, input) }) It("verify DPUDeployment and DPUServiceInterface metrics", func() { - ValidateDPUDeploymentMetrics(ctx, input) + ValidateDPUDeploymentMetrics(Ctx, input) }) It("verify deletion on a disruptive upgrade with bad parameters so that the up to date DPUService never becomes ready", func() { - ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(ctx, input) + ValidateDPUDeploymentDeletionWhileDisruptiveUpgradeInProgress(Ctx, input) }) }) Context("DPU Service IPAM", Labels{Domain.ZeroTrust}, func() { It("create an invalid DPUServiceIPAM and ensure that the webhook rejects the request", func() { - ValidateDPUServiceIPAMCreationInvalid(ctx, input) + ValidateDPUServiceIPAMCreationInvalid(Ctx, input) }) It("create a DPUServiceIPAM with subnet split per node configuration and check NVIPAM IPPool is created to each cluster", func() { - ValidateDPUServiceIPAMCreationSubnetSplit(ctx, input) + ValidateDPUServiceIPAMCreationSubnetSplit(Ctx, input) }) It("verify DPUServiceIPAM metrics", func() { - ValidateDPUServiceIPAMMetrics(ctx, input) + ValidateDPUServiceIPAMMetrics(Ctx, input) }) It("delete the DPUServiceIPAM with subnet split per node configuration and check NVIPAM IPPool is deleted in each cluster", func() { - ValidateDPUServiceIPAMMetricsDeletion(ctx, input) + ValidateDPUServiceIPAMMetricsDeletion(Ctx, input) }) It("create a DPUServiceIPAM with cidr split in subnet per node configuration and check NVIPAM CIDRPool is created to each cluster", func() { - ValidateDPUServiceIPAMCreationCidrSplit(ctx, input) + ValidateDPUServiceIPAMCreationCidrSplit(Ctx, input) }) It("delete the DPUServiceIPAM with cidr split in subnet per node configuration and check NVIPAM CIDRPool is deleted in each cluster", func() { - ValidateDPUServiceIPAMDeletionCidrSplit(ctx, input) + ValidateDPUServiceIPAMDeletionCidrSplit(Ctx, input) }) }) Context("DPU Service Chain", Labels{Domain.ZeroTrust}, func() { It("create DPUServiceInterface and check that it is mirrored to each cluster", func() { - ValidateDPUServiceInterfaceCreation(ctx, input) + ValidateDPUServiceInterfaceCreation(Ctx, input) }) It("create DPUServiceChain and check that it is mirrored to each cluster", func() { - ValidateDPUServiceChainCreation(ctx, input) + ValidateDPUServiceChainCreation(Ctx, input) }) It("verify DPUServiceChain metrics", func() { - ValidateDPUServiceChainMetrics(ctx, input) + ValidateDPUServiceChainMetrics(Ctx, input) }) It("delete the DPUServiceChain & DPUServiceInterface and check that the Sets are cleaned up", func() { - ValidateDPUServiceChainDeletion(ctx, input) + ValidateDPUServiceChainDeletion(Ctx, input) }) }) Context("DPU Service Credential Request", Labels{Domain.ZeroTrust}, func() { It("create a DPUServiceCredentialRequest and check that the credentials are created", func() { - ValidateDPUServiceCredentialRequestCreation(ctx, input) + ValidateDPUServiceCredentialRequestCreation(Ctx, input) }) It("verify DPUServiceCredentialRequest metrics", func() { - ValidateDPUServiceCredentialRequestMetrics(ctx, input) + ValidateDPUServiceCredentialRequestMetrics(Ctx, input) }) It("delete the DPUServiceCredentialRequest and check that the credentials are deleted", func() { - ValidateDPUServiceCredentialRequestDeletion(ctx, input) + ValidateDPUServiceCredentialRequestDeletion(Ctx, input) }) }) Context("DPU Service", func() { It("verify DPUService and DPUServiceInterface metrics", Labels{Domain.ZeroTrust}, func() { - ValidateDPUServiceMetrics(ctx, input) + ValidateDPUServiceMetrics(Ctx, input) }) It("delete the DPUServices and check that the applications are cleaned up", Labels{Domain.ZeroTrust}, func() { - ValidateDPUServiceDeletion(ctx, input) + ValidateDPUServiceDeletion(Ctx, input) }) It("verify that the ImagePullSecrets have been synced correctly and cleaned up", Labels{Domain.ZeroTrust, Domain.ImagePullSecretsSync}, func() { - ValidateImagePullSecretsSync(ctx, input) + ValidateImagePullSecretsSync(Ctx, input) }) }) Context("DPU Service Template", Labels{Domain.ZeroTrust}, func() { It("create a DPUServiceTemplate with a chart that doesn't include annotations and expect no versions in status", func() { - ValidateDPUServiceTemplateCreationNoAnnotations(ctx, input) + ValidateDPUServiceTemplateCreationNoAnnotations(Ctx, input) }) It("create a DPUServiceTemplate with a chart that includes annotations and expect versions in status", func() { - VerifyDPUServiceTemplateCreationWithAnnotations(ctx, input) + VerifyDPUServiceTemplateCreationWithAnnotations(Ctx, input) }) It("verify DPUServiceTemplate metrics", func() { - VerifyDPUServiceTemplateMetrics(ctx, input) + VerifyDPUServiceTemplateMetrics(Ctx, input) }) }) Context("Validate General DPF Metrics", Labels{Domain.ZeroTrust}, func() { It("should validate general DPF Metrics ", func() { - ValidateGeneralDPFMetrics(ctx, input) + ValidateGeneralDPFMetrics(Ctx, input) }) }) Context("VAP Deprecation Warnings", Labels{Domain.ZeroTrust}, func() { It("verify VAP emits a warning when a deprecated field is set", func() { - ValidateVAPDeprecationWarnings(ctx, input) + ValidateVAPDeprecationWarnings(Ctx, input) }) }) @@ -454,25 +149,25 @@ var _ = Describe("DPF System tests - Core", SpecPriority(CoreTestPriority), Labe Context("Monitoring", func() { Context("KSM Metrics Collection", Labels{Domain.ZeroTrust}, func() { It("validate host cluster kube-state-metrics is accessible", func() { - VerifyHostKSMMetricsCollection(ctx) + VerifyHostKSMMetricsCollection(Ctx) }) It("validate DPU cluster kube-state-metrics is accessible", func() { By("Waiting for DPU cluster kube-state-metrics to be ready") - VerifyClusterPods(ctx, input.client, []string{"in-cluster-kube-state-metrics"}) + VerifyClusterPods(Ctx, input.Client, []string{"in-cluster-kube-state-metrics"}) By("Validating DPU cluster kube-state-metrics accessibility") - VerifyDPUKSMMetricsCollection(ctx, input) + VerifyDPUKSMMetricsCollection(Ctx, input) }) }) Context("Node Problem Detector", Labels{Domain.ZeroTrust, Domain.RequiresNodes}, func() { It("validate node-problem-detector is reporting DPU-specific node conditions", func() { - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { Skip("Skip Node Problem Detector test as there are no DPU nodes") } By("Waiting for node-problem-detector to be ready") - VerifyClusterPods(ctx, dpuClusterClient[0], []string{"node-problem-detector"}) + VerifyClusterPods(Ctx, DPUClusterClient[0], []string{"node-problem-detector"}) By("Validating node-problem-detector conditions for DPU nodes") - VerifyNodeProblemDetectorConditions(ctx, input) + VerifyNodeProblemDetectorConditions(Ctx, input) }) }) }) @@ -480,43 +175,43 @@ var _ = Describe("DPF System tests - Core", SpecPriority(CoreTestPriority), Labe Context("Component Deployment", func() { It("should verify OpenTelemetry Collector DaemonSets running in host cluster", func() { By("Running in host cluster") - VerifyClusterPods(ctx, input.client, []string{"opentelemetry-collector"}) + VerifyClusterPods(Ctx, input.Client, []string{"opentelemetry-collector"}) }) It("should verify OpenTelemetry Collector DaemonSets running in DPU cluster", Labels{Domain.RequiresNodes}, func() { - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { Skip("Skip test as there are no DPU nodes") } By("Running in DPUCluster") - VerifyClusterPods(ctx, dpuClusterClient[0], []string{"opentelemetry-collector"}) + VerifyClusterPods(Ctx, DPUClusterClient[0], []string{"opentelemetry-collector"}) }) }) Context("Configuration", func() { It("should verify DPU cluster collector configuration", Labels{Domain.RequiresNodes}, func() { - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { Skip("Skip test as there are no DPU nodes") } - ValidateDPUClusterOpenTelemetryConfiguration(ctx, input) + ValidateDPUClusterOpenTelemetryConfiguration(Ctx, input) }) }) Context("Log Flow", func() { It("should collect and forward logs from management cluster to Loki", func() { - ValidateManagementClusterLogFlow(ctx, input) + ValidateManagementClusterLogFlow(Ctx, input) }) It("should collect and forward logs from DPU cluster to Loki", Labels{Domain.RequiresNodes}, func() { - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { Skip("Skip test as there are no DPU nodes") } - ValidateDPUClusterLogFlow(ctx, input) + ValidateDPUClusterLogFlow(Ctx, input) }) }) }) }) Context("DPU Agent", Labels{Domain.ZeroTrust, Domain.RequiresNodes}, func() { It("validate DPU agent has reported status on all provisioned DPUs", func() { - ValidateDPUAgentStatus(ctx, input, provisioningv1.AgentStatus{ + ValidateDPUAgentStatus(Ctx, input, provisioningv1.AgentStatus{ RebootMethod: ptr.To(provisioningv1.RebootMethodNoAction), RebootSequenceCount: ptr.To(int32(0)), Conditions: []metav1.Condition{ @@ -548,57 +243,57 @@ var _ = Describe("DPF System tests - Core", SpecPriority(CoreTestPriority), Labe Context("DPU Service Kata Containers", Labels{Domain.RequiresNodes}, func() { It("deploy a DPUService pod with kata-qemu RuntimeClass and an SF", func() { - ValidateDPUServiceKataRuntimeClass(ctx, input) + ValidateDPUServiceKataRuntimeClass(Ctx, input) }) }) // Config Ports check is not valid for ZeroTrust Context("DPU Service Config Ports", Labels{Domain.RequiresNodes}, Serial, func() { It("expose ConfigPorts via DPUService and test reachability", func() { - ValidateDPUServiceConfigPorts(ctx, input) + ValidateDPUServiceConfigPorts(Ctx, input) }) }) Context("NodeSRIOVDevicePluginController", func() { It("verify the webhook rejects invalid NodeSRIOVDevicePluginConfig", func() { - ValidateNodeSRIOVDevicePluginWebhookRejectsInvalid(ctx, input) + ValidateNodeSRIOVDevicePluginWebhookRejectsInvalid(Ctx, input) }) It("verify a valid NodeSRIOVDevicePluginConfig is accepted and can be deleted", func() { - ValidateNodeSRIOVDevicePluginConfigValidCreate(ctx, input) + ValidateNodeSRIOVDevicePluginConfigValidCreate(Ctx, input) }) }) Context("NodeSRIOVDevicePluginController Managed Pods", Labels{Domain.RequiresNodes}, Serial, Ordered, func() { It("verify node SRIOV device plugin management", func() { - ValidateNodeSRIOVDevicePluginManagement(ctx, input) + ValidateNodeSRIOVDevicePluginManagement(Ctx, input) }) }) Context("Validate DPU Operator Config", Serial, Ordered, func() { It("verify system component overrides", Labels{Domain.ZeroTrust}, func() { - ValidateDPFOperatorBaseConfiguration(ctx, input) + ValidateDPFOperatorBaseConfiguration(Ctx, input) }) It("verify that the current MTU in the DPU clusters flannel configmap is 1500", Labels{Domain.ZeroTrust}, func() { - ValidateDPFOperatorMTUCurrentConfiguration(ctx, input) + ValidateDPFOperatorMTUCurrentConfiguration(Ctx, input) }) It("change the MTUs in the operatorConfig and verify that DPU Clusters are updated", Labels{Domain.ZeroTrust}, func() { - ValidateDPFOperatorMTUConfigurationChange(ctx, input) + ValidateDPFOperatorMTUConfigurationChange(Ctx, input) }) It("verify overrides path setting for system DPUServices", Labels{Domain.ZeroTrust}, func() { - ValidateDPFOperatorPathConfiguration(ctx, input) + ValidateDPFOperatorPathConfiguration(Ctx, input) }) It("change the MaxDPUParallelInstallations in the operatorConfig and verify that the provisioning controller is restarted", Labels{Domain.ZeroTrust}, func() { - ValidateDPFOperatorMaxDPUParallelInstallations(ctx, input) + ValidateDPFOperatorMaxDPUParallelInstallations(Ctx, input) }) It("change the flannel podCIDR in the operatorConfig and check that it is set", Labels{Domain.ZeroTrust}, func() { - ValidateDPFOperatorFlannelPodCIDRChange(ctx, input) + ValidateDPFOperatorFlannelPodCIDRChange(Ctx, input) }) // This test triggers reprovisioning, which might disrupt other tests relying on provisioned nodes. // Added BeforeEach wait for the nodes to be provisioned for the test with Domain.RequiresNodes // DMS check is not valid for ZeroTrust It("verify Kubernetes API related variables are propagated correctly", Labels{Domain.RequiresNodes}, func() { - ValidateDPFOperatorKubernetesAPIServerVIPAndPort(ctx, input) + ValidateDPFOperatorKubernetesAPIServerVIPAndPort(Ctx, input) }) }) @@ -608,33 +303,33 @@ var _ = Describe("DPF System tests - Core", SpecPriority(CoreTestPriority), Labe Context("Validate DPUDeployment full creation", Serial, Ordered, func() { BeforeAll(func() { By("Should validate DPUDeployment and underlying objects creation") - ValidateDPUDeploymentFullCreation(ctx, input) + ValidateDPUDeploymentFullCreation(Ctx, input) }) It("should validate DPUDeployment becomes ready", Labels{Domain.ZeroTrust}, func() { - VerifyDPUDeploymentIsReady(ctx, input) + VerifyDPUDeploymentIsReady(Ctx, input) }) It("should validate DPUDeployment disruptive upgrade of standard DPUServices", Labels{Domain.ZeroTrust}, func() { - if isGinkgoLabelApplied(Domain.ZeroTrust) { - ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(ctx, input) + if IsGinkgoLabelApplied(Domain.ZeroTrust) { + ValidateDPUDeploymentDPUServiceDisruptiveUpgradeHold(Ctx, input) } else { - ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(ctx, input) + ValidateDPUDeploymentDPUServiceDisruptiveUpgradeDrain(Ctx, input) } }) It("should validate DPUDeployment disruptive upgrade of standard DPUServices with bad configuration", func() { - ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(ctx, input) + ValidateDPUDeploymentDPUServiceDisruptiveUpgradeBadConfigurationAndBack(Ctx, input) }) It("should validate DPUDeployment disruptive upgrade of in-cluster DPUServices", func() { - ValidateDPUDeploymentInClusterDPUServiceDisruptiveUpgrade(ctx, input) + ValidateDPUDeploymentInClusterDPUServiceDisruptiveUpgrade(Ctx, input) }) It("should validate DPUDeployment disruptive upgrade of DPUServiceChain", Labels{Domain.ZeroTrust}, func() { - if isGinkgoLabelApplied(Domain.ZeroTrust) { - ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(ctx, input) + if IsGinkgoLabelApplied(Domain.ZeroTrust) { + ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeHold(Ctx, input) } else { - ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeDrain(ctx, input) + ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeDrain(Ctx, input) } }) It("should validate DPUDeployment disruptive upgrade of DPUServiceChain with bad configuration", func() { - ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBack(ctx, input) + ValidateDPUDeploymentDPUServiceChainDisruptiveUpgradeBadConfigurationAndBack(Ctx, input) }) }) @@ -642,27 +337,7 @@ var _ = Describe("DPF System tests - Core", SpecPriority(CoreTestPriority), Labe // proceed with the removal. Context("Validate DPFOperatorConfig deletion", Serial, Labels{Domain.ZeroTrust}, func() { It("should validate the expected objects exist before leaving the Container node", func() { - ValidateDPFOperatorConfigCleanupPrerequisites(ctx, input) + ValidateDPFOperatorConfigCleanupPrerequisites(Ctx, input) }) }) }) - -func getProvisionDPUClustersInput() ProvisionDPUClustersInput { - return ProvisionDPUClustersInput{ - numberOfDPUNodes: input.numberOfDPUNodes, - numberOfDPUsPerNode: input.numberOfDPUsPerNode, - dpuClusterPrerequisites: input.dpuClusterPrerequisites, - dpuClusters: input.dpuClusters, - dpuSet: input.dpuSet, - bfb: input.bfb, - blueFieldSoftware: input.blueFieldSoftware, - dpuFlavor: input.dpuFlavor, - client: input.client, - bfbImageURL: input.bfbImageURL, - bfsOsIsoURL: input.bfsOsIsoURL, - bfsPldmFwBundleURL: input.bfsPldmFwBundleURL, - restConfig: restConfig, - NodeRebootConfigMap: input.nodeRebootConfigMap, - DPUNodeBMCs: input.dpuNodeBMCs, - } -} diff --git a/test/e2e/upgrade_apply.go b/test/e2e/upgrade_apply.go index cbf82d99..72c0973d 100644 --- a/test/e2e/upgrade_apply.go +++ b/test/e2e/upgrade_apply.go @@ -28,69 +28,69 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// createDPUServiceTemplate creates a DPUServiceTemplate from a config-loaded +// CreateDPUServiceTemplate creates a DPUServiceTemplate from a config-loaded // manifest, with the dummy chart override that Phase 1 needs. No-op if the // manifest pointer is nil. -func createDPUServiceTemplate(ctx context.Context, input *systemTestInput, manifest *dpuservicev1.DPUServiceTemplate) { +func CreateDPUServiceTemplate(ctx context.Context, input *SystemTestInput, manifest *dpuservicev1.DPUServiceTemplate) { Expect(manifest).ToNot(BeNil()) obj := manifest.DeepCopy() obj.SetLabels(CleanupScope.Suite) useDummyDPUServiceChart(obj) By(fmt.Sprintf("Creating DPUServiceTemplate %s", obj.Name)) - Expect(input.client.Create(ctx, obj)).To(Succeed()) + Expect(input.Client.Create(ctx, obj)).To(Succeed()) } -// createDPUServiceConfiguration creates a DPUServiceConfiguration from a +// CreateDPUServiceConfiguration creates a DPUServiceConfiguration from a // config-loaded manifest. No-op if nil. -func createDPUServiceConfiguration(ctx context.Context, input *systemTestInput, manifest *dpuservicev1.DPUServiceConfiguration) { +func CreateDPUServiceConfiguration(ctx context.Context, input *SystemTestInput, manifest *dpuservicev1.DPUServiceConfiguration) { Expect(manifest).ToNot(BeNil()) obj := manifest.DeepCopy() obj.SetLabels(CleanupScope.Suite) By(fmt.Sprintf("Creating DPUServiceConfiguration %s", obj.Name)) - Expect(input.client.Create(ctx, obj)).To(Succeed()) + Expect(input.Client.Create(ctx, obj)).To(Succeed()) } -func createAdditionalDPUServiceDependencies(ctx context.Context, input *systemTestInput) { - if input.additionalDPUServiceTemplate == nil && input.additionalDPUServiceConfiguration == nil { +func CreateAdditionalDPUServiceDependencies(ctx context.Context, input *SystemTestInput) { + if input.AdditionalDPUServiceTemplate == nil && input.AdditionalDPUServiceConfiguration == nil { return } - Expect(input.additionalDPUServiceTemplate).NotTo(BeNil(), + Expect(input.AdditionalDPUServiceTemplate).NotTo(BeNil(), "additional DPUService configuration requires additional DPUService template") - Expect(input.additionalDPUServiceConfiguration).NotTo(BeNil(), + Expect(input.AdditionalDPUServiceConfiguration).NotTo(BeNil(), "additional DPUService template requires additional DPUService configuration") - createDPUServiceTemplate(ctx, input, input.additionalDPUServiceTemplate) - createDPUServiceConfiguration(ctx, input, input.additionalDPUServiceConfiguration) + CreateDPUServiceTemplate(ctx, input, input.AdditionalDPUServiceTemplate) + CreateDPUServiceConfiguration(ctx, input, input.AdditionalDPUServiceConfiguration) } -// createDPUServiceIPAMPool1 creates the dpudeployment-ipam-pool1 IPAM resource +// CreateDPUServiceIPAMPool1 creates the dpudeployment-ipam-pool1 IPAM resource // the upgrade tests reference. The base IPAM manifest is reused across many // test suites, so the upgrade-specific bits (name override, no NodeSelector) // are set in code here. -func createDPUServiceIPAMPool1(ctx context.Context, input *systemTestInput) { - dpuServiceIPAM := input.ipPoolDPUServiceIPAM.DeepCopy() +func CreateDPUServiceIPAMPool1(ctx context.Context, input *SystemTestInput) { + dpuServiceIPAM := input.IPPoolDPUServiceIPAM.DeepCopy() dpuServiceIPAM.SetLabels(CleanupScope.Suite) dpuServiceIPAM.SetName("dpudeployment-ipam-pool1") - dpuServiceIPAM.SetNamespace(dpfOperatorSystemNamespace) + dpuServiceIPAM.SetNamespace(DPFOperatorSystemNamespace) dpuServiceIPAM.Spec.NodeSelector = nil By("Creating DPUServiceIPAM dpudeployment-ipam-pool1") - Expect(input.client.Create(ctx, dpuServiceIPAM)).To(Succeed()) + Expect(input.Client.Create(ctx, dpuServiceIPAM)).To(Succeed()) } -// patchDPFOperatorConfigForSpecDeploymentMode supports the breaking change that +// PatchDPFOperatorConfigForSpecDeploymentMode supports the breaking change that // introduced DPFOperatorConfig.spec.deploymentMode as a required field. Upgrade // validation runs preserve resources from the previous phase, so a cluster // upgraded from an older build can still have no deploymentMode. -func patchDPFOperatorConfigForSpecDeploymentMode(ctx context.Context, input *systemTestInput) { +func PatchDPFOperatorConfigForSpecDeploymentMode(ctx context.Context, input *SystemTestInput) { cfg := &operatorv1.DPFOperatorConfig{} - Expect(input.client.Get(ctx, client.ObjectKey{ - Name: configName, - Namespace: dpfOperatorSystemNamespace, + Expect(input.Client.Get(ctx, client.ObjectKey{ + Name: ConfigName, + Namespace: DPFOperatorSystemNamespace, }, cfg)).To(Succeed()) if cfg.Spec.DeploymentMode != "" { return } original := cfg.DeepCopy() - cfg.Spec.DeploymentMode = input.config.Spec.DeploymentMode + cfg.Spec.DeploymentMode = input.Config.Spec.DeploymentMode By("Patching DPFOperatorConfig for required spec.deploymentMode") - Expect(input.client.Patch(ctx, cfg, client.MergeFrom(original))).To(Succeed()) + Expect(input.Client.Patch(ctx, cfg, client.MergeFrom(original))).To(Succeed()) } diff --git a/test/e2e/upgrade_artifacts_test.go b/test/e2e/upgrade_artifacts.go similarity index 76% rename from test/e2e/upgrade_artifacts_test.go rename to test/e2e/upgrade_artifacts.go index 9f771a38..8240c1b2 100644 --- a/test/e2e/upgrade_artifacts_test.go +++ b/test/e2e/upgrade_artifacts.go @@ -37,14 +37,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// upgradeArtifactsFile returns the on-disk path for the snapshot identified +// UpgradeArtifactsFile returns the on-disk path for the snapshot identified // by key. Files live one level above artifactsDir so all phases in a run // share the same parent and later phases can read earlier ones. -func upgradeArtifactsFile(key string) string { - return filepath.Join(artifactsDir, "..", "upgrade-artifacts-"+key+".json") +func UpgradeArtifactsFile(key string) string { + return filepath.Join(ArtifactsDir, "..", "upgrade-artifacts-"+key+".json") } -// upgradeExpectedChange describes a known spec change introduced by an upgrade. +// UpgradeExpectedChange describes a known spec change introduced by an upgrade. // Objects that are recreated can't be handled by this struct and need to be // handled in a different way. transform is applied only to the after artifact // of the matching GVK before comparison, resetting the changed field(s) back @@ -58,15 +58,15 @@ func upgradeArtifactsFile(key string) string { // spec, _ := a["spec"].(map[string]interface{}) // spec["something"] = false // }} -type upgradeExpectedChange struct { - gvk schema.GroupVersionKind - transform func(artifact map[string]interface{}) +type UpgradeExpectedChange struct { + GVK schema.GroupVersionKind + Transform func(artifact map[string]interface{}) } -// applyUpgradeExpectedChanges mutates `after` to reset the fields touched by +// ApplyUpgradeExpectedChanges mutates `after` to reset the fields touched by // each registered transform, and bumps the matching `before` artifact's // generation by one (since the upgrade necessarily bumped it once). -func applyUpgradeExpectedChanges(before, after []map[string]interface{}, expectedChanges []upgradeExpectedChange) { +func ApplyUpgradeExpectedChanges(before, after []map[string]interface{}, expectedChanges []UpgradeExpectedChange) { type artifactKey struct{ apiVersion, kind, name, namespace string } beforeIdx := make(map[artifactKey]int, len(before)) for i, b := range before { @@ -86,10 +86,10 @@ func applyUpgradeExpectedChanges(before, after []map[string]interface{}, expecte Expect(err).ToNot(HaveOccurred()) artifactGVK := gv.WithKind(kind) for _, change := range expectedChanges { - if change.gvk != artifactGVK { + if change.GVK != artifactGVK { continue } - change.transform(after[i]) + change.Transform(after[i]) // The spec change introduced by the upgrade bumped the generation once. // Increment the matching before artifact's generation so the comparison holds. k := artifactKey{ @@ -107,11 +107,11 @@ func applyUpgradeExpectedChanges(before, after []map[string]interface{}, expecte } } -// collectArtifacts writes a snapshot of all tracked objects (DPUs, +// CollectArtifacts writes a snapshot of all tracked objects (DPUs, // DPUDeployment-owned DPUServices, DPUServiceChains, DPUSets, // DPUServiceInterfaces, plus DPU-cluster-side ServiceChains, ServiceInterfaces, // and service Pods) to filePath as JSON. -func collectArtifacts(filePath string) { +func CollectArtifacts(filePath string) { By("Collecting artifacts to: " + filePath) Expect(os.MkdirAll(filepath.Dir(filePath), 0755)).To(Succeed()) @@ -119,40 +119,40 @@ func collectArtifacts(filePath string) { By("Capturing DPU artifacts") dpuList := &provisioningv1.DPUList{} - Expect(input.client.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - allArtifacts = append(allArtifacts, extractArtifacts(ToClientObjectSlice(dpuList.Items))...) + Expect(input.Client.List(Ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + allArtifacts = append(allArtifacts, ExtractArtifacts(ToClientObjectSlice(dpuList.Items))...) By("Capturing DPUService artifacts with owned-by-dpudeployment label") dpuServiceList := &dpuservicev1.DPUServiceList{} - Expect(input.client.List(ctx, dpuServiceList, - client.InNamespace(dpfOperatorSystemNamespace), + Expect(input.Client.List(Ctx, dpuServiceList, + client.InNamespace(DPFOperatorSystemNamespace), client.HasLabels{dpuservicev1.ParentDPUDeploymentNameLabel})).To(Succeed()) - allArtifacts = append(allArtifacts, extractArtifacts(ToClientObjectSlice(dpuServiceList.Items))...) + allArtifacts = append(allArtifacts, ExtractArtifacts(ToClientObjectSlice(dpuServiceList.Items))...) By("Capturing DPUServiceChain artifacts") dpuServiceChainList := &dpuservicev1.DPUServiceChainList{} - Expect(input.client.List(ctx, dpuServiceChainList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - allArtifacts = append(allArtifacts, extractArtifacts(ToClientObjectSlice(dpuServiceChainList.Items))...) + Expect(input.Client.List(Ctx, dpuServiceChainList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + allArtifacts = append(allArtifacts, ExtractArtifacts(ToClientObjectSlice(dpuServiceChainList.Items))...) By("Capturing DPUSet artifacts") dpuSetList := &provisioningv1.DPUSetList{} - Expect(input.client.List(ctx, dpuSetList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - allArtifacts = append(allArtifacts, extractArtifacts(ToClientObjectSlice(dpuSetList.Items))...) + Expect(input.Client.List(Ctx, dpuSetList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + allArtifacts = append(allArtifacts, ExtractArtifacts(ToClientObjectSlice(dpuSetList.Items))...) By("Capturing DPUServiceInterface artifacts") dpuServiceInterfaceList := &dpuservicev1.DPUServiceInterfaceList{} - Expect(input.client.List(ctx, dpuServiceInterfaceList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - allArtifacts = append(allArtifacts, extractArtifacts(ToClientObjectSlice(dpuServiceInterfaceList.Items))...) + Expect(input.Client.List(Ctx, dpuServiceInterfaceList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + allArtifacts = append(allArtifacts, ExtractArtifacts(ToClientObjectSlice(dpuServiceInterfaceList.Items))...) By("Capturing ServiceChain artifacts from DPU cluster") serviceChainList := &dpuservicev1.ServiceChainList{} - Expect(dpuClusterClient[0].List(ctx, serviceChainList)).To(Succeed()) - allArtifacts = append(allArtifacts, extractArtifacts(ToClientObjectSlice(serviceChainList.Items))...) + Expect(DPUClusterClient[0].List(Ctx, serviceChainList)).To(Succeed()) + allArtifacts = append(allArtifacts, ExtractArtifacts(ToClientObjectSlice(serviceChainList.Items))...) By("Capturing ServiceInterface artifacts from DPU cluster") serviceInterfaceList := &dpuservicev1.ServiceInterfaceList{} - Expect(dpuClusterClient[0].List(ctx, serviceInterfaceList)).To(Succeed()) - allArtifacts = append(allArtifacts, extractArtifacts(ToClientObjectSlice(serviceInterfaceList.Items))...) + Expect(DPUClusterClient[0].List(Ctx, serviceInterfaceList)).To(Succeed()) + allArtifacts = append(allArtifacts, ExtractArtifacts(ToClientObjectSlice(serviceInterfaceList.Items))...) By("Capturing Pod artifacts from DPU cluster with service label but not system component label") podList := &corev1.PodList{} @@ -161,8 +161,8 @@ func collectArtifacts(filePath string) { notSystemComponentReq, reqErr := labels.NewRequirement(operatorv1.DPFComponentLabelKey, selection.DoesNotExist, nil) Expect(reqErr).ToNot(HaveOccurred()) podSelector := labels.NewSelector().Add(*hasServiceLabelReq, *notSystemComponentReq) - Expect(dpuClusterClient[0].List(ctx, podList, &client.MatchingLabelsSelector{Selector: podSelector})).To(Succeed()) - allArtifacts = append(allArtifacts, extractArtifacts(ToClientObjectSlice(podList.Items))...) + Expect(DPUClusterClient[0].List(Ctx, podList, &client.MatchingLabelsSelector{Selector: podSelector})).To(Succeed()) + allArtifacts = append(allArtifacts, ExtractArtifacts(ToClientObjectSlice(podList.Items))...) artifactData, err := json.MarshalIndent(allArtifacts, "", " ") Expect(err).ToNot(HaveOccurred()) @@ -171,8 +171,8 @@ func collectArtifacts(filePath string) { Expect(os.WriteFile(filePath, artifactData, 0644)).To(Succeed()) } -// getArtifacts reads a snapshot previously written by collectArtifacts. -func getArtifacts(filePath string) []map[string]interface{} { +// GetArtifacts reads a snapshot previously written by collectArtifacts. +func GetArtifacts(filePath string) []map[string]interface{} { By("Reading artifacts from: " + filePath) data, err := os.ReadFile(filePath) Expect(err).ToNot(HaveOccurred()) @@ -182,13 +182,13 @@ func getArtifacts(filePath string) []map[string]interface{} { return artifacts } -// compareArtifactSnapshots loads the two named snapshots, applies the given +// CompareArtifactSnapshots loads the two named snapshots, applies the given // expected-change transforms, and asserts they match (modulo sorting). The // phaseDescription is used in assertion messages. -func compareArtifactSnapshots(prevKey, currKey, phaseDescription string, expectedChanges []upgradeExpectedChange) { - prev := getArtifacts(upgradeArtifactsFile(prevKey)) - curr := getArtifacts(upgradeArtifactsFile(currKey)) - applyUpgradeExpectedChanges(prev, curr, expectedChanges) +func CompareArtifactSnapshots(prevKey, currKey, phaseDescription string, expectedChanges []UpgradeExpectedChange) { + prev := GetArtifacts(UpgradeArtifactsFile(prevKey)) + curr := GetArtifacts(UpgradeArtifactsFile(currKey)) + ApplyUpgradeExpectedChanges(prev, curr, expectedChanges) By(fmt.Sprintf("Comparing artifacts: %s vs %s", prevKey, currKey)) Expect(curr).To(HaveLen(len(prev)), "Number of tracked objects should be unchanged after %s upgrade", phaseDescription) @@ -208,12 +208,12 @@ func ToClientObjectSlice[T any](in []T) []client.Object { return out } -// extractArtifacts extracts the GVK, name, namespace, UID, generation, and +// ExtractArtifacts extracts the GVK, name, namespace, UID, generation, and // spec of each object — the stable subset we care about for upgrade // comparison. All other fields (status, volatile metadata) are excluded. // GVK is resolved via the scheme because List calls do not populate TypeMeta // on individual items. -func extractArtifacts(objects []client.Object) []map[string]interface{} { +func ExtractArtifacts(objects []client.Object) []map[string]interface{} { artifacts := make([]map[string]interface{}, 0, len(objects)) for _, obj := range objects { data, err := json.Marshal(obj) diff --git a/test/e2e/upgrade_framework_test.go b/test/e2e/upgrade_framework.go similarity index 69% rename from test/e2e/upgrade_framework_test.go rename to test/e2e/upgrade_framework.go index a56318e9..2c8b3684 100644 --- a/test/e2e/upgrade_framework_test.go +++ b/test/e2e/upgrade_framework.go @@ -54,112 +54,112 @@ import ( const reconciliationWaitAfterRollout time.Duration = 30 * time.Second -// installPhaseInput configures one install phase of an upgrade path: provision +// InstallPhaseInput configures one install phase of an upgrade path: provision // DPUs and create the dependency resources from the phase's config manifests, // then capture the initial artifact snapshot. All booleans default to false and // most fields are optional. -type installPhaseInput struct { - // label is the Ginkgo label used to filter this phase in CI. - label string - // skipBFBImageURL clears provInput.bfbImageURL before +type InstallPhaseInput struct { + // Label is the Ginkgo Label used to filter this phase in CI. + Label string + // SkipBFBImageURL clears provInput.bfbImageURL before // ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor, so the pre-upgrade state // reflects the hardcoded URL from the phase's BFB manifest regardless of // BFB_IMAGE_URL. - skipBFBImageURL bool - // skipSystemComponentValidation skips the current-shape system-component + SkipBFBImageURL bool + // SkipSystemComponentValidation skips the current-shape system-component // checks during setup. Set for previous-release installs (e.g. BFB LTS // v25.10) whose deployed component shape differs from the current release. - skipSystemComponentValidation bool - // expectedKubernetesVersion, if set, is the DPUCluster Status.Version this + SkipSystemComponentValidation bool + // ExpectedKubernetesVersion, if set, is the DPUCluster Status.Version this // install should report instead of util.KubernetesVersion. Set for // previous-release installs (e.g. BFB LTS v25.10) on an older Kubernetes // version than HEAD. - expectedKubernetesVersion string - // artifactsKey, if set, captures a snapshot to upgrade-artifacts-.json. - artifactsKey string - // expectedDPUServices returns the DPUService names verifySystemReady expects + ExpectedKubernetesVersion string + // ArtifactsKey, if set, captures a snapshot to upgrade-artifacts-.json. + ArtifactsKey string + // ExpectedDPUServices returns the DPUService names verifySystemReady expects // on the DPU cluster at this phase's DPF release. - expectedDPUServices func(input *systemTestInput) []string + ExpectedDPUServices func(input *SystemTestInput) []string } -// validationPhaseInput configures one validation phase of an upgrade path: +// ValidationPhaseInput configures one validation phase of an upgrade path: // validate existing resources after the operator has been upgraded externally. // All booleans default to false and most fields are optional. -type validationPhaseInput struct { - // label is the Ginkgo label used to filter this phase in CI. - label string - // expectedDPFVersion, if set, overrides TAG for this validation phase. +type ValidationPhaseInput struct { + // Label is the Ginkgo Label used to filter this phase in CI. + Label string + // ExpectedDPFVersion, if set, overrides TAG for this validation phase. // This is needed for intermediate released hops in multi-step upgrade paths. - expectedDPFVersion string - // patchDeploymentMode, if true, sets DPFOperatorConfig.spec.deploymentMode + ExpectedDPFVersion string + // PatchDeploymentMode, if true, sets DPFOperatorConfig.spec.deploymentMode // when the deployed config leaves it empty. Only set it on hops that upgrade // a config predating the field (the patch is a no-op when the field is already // set). - patchDeploymentMode bool - // captureBeforeRollout makes capture+compare happen BEFORE rollout steps. + PatchDeploymentMode bool + // CaptureBeforeRollout makes capture+compare happen BEFORE rollout steps. // Set true for the regular upgrade, where artifact validation precedes the // post-upgrade rollout exercise. - captureBeforeRollout bool - // rolloutAllDPUs deletes every DPU and waits for them to be recreated with + CaptureBeforeRollout bool + // RolloutAllDPUs deletes every DPU and waits for them to be recreated with // the new DPFVersion. Used by BFB LTS phases that bump major.minor. - rolloutAllDPUs bool - // rolloutDPFVersionMinor is the major.minor expected in DPFVersion after + RolloutAllDPUs bool + // RolloutDPFVersionMinor is the major.minor expected in DPFVersion after // rolloutAllDPUs (e.g. "v26.4"). - rolloutDPFVersionMinor string - // rolloutDependencies updates one DPUDeployment to a new dependency set + RolloutDPFVersionMinor string + // RolloutDependencies updates one DPUDeployment to a new dependency set // (BFB, DPUFlavor, DPUServiceTemplate, DPUServiceConfiguration) and waits // for reconciliation. Used by regular and BFB LTS upgrades. - rolloutDependencies bool - // verifyKubeletVersion asserts every DPU reports a non-empty KubeletVersion. + RolloutDependencies bool + // VerifyKubeletVersion asserts every DPU reports a non-empty KubeletVersion. // Required after DPUs are reprovisioned with DPF v26.4+. - verifyKubeletVersion bool - // removeStaleDPUDeviceFinalizers, if true, clears the dpudevice-protection + VerifyKubeletVersion bool + // RemoveStaleDPUDeviceFinalizers, if true, clears the dpudevice-protection // finalizer from unreferenced DPUDevices in an AfterAll. Set for phases that // preserve the cluster with -e2e.skip-cleanup (so the AfterSuite teardown, // which would otherwise clear them via DeleteDPFOperatorConfig, never runs): // after a v25.10 → v26.4 upgrade non-selected DPUDevices can retain the legacy // finalizer and stall the eventual teardown (#5048585). - removeStaleDPUDeviceFinalizers bool - // artifactsKey captures a snapshot to upgrade-artifacts-.json. - artifactsKey string - // prevArtifactsKey compares the current snapshot against this previously + RemoveStaleDPUDeviceFinalizers bool + // ArtifactsKey captures a snapshot to upgrade-artifacts-.json. + ArtifactsKey string + // PrevArtifactsKey compares the current snapshot against this previously // captured one. - prevArtifactsKey string - // preRolloutArtifactsKey, if set, captures a validation snapshot before any + PrevArtifactsKey string + // PreRolloutArtifactsKey, if set, captures a validation snapshot before any // rollout step, so a multi-hop path can prove the operator hop itself did not // recreate objects while keeping a separate post-rollout snapshot as the next // hop's baseline. - preRolloutArtifactsKey string - // preRolloutPrevArtifactsKey, if set, compares the pre-rollout snapshot + PreRolloutArtifactsKey string + // PreRolloutPrevArtifactsKey, if set, compares the pre-rollout snapshot // against a previous phase's snapshot. - preRolloutPrevArtifactsKey string - // expectedChanges lists spec changes this hop intentionally introduces (e.g. + PreRolloutPrevArtifactsKey string + // ExpectedChanges lists spec changes this hop intentionally introduces (e.g. // a newly defaulted field) so the artifact comparison ignores them. Empty for // hops that introduce no such change. Applied to every snapshot comparison // this phase runs. - expectedChanges []upgradeExpectedChange - // expectedDPUServices returns the DPUService names verifySystemReady expects + ExpectedChanges []UpgradeExpectedChange + // ExpectedDPUServices returns the DPUService names verifySystemReady expects // on the DPU cluster at this phase's DPF release. Required; the shape might // differ between releases. - expectedDPUServices func(input *systemTestInput) []string - // expectedKubernetesVersion, if set, is the DPUCluster Status.Version this + ExpectedDPUServices func(input *SystemTestInput) []string + // ExpectedKubernetesVersion, if set, is the DPUCluster Status.Version this // install should report instead of util.KubernetesVersion. Set for // previous-release installs (e.g. BFB LTS v25.10) on an older Kubernetes // version than HEAD. - expectedKubernetesVersion string + ExpectedKubernetesVersion string } -// validationPhaseLabels collects the Ginkgo label of every registered +// ValidationPhaseLabels collects the Ginkgo label of every registered // validation phase as a side effect of validationPhase. BeforeSuite consults it // (via isUpgradeValidationPhase) to skip cleanup between phases. -var validationPhaseLabels []string +var ValidationPhaseLabels []string -// isUpgradeValidationPhase reports whether the active Ginkgo label filter +// IsUpgradeValidationPhase reports whether the active Ginkgo label filter // matches any upgrade *validation* phase. Used by BeforeSuite to skip cleanup // between phases. Install phases are NOT covered here because Phase 1 needs // normal pre-test cleanup. -func isUpgradeValidationPhase() bool { - for _, label := range validationPhaseLabels { +func IsUpgradeValidationPhase() bool { + for _, label := range ValidationPhaseLabels { if Label(label).MatchesLabelFilter(GinkgoLabelFilter()) { return true } @@ -167,49 +167,49 @@ func isUpgradeValidationPhase() bool { return false } -// installPhase emits the Ginkgo container for one install phase: provision DPU +// InstallPhase emits the Ginkgo container for one install phase: provision DPU // clusters + BFB + DPUFlavor, create DPUService dependencies (templates, // configurations, IPAM, optional additional service variants), create // DPUDeployments per worker node, and capture the initial artifact snapshot. // Call from inside the upgrade path's Describe block. -func installPhase(description string, in installPhaseInput) { - if in.expectedDPUServices == nil { +func InstallPhase(description string, in InstallPhaseInput) { + if in.ExpectedDPUServices == nil { panic(fmt.Sprintf("install phase %q must set expectedDPUServices", description)) } - Context("install: "+description, Labels{in.label, Domain.RequiresNodes}, Serial, Ordered, func() { + Context("install: "+description, Labels{in.Label, Domain.RequiresNodes}, Serial, Ordered, func() { It("create DPFOperatorConfig", func() { - SystemSetupBeforeSuite(in.skipSystemComponentValidation) + SystemSetupBeforeSuite(in.SkipSystemComponentValidation) By("Pre provisioning DPU cluster setup") - provInput := getProvisionDPUClustersInput() - provInput.expectedKubernetesVersion = in.expectedKubernetesVersion - ProvisionDPUClusters(ctx, provInput) - if in.skipBFBImageURL { + provInput := GetProvisionDPUClustersInput() + provInput.ExpectedKubernetesVersion = in.ExpectedKubernetesVersion + ProvisionDPUClusters(Ctx, provInput) + if in.SkipBFBImageURL { // Use the hardcoded URL from the BFB manifest regardless of // BFB_IMAGE_URL — pre-upgrade state reflects the known // previous-release BFB. - provInput.bfbImageURL = "" + provInput.BFBImageURL = "" } - ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(ctx, provInput) + ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(Ctx, provInput) }) It("create DPUDeployment dependencies", func() { - createDPUServiceTemplate(ctx, input, input.dpuServiceTemplate) - createDPUServiceConfiguration(ctx, input, input.dpuServiceConfiguration) - createAdditionalDPUServiceDependencies(ctx, input) - createDPUServiceIPAMPool1(ctx, input) + CreateDPUServiceTemplate(Ctx, input, input.DPUServiceTemplate) + CreateDPUServiceConfiguration(Ctx, input, input.DPUServiceConfiguration) + CreateAdditionalDPUServiceDependencies(Ctx, input) + CreateDPUServiceIPAMPool1(Ctx, input) }) It("create DPUDeployment objects", func() { By("Get worker nodes") nodes := &corev1.NodeList{} - Expect(input.client.List(ctx, nodes, + Expect(input.Client.List(Ctx, nodes, client.MatchingLabels{"node-role.kubernetes.io/worker": ""})).To(Succeed()) By("Creating DPUDeployment objects for each DPU node") - for i := 0; i < input.numberOfDPUNodes; i++ { + for i := 0; i < input.NumberOfDPUNodes; i++ { node := &nodes.Items[i] - dpuDeployment := input.dpuDeployment.DeepCopy() + dpuDeployment := input.DPUDeployment.DeepCopy() dpuDeployment.SetLabels(CleanupScope.Suite) dpuDeployment.SetName(node.GetName()) // Per-node hostname selector — the only field that has to be @@ -221,67 +221,67 @@ func installPhase(description string, in installPhaseInput) { dpuDeployment.Spec.DPUs.DPUSets[0].NodeSelector = &metav1.LabelSelector{ MatchLabels: map[string]string{"kubernetes.io/hostname": node.GetName()}, } - Expect(input.client.Create(ctx, dpuDeployment)).To(Succeed()) + Expect(input.Client.Create(Ctx, dpuDeployment)).To(Succeed()) } }) It("get DPUCluster client", func() { - getDPUClusterClients(ctx, getProvisionDPUClustersInput()) + GetDPUClusterClients(Ctx, GetProvisionDPUClustersInput()) }) It("wait for DPUs to be provisioned", func() { By("Waiting for provisioning") - VerifyDPUClusterWithNodes(ctx, getProvisionDPUClustersInput()) + VerifyDPUClusterWithNodes(Ctx, GetProvisionDPUClustersInput()) By("Waiting for system components to be ready") - verifySystemReady(in.expectedDPUServices(input)) + VerifySystemReady(in.ExpectedDPUServices(input)) }) - if in.artifactsKey != "" { + if in.ArtifactsKey != "" { It("capture DPU and DPUService artifacts after install", func() { - collectArtifacts(upgradeArtifactsFile(in.artifactsKey)) + CollectArtifacts(UpgradeArtifactsFile(in.ArtifactsKey)) }) } }) } -// validationPhase emits the Ginkgo container for one validation phase. The +// ValidationPhase emits the Ginkgo container for one validation phase. The // common steps (pre-upgrade check, DPF version, DPUCluster client, cluster // health, DMS image tag) always run; everything else is gated on the input // fields. Call from inside the upgrade path's Describe block. -func validationPhase(description string, in validationPhaseInput) { - if in.expectedDPUServices == nil { +func ValidationPhase(description string, in ValidationPhaseInput) { + if in.ExpectedDPUServices == nil { panic(fmt.Sprintf("validation phase %q must set expectedDPUServices", description)) } - if in.rolloutAllDPUs && in.rolloutDPFVersionMinor == "" { + if in.RolloutAllDPUs && in.RolloutDPFVersionMinor == "" { panic(fmt.Sprintf("validation phase %q sets rolloutAllDPUs but not rolloutDPFVersionMinor", description)) } - validationPhaseLabels = append(validationPhaseLabels, in.label) - Context("validation: "+description, Labels{in.label, Domain.RequiresNodes}, Serial, Ordered, func() { + ValidationPhaseLabels = append(ValidationPhaseLabels, in.Label) + Context("validation: "+description, Labels{in.Label, Domain.RequiresNodes}, Serial, Ordered, func() { - if in.removeStaleDPUDeviceFinalizers { + if in.RemoveStaleDPUDeviceFinalizers { // Runs even when this phase preserves the cluster (-e2e.skip-cleanup), // so the stale dpudevice-protection finalizers from a v25.10 → v26.4 // upgrade are cleared before the cluster is eventually torn down // (#5048585). DeleteDPFOperatorConfig clears them for non-skip-cleanup // phases; this AfterAll covers the skip-cleanup ones. AfterAll(func() { - removeStaleDPUDeviceProtectionFinalizers(ctx, input.client) + RemoveStaleDPUDeviceProtectionFinalizers(Ctx, input.Client) }) } - if in.patchDeploymentMode { + if in.PatchDeploymentMode { It("patch DPFOperatorConfig schema bridge fields", func() { - patchDPFOperatorConfigForSpecDeploymentMode(ctx, input) + PatchDPFOperatorConfigForSpecDeploymentMode(Ctx, input) }) } It("validate pre-upgrade conditions pass", func() { - validatePreUpgradeConditions(ctx, input) + ValidatePreUpgradeConditions(Ctx, input) }) It("validate the DPF version", func() { - validateDPFVersionUpgrade(in.expectedDPFVersion) + ValidateDPFVersionUpgrade(in.ExpectedDPFVersion) }) It("validate DPUCluster ready", func() { - validateDPUClusterUpgrade(ctx, getProvisionDPUClustersInput(), in.expectedKubernetesVersion) + ValidateDPUClusterUpgrade(Ctx, GetProvisionDPUClustersInput(), in.ExpectedKubernetesVersion) }) // Create the DPUCluster client only after the DPUCluster upgrade is // confirmed complete. The control-plane roll during the upgrade tears @@ -290,15 +290,15 @@ func validationPhase(description string, in validationPhaseInput) { // so it never re-binds, and later DPU-cluster calls (e.g. artifact // capture) fail with "connection refused". It("get DPUCluster client", func() { - getDPUClusterClients(ctx, getProvisionDPUClustersInput()) + GetDPUClusterClients(Ctx, GetProvisionDPUClustersInput()) }) It("validate DPUCluster is healthy", func() { - VerifyDPUClusterWithNodes(ctx, getProvisionDPUClustersInput()) + VerifyDPUClusterWithNodes(Ctx, GetProvisionDPUClustersInput()) By("Waiting for system components to be ready") - verifySystemReady(in.expectedDPUServices(input)) + VerifySystemReady(in.ExpectedDPUServices(input)) }) It("validate that DMS Pods are upgraded", func() { - VerifyHostAgentPodsImageTag(ctx, input) + VerifyHostAgentPodsImageTag(Ctx, input) }) It("wait for controllers to reconcile", func() { @@ -308,53 +308,53 @@ func validationPhase(description string, in validationPhaseInput) { // Capture before any rollout step when the phase compares the // operator upgrade itself separately from an intentional rollout. - if in.captureBeforeRollout { - registerArtifactCaptureStep(description, "", in.artifactsKey, in.prevArtifactsKey, in.expectedChanges) + if in.CaptureBeforeRollout { + RegisterArtifactCaptureStep(description, "", in.ArtifactsKey, in.PrevArtifactsKey, in.ExpectedChanges) } // Capture a pre-rollout snapshot for multi-hop paths that prove the // operator hop itself recreated nothing, kept separate from the // post-rollout snapshot that becomes the next hop's baseline. - if in.preRolloutArtifactsKey != "" { - registerArtifactCaptureStep(description, "before rollout", in.preRolloutArtifactsKey, in.preRolloutPrevArtifactsKey, in.expectedChanges) + if in.PreRolloutArtifactsKey != "" { + RegisterArtifactCaptureStep(description, "before rollout", in.PreRolloutArtifactsKey, in.PreRolloutPrevArtifactsKey, in.ExpectedChanges) } - if in.rolloutAllDPUs { + if in.RolloutAllDPUs { It(fmt.Sprintf("roll out all DPUs with BFB LTS under %s", description), func() { - rolloutAllDPUs(ctx, input, in.rolloutDPFVersionMinor) + RolloutAllDPUs(Ctx, input, in.RolloutDPFVersionMinor) }) } - if in.rolloutDependencies { + if in.RolloutDependencies { It("perform DPU and DPUService rollout test", func() { - rolloutDependencies(ctx, input) + RolloutDependencies(Ctx, input) }) } It("wait for DPUs to be ready and system healthy after rollout", func() { - VerifyDPUClusterWithNodes(ctx, getProvisionDPUClustersInput()) + VerifyDPUClusterWithNodes(Ctx, GetProvisionDPUClustersInput()) By("Waiting for system components to be ready after rollout") - verifySystemReady(in.expectedDPUServices(input)) + VerifySystemReady(in.ExpectedDPUServices(input)) }) - if in.verifyKubeletVersion { + if in.VerifyKubeletVersion { It("verify all DPUs report KubeletVersion", func() { - verifyDPUsHaveKubeletVersion(ctx, input) + VerifyDPUsHaveKubeletVersion(Ctx, input) }) } // Capture position #2 (BFB LTS): after rollout steps complete. - if !in.captureBeforeRollout { - registerArtifactCaptureStep(description, "", in.artifactsKey, in.prevArtifactsKey, in.expectedChanges) + if !in.CaptureBeforeRollout { + RegisterArtifactCaptureStep(description, "", in.ArtifactsKey, in.PrevArtifactsKey, in.ExpectedChanges) } }) } -// registerArtifactCaptureStep emits an It block that captures a snapshot and +// RegisterArtifactCaptureStep emits an It block that captures a snapshot and // optionally compares it against a previous one. No-op if artifactsKey is // empty. See upgrade_artifacts_test.go for the underlying capture/compare // machinery. -func registerArtifactCaptureStep(phaseDescription, stepSuffix, artifactsKey, prevArtifactsKey string, expectedChanges []upgradeExpectedChange) { +func RegisterArtifactCaptureStep(phaseDescription, stepSuffix, artifactsKey, prevArtifactsKey string, expectedChanges []UpgradeExpectedChange) { if artifactsKey == "" { return } @@ -365,22 +365,22 @@ func registerArtifactCaptureStep(phaseDescription, stepSuffix, artifactsKey, pre itName += " " + stepSuffix } It(itName, func() { - collectArtifacts(upgradeArtifactsFile(artifactsKey)) + CollectArtifacts(UpgradeArtifactsFile(artifactsKey)) if prevArtifactsKey == "" { return } - compareArtifactSnapshots(prevArtifactsKey, artifactsKey, phaseDescription, expectedChanges) + CompareArtifactSnapshots(prevArtifactsKey, artifactsKey, phaseDescription, expectedChanges) }) } -// validatePreUpgradeConditions waits for DPFOperatorConfig to report +// ValidatePreUpgradeConditions waits for DPFOperatorConfig to report // PreUpgradeValidationReady=True and asserts the condition remains stable. -func validatePreUpgradeConditions(ctx context.Context, input *systemTestInput) { +func ValidatePreUpgradeConditions(ctx context.Context, input *SystemTestInput) { By("Validating pre-upgrade conditions of dpfoperatorconfig with stability verification") checkConditionReady := func(g Gomega) { dpfOperatorConfig := &operatorv1.DPFOperatorConfig{} - g.Expect(input.client.Get(ctx, client.ObjectKey{Name: configName, Namespace: dpfOperatorSystemNamespace}, dpfOperatorConfig)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKey{Name: ConfigName, Namespace: DPFOperatorSystemNamespace}, dpfOperatorConfig)).To(Succeed()) g.Expect(dpfOperatorConfig.Status.ObservedGeneration).To(Equal(dpfOperatorConfig.GetGeneration())) g.Expect(dpfOperatorConfig.Status.Conditions).NotTo(BeEmpty()) g.Expect(conditions.IsTrue(dpfOperatorConfig, operatorv1.PreUpgradeValidationReadyCondition)).To(BeTrue()) @@ -398,17 +398,17 @@ func validatePreUpgradeConditions(ctx context.Context, input *systemTestInput) { "PreUpgradeValidationReady condition should be ready and stable") } -// validateDPFVersionUpgrade asserts the operator has reached the expected +// ValidateDPFVersionUpgrade asserts the operator has reached the expected // version. Empty expectedVersion falls back to TAG, the version under test. -func validateDPFVersionUpgrade(expectedVersion string) { +func ValidateDPFVersionUpgrade(expectedVersion string) { if expectedVersion == "" { expectedVersion = tag } Eventually(func(g Gomega) { dpfOperatorConfig := &operatorv1.DPFOperatorConfig{} - g.Expect(input.client.Get(ctx, client.ObjectKey{ - Name: configName, - Namespace: dpfOperatorSystemNamespace, + g.Expect(input.Client.Get(Ctx, client.ObjectKey{ + Name: ConfigName, + Namespace: DPFOperatorSystemNamespace, }, dpfOperatorConfig)).To(Succeed()) g.Expect(dpfOperatorConfig.Status.Version).NotTo(BeNil(), "DPFOperatorConfig.Status.Version must be set before comparing") @@ -417,19 +417,19 @@ func validateDPFVersionUpgrade(expectedVersion string) { "DPF version should be upgraded to the expected version") } -// validateDPUClusterUpgrade asserts that, after the operator upgrade, every +// ValidateDPUClusterUpgrade asserts that, after the operator upgrade, every // Kamaji DPUCluster is in the Ready phase, carries a True Ready condition, and // reports the expected Kubernetes version (expectedKubernetesVersion, defaulting // to util.KubernetesVersion). Non-Kamaji clusters are skipped because their // upgrade is not handled here. This complements the operator (DPF) version check // in validateDPFVersionUpgrade. -func validateDPUClusterUpgrade(ctx context.Context, input ProvisionDPUClustersInput, expectedKubernetesVersion string) { +func ValidateDPUClusterUpgrade(ctx context.Context, input ProvisionDPUClustersInput, expectedKubernetesVersion string) { if expectedKubernetesVersion == "" { expectedKubernetesVersion = util.KubernetesVersion } - Expect(input.dpuClusters).ToNot(BeEmpty(), "expected at least one DPUCluster to validate after upgrade") + Expect(input.DPUClusters).ToNot(BeEmpty(), "expected at least one DPUCluster to validate after upgrade") hasKamajiDPUCluster := false - for _, dpuCluster := range input.dpuClusters { + for _, dpuCluster := range input.DPUClusters { if dpuCluster.Spec.Type == string(provisioningv1.KamajiCluster) { hasKamajiDPUCluster = true break @@ -438,9 +438,9 @@ func validateDPUClusterUpgrade(ctx context.Context, input ProvisionDPUClustersIn Expect(hasKamajiDPUCluster).To(BeTrue(), "expected at least one Kamaji DPUCluster to validate after upgrade") Eventually(func(g Gomega) { - for _, expectedDPUCluster := range input.dpuClusters { + for _, expectedDPUCluster := range input.DPUClusters { dpuCluster := &provisioningv1.DPUCluster{} - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(expectedDPUCluster), dpuCluster)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(expectedDPUCluster), dpuCluster)).To(Succeed()) // Ignore non-Kamaji clusters for now, because we do not handle their upgrade. if dpuCluster.Spec.Type != string(provisioningv1.KamajiCluster) { @@ -467,21 +467,21 @@ func validateDPUClusterUpgrade(ctx context.Context, input ProvisionDPUClustersIn // rather than the test binary's TAG env var, so the check is correct across // every phase of every upgrade path, including intermediates where the running // operator is not yet the version under test. -func VerifyHostAgentPodsImageTag(ctx context.Context, input *systemTestInput) { +func VerifyHostAgentPodsImageTag(ctx context.Context, input *SystemTestInput) { By("Verifying HostAgent Pods have the same image tag as the deployed operator") Eventually(func(g Gomega) { cfg := &operatorv1.DPFOperatorConfig{} - g.Expect(input.client.Get(ctx, client.ObjectKey{ - Name: configName, - Namespace: dpfOperatorSystemNamespace, + g.Expect(input.Client.Get(ctx, client.ObjectKey{ + Name: ConfigName, + Namespace: DPFOperatorSystemNamespace, }, cfg)).To(Succeed()) g.Expect(cfg.Status.Version).ToNot(BeNil(), "DPFOperatorConfig.Status.Version must be set before checking DMS tags") operatorVersion := *cfg.Status.Version dmsPods := &corev1.PodList{} - g.Expect(input.client.List(ctx, dmsPods, - client.InNamespace(dpfOperatorSystemNamespace), + g.Expect(input.Client.List(ctx, dmsPods, + client.InNamespace(DPFOperatorSystemNamespace), client.MatchingLabels{util.ProvisioningComponentLabelKey: "hostagent"}, )).To(Succeed()) @@ -496,13 +496,13 @@ func VerifyHostAgentPodsImageTag(ctx context.Context, input *systemTestInput) { "DMS Pods should have the same image tag as the deployed operator") } -// verifySystemReady checks that the DPF system components are healthy on the +// VerifySystemReady checks that the DPF system components are healthy on the // DPU cluster. The pod-name list is intentionally a minimum viable subset of // the important ones. The expected DPUService names are supplied by the caller // (phase.expectedDPUServices) so each upgrade phase asserts the DPUService // shape that matches its DPF release. -func verifySystemReady(dpuServiceNames []string) { - VerifyClusterPods(ctx, dpuClusterClient[0], []string{ +func VerifySystemReady(dpuServiceNames []string) { + VerifyClusterPods(Ctx, DPUClusterClient[0], []string{ // Kubernetes system pods "kube-flannel-ds", "coredns", "kube-proxy", // DPF system components @@ -511,34 +511,34 @@ func verifySystemReady(dpuServiceNames []string) { "example", }) - verifyDPUServicesReady(ctx, input, dpfOperatorSystemNamespace, dpuServiceNames) + verifyDPUServicesReady(Ctx, input, DPFOperatorSystemNamespace, dpuServiceNames) } -// rolloutDependencies simulates a post-upgrade dependency rollout by creating +// RolloutDependencies simulates a post-upgrade dependency rollout by creating // the current BFB, DPUFlavor, "-rollout"-suffixed DPUServiceTemplate, and // DPUServiceConfiguration objects from the current manifests and updating one // DPUDeployment to reference them. -func rolloutDependencies(ctx context.Context, input *systemTestInput) { +func RolloutDependencies(ctx context.Context, input *SystemTestInput) { By("Creating current BFB and DPUFlavor") - ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(ctx, getProvisionDPUClustersInput()) + ProvisionBFBOrBlueFieldSoftwareAndDPUFlavor(ctx, GetProvisionDPUClustersInput()) By("Creating current DPUServiceTemplate") - currentTemplate := input.dpuServiceTemplate.DeepCopy() + currentTemplate := input.DPUServiceTemplate.DeepCopy() currentTemplate.SetLabels(CleanupScope.Suite) - currentTemplate.SetName(input.dpuServiceTemplate.Name + "-rollout") + currentTemplate.SetName(input.DPUServiceTemplate.Name + "-rollout") useDummyDPUServiceChart(currentTemplate) - Expect(input.client.Create(ctx, currentTemplate)).To(Succeed()) + Expect(input.Client.Create(ctx, currentTemplate)).To(Succeed()) By("Creating current DPUServiceConfiguration") - currentConfig := input.dpuServiceConfiguration.DeepCopy() + currentConfig := input.DPUServiceConfiguration.DeepCopy() currentConfig.SetLabels(CleanupScope.Suite) - currentConfig.SetName(input.dpuServiceConfiguration.Name + "-rollout") - Expect(input.client.Create(ctx, currentConfig)).To(Succeed()) + currentConfig.SetName(input.DPUServiceConfiguration.Name + "-rollout") + Expect(input.Client.Create(ctx, currentConfig)).To(Succeed()) By("Selecting one DPUDeployment to update") dpuDeploymentList := &dpuservicev1.DPUDeploymentList{} - Expect(input.client.List(ctx, dpuDeploymentList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - Expect(dpuDeploymentList.Items).To(HaveLen(input.numberOfDPUNodes), "expected one DPUDeployment per DPU node") + Expect(input.Client.List(ctx, dpuDeploymentList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + Expect(dpuDeploymentList.Items).To(HaveLen(input.NumberOfDPUNodes), "expected one DPUDeployment per DPU node") // Re-apply the target manifest so fields a new release adds (e.g. v26.4's // required dpuSetStrategy) reach DPUDeployments created under older releases. @@ -546,7 +546,7 @@ func rolloutDependencies(ctx context.Context, input *systemTestInput) { for i := range dpuDeploymentList.Items { dpuDeployment := &dpuDeploymentList.Items[i] patchBase := dpuDeployment.DeepCopy() - desiredSpec := input.dpuDeployment.DeepCopy().Spec + desiredSpec := input.DPUDeployment.DeepCopy().Spec for j := range desiredSpec.DPUs.DPUSets { if j < len(dpuDeployment.Spec.DPUs.DPUSets) { // NodeSelector is filled in per worker node at install, not in the manifest. @@ -555,7 +555,7 @@ func rolloutDependencies(ctx context.Context, input *systemTestInput) { } } dpuDeployment.Spec = desiredSpec - Expect(input.client.Patch(ctx, dpuDeployment, client.MergeFrom(patchBase))).To(Succeed()) + Expect(input.Client.Patch(ctx, dpuDeployment, client.MergeFrom(patchBase))).To(Succeed()) } selectedDPUDeployment := &dpuDeploymentList.Items[0] @@ -563,19 +563,19 @@ func rolloutDependencies(ctx context.Context, input *systemTestInput) { By("Updating selected DPUDeployment to reference current BFB, DPUFlavor, DPUServiceTemplate and DPUServiceConfiguration") original := selectedDPUDeployment.DeepCopy() - selectedDPUDeployment.Spec.DPUs.BFB = ptr.To(input.bfb.Name) - selectedDPUDeployment.Spec.DPUs.Flavor = input.dpuFlavor.Name - primaryServiceName := input.dpuServiceTemplate.Name + selectedDPUDeployment.Spec.DPUs.BFB = ptr.To(input.BFB.Name) + selectedDPUDeployment.Spec.DPUs.Flavor = input.DPUFlavor.Name + primaryServiceName := input.DPUServiceTemplate.Name svc, ok := selectedDPUDeployment.Spec.Services[primaryServiceName] Expect(ok).To(BeTrue(), "DPUDeployment %s should contain service %s", selectedDPUDeployment.Name, primaryServiceName) svc.ServiceTemplate = currentTemplate.Name svc.ServiceConfiguration = currentConfig.Name selectedDPUDeployment.Spec.Services[primaryServiceName] = svc - Expect(input.client.Patch(ctx, selectedDPUDeployment, client.MergeFrom(original))).To(Succeed()) + Expect(input.Client.Patch(ctx, selectedDPUDeployment, client.MergeFrom(original))).To(Succeed()) By("Waiting for selected DPUDeployment Reconciled conditions to become True") Eventually(func(g Gomega) { - g.Expect(input.client.Get(ctx, client.ObjectKeyFromObject(selectedDPUDeployment), selectedDPUDeployment)).To(Succeed()) + g.Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(selectedDPUDeployment), selectedDPUDeployment)).To(Succeed()) for _, condType := range []conditions.ConditionType{ dpuservicev1.ConditionDPUSetsReconciled, dpuservicev1.ConditionDPUServicesReconciled, @@ -587,10 +587,10 @@ func rolloutDependencies(ctx context.Context, input *systemTestInput) { } }).WithTimeout(20 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) - verifyDPUDeploymentDependencyTracking(ctx, input) + VerifyDPUDeploymentDependencyTracking(ctx, input) } -// verifyDPUDeploymentDependencyTracking asserts that consumed-by-DPUDeployment +// VerifyDPUDeploymentDependencyTracking asserts that consumed-by-DPUDeployment // labels on dependency resources accurately reflect current DPUDeployment // references. Referenced dependencies must carry dependency labels; unreferenced // dependencies in the namespace must have been released. Regression coverage for @@ -598,7 +598,7 @@ func rolloutDependencies(ctx context.Context, input *systemTestInput) { // // Wrapped in Eventually because dependency tracking is reconciler-driven and // may lag the DPUDeployment patch by a few seconds. -func verifyDPUDeploymentDependencyTracking(ctx context.Context, input *systemTestInput) { +func VerifyDPUDeploymentDependencyTracking(ctx context.Context, input *SystemTestInput) { By("Verifying dependency consumed-by-DPUDeployment labels match current references") Eventually(func(g Gomega) { activeBFBs := map[string]bool{} @@ -606,7 +606,7 @@ func verifyDPUDeploymentDependencyTracking(ctx context.Context, input *systemTes activeServiceConfigurations := map[string]bool{} activeServiceTemplates := map[string]bool{} deployments := &dpuservicev1.DPUDeploymentList{} - g.Expect(input.client.List(ctx, deployments, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) + g.Expect(input.Client.List(ctx, deployments, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) g.Expect(deployments.Items).NotTo(BeEmpty()) for i := range deployments.Items { deployment := &deployments.Items[i] @@ -625,30 +625,30 @@ func verifyDPUDeploymentDependencyTracking(ctx context.Context, input *systemTes } bfbs := &provisioningv1.BFBList{} - g.Expect(input.client.List(ctx, bfbs, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - assertDependencyLabels(g, "BFB", activeBFBs, ToClientObjectSlice(bfbs.Items)) + g.Expect(input.Client.List(ctx, bfbs, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + AssertDependencyLabels(g, "BFB", activeBFBs, ToClientObjectSlice(bfbs.Items)) flavors := &provisioningv1.DPUFlavorList{} - g.Expect(input.client.List(ctx, flavors, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - assertDependencyLabels(g, "DPUFlavor", activeFlavors, ToClientObjectSlice(flavors.Items)) + g.Expect(input.Client.List(ctx, flavors, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + AssertDependencyLabels(g, "DPUFlavor", activeFlavors, ToClientObjectSlice(flavors.Items)) configurations := &dpuservicev1.DPUServiceConfigurationList{} - g.Expect(input.client.List(ctx, configurations, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - assertDependencyLabels(g, "DPUServiceConfiguration", activeServiceConfigurations, ToClientObjectSlice(configurations.Items)) + g.Expect(input.Client.List(ctx, configurations, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + AssertDependencyLabels(g, "DPUServiceConfiguration", activeServiceConfigurations, ToClientObjectSlice(configurations.Items)) templates := &dpuservicev1.DPUServiceTemplateList{} - g.Expect(input.client.List(ctx, templates, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - assertDependencyLabels(g, "DPUServiceTemplate", activeServiceTemplates, ToClientObjectSlice(templates.Items)) + g.Expect(input.Client.List(ctx, templates, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + AssertDependencyLabels(g, "DPUServiceTemplate", activeServiceTemplates, ToClientObjectSlice(templates.Items)) }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) } -func assertDependencyLabels(g Gomega, kind string, activeNames map[string]bool, objects []client.Object) { +func AssertDependencyLabels(g Gomega, kind string, activeNames map[string]bool, objects []client.Object) { g.Expect(objects).NotTo(BeEmpty(), "expected %s objects to exist", kind) seen := map[string]bool{} for _, obj := range objects { seen[obj.GetName()] = true - hasConsumedByLabel := hasDPUDeploymentDependencyLabel(obj) - hasFinalizer := hasDPUDeploymentFinalizer(obj) + hasConsumedByLabel := HasDPUDeploymentDependencyLabel(obj) + hasFinalizer := HasDPUDeploymentFinalizer(obj) if activeNames[obj.GetName()] { g.Expect(hasConsumedByLabel).To(BeTrue(), "referenced %s %s should have consumed-by-DPUDeployment labels", kind, obj.GetName()) @@ -666,7 +666,7 @@ func assertDependencyLabels(g Gomega, kind string, activeNames map[string]bool, } } -func hasDPUDeploymentDependencyLabel(obj client.Object) bool { +func HasDPUDeploymentDependencyLabel(obj client.Object) bool { for key := range obj.GetLabels() { if strings.HasPrefix(key, dpuservicev1.DependentDPUDeploymentLabelKeyPrefix) { return true @@ -675,7 +675,7 @@ func hasDPUDeploymentDependencyLabel(obj client.Object) bool { return false } -func hasDPUDeploymentFinalizer(obj client.Object) bool { +func HasDPUDeploymentFinalizer(obj client.Object) bool { for _, finalizer := range obj.GetFinalizers() { if finalizer == dpuservicev1.DPUDeploymentFinalizer { return true diff --git a/test/e2e/upgrade_lts.go b/test/e2e/upgrade_lts.go new file mode 100644 index 00000000..b2d6e586 --- /dev/null +++ b/test/e2e/upgrade_lts.go @@ -0,0 +1,135 @@ +/* +Copyright 2026 NVIDIA + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "slices" + "time" + + provisioningv1 "github.com/nvidia/doca-platform/api/provisioning/v1alpha1" + "github.com/nvidia/doca-platform/internal/provisioning/controllers/util" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// RolloutAllDPUs deletes every DPU in the system namespace and waits for all +// to be recreated with the given expectedDPFVersion. Used in the BFB LTS +// upgrade path to reprovision all DPUs so they report their kubelet version. +func RolloutAllDPUs(ctx context.Context, input *SystemTestInput, expectedDPFVersionMajorMinor string) { + By("Listing all DPUs before rollout") + dpuList := &provisioningv1.DPUList{} + Expect(input.Client.List(ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + Expect(dpuList.Items).NotTo(BeEmpty(), "expected DPUs to be present before rollout") + + type dpuRecord struct { + oldUID string + deviceLabel string + } + dpusBefore := make([]dpuRecord, len(dpuList.Items)) + for i, dpu := range dpuList.Items { + deviceLabel := dpu.GetLabels()[util.DPUDeviceNameLabel] + Expect(deviceLabel).NotTo(BeEmpty(), "DPU %s must have device name label", dpu.Name) + dpusBefore[i] = dpuRecord{oldUID: string(dpu.GetUID()), deviceLabel: deviceLabel} + } + + By(fmt.Sprintf("Deleting all %d DPUs to trigger rollout", len(dpusBefore))) + for i := range dpuList.Items { + Expect(client.IgnoreNotFound(input.Client.Delete(ctx, &dpuList.Items[i]))).To(Succeed()) + } + + By("Waiting for all DPUs to be recreated with DPFVersion matching " + expectedDPFVersionMajorMinor) + Eventually(func(g Gomega) { + for _, before := range dpusBefore { + updated := &provisioningv1.DPUList{} + g.Expect(input.Client.List(ctx, updated, + client.InNamespace(DPFOperatorSystemNamespace), + client.MatchingLabels{util.DPUDeviceNameLabel: before.deviceLabel}, + )).To(Succeed()) + g.Expect(updated.Items).To(HaveLen(1), "DPU for device %s should be recreated", before.deviceLabel) + dpu := &updated.Items[0] + g.Expect(string(dpu.GetUID())).NotTo(Equal(before.oldUID), "DPU for device %s should have a new UID", before.deviceLabel) + g.Expect(dpu.Status.DPFVersion).NotTo(BeNil()) + g.Expect(*dpu.Status.DPFVersion).To(ContainSubstring(expectedDPFVersionMajorMinor), + "DPU for device %s should have DPFVersion containing %s", before.deviceLabel, expectedDPFVersionMajorMinor) + } + }).WithTimeout(20 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) +} + +// VerifyDPUsHaveKubeletVersion asserts that every DPU in the system namespace +// has a non-empty KubeletVersion in its AgentStatus. Required after DPUs are +// reprovisioned with DPF v26.4+. +func VerifyDPUsHaveKubeletVersion(ctx context.Context, input *SystemTestInput) { + By("Verifying all DPUs report KubeletVersion") + Eventually(func(g Gomega) { + dpuList := &provisioningv1.DPUList{} + g.Expect(input.Client.List(ctx, dpuList, client.InNamespace(DPFOperatorSystemNamespace))).To(Succeed()) + g.Expect(dpuList.Items).NotTo(BeEmpty()) + for _, dpu := range dpuList.Items { + g.Expect(dpu.Status.AgentStatus).NotTo(BeNil(), "DPU %s should have AgentStatus", dpu.Name) + g.Expect(dpu.Status.AgentStatus.KubeletVersion).NotTo(BeNil(), "DPU %s should have KubeletVersion", dpu.Name) + g.Expect(*dpu.Status.AgentStatus.KubeletVersion).NotTo(BeEmpty(), "DPU %s KubeletVersion should not be empty", dpu.Name) + } + }).WithTimeout(5 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) +} + +// RemoveStaleDPUDeviceProtectionFinalizers clears provisioning.dpu.nvidia.com/dpudevice-protection +// from DPUDevice objects that are not referenced by any active DPU. +// +// Workaround for v25.10 → v26.4 upgrade (#5048585): non-selected DPUDevices can retain the +// legacy finalizer after upgrade, which blocks DPUDevice deletion and stalls DPFOperatorConfig +// teardown. Only the finalizer is removed; DPUDevice objects are kept. +func RemoveStaleDPUDeviceProtectionFinalizers(ctx context.Context, testClient client.Client) { + By("Removing stale dpudevice-protection finalizers from unreferenced DPUDevices (v25.10→v26.4 upgrade workaround)") + + dpuList := &provisioningv1.DPUList{} + Expect(testClient.List(ctx, dpuList)).To(Succeed()) + + referencedDPUDevices := make(map[string]struct{}, len(dpuList.Items)) + for i := range dpuList.Items { + dpu := &dpuList.Items[i] + if name := dpu.Spec.DPUDeviceName; name != "" { + referencedDPUDevices[name] = struct{}{} + } + if name := dpu.GetLabels()[util.DPUDeviceNameLabel]; name != "" { + referencedDPUDevices[name] = struct{}{} + } + } + + dpuDeviceList := &provisioningv1.DPUDeviceList{} + Expect(testClient.List(ctx, dpuDeviceList)).To(Succeed()) + + for i := range dpuDeviceList.Items { + device := &dpuDeviceList.Items[i] + if _, referenced := referencedDPUDevices[device.Name]; referenced { + continue + } + if !slices.Contains(device.Finalizers, provisioningv1.DPUDeviceFinalizer) { + continue + } + By(fmt.Sprintf("Patching DPUDevice %s/%s: remove %s finalizer", + device.Namespace, device.Name, provisioningv1.DPUDeviceFinalizer)) + original := device.DeepCopy() + device.Finalizers = slices.DeleteFunc(device.Finalizers, func(finalizer string) bool { + return finalizer == provisioningv1.DPUDeviceFinalizer + }) + Expect(testClient.Patch(ctx, device, client.MergeFrom(original))).To(Succeed()) + } +} diff --git a/test/e2e/upgrade_lts_test.go b/test/e2e/upgrade_lts_test.go index 64480bc0..576daf25 100644 --- a/test/e2e/upgrade_lts_test.go +++ b/test/e2e/upgrade_lts_test.go @@ -17,19 +17,11 @@ limitations under the License. package e2e import ( - "context" - "fmt" "os" - "slices" - "time" operatorv1 "github.com/nvidia/doca-platform/api/operator/v1alpha1" - provisioningv1 "github.com/nvidia/doca-platform/api/provisioning/v1alpha1" - "github.com/nvidia/doca-platform/internal/provisioning/controllers/util" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/client" ) // expectedDPUServicesV2510 returns the pre-v26.04 DPUService shape: singleton @@ -39,7 +31,7 @@ import ( // is upgraded to v26.4 the controller reshapes DPUServices to the current // layout (see expectedDPUServicesCurrent in upgrade_test.go) without needing // a DPU reprovision. -func expectedDPUServicesV2510(_ *systemTestInput) []string { +func expectedDPUServicesV2510(_ *SystemTestInput) []string { return []string{ operatorv1.FlannelName.String(), operatorv1.MultusName.String(), @@ -59,58 +51,58 @@ func expectedDPUServicesV2510(_ *systemTestInput) []string { // labeled Ginkgo container, selected by CI via its label. Append a new // validationPhase for each future hop (v26.10 → …). var _ = Describe("DPF Upgrade LTS", func() { - installPhase("BFB LTS v25.10", installPhaseInput{ - label: Domain.DPFBFBLTSUpgrade, + InstallPhase("BFB LTS v25.10", InstallPhaseInput{ + Label: Domain.DPFBFBLTSUpgrade, // Pin to the LTS BFB manifest even when CI exports BFB_IMAGE_URL. - skipBFBImageURL: true, + SkipBFBImageURL: true, // v25.10's servicechainset-controller creates a DPUServiceCredentialRequest with an // empty spec.targetCluster.name that the current CRD rejects. Provisioning works // without it being Ready, so skip the DPFOperatorConfig.Ready wait. - skipSystemComponentValidation: true, + SkipSystemComponentValidation: true, - expectedKubernetesVersion: "v1.34.0", - artifactsKey: "v25.10", - expectedDPUServices: expectedDPUServicesV2510, + ExpectedKubernetesVersion: "v1.34.0", + ArtifactsKey: "v25.10", + ExpectedDPUServices: expectedDPUServicesV2510, }) - validationPhase("v26.4", validationPhaseInput{ - label: Domain.DPFBFBLTSUpgradeV264, + ValidationPhase("v26.4", ValidationPhaseInput{ + Label: Domain.DPFBFBLTSUpgradeV264, // Reprovision all DPUs under v26.4 so they start reporting KubeletVersion // (required for the v26.7 skew check), then exercise a dependency rollout. - rolloutAllDPUs: true, - rolloutDPFVersionMinor: "v26.4", - rolloutDependencies: true, - verifyKubeletVersion: true, + RolloutAllDPUs: true, + RolloutDPFVersionMinor: "v26.4", + RolloutDependencies: true, + VerifyKubeletVersion: true, - expectedDPFVersion: envOrDefault("DPF_V264_VERSION", "v26.4.0"), - expectedKubernetesVersion: "v1.34.0", + ExpectedDPFVersion: envOrDefault("DPF_V264_VERSION", "v26.4.0"), + ExpectedKubernetesVersion: "v1.34.0", // Phase runs with -e2e.skip-cleanup, so clear the stale dpudevice-protection // finalizers here rather than at teardown (#5048585). - removeStaleDPUDeviceFinalizers: true, + RemoveStaleDPUDeviceFinalizers: true, // v26.4 post-rollout artifacts become the v26.7 comparison baseline. - artifactsKey: "v26.4", - preRolloutArtifactsKey: "v26.4-pre-rollout", - preRolloutPrevArtifactsKey: "v25.10", + ArtifactsKey: "v26.4", + PreRolloutArtifactsKey: "v26.4-pre-rollout", + PreRolloutPrevArtifactsKey: "v25.10", - expectedDPUServices: expectedDPUServicesCurrent, + ExpectedDPUServices: expectedDPUServicesCurrent, }) - validationPhase("current", validationPhaseInput{ - label: Domain.DPFBFBLTSUpgradeCurrent, + ValidationPhase("current", ValidationPhaseInput{ + Label: Domain.DPFBFBLTSUpgradeCurrent, // No rollout. BFB stays at LTS 3.2.1 and DPUs are not reprovisioned. - rolloutAllDPUs: false, - verifyKubeletVersion: true, - patchDeploymentMode: true, + RolloutAllDPUs: false, + VerifyKubeletVersion: true, + PatchDeploymentMode: true, - artifactsKey: "current", - prevArtifactsKey: "v26.4", - expectedChanges: expectedChangesCurrent, + ArtifactsKey: "current", + PrevArtifactsKey: "v26.4", + ExpectedChanges: expectedChangesCurrent, - expectedDPUServices: expectedDPUServicesCurrent, + ExpectedDPUServices: expectedDPUServicesCurrent, }) }) @@ -120,107 +112,3 @@ func envOrDefault(name, fallback string) string { } return fallback } - -// rolloutAllDPUs deletes every DPU in the system namespace and waits for all -// to be recreated with the given expectedDPFVersion. Used in the BFB LTS -// upgrade path to reprovision all DPUs so they report their kubelet version. -func rolloutAllDPUs(ctx context.Context, input *systemTestInput, expectedDPFVersionMajorMinor string) { - By("Listing all DPUs before rollout") - dpuList := &provisioningv1.DPUList{} - Expect(input.client.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - Expect(dpuList.Items).NotTo(BeEmpty(), "expected DPUs to be present before rollout") - - type dpuRecord struct { - oldUID string - deviceLabel string - } - dpusBefore := make([]dpuRecord, len(dpuList.Items)) - for i, dpu := range dpuList.Items { - deviceLabel := dpu.GetLabels()[util.DPUDeviceNameLabel] - Expect(deviceLabel).NotTo(BeEmpty(), "DPU %s must have device name label", dpu.Name) - dpusBefore[i] = dpuRecord{oldUID: string(dpu.GetUID()), deviceLabel: deviceLabel} - } - - By(fmt.Sprintf("Deleting all %d DPUs to trigger rollout", len(dpusBefore))) - for i := range dpuList.Items { - Expect(client.IgnoreNotFound(input.client.Delete(ctx, &dpuList.Items[i]))).To(Succeed()) - } - - By("Waiting for all DPUs to be recreated with DPFVersion matching " + expectedDPFVersionMajorMinor) - Eventually(func(g Gomega) { - for _, before := range dpusBefore { - updated := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, updated, - client.InNamespace(dpfOperatorSystemNamespace), - client.MatchingLabels{util.DPUDeviceNameLabel: before.deviceLabel}, - )).To(Succeed()) - g.Expect(updated.Items).To(HaveLen(1), "DPU for device %s should be recreated", before.deviceLabel) - dpu := &updated.Items[0] - g.Expect(string(dpu.GetUID())).NotTo(Equal(before.oldUID), "DPU for device %s should have a new UID", before.deviceLabel) - g.Expect(dpu.Status.DPFVersion).NotTo(BeNil()) - g.Expect(*dpu.Status.DPFVersion).To(ContainSubstring(expectedDPFVersionMajorMinor), - "DPU for device %s should have DPFVersion containing %s", before.deviceLabel, expectedDPFVersionMajorMinor) - } - }).WithTimeout(20 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) -} - -// verifyDPUsHaveKubeletVersion asserts that every DPU in the system namespace -// has a non-empty KubeletVersion in its AgentStatus. Required after DPUs are -// reprovisioned with DPF v26.4+. -func verifyDPUsHaveKubeletVersion(ctx context.Context, input *systemTestInput) { - By("Verifying all DPUs report KubeletVersion") - Eventually(func(g Gomega) { - dpuList := &provisioningv1.DPUList{} - g.Expect(input.client.List(ctx, dpuList, client.InNamespace(dpfOperatorSystemNamespace))).To(Succeed()) - g.Expect(dpuList.Items).NotTo(BeEmpty()) - for _, dpu := range dpuList.Items { - g.Expect(dpu.Status.AgentStatus).NotTo(BeNil(), "DPU %s should have AgentStatus", dpu.Name) - g.Expect(dpu.Status.AgentStatus.KubeletVersion).NotTo(BeNil(), "DPU %s should have KubeletVersion", dpu.Name) - g.Expect(*dpu.Status.AgentStatus.KubeletVersion).NotTo(BeEmpty(), "DPU %s KubeletVersion should not be empty", dpu.Name) - } - }).WithTimeout(5 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) -} - -// removeStaleDPUDeviceProtectionFinalizers clears provisioning.dpu.nvidia.com/dpudevice-protection -// from DPUDevice objects that are not referenced by any active DPU. -// -// Workaround for v25.10 → v26.4 upgrade (#5048585): non-selected DPUDevices can retain the -// legacy finalizer after upgrade, which blocks DPUDevice deletion and stalls DPFOperatorConfig -// teardown. Only the finalizer is removed; DPUDevice objects are kept. -func removeStaleDPUDeviceProtectionFinalizers(ctx context.Context, testClient client.Client) { - By("Removing stale dpudevice-protection finalizers from unreferenced DPUDevices (v25.10→v26.4 upgrade workaround)") - - dpuList := &provisioningv1.DPUList{} - Expect(testClient.List(ctx, dpuList)).To(Succeed()) - - referencedDPUDevices := make(map[string]struct{}, len(dpuList.Items)) - for i := range dpuList.Items { - dpu := &dpuList.Items[i] - if name := dpu.Spec.DPUDeviceName; name != "" { - referencedDPUDevices[name] = struct{}{} - } - if name := dpu.GetLabels()[util.DPUDeviceNameLabel]; name != "" { - referencedDPUDevices[name] = struct{}{} - } - } - - dpuDeviceList := &provisioningv1.DPUDeviceList{} - Expect(testClient.List(ctx, dpuDeviceList)).To(Succeed()) - - for i := range dpuDeviceList.Items { - device := &dpuDeviceList.Items[i] - if _, referenced := referencedDPUDevices[device.Name]; referenced { - continue - } - if !slices.Contains(device.Finalizers, provisioningv1.DPUDeviceFinalizer) { - continue - } - By(fmt.Sprintf("Patching DPUDevice %s/%s: remove %s finalizer", - device.Namespace, device.Name, provisioningv1.DPUDeviceFinalizer)) - original := device.DeepCopy() - device.Finalizers = slices.DeleteFunc(device.Finalizers, func(finalizer string) bool { - return finalizer == provisioningv1.DPUDeviceFinalizer - }) - Expect(testClient.Patch(ctx, device, client.MergeFrom(original))).To(Succeed()) - } -} diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index f63e6d11..dcc87919 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -29,8 +29,8 @@ import ( // each split into a per-cluster controller service plus a node/RBAC // companion service. Every phase of the regular GA upgrade path runs against // this shape; a future LTS path can reuse it for its v26.04+ phases. -func expectedDPUServicesCurrent(input *systemTestInput) []string { - c := input.dpuClusters[0] +func expectedDPUServicesCurrent(input *SystemTestInput) []string { + c := input.DPUClusters[0] return []string{ operatorv1.FlannelName.String(), operatorv1.MultusName.String(), @@ -49,12 +49,12 @@ func expectedDPUServicesCurrent(input *systemTestInput) []string { // expectedChangesCurrent lists the spec changes an upgrade to the current HEAD // release intentionally introduces. Shared by every hop that lands on HEAD: the // regular previous-GA → HEAD upgrade and the BFB LTS v26.4 → v26.7 hop. -var expectedChangesCurrent = []upgradeExpectedChange{ +var expectedChangesCurrent = []UpgradeExpectedChange{ // DPUService .spec.security is newly defaulted at HEAD: "before" lacks it while // "after" has it, so strip it from "after" (before's generation is bumped by one). { - gvk: dpuservicev1.GroupVersion.WithKind("DPUService"), - transform: func(artifact map[string]interface{}) { + GVK: dpuservicev1.GroupVersion.WithKind("DPUService"), + Transform: func(artifact map[string]interface{}) { unstructured.RemoveNestedField(artifact, "spec", "security") }, }, @@ -65,24 +65,24 @@ var expectedChangesCurrent = []upgradeExpectedChange{ // operator has been upgraded externally. Each phase is its own labeled Ginkgo // container, selected by CI via its label. var _ = Describe("DPF Upgrade", func() { - installPhase("previous GA", installPhaseInput{ - label: Domain.DPFUpgrade, - skipBFBImageURL: true, + InstallPhase("previous GA", InstallPhaseInput{ + Label: Domain.DPFUpgrade, + SkipBFBImageURL: true, // The previous GA (LAST_STABLE_DPF_VERSION, default v26.4.0) pins its own // Kubernetes version, which differs from HEAD's util.KubernetesVersion. - expectedKubernetesVersion: "v1.34.0", - artifactsKey: "before", - expectedDPUServices: expectedDPUServicesCurrent, + ExpectedKubernetesVersion: "v1.34.0", + ArtifactsKey: "before", + ExpectedDPUServices: expectedDPUServicesCurrent, }) - validationPhase("GA-to-current", validationPhaseInput{ - label: Domain.DPFUpgradeValidation, - patchDeploymentMode: true, - captureBeforeRollout: true, - artifactsKey: "after", - prevArtifactsKey: "before", - rolloutDependencies: true, - expectedChanges: expectedChangesCurrent, - expectedDPUServices: expectedDPUServicesCurrent, + ValidationPhase("GA-to-current", ValidationPhaseInput{ + Label: Domain.DPFUpgradeValidation, + PatchDeploymentMode: true, + CaptureBeforeRollout: true, + ArtifactsKey: "after", + PrevArtifactsKey: "before", + RolloutDependencies: true, + ExpectedChanges: expectedChangesCurrent, + ExpectedDPUServices: expectedDPUServicesCurrent, }) }) diff --git a/test/e2e/utils.go b/test/e2e/utils.go index 64c2919b..39ee7238 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -55,28 +55,28 @@ const ( // SDNTestPriority is the test priority for the "DPF System tests - SDN" test suite. SDNTestPriority = 100 - // kubeStateMetricsPort is the port used by kube-state-metrics across host and DPU clusters. - kubeStateMetricsPort = 8080 + // KubeStateMetricsPort is the port used by kube-state-metrics across host and DPU clusters. + KubeStateMetricsPort = 8080 // testMTUValue is the MTU value used across e2e tests to trigger configuration changes. testMTUValue = 1300 - // defaultAPIServerPort is the default Kubernetes API server port used in performance tests. - defaultAPIServerPort = 6443 - // performanceMTU is the MTU configured for both the control plane and high-speed networks in performance tests. - performanceMTU = 9000 + // DefaultAPIServerPort is the default Kubernetes API server port used in performance tests. + DefaultAPIServerPort = 6443 + // PerformanceMTU is the MTU configured for both the control plane and high-speed networks in performance tests. + PerformanceMTU = 9000 - // provisioningTimeout is the Eventually budget for provisioning-side waits in + // ProvisioningTimeout is the Eventually budget for provisioning-side waits in // the e2e suite (DPUs being installed and joining the DPU cluster as K8s // Nodes). Sized to absorb a first-install BFB run that includes a full BMC + // CEC + NIC firmware update cycle plus host power-cycle, which can take // ~45-55 minutes per DPU. - provisioningTimeout = 60 * time.Minute + ProvisioningTimeout = 60 * time.Minute - // dpuDeploymentReadyTimeout is the Eventually budget for waits that gate on + // DPUDeploymentReadyTimeout is the Eventually budget for waits that gate on // DPUDeployment.Status.Ready=True when DPU provisioning has not been awaited // separately upstream. Such waits must absorb the full provisioning chain // plus the dpuservice / ArgoCD / ServiceChain layer settling on top, so this // is intentionally larger than provisioningTimeout. - dpuDeploymentReadyTimeout = 75 * time.Minute + DPUDeploymentReadyTimeout = 75 * time.Minute ) // CleanupScope is an alias for cleanup.CleanupLabels for ease of use @@ -139,12 +139,12 @@ var Domain = TestDomain{ } var ( - dpuClusterClient []client.Client - dpuClusterRestConfig []*rest.Config - dpuClusterRestClient []*rest.RESTClient + DPUClusterClient []client.Client + DPUClusterRestConfig []*rest.Config + DPUClusterRestClient []*rest.RESTClient dpuClusterClientsInitialized bool // tracks if getDPUClusterClients was called (must only be called once) - hostClusterRESTClient *rest.RESTClient - metricsURI string + HostClusterRESTClient *rest.RESTClient + MetricsURI string // helmRegistry holds the Helm registry in which the artifacts used in e2e are pushed helmRegistry = "" // dockerIORegistry is a DockerHub mirror registry used to pull mirrored images to avoid rate-limiting. @@ -231,13 +231,13 @@ var ( ) const ( - configName = "dpfoperatorconfig" - dpfOperatorSystemNamespace = "dpf-operator-system" - argoCDTrackingIDAnnotation = "argocd.argoproj.io/tracking-id" - // ngcPullSecretName is the name of the secret used to pull images from NGC - ngcPullSecretName = "ngc-pull-secret" - // dpfPullSecretName is the name of the secret that is set in hack/scripts/create-artefact-secrets.sh - dpfPullSecretName = "dpf-pull-secret" + ConfigName = "dpfoperatorconfig" + DPFOperatorSystemNamespace = "dpf-operator-system" + ArgoCDTrackingIDAnnotation = "argocd.argoproj.io/tracking-id" + // NGCPullSecretName is the name of the secret used to pull images from NGC + NGCPullSecretName = "ngc-pull-secret" + // DPFPullSecretName is the name of the secret that is set in hack/scripts/create-artefact-secrets.sh + DPFPullSecretName = "dpf-pull-secret" ) // EventuallyCheckReadyStatusCondition waits until obj has a Ready condition with Status True and @@ -380,53 +380,53 @@ func getDPUNodesInOrder(ctx context.Context, hostClient, dpuClusterClient client } // isGinkgoLabel returns if a label is passed while running ginkgo and is not excluded -func isGinkgoLabelApplied(ginkgoLabel string) bool { +func IsGinkgoLabelApplied(ginkgoLabel string) bool { return strings.Contains(GinkgoLabelFilter(), ginkgoLabel) && !strings.Contains(GinkgoLabelFilter(), "!"+ginkgoLabel) } // VerifyPerformancePodToPodSameNode verifies performance between pods on the same node -func VerifyPerformancePodToPodSameNode(ctx context.Context, input *systemTestInput, namespacePrefix string) { - if !input.hasDpuNodes() { +func VerifyPerformancePodToPodSameNode(ctx context.Context, input *SystemTestInput, namespacePrefix string) { + if !input.HasDpuNodes() { Skip("Skip test as there are not multiple nodes") } hostNamespace := namespacePrefix + "-same-node" - createTestNamespace(ctx, input.client, hostNamespace) + createTestNamespace(ctx, input.Client, hostNamespace) By("Creating test pods") pod1Config, pod2Config := getPodSameNodeConfigs(ctx, input, hostNamespace) - netshoot.CreateAndWaitForPods(ctx, input.client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) + netshoot.CreateAndWaitForPods(ctx, input.Client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) By("Get pod2 IP") - pod2IP := netshoot.GetPodIP(ctx, input.client, hostNamespace, pod2Config.Name) + pod2IP := netshoot.GetPodIP(ctx, input.Client, hostNamespace, pod2Config.Name) By("Running traffic test between pods") - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2IP) } // VerifyPerformancePodToPodDifferentNode verifies performance between pods on different nodes -func VerifyPerformancePodToPodDifferentNode(ctx context.Context, input *systemTestInput, namespacePrefix string) { - if !input.hasDpuNodes() { +func VerifyPerformancePodToPodDifferentNode(ctx context.Context, input *SystemTestInput, namespacePrefix string) { + if !input.HasDpuNodes() { Skip("Skip test as there are not multiple nodes") } hostNamespace := namespacePrefix + "-different-node" - createTestNamespace(ctx, input.client, hostNamespace) + createTestNamespace(ctx, input.Client, hostNamespace) By("Creating test pods") pod1Config, pod2Config := getPodDifferentNodeConfigs(ctx, input, hostNamespace) - netshoot.CreateAndWaitForPods(ctx, input.client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) + netshoot.CreateAndWaitForPods(ctx, input.Client, []*netshoot.TestPodConfig{&pod1Config, &pod2Config}) By("Get pod2 IP") - pod2IP := netshoot.GetPodIP(ctx, input.client, hostNamespace, pod2Config.Name) + pod2IP := netshoot.GetPodIP(ctx, input.Client, hostNamespace, pod2Config.Name) By("Running traffic test between pods") - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, hostNamespace, pod1Config.Name, pod2Config.Name, pod2IP) } // getPodDifferentNodeConfigs returns two pod configs for different nodes -func getPodDifferentNodeConfigs(ctx context.Context, input *systemTestInput, namespace string) (netshoot.TestPodConfig, netshoot.TestPodConfig) { - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) +func getPodDifferentNodeConfigs(ctx context.Context, input *SystemTestInput, namespace string) (netshoot.TestPodConfig, netshoot.TestPodConfig) { + workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.Client) pod1Config := netshoot.TestPodConfig{ Name: "pod1", @@ -443,8 +443,8 @@ func getPodDifferentNodeConfigs(ctx context.Context, input *systemTestInput, nam } // getPodSameNodeConfigs returns two pod configs for the same node -func getPodSameNodeConfigs(ctx context.Context, input *systemTestInput, namespace string) (netshoot.TestPodConfig, netshoot.TestPodConfig) { - workerNode1, _ := getTwoWorkerNodeNames(ctx, input.client) +func getPodSameNodeConfigs(ctx context.Context, input *SystemTestInput, namespace string) (netshoot.TestPodConfig, netshoot.TestPodConfig) { + workerNode1, _ := getTwoWorkerNodeNames(ctx, input.Client) pod1Config := netshoot.TestPodConfig{ Name: "pod1", diff --git a/test/e2e/vpcovn.go b/test/e2e/vpcovn.go index 654b1b66..59a149a1 100644 --- a/test/e2e/vpcovn.go +++ b/test/e2e/vpcovn.go @@ -68,87 +68,87 @@ type TestVPCConfig struct { Labels map[string]string } -type vpcOvnTestInput struct { - dpuServiceOVNCentral *dpuservicev1.DPUService - dpuServiceOVNController *dpuservicev1.DPUService - dpuServiceVPCOVNController *dpuservicev1.DPUService - dpuServiceVPCOVNNode *dpuservicev1.DPUService - dpuServiceIPAMTemplate *dpuservicev1.DPUServiceIPAM - dpuServiceInterfaceTemplate *dpuservicev1.DPUServiceInterface - dpuServiceChainTemplate *dpuservicev1.DPUServiceChain - dhcpDaemonSet *appsv1.DaemonSet +type VPCOVNTestInput struct { + DPUServiceOVNCentral *dpuservicev1.DPUService + DPUServiceOVNController *dpuservicev1.DPUService + DPUServiceVPCOVNController *dpuservicev1.DPUService + DPUServiceVPCOVNNode *dpuservicev1.DPUService + DPUServiceIPAMTemplate *dpuservicev1.DPUServiceIPAM + DPUServiceInterfaceTemplate *dpuservicev1.DPUServiceInterface + DPUServiceChainTemplate *dpuservicev1.DPUServiceChain + DHCPDaemonSet *appsv1.DaemonSet } -func (t *vpcOvnTestInput) applyVPCOVNConfig(conf config) { +func (t *VPCOVNTestInput) ApplyVPCOVNConfig(conf Config) { dpuServiceIPAMTemplate := &dpuservicev1.DPUServiceIPAM{} - ipam := unstructuredFromFile(conf.DPUServiceIPAMTemplatePath) + ipam := UnstructuredFromFile(conf.DPUServiceIPAMTemplatePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(ipam.Object, dpuServiceIPAMTemplate)).To(Succeed()) - t.dpuServiceIPAMTemplate = dpuServiceIPAMTemplate + t.DPUServiceIPAMTemplate = dpuServiceIPAMTemplate dpuServiceInterfaceTemplate := &dpuservicev1.DPUServiceInterface{} - dsiTemplate := unstructuredFromFile(conf.DPUServiceInterfaceTemplatePath) + dsiTemplate := UnstructuredFromFile(conf.DPUServiceInterfaceTemplatePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(dsiTemplate.Object, dpuServiceInterfaceTemplate)).To(Succeed()) - t.dpuServiceInterfaceTemplate = dpuServiceInterfaceTemplate + t.DPUServiceInterfaceTemplate = dpuServiceInterfaceTemplate dpuServiceChainTemplate := &dpuservicev1.DPUServiceChain{} - chainTemplate := unstructuredFromFile(conf.DPUServiceChainTemplatePath) + chainTemplate := UnstructuredFromFile(conf.DPUServiceChainTemplatePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(chainTemplate.Object, dpuServiceChainTemplate)).To(Succeed()) - t.dpuServiceChainTemplate = dpuServiceChainTemplate + t.DPUServiceChainTemplate = dpuServiceChainTemplate dpuServiceOVNCentral := &dpuservicev1.DPUService{} - svcOVNCentral := unstructuredFromFile(conf.DPUServiceOVNCentralPath) + svcOVNCentral := UnstructuredFromFile(conf.DPUServiceOVNCentralPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(svcOVNCentral.Object, dpuServiceOVNCentral)).To(Succeed()) - t.dpuServiceOVNCentral = dpuServiceOVNCentral + t.DPUServiceOVNCentral = dpuServiceOVNCentral dpuServiceOVNController := &dpuservicev1.DPUService{} - svcOVNController := unstructuredFromFile(conf.DPUServiceOVNControllerPath) + svcOVNController := UnstructuredFromFile(conf.DPUServiceOVNControllerPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(svcOVNController.Object, dpuServiceOVNController)).To(Succeed()) - t.dpuServiceOVNController = dpuServiceOVNController + t.DPUServiceOVNController = dpuServiceOVNController dpuServiceVPCOVNController := &dpuservicev1.DPUService{} - svcVPCOVNController := unstructuredFromFile(conf.DPUServiceVPCOVNControllerPath) + svcVPCOVNController := UnstructuredFromFile(conf.DPUServiceVPCOVNControllerPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(svcVPCOVNController.Object, dpuServiceVPCOVNController)).To(Succeed()) - t.dpuServiceVPCOVNController = dpuServiceVPCOVNController + t.DPUServiceVPCOVNController = dpuServiceVPCOVNController dpuServiceVPCOVNNode := &dpuservicev1.DPUService{} - svcVPCOVNNode := unstructuredFromFile(conf.DPUServiceVPCOVNNodePath) + svcVPCOVNNode := UnstructuredFromFile(conf.DPUServiceVPCOVNNodePath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(svcVPCOVNNode.Object, dpuServiceVPCOVNNode)).To(Succeed()) - t.dpuServiceVPCOVNNode = dpuServiceVPCOVNNode + t.DPUServiceVPCOVNNode = dpuServiceVPCOVNNode dhcpDaemonSet := &appsv1.DaemonSet{} - dhcpObj := unstructuredFromFile(conf.DHCPDaemonSetPath) + dhcpObj := UnstructuredFromFile(conf.DHCPDaemonSetPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(dhcpObj.Object, dhcpDaemonSet)).To(Succeed()) - t.dhcpDaemonSet = dhcpDaemonSet + t.DHCPDaemonSet = dhcpDaemonSet } -func createVtepDPUServiceIPAM(ctx context.Context, input *systemTestInput) { +func CreateVtepDPUServiceIPAM(ctx context.Context, input *SystemTestInput) { vpcVtepIPAMLabels := map[string]string{ ovnutils.PoolLabelKey: ovnutils.VtepIPPoolName, } - vtepDpuServiceIPAM := generateVPCDPUObj(ovnutils.VtepIPPoolName, dpfOperatorSystemNamespace, input.dpuServiceIPAMTemplate.DeepCopy(), cleanup.MergeMaps(vpcPrerequisiteScope.CleanupLabels, vpcVtepIPAMLabels)) + vtepDpuServiceIPAM := GenerateVPCDPUObj(ovnutils.VtepIPPoolName, DPFOperatorSystemNamespace, input.DPUServiceIPAMTemplate.DeepCopy(), cleanup.MergeMaps(vpcPrerequisiteScope.CleanupLabels, vpcVtepIPAMLabels)) ovnutils.SetVPCDPUServiceIPAM(vtepDpuServiceIPAM, ovnutils.VtepIPPoolSubnet, ovnutils.VtepIPPoolGateway, ovnutils.IPPoolPerNodeCount) By("Creating VTEP DPU service IPAM") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, vtepDpuServiceIPAM))).ToNot(HaveOccurred()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, vtepDpuServiceIPAM))).ToNot(HaveOccurred()) } -func createGatewayDPUServiceIPAM(ctx context.Context, input *systemTestInput) { +func CreateGatewayDPUServiceIPAM(ctx context.Context, input *SystemTestInput) { vpcGatewayIPAMLabels := map[string]string{ ovnutils.PoolLabelKey: ovnutils.GatewayIPPoolName, } - gatewayDpuServiceIPAM := generateVPCDPUObj(ovnutils.GatewayIPPoolName, dpfOperatorSystemNamespace, input.dpuServiceIPAMTemplate.DeepCopy(), cleanup.MergeMaps(vpcPrerequisiteScope.CleanupLabels, vpcGatewayIPAMLabels)) + gatewayDpuServiceIPAM := GenerateVPCDPUObj(ovnutils.GatewayIPPoolName, DPFOperatorSystemNamespace, input.DPUServiceIPAMTemplate.DeepCopy(), cleanup.MergeMaps(vpcPrerequisiteScope.CleanupLabels, vpcGatewayIPAMLabels)) ovnutils.SetVPCDPUServiceIPAM(gatewayDpuServiceIPAM, ovnutils.GatewayIPPoolSubnet, ovnutils.GatewayIPPoolGateway, ovnutils.IPPoolPerNodeCount) By("Creating Gateway DPU service IPAM") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, gatewayDpuServiceIPAM))).ToNot(HaveOccurred()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, gatewayDpuServiceIPAM))).ToNot(HaveOccurred()) } -// createDPUService creates a generic DPU service with the given name -func createDPUService(ctx context.Context, testClient client.Client, serviceName, namespace string, dpuService *dpuservicev1.DPUService, cleanupLabels map[string]string) { - dpuService = generateVPCDPUObj(serviceName, namespace, dpuService, cleanupLabels) +// CreateDPUService creates a generic DPU service with the given name +func CreateDPUService(ctx context.Context, testClient client.Client, serviceName, namespace string, dpuService *dpuservicev1.DPUService, cleanupLabels map[string]string) { + dpuService = GenerateVPCDPUObj(serviceName, namespace, dpuService, cleanupLabels) Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, dpuService))).To(Succeed()) } -// createOVNCentralDPUService creates an OVN central DPU service -func createOVNCentralDPUService(ctx context.Context, testClient client.Client, namespace string, dpuServiceTemplate *dpuservicev1.DPUService) { +// CreateOVNCentralDPUService creates an OVN central DPU service +func CreateOVNCentralDPUService(ctx context.Context, testClient client.Client, namespace string, dpuServiceTemplate *dpuservicev1.DPUService) { dpuService := dpuServiceTemplate.DeepCopy() dpuService.Spec.HelmChart.Source = dpuservicev1.ApplicationSource{ Chart: ovnChartName, @@ -156,11 +156,11 @@ func createOVNCentralDPUService(ctx context.Context, testClient client.Client, n RepoURL: helmRegistry, } By("Creating OVN central service") - createDPUService(ctx, testClient, ovnutils.OvnCentralService, namespace, dpuService, vpcPrerequisiteScope.CleanupLabels) + CreateDPUService(ctx, testClient, ovnutils.OvnCentralService, namespace, dpuService, vpcPrerequisiteScope.CleanupLabels) } -// createOVNControllerDPUService creates an OVN controller DPU service -func createOVNControllerDPUService(ctx context.Context, testClient client.Client, namespace string, dpuServiceTemplate *dpuservicev1.DPUService) { +// CreateOVNControllerDPUService creates an OVN controller DPU service +func CreateOVNControllerDPUService(ctx context.Context, testClient client.Client, namespace string, dpuServiceTemplate *dpuservicev1.DPUService) { dpuService := dpuServiceTemplate.DeepCopy() dpuService.Spec.HelmChart.Source = dpuservicev1.ApplicationSource{ Chart: ovnChartName, @@ -168,11 +168,11 @@ func createOVNControllerDPUService(ctx context.Context, testClient client.Client RepoURL: helmRegistry, } By("Creating OVN controller service") - createDPUService(ctx, testClient, ovnutils.OvnControllerService, namespace, dpuService, vpcPrerequisiteScope.CleanupLabels) + CreateDPUService(ctx, testClient, ovnutils.OvnControllerService, namespace, dpuService, vpcPrerequisiteScope.CleanupLabels) } -// createVPCOVNControllerDPUService creates a VPC controller DPU service -func createVPCOVNControllerDPUService(ctx context.Context, testClient client.Client, namespace string, dpuServiceTemplate *dpuservicev1.DPUService) { +// CreateVPCOVNControllerDPUService creates a VPC controller DPU service +func CreateVPCOVNControllerDPUService(ctx context.Context, testClient client.Client, namespace string, dpuServiceTemplate *dpuservicev1.DPUService) { dpuService := dpuServiceTemplate.DeepCopy() dpuService.Spec.HelmChart.Source = dpuservicev1.ApplicationSource{ Chart: vpcOvnChartName, @@ -180,18 +180,18 @@ func createVPCOVNControllerDPUService(ctx context.Context, testClient client.Cli RepoURL: helmRegistry, } By("Creating VPC OVN controller service") - createDPUService(ctx, testClient, ovnutils.VpcOVNControllerService, namespace, dpuService, vpcPrerequisiteScope.CleanupLabels) + CreateDPUService(ctx, testClient, ovnutils.VpcOVNControllerService, namespace, dpuService, vpcPrerequisiteScope.CleanupLabels) } -// createVPCOVNNodeDPUService creates a VPC OVN Node DPU service -func createVPCOVNNodeDPUService(ctx context.Context, testClient client.Client, namespace string, dpuServiceTemplate *dpuservicev1.DPUService) { +// CreateVPCOVNNodeDPUService creates a VPC OVN Node DPU service +func CreateVPCOVNNodeDPUService(ctx context.Context, testClient client.Client, namespace string, dpuServiceTemplate *dpuservicev1.DPUService) { dpuService := dpuServiceTemplate.DeepCopy() dpuService.Spec.HelmChart.Source = dpuservicev1.ApplicationSource{ Chart: vpcOvnChartName, Version: tag, RepoURL: helmRegistry, } - dpuService = generateVPCDPUObj(ovnutils.VpcOVNNodeService, namespace, dpuService, vpcPrerequisiteScope.CleanupLabels) + dpuService = GenerateVPCDPUObj(ovnutils.VpcOVNNodeService, namespace, dpuService, vpcPrerequisiteScope.CleanupLabels) // configure OVN SB endpoint existingData := make(map[string]any) @@ -223,9 +223,9 @@ func createVPCOVNNodeDPUService(ctx context.Context, testClient client.Client, n Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, dpuService))).To(Succeed()) } -// createVPCDPUServiceInterface creates a DPU service interface with the given name, type and namespace -func createVPCDPUServiceInterface(ctx context.Context, input *systemTestInput, config dpuservice.TestDPUServiceInterfaceConfig) { - dpuServiceInterface := generateVPCDPUObj(config.Name, config.Namespace, input.dpuServiceInterfaceTemplate.DeepCopy(), config.Labels) +// CreateVPCDPUServiceInterface creates a DPU service interface with the given name, type and namespace +func CreateVPCDPUServiceInterface(ctx context.Context, input *SystemTestInput, config dpuservice.TestDPUServiceInterfaceConfig) { + dpuServiceInterface := GenerateVPCDPUObj(config.Name, config.Namespace, input.DPUServiceInterfaceTemplate.DeepCopy(), config.Labels) if config.NodeName != nil { dpuServiceInterface.Spec.Template.Spec.NodeSelector = &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -250,16 +250,16 @@ func createVPCDPUServiceInterface(ctx context.Context, input *systemTestInput, c Fail(fmt.Sprintf("invalid interface type: %s", config.Type)) } By(fmt.Sprintf("Creating %s/%s DPUServiceInterface with interface name %s", config.Name, config.Namespace, config.InterfaceName)) - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuServiceInterface))).To(Succeed()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuServiceInterface))).To(Succeed()) } -func createVPCPrerequisiteDPUServiceInterfaces(ctx context.Context, input *systemTestInput) { +func CreateVPCPrerequisiteDPUServiceInterfaces(ctx context.Context, input *SystemTestInput) { By("Creating physical service interface") - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: ovnutils.PhysicalInterface0, InterfaceName: ovnutils.PhysicalInterface0, Type: dpuservicev1.InterfaceTypePhysical, - Namespace: input.namespace, + Namespace: input.Namespace, Labels: cleanup.MergeMaps(vpcPrerequisiteScope.CleanupLabels, physicalInterfaceLabels), Annotations: map[string]string{ "svc.dpu.nvidia.com/noop-physical-removal": "", @@ -269,19 +269,19 @@ func createVPCPrerequisiteDPUServiceInterfaces(ctx context.Context, input *syste }) By("Creating OVN ext service interface") - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: ovnutils.OvnExtPatchName, InterfaceName: ovnutils.OvnExtPatchName, Type: dpuservicev1.InterfaceTypePatch, - Namespace: input.namespace, + Namespace: input.Namespace, Labels: cleanup.MergeMaps(vpcPrerequisiteScope.CleanupLabels, brOVNExtLabels), PeerBridge: ovnutils.BrOVNExt, }) } -func createOrUpdateVPCDPUServiceChain(ctx context.Context, input *systemTestInput, nodeName *string) { +func CreateOrUpdateVPCDPUServiceChain(ctx context.Context, input *SystemTestInput, nodeName *string) { // Build desired object from template - desired := generateVPCDPUObj(ovnutils.VpcOVNServiceChain, input.namespace, input.dpuServiceChainTemplate.DeepCopy(), vpcPrerequisiteScope.CleanupLabels) + desired := GenerateVPCDPUObj(ovnutils.VpcOVNServiceChain, input.Namespace, input.DPUServiceChainTemplate.DeepCopy(), vpcPrerequisiteScope.CleanupLabels) if nodeName != nil { desired.Spec.Template.Spec.NodeSelector = &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -312,20 +312,20 @@ func createOrUpdateVPCDPUServiceChain(ctx context.Context, input *systemTestInpu By("Creating or updating VPC OVN service chain") existing := &dpuservicev1.DPUServiceChain{} - err := input.client.Get(ctx, client.ObjectKey{Namespace: input.namespace, Name: ovnutils.VpcOVNServiceChain}, existing) + err := input.Client.Get(ctx, client.ObjectKey{Namespace: input.Namespace, Name: ovnutils.VpcOVNServiceChain}, existing) if apierrors.IsNotFound(err) { - Expect(input.client.Create(ctx, desired)).To(Succeed()) + Expect(input.Client.Create(ctx, desired)).To(Succeed()) return } Expect(err).NotTo(HaveOccurred()) original := existing.DeepCopy() existing.SetLabels(desired.GetLabels()) existing.Spec = desired.Spec - Expect(input.client.Patch(ctx, existing, client.MergeFrom(original))).To(Succeed()) + Expect(input.Client.Patch(ctx, existing, client.MergeFrom(original))).To(Succeed()) } -func createDPUServiceChainP0ToInterfaceMatchingLabels(ctx context.Context, input *systemTestInput, name string, matchingInterfaceLabels map[string]string, nodeName *string, labels map[string]string) { - dpuServiceChain := generateVPCDPUObj(name, input.namespace, input.dpuServiceChainTemplate.DeepCopy(), labels) +func CreateDPUServiceChainP0ToInterfaceMatchingLabels(ctx context.Context, input *SystemTestInput, name string, matchingInterfaceLabels map[string]string, nodeName *string, labels map[string]string) { + dpuServiceChain := GenerateVPCDPUObj(name, input.Namespace, input.DPUServiceChainTemplate.DeepCopy(), labels) if nodeName != nil { dpuServiceChain.Spec.Template.Spec.NodeSelector = &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -354,30 +354,30 @@ func createDPUServiceChainP0ToInterfaceMatchingLabels(ctx context.Context, input }, } By("Creating VPC OVN service chain") - Expect(client.IgnoreAlreadyExists(input.client.Create(ctx, dpuServiceChain))).To(Succeed()) + Expect(client.IgnoreAlreadyExists(input.Client.Create(ctx, dpuServiceChain))).To(Succeed()) } -// generateVPCDPUObj generates a DPU object with the given name, namespace and labels -func generateVPCDPUObj[T client.Object](name, ns string, obj T, labels map[string]string) T { +// GenerateVPCDPUObj generates a DPU object with the given name, namespace and labels +func GenerateVPCDPUObj[T client.Object](name, ns string, obj T, labels map[string]string) T { obj.SetName(name) obj.SetNamespace(ns) obj.SetLabels(labels) return obj } -// cleanupDPUClusterNodeLabels cleans up the DPU cluster node labels -func cleanupDPUClusterNodeLabels(ctx context.Context) { - dpuNodes := getDPUClusterNodes(ctx, dpuClusterClient[0]) +// CleanupDPUClusterNodeLabels cleans up the DPU cluster node labels +func CleanupDPUClusterNodeLabels(ctx context.Context) { + dpuNodes := getDPUClusterNodes(ctx, DPUClusterClient[0]) Expect(dpuNodes).To(HaveLen(2)) // Delete the specific labels for _, dpuNode := range dpuNodes { - vpc.UpdateDPUNodeLabelsMerge(ctx, dpuClusterClient[0], dpuNode.Name, nil, []string{ovnutils.TenantNodeLabelKey, ovnutils.TenantLabelKey}) + vpc.UpdateDPUNodeLabelsMerge(ctx, DPUClusterClient[0], dpuNode.Name, nil, []string{ovnutils.TenantNodeLabelKey, ovnutils.TenantLabelKey}) } } -// createOVNIsolationClass creates an OVN isolation class -func createOVNIsolationClass(ctx context.Context, testClient client.Client, name string, labels map[string]string) { +// CreateOVNIsolationClass creates an OVN isolation class +func CreateOVNIsolationClass(ctx context.Context, testClient client.Client, name string, labels map[string]string) { controlPlaneIP := getClusterControlPlaneIP(ctx, testClient) ovnNbEndpoint := fmt.Sprintf("tcp:%s:%d", controlPlaneIP, ovnutils.OvnNbPort) ovnSbEndpoint := fmt.Sprintf("tcp:%s:%d", controlPlaneIP, ovnutils.OvnSbPort) @@ -399,12 +399,12 @@ func createOVNIsolationClass(ctx context.Context, testClient client.Client, name Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, ovni))).To(Succeed()) } -// createDPUVPC creates a DPU VPC -func createDPUVPC(ctx context.Context, testClient client.Client, name, tenant, isolationClassName string, labels map[string]string) { +// CreateDPUVPC creates a DPU VPC +func CreateDPUVPC(ctx context.Context, testClient client.Client, name, tenant, isolationClassName string, labels map[string]string) { dpuVPC := &vpcv1.DPUVPC{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: labels, }, Spec: vpcv1.DPUVPCSpec{ @@ -421,12 +421,12 @@ func createDPUVPC(ctx context.Context, testClient client.Client, name, tenant, i Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, dpuVPC))).To(Succeed()) } -// createDPUVirtualNetwork creates a DPU virtual network -func createDPUVirtualNetwork(ctx context.Context, testClient client.Client, name, vpcName, tenant, subnet string, labels map[string]string) { +// CreateDPUVirtualNetwork creates a DPU virtual network +func CreateDPUVirtualNetwork(ctx context.Context, testClient client.Client, name, vpcName, tenant, subnet string, labels map[string]string) { dpuVirtualNetwork := &vpcv1.DPUVirtualNetwork{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: labels, }, Spec: vpcv1.DPUVirtualNetworkSpec{ @@ -452,8 +452,8 @@ func createDPUVirtualNetwork(ctx context.Context, testClient client.Client, name Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, dpuVirtualNetwork))).To(Succeed()) } -// labelDPUNodesWithTenantAndTenantNode labels DPU nodes with tenant and tenant-node labels -func labelDPUNodesWithTenantAndTenantNode(ctx context.Context, dpuClusterClient client.Client, dpuNode1, dpuNode2 corev1.Node, tenant1Label, tenant2Label string) { +// LabelDPUNodesWithTenantAndTenantNode labels DPU nodes with tenant and tenant-node labels +func LabelDPUNodesWithTenantAndTenantNode(ctx context.Context, dpuClusterClient client.Client, dpuNode1, dpuNode2 corev1.Node, tenant1Label, tenant2Label string) { labelsDPUNode1 := map[string]string{ ovnutils.TenantNodeLabelKey: dpuNode1.Name, ovnutils.TenantLabelKey: tenant1Label, @@ -467,8 +467,8 @@ func labelDPUNodesWithTenantAndTenantNode(ctx context.Context, dpuClusterClient vpc.UpdateDPUNodeLabelsMerge(ctx, dpuClusterClient, dpuNode2.Name, labelsDPUNode2, nil) } -// createDummyDPUService creates a dummy DPU service -func createDummyDPUService(ctx context.Context, testClient client.Client, namespace, name string, labels map[string]string, tenantNode *string, serviceID, network, interfaceName string) { +// CreateDummyDPUService creates a dummy DPU service +func CreateDummyDPUService(ctx context.Context, testClient client.Client, namespace, name string, labels map[string]string, tenantNode *string, serviceID, network, interfaceName string) { dpuService := &dpuservicev1.DPUService{} dpuService.Spec.HelmChart.Source = dpuservicev1.ApplicationSource{ Chart: "dummydpuservice-chart", @@ -477,7 +477,7 @@ func createDummyDPUService(ctx context.Context, testClient client.Client, namesp } if ngcAPIKey != "" { dpuService.Spec.HelmChart.Values = &machineryruntime.RawExtension{ - Raw: []byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, ngcPullSecretName)), + Raw: []byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, NGCPullSecretName)), } } dpuService.Spec.ServiceID = ptr.To(serviceID) @@ -507,12 +507,12 @@ func createDummyDPUService(ctx context.Context, testClient client.Client, namesp dpuService.Spec.ServiceDaemonSet.Annotations = map[string]string{ "k8s.v1.cni.cncf.io/networks": fmt.Sprintf(`[{"name": "%s", "interface": "%s"}]`, network, interfaceName), } - dpuService = generateVPCDPUObj(name, namespace, dpuService, labels) + dpuService = GenerateVPCDPUObj(name, namespace, dpuService, labels) Expect(client.IgnoreAlreadyExists(testClient.Create(ctx, dpuService))).To(Succeed()) } -func validateVPCMetrics(ctx context.Context) { +func ValidateVPCMetrics(ctx context.Context) { By("Verify DPUVPC and DPUVirtualNetwork metrics in KSM") expectedMetricsNames := map[string][]string{ "dpuvpc": {"created", "info", "inter_network_access", "status_conditions", "status_condition_last_transition_time"}, @@ -520,8 +520,13 @@ func validateVPCMetrics(ctx context.Context) { } Eventually(func(g Gomega) { - actualMetricsNames := metrics.GetKSMMetrics(g, ctx, hostClusterRESTClient, metricsURI) + actualMetricsNames := metrics.GetKSMMetrics(g, ctx, HostClusterRESTClient, MetricsURI) g.Expect(actualMetricsNames).NotTo(BeEmpty(), "Actual metrics are empty") g.Expect(metrics.VerifyMetrics(expectedMetricsNames, actualMetricsNames)).To(BeEmpty()) }).WithTimeout(5 * time.Second).Should(Succeed()) } + +func VPCOVNBeforeSuite() { + By("Setting VPC OVN configs for the test") + VPCOVNInput.ApplyVPCOVNConfig(*Conf) +} diff --git a/test/e2e/vpcovn_test.go b/test/e2e/vpcovn_test.go index 372e4888..31859651 100644 --- a/test/e2e/vpcovn_test.go +++ b/test/e2e/vpcovn_test.go @@ -36,11 +36,6 @@ import ( const nadNamePrefix = "nad-" -func VPCOVNBeforeSuite() { - By("Setting VPC OVN configs for the test") - vpcOvnInput.applyVPCOVNConfig(*conf) -} - //nolint:dupl var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN}, Ordered, func() { var ( @@ -50,32 +45,32 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} BeforeAll(func() { // Register scopes for VPC OVN tests (using global scope variables from vpcovn.go) - vpcPrerequisiteScope = cleanupTracker.RegisterScope(cleanup.NamedScopeManual("vpc-ovn-prerequisites")) - vpcOvnContextScope = cleanupTracker.RegisterScope(cleanup.NamedScopeManual("vpc-ovn-tests")) + vpcPrerequisiteScope = CleanupTracker.RegisterScope(cleanup.NamedScopeManual("vpc-ovn-prerequisites")) + vpcOvnContextScope = CleanupTracker.RegisterScope(cleanup.NamedScopeManual("vpc-ovn-tests")) for _, label := range CurrentSpecReport().Labels() { if label != Domain.RequiresNodes { continue } - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { Skip("Skip test as there are not multiple nodes") } // Provisioning is skipped if the test is labels with !Domain.Provisioning if !strings.Contains(GinkgoLabelFilter(), "!"+Domain.Provisioning) { By("Waiting for provisioning") - VerifyDPUClusterWithNodes(ctx, getProvisionDPUClustersInput()) + VerifyDPUClusterWithNodes(Ctx, GetProvisionDPUClustersInput()) By("Waiting for DPU cluster pods to be ready") - VerifyClusterPods(ctx, dpuClusterClient[0], systemPodsToVerify) + VerifyClusterPods(Ctx, DPUClusterClient[0], systemPodsToVerify) By("Waiting for DPFOperatorConfig to be ready") - VerifyDPFOperatorConfigReady(ctx, input.client, 20*time.Minute) + VerifyDPFOperatorConfigReady(Ctx, input.Client, 20*time.Minute) } // Cleanup any VPC-related resources from previous test runs (when tests were run with skip cleanup) vpcPrerequisiteScope.CleanupBefore() vpcOvnContextScope.CleanupBefore() - getDPUClusterClients(ctx, getProvisionDPUClustersInput()) + GetDPUClusterClients(Ctx, GetProvisionDPUClustersInput()) } }) @@ -94,61 +89,61 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} var dhcpDS *appsv1.DaemonSet It("create DPU VPC OVN VTEP DPUServiceIPAM", func() { - createVtepDPUServiceIPAM(ctx, input) + CreateVtepDPUServiceIPAM(Ctx, input) }) It("create DPU VPC OVN gateway DPUServiceIPAM", func() { - createGatewayDPUServiceIPAM(ctx, input) + CreateGatewayDPUServiceIPAM(Ctx, input) }) It("create DPU VPC OVN central DPUService", func() { - createOVNCentralDPUService(ctx, input.client, dpfOperatorSystemNamespace, vpcOvnInput.dpuServiceOVNCentral) + CreateOVNCentralDPUService(Ctx, input.Client, DPFOperatorSystemNamespace, VPCOVNInput.DPUServiceOVNCentral) }) It("create DPU OVN controller service", func() { - createOVNControllerDPUService(ctx, input.client, dpfOperatorSystemNamespace, vpcOvnInput.dpuServiceOVNController) + CreateOVNControllerDPUService(Ctx, input.Client, DPFOperatorSystemNamespace, VPCOVNInput.DPUServiceOVNController) }) It("create DPU VPC OVN controller service", func() { - createVPCOVNControllerDPUService(ctx, input.client, dpfOperatorSystemNamespace, vpcOvnInput.dpuServiceVPCOVNController) + CreateVPCOVNControllerDPUService(Ctx, input.Client, DPFOperatorSystemNamespace, VPCOVNInput.DPUServiceVPCOVNController) }) It("create DPU VPC OVN node service", func() { - createVPCOVNNodeDPUService(ctx, input.client, dpfOperatorSystemNamespace, vpcOvnInput.dpuServiceVPCOVNNode) + CreateVPCOVNNodeDPUService(Ctx, input.Client, DPFOperatorSystemNamespace, VPCOVNInput.DPUServiceVPCOVNNode) }) It("wait for pre-requisite DPU services to be ready", func() { - dpuservice.WaitForDPUServices(ctx, input.client, dpfOperatorSystemNamespace, []string{"ovn-central", "ovn-controller", "vpc-ovn-controller", "vpc-ovn-node"}) + dpuservice.WaitForDPUServices(Ctx, input.Client, DPFOperatorSystemNamespace, []string{"ovn-central", "ovn-controller", "vpc-ovn-controller", "vpc-ovn-node"}) }) It("create DPU service interfaces", func() { - createVPCPrerequisiteDPUServiceInterfaces(ctx, input) + CreateVPCPrerequisiteDPUServiceInterfaces(Ctx, input) }) It("wait for DPU service interfaces to be ready", func() { dpuServiceInterfaceNames := []string{ovnutils.PhysicalInterface0, ovnutils.OvnExtPatchName} - dpuservice.WaitForDPUServiceInterfacesReady(ctx, input.client, dpuClusterClient[0], dpuServiceInterfaceNames, dpfOperatorSystemNamespace) + dpuservice.WaitForDPUServiceInterfacesReady(Ctx, input.Client, DPUClusterClient[0], dpuServiceInterfaceNames, DPFOperatorSystemNamespace) }) It("create DPU service chain", func() { - createOrUpdateVPCDPUServiceChain(ctx, input, nil) + CreateOrUpdateVPCDPUServiceChain(Ctx, input, nil) }) It("wait for DPU service chain to be ready", func() { - dpuservice.WaitForDPUServiceChainsReady(ctx, input.client, dpuClusterClient[0], []string{ovnutils.VpcOVNServiceChain}, dpfOperatorSystemNamespace, vpcutils.DefaultTimeout) + dpuservice.WaitForDPUServiceChainsReady(Ctx, input.Client, DPUClusterClient[0], []string{ovnutils.VpcOVNServiceChain}, DPFOperatorSystemNamespace, vpcutils.DefaultTimeout) }) It("create dhcp daemon", func() { - dhcpDS = vpcutils.DeployDHCPDaemon(ctx, input.client, vpcOvnInput.dhcpDaemonSet, vpcPrerequisiteScope.CleanupLabels) + dhcpDS = vpcutils.DeployDHCPDaemon(Ctx, input.Client, VPCOVNInput.DHCPDaemonSet, vpcPrerequisiteScope.CleanupLabels) }) It("wait for dhcp daemon pods to be ready", func() { - vpcutils.WaitForDHCPDaemonReady(ctx, input.client, dhcpDS) + vpcutils.WaitForDHCPDaemonReady(Ctx, input.Client, dhcpDS) }) It("get DPU nodes", func() { By("Getting DPU cluster nodes in order") - dpuNode1, dpuNode2 = getDPUNodesInOrder(ctx, input.client, dpuClusterClient[0]) + dpuNode1, dpuNode2 = getDPUNodesInOrder(Ctx, input.Client, DPUClusterClient[0]) Expect(dpuNode1.Name).ToNot(BeEmpty()) Expect(dpuNode2.Name).ToNot(BeEmpty()) }) @@ -203,35 +198,35 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} reportAfterEach(CurrentSpecReport()) } vpcOvnContextScope.CleanupAfter() - cleanupDPUClusterNodeLabels(ctx) + CleanupDPUClusterNodeLabels(Ctx) }) It("label DPU nodes with tenant and tenant-node labels", func() { - labelDPUNodesWithTenantAndTenantNode(ctx, dpuClusterClient[0], dpuNode1, dpuNode2, defaultTenant, defaultTenant) + LabelDPUNodesWithTenantAndTenantNode(Ctx, DPUClusterClient[0], dpuNode1, dpuNode2, defaultTenant, defaultTenant) }) It("create OVNIsolationClass object", func() { - createOVNIsolationClass(ctx, input.client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateOVNIsolationClass(Ctx, input.Client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVPC object", func() { - createDPUVPC(ctx, input.client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateDPUVPC(Ctx, input.Client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVirtualNetwork object", func() { - createDPUVirtualNetwork(ctx, input.client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) + CreateDPUVirtualNetwork(Ctx, input.Client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) }) It("verify DPUVirtualNetwork is ready", func() { - ovnutils.WaitForDPUServiceVirtualNetworkReady(ctx, input.client, testnet1, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUServiceVirtualNetworkReady(Ctx, input.Client, testnet1, DPFOperatorSystemNamespace) }) It("verify DPUVPC is ready", func() { - ovnutils.WaitForDPUVPCReady(ctx, input.client, vpcName, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUVPCReady(Ctx, input.Client, vpcName, DPFOperatorSystemNamespace) }) It("verify DPUVPC and DPUVirtualNetwork metrics", func() { - validateVPCMetrics(ctx) + ValidateVPCMetrics(Ctx) }) It("create DPUServiceInterfaces on the nodes, same virtual network", func() { @@ -245,9 +240,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} ovnutils.InterfaceLabelKey: pf0vf3Worker1, } - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf2Worker1, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf2Worker1Labels), NodeName: &dpuNode1.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -256,9 +251,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} VFIndex: 2, VirtualNetwork: &testnet1, }) - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf2Worker2, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf2Worker2Labels), NodeName: &dpuNode2.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -268,9 +263,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} VirtualNetwork: &testnet1, }) - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf3Worker1, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf3Worker1Labels), NodeName: &dpuNode1.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -283,7 +278,7 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} It("verify DPUServiceInterfaces are ready", func() { dpuServiceInterfaceNames := []string{pf0vf2Worker1, pf0vf2Worker2, pf0vf3Worker1} - dpuservice.WaitForDPUServiceInterfacesReady(ctx, input.client, dpuClusterClient[0], dpuServiceInterfaceNames, dpfOperatorSystemNamespace) + dpuservice.WaitForDPUServiceInterfacesReady(Ctx, input.Client, DPUClusterClient[0], dpuServiceInterfaceNames, DPFOperatorSystemNamespace) }) It("get the MAC addresses of the ServiceInterface objects", func() { @@ -296,9 +291,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} pf0vf3Worker1Labels := map[string]string{ ovnutils.InterfaceLabelKey: pf0vf3Worker1, } - pf0vf2Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf2Worker1Labels) - pf0vf2Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf2Worker2Labels) - pf0vf3Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf3Worker1Labels) + pf0vf2Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf2Worker1Labels) + pf0vf2Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf2Worker2Labels) + pf0vf3Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf3Worker1Labels) Expect(pf0vf2Worker1MacAddressesMap).To(HaveLen(1)) pf0vf2Worker1MacAddress = pf0vf2Worker1MacAddressesMap[dpuNode1.Name] @@ -311,23 +306,23 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} // Note: This is a workaround for testing to avoid rebooting the hosts. // MAC addresses will be set as part of the BFB, then when the host boots up, the mac address will be set. It("set host VF MAC addresses", func() { - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) - workerNode1IP := GetNodeInternalIP(ctx, input.client, workerNode1) - workerNode2IP := GetNodeInternalIP(ctx, input.client, workerNode2) + workerNode1, workerNode2 := getTwoWorkerNodeNames(Ctx, input.Client) + workerNode1IP := GetNodeInternalIP(Ctx, input.Client, workerNode1) + workerNode2IP := GetNodeInternalIP(Ctx, input.Client, workerNode2) vpcutils.SetLinkMacAddress(workerNode1IP, hostPf0Vf2, pf0vf2Worker1MacAddress) vpcutils.SetLinkMacAddress(workerNode2IP, hostPf0Vf2, pf0vf2Worker2MacAddress) vpcutils.SetLinkMacAddress(workerNode1IP, hostPf0Vf3, pf0vf3Worker1MacAddress) }) It("create netshoot pods and NetworkAttachmentDefinitions", func() { - vpcutils.CreateTestNamespace(ctx, input.client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateTestNamespace(Ctx, input.Client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) nadName1 := nadNamePrefix + podName1 nadName2 := nadNamePrefix + podName2 nadName3 := nadNamePrefix + podName3 - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName2, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName3, hostPf0Vf3, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName2, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName3, hostPf0Vf3, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + workerNode1, workerNode2 := getTwoWorkerNodeNames(Ctx, input.Client) testPodConfigs = []*netshoot.TestPodConfig{ { Namespace: vpcTrafficTestNS, @@ -354,40 +349,40 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} CommandArgs: []string{deleteFlannelDefaultRouteCmd}, }, } - netshoot.CreatePods(ctx, input.client, testPodConfigs) + netshoot.CreatePods(Ctx, input.Client, testPodConfigs) }) It("verify netshoot pods are running", func() { - netshoot.WaitForPodsReady(ctx, input.client, testPodConfigs, vpcutils.LongTimeout) + netshoot.WaitForPodsReady(Ctx, input.Client, testPodConfigs, vpcutils.LongTimeout) }) It("get pod IP addresses", func() { - pod1IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName1, vfDefaultInterfaceName) - pod2IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName2, vfDefaultInterfaceName) - pod3IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName3, vfDefaultInterfaceName) + pod1IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName1, vfDefaultInterfaceName) + pod2IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName2, vfDefaultInterfaceName) + pod3IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName3, vfDefaultInterfaceName) Expect(pod1IP).ToNot(BeEmpty()) Expect(pod2IP).ToNot(BeEmpty()) Expect(pod3IP).ToNot(BeEmpty()) }) It("verify netshoot pods can ping each other in the same node", func() { - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, pod3IP) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName3, pod1IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, pod3IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName3, pod1IP) }) It("verify netshoot pods can ping each other cross nodes", func() { - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, pod2IP) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName2, pod1IP) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName3, pod2IP) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName2, pod3IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, pod2IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName2, pod1IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName3, pod2IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName2, pod3IP) }) It("verify performance with iperf same node traffic", func() { - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, podName3, pod3IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, podName3, pod3IP) }) It("verify performance with iperf cross node traffic", func() { - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, podName2, pod2IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, podName2, pod2IP) }) }) @@ -416,33 +411,33 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} reportAfterEach(CurrentSpecReport()) } vpcOvnContextScope.CleanupAfter() - cleanupDPUClusterNodeLabels(ctx) + CleanupDPUClusterNodeLabels(Ctx) }) It("label DPU nodes with tenant and tenant-node labels", func() { - labelDPUNodesWithTenantAndTenantNode(ctx, dpuClusterClient[0], dpuNode1, dpuNode2, defaultTenant, defaultTenant) + LabelDPUNodesWithTenantAndTenantNode(Ctx, DPUClusterClient[0], dpuNode1, dpuNode2, defaultTenant, defaultTenant) }) It("create OVNIsolationClass object", func() { - createOVNIsolationClass(ctx, input.client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateOVNIsolationClass(Ctx, input.Client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVPC object", func() { - createDPUVPC(ctx, input.client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateDPUVPC(Ctx, input.Client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVirtualNetwork objects", func() { - createDPUVirtualNetwork(ctx, input.client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) - createDPUVirtualNetwork(ctx, input.client, testnet2, vpcName, defaultTenant, vnet2DefaultSubnet, vpcOvnContextScope.CleanupLabels) + CreateDPUVirtualNetwork(Ctx, input.Client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) + CreateDPUVirtualNetwork(Ctx, input.Client, testnet2, vpcName, defaultTenant, vnet2DefaultSubnet, vpcOvnContextScope.CleanupLabels) }) It("verify DPUVirtualNetwork is ready", func() { - ovnutils.WaitForDPUServiceVirtualNetworkReady(ctx, input.client, testnet1, dpfOperatorSystemNamespace) - ovnutils.WaitForDPUServiceVirtualNetworkReady(ctx, input.client, testnet2, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUServiceVirtualNetworkReady(Ctx, input.Client, testnet1, DPFOperatorSystemNamespace) + ovnutils.WaitForDPUServiceVirtualNetworkReady(Ctx, input.Client, testnet2, DPFOperatorSystemNamespace) }) It("verify DPUVPC is ready", func() { - ovnutils.WaitForDPUVPCReady(ctx, input.client, vpcName, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUVPCReady(Ctx, input.Client, vpcName, DPFOperatorSystemNamespace) }) It("create DPUServiceInterfaces on the nodes, different virtual networks", func() { @@ -459,9 +454,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} ovnutils.InterfaceLabelKey: pf0vf3Worker1, } - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf2Worker1, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf2Worker1Labels), NodeName: &dpuNode1.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -471,9 +466,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} VirtualNetwork: &testnet1, }) - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf2Worker2, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf2Worker2Labels), NodeName: &dpuNode2.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -483,9 +478,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} VirtualNetwork: &testnet2, }) - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf3Worker1, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf3Worker1Labels), NodeName: &dpuNode1.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -498,7 +493,7 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} It("verify DPUServiceInterface is ready", func() { dpuServiceInterfaceNames := []string{pf0vf2Worker1, pf0vf2Worker2, pf0vf3Worker1} - dpuservice.WaitForDPUServiceInterfacesReady(ctx, input.client, dpuClusterClient[0], dpuServiceInterfaceNames, dpfOperatorSystemNamespace) + dpuservice.WaitForDPUServiceInterfacesReady(Ctx, input.Client, DPUClusterClient[0], dpuServiceInterfaceNames, DPFOperatorSystemNamespace) }) It("get the MAC addresses of the ServiceInterface objects", func() { @@ -512,9 +507,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} ovnutils.InterfaceLabelKey: pf0vf3Worker1, } - pf0vf2Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf2Worker1Labels) - pf0vf2Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf2Worker2Labels) - pf0vf3Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf3Worker1Labels) + pf0vf2Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf2Worker1Labels) + pf0vf2Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf2Worker2Labels) + pf0vf3Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf3Worker1Labels) Expect(pf0vf2Worker1MacAddressesMap).To(HaveLen(1)) pf0vf2Worker1MacAddress = pf0vf2Worker1MacAddressesMap[dpuNode1.Name] @@ -527,23 +522,23 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} // Note: This is a workaround for testing to avoid rebooting the hosts. // MAC addresses will be set as part of the BFB, then when the host boots up, the mac address will be set. It("set host VF MAC addresses", func() { - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) - workerNode1IP := GetNodeInternalIP(ctx, input.client, workerNode1) - workerNode2IP := GetNodeInternalIP(ctx, input.client, workerNode2) + workerNode1, workerNode2 := getTwoWorkerNodeNames(Ctx, input.Client) + workerNode1IP := GetNodeInternalIP(Ctx, input.Client, workerNode1) + workerNode2IP := GetNodeInternalIP(Ctx, input.Client, workerNode2) vpcutils.SetLinkMacAddress(workerNode1IP, hostPf0Vf2, pf0vf2Worker1MacAddress) vpcutils.SetLinkMacAddress(workerNode2IP, hostPf0Vf2, pf0vf2Worker2MacAddress) vpcutils.SetLinkMacAddress(workerNode1IP, hostPf0Vf3, pf0vf3Worker1MacAddress) }) It("create netshoot pods and NetworkAttachmentDefinitions", func() { - vpcutils.CreateTestNamespace(ctx, input.client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateTestNamespace(Ctx, input.Client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) nadName1 := nadNamePrefix + podName1 nadName2 := nadNamePrefix + podName2 nadName3 := nadNamePrefix + podName3 - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName2, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName3, hostPf0Vf3, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName2, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName3, hostPf0Vf3, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + workerNode1, workerNode2 := getTwoWorkerNodeNames(Ctx, input.Client) testPodConfigs = []*netshoot.TestPodConfig{ { Namespace: vpcTrafficTestNS, @@ -570,40 +565,40 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} CommandArgs: []string{deleteFlannelDefaultRouteCmd}, }, } - netshoot.CreatePods(ctx, input.client, testPodConfigs) + netshoot.CreatePods(Ctx, input.Client, testPodConfigs) }) It("verify netshoot pods are running", func() { - netshoot.WaitForPodsReady(ctx, input.client, testPodConfigs, vpcutils.LongTimeout) + netshoot.WaitForPodsReady(Ctx, input.Client, testPodConfigs, vpcutils.LongTimeout) }) It("get pod IP addresses", func() { - pod1IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName1, vfDefaultInterfaceName) - pod2IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName2, vfDefaultInterfaceName) - pod3IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName3, vfDefaultInterfaceName) + pod1IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName1, vfDefaultInterfaceName) + pod2IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName2, vfDefaultInterfaceName) + pod3IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName3, vfDefaultInterfaceName) Expect(pod1IP).ToNot(BeEmpty()) Expect(pod2IP).ToNot(BeEmpty()) Expect(pod3IP).ToNot(BeEmpty()) }) It("verify netshoot pods can ping each other in the same node", func() { - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, pod3IP) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName3, pod1IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, pod3IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName3, pod1IP) }) It("verify netshoot pods can ping each other cross nodes", func() { - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, pod2IP) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName2, pod1IP) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName3, pod2IP) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName2, pod3IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, pod2IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName2, pod1IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName3, pod2IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName2, pod3IP) }) It("verify performance with iperf same node traffic", func() { - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, podName3, pod3IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, podName3, pod3IP) }) It("verify performance with iperf cross node traffic", func() { - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, podName2, pod2IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, podName2, pod2IP) }) }) @@ -634,36 +629,36 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} reportAfterEach(CurrentSpecReport()) } vpcOvnContextScope.CleanupAfter() - cleanupDPUClusterNodeLabels(ctx) + CleanupDPUClusterNodeLabels(Ctx) }) It("label DPU nodes with tenant and tenant-node labels", func() { - labelDPUNodesWithTenantAndTenantNode(ctx, dpuClusterClient[0], dpuNode1, dpuNode2, defaultTenant, alternateTenant) + LabelDPUNodesWithTenantAndTenantNode(Ctx, DPUClusterClient[0], dpuNode1, dpuNode2, defaultTenant, alternateTenant) }) It("create OVNIsolationClass object", func() { - createOVNIsolationClass(ctx, input.client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateOVNIsolationClass(Ctx, input.Client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVPC object", func() { - createDPUVPC(ctx, input.client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) - createDPUVPC(ctx, input.client, vpcName2, alternateTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateDPUVPC(Ctx, input.Client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateDPUVPC(Ctx, input.Client, vpcName2, alternateTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVirtualNetwork objects", func() { - createDPUVirtualNetwork(ctx, input.client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) - createDPUVirtualNetwork(ctx, input.client, testnet2, vpcName2, alternateTenant, vnet2DefaultSubnet, vpcOvnContextScope.CleanupLabels) + CreateDPUVirtualNetwork(Ctx, input.Client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) + CreateDPUVirtualNetwork(Ctx, input.Client, testnet2, vpcName2, alternateTenant, vnet2DefaultSubnet, vpcOvnContextScope.CleanupLabels) }) It("verify DPUVirtualNetwork is ready", func() { - ovnutils.WaitForDPUServiceVirtualNetworkReady(ctx, input.client, testnet1, dpfOperatorSystemNamespace) - ovnutils.WaitForDPUServiceVirtualNetworkReady(ctx, input.client, testnet2, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUServiceVirtualNetworkReady(Ctx, input.Client, testnet1, DPFOperatorSystemNamespace) + ovnutils.WaitForDPUServiceVirtualNetworkReady(Ctx, input.Client, testnet2, DPFOperatorSystemNamespace) }) It("verify DPUVPC is ready", func() { - ovnutils.WaitForDPUVPCReady(ctx, input.client, vpcName, dpfOperatorSystemNamespace) - ovnutils.WaitForDPUVPCReady(ctx, input.client, vpcName2, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUVPCReady(Ctx, input.Client, vpcName, DPFOperatorSystemNamespace) + ovnutils.WaitForDPUVPCReady(Ctx, input.Client, vpcName2, DPFOperatorSystemNamespace) }) It("create DPUServiceInterfaces on the nodes, different virtual networks, different vpcs", func() { @@ -677,9 +672,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} ovnutils.InterfaceLabelKey: pf0vf3Worker2, } - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf2Worker1, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf2Worker1Labels), NodeName: &dpuNode1.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -689,9 +684,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} VirtualNetwork: &testnet1, }) - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf2Worker2, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf2Worker2Labels), NodeName: &dpuNode2.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -701,9 +696,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} VirtualNetwork: &testnet2, }) - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf3Worker2, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf3Worker2Labels), NodeName: &dpuNode2.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -716,7 +711,7 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} It("verify DPUServiceInterface is ready", func() { dpuServiceInterfaceNames := []string{pf0vf2Worker1, pf0vf2Worker2, pf0vf3Worker2} - dpuservice.WaitForDPUServiceInterfacesReady(ctx, input.client, dpuClusterClient[0], dpuServiceInterfaceNames, dpfOperatorSystemNamespace) + dpuservice.WaitForDPUServiceInterfacesReady(Ctx, input.Client, DPUClusterClient[0], dpuServiceInterfaceNames, DPFOperatorSystemNamespace) }) It("get the MAC addresses of the ServiceInterface objects", func() { @@ -730,9 +725,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} ovnutils.InterfaceLabelKey: pf0vf3Worker2, } - pf0vf2Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf2Worker1Labels) - pf0vf2Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf2Worker2Labels) - pf0vf3Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf3Worker2Labels) + pf0vf2Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf2Worker1Labels) + pf0vf2Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf2Worker2Labels) + pf0vf3Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf3Worker2Labels) Expect(pf0vf2Worker1MacAddressesMap).To(HaveLen(1)) pf0vf2Worker1MacAddress = pf0vf2Worker1MacAddressesMap[dpuNode1.Name] @@ -749,23 +744,23 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} // Note: This is a workaround for testing to avoid rebooting the hosts. // MAC addresses will be set as part of the BFB, then when the host boots up, the mac address will be set. It("set host VF MAC addresses", func() { - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) - workerNode1IP := GetNodeInternalIP(ctx, input.client, workerNode1) - workerNode2IP := GetNodeInternalIP(ctx, input.client, workerNode2) + workerNode1, workerNode2 := getTwoWorkerNodeNames(Ctx, input.Client) + workerNode1IP := GetNodeInternalIP(Ctx, input.Client, workerNode1) + workerNode2IP := GetNodeInternalIP(Ctx, input.Client, workerNode2) vpcutils.SetLinkMacAddress(workerNode1IP, hostPf0Vf2, pf0vf2Worker1MacAddress) vpcutils.SetLinkMacAddress(workerNode2IP, hostPf0Vf2, pf0vf2Worker2MacAddress) vpcutils.SetLinkMacAddress(workerNode2IP, hostPf0Vf3, pf0vf3Worker2MacAddress) }) It("create netshoot pods and NetworkAttachmentDefinitions", func() { - vpcutils.CreateTestNamespace(ctx, input.client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateTestNamespace(Ctx, input.Client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) nadName1 := nadNamePrefix + podName1 nadName2 := nadNamePrefix + podName2 nadName3 := nadNamePrefix + podName3 - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName2, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName3, hostPf0Vf3, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName2, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName3, hostPf0Vf3, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + workerNode1, workerNode2 := getTwoWorkerNodeNames(Ctx, input.Client) testPodConfigs = []*netshoot.TestPodConfig{ { Namespace: vpcTrafficTestNS, @@ -792,32 +787,32 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} CommandArgs: []string{deleteFlannelDefaultRouteCmd}, }, } - netshoot.CreatePods(ctx, input.client, testPodConfigs) + netshoot.CreatePods(Ctx, input.Client, testPodConfigs) }) It("verify netshoot pods are running", func() { - netshoot.WaitForPodsReady(ctx, input.client, testPodConfigs, vpcutils.LongTimeout) + netshoot.WaitForPodsReady(Ctx, input.Client, testPodConfigs, vpcutils.LongTimeout) }) It("get pod IP addresses", func() { - pod1IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName1, vfDefaultInterfaceName) - pod2IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName2, vfDefaultInterfaceName) - pod3IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName3, vfDefaultInterfaceName) + pod1IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName1, vfDefaultInterfaceName) + pod2IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName2, vfDefaultInterfaceName) + pod3IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName3, vfDefaultInterfaceName) Expect(pod1IP).ToNot(BeEmpty()) Expect(pod2IP).ToNot(BeEmpty()) Expect(pod3IP).ToNot(BeEmpty()) }) It("verify netshoot pods within the same VPC can ping each other", func() { - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName2, pod3IP) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName3, pod2IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName2, pod3IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName3, pod2IP) }) It("verify netshoot pods different vpcs cannot ping each other", func() { - netshoot.AssertPingFailure(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, pod2IP) - netshoot.AssertPingFailure(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName2, pod1IP) - netshoot.AssertPingFailure(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, pod3IP) - netshoot.AssertPingFailure(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName3, pod1IP) + netshoot.AssertPingFailure(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, pod2IP) + netshoot.AssertPingFailure(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName2, pod1IP) + netshoot.AssertPingFailure(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, pod3IP) + netshoot.AssertPingFailure(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName3, pod1IP) }) }) @@ -845,7 +840,7 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} ) BeforeAll(func() { - hostWorkerNode1, hostWorkerNode2 = getTwoWorkerNodeNames(ctx, input.client) + hostWorkerNode1, hostWorkerNode2 = getTwoWorkerNodeNames(Ctx, input.Client) }) AfterEach(func() { @@ -860,31 +855,31 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} reportAfterEach(CurrentSpecReport()) } vpcOvnContextScope.CleanupAfter() - cleanupDPUClusterNodeLabels(ctx) + CleanupDPUClusterNodeLabels(Ctx) }) It("label DPU nodes with tenant and tenant-node labels", func() { - labelDPUNodesWithTenantAndTenantNode(ctx, dpuClusterClient[0], dpuNode1, dpuNode2, defaultTenant, defaultTenant) + LabelDPUNodesWithTenantAndTenantNode(Ctx, DPUClusterClient[0], dpuNode1, dpuNode2, defaultTenant, defaultTenant) }) It("create OVNIsolationClass object", func() { - createOVNIsolationClass(ctx, input.client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateOVNIsolationClass(Ctx, input.Client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVPC object", func() { - createDPUVPC(ctx, input.client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateDPUVPC(Ctx, input.Client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVirtualNetwork object", func() { - createDPUVirtualNetwork(ctx, input.client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) + CreateDPUVirtualNetwork(Ctx, input.Client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) }) It("verify DPUVirtualNetwork is ready", func() { - ovnutils.WaitForDPUServiceVirtualNetworkReady(ctx, input.client, testnet1, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUServiceVirtualNetworkReady(Ctx, input.Client, testnet1, DPFOperatorSystemNamespace) }) It("verify DPUVPC is ready", func() { - ovnutils.WaitForDPUVPCReady(ctx, input.client, vpcName, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUVPCReady(Ctx, input.Client, vpcName, DPFOperatorSystemNamespace) }) It("create DPUServiceInterfaces on the nodes, same virtual network", func() { @@ -900,19 +895,19 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} } // Create SFs on both nodes. - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: sfName, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, sfLabels), Type: dpuservicev1.InterfaceTypeService, InterfaceName: sfInterfaceName, ServiceID: serviceID, - Network: fmt.Sprintf("%s/%s", dpfOperatorSystemNamespace, brIntNetwork), + Network: fmt.Sprintf("%s/%s", DPFOperatorSystemNamespace, brIntNetwork), VirtualNetwork: &testnet1, }) - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf2Worker1, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf2Worker1Labels), NodeName: &dpuNode1.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -921,9 +916,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} VFIndex: 2, VirtualNetwork: &testnet1, }) - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf2Worker2, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf2Worker2Labels), NodeName: &dpuNode2.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -935,7 +930,7 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} }) It("verify VF DPUServiceInterface is ready", func() { - dpuservice.WaitForDPUServiceInterfacesReady(ctx, input.client, dpuClusterClient[0], []string{pf0vf2Worker1, pf0vf2Worker2}, dpfOperatorSystemNamespace) + dpuservice.WaitForDPUServiceInterfacesReady(Ctx, input.Client, DPUClusterClient[0], []string{pf0vf2Worker1, pf0vf2Worker2}, DPFOperatorSystemNamespace) }) It("get the MAC addresses of the ServiceInterface objects", func() { @@ -946,12 +941,12 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} ovnutils.InterfaceLabelKey: pf0vf2Worker2, } - pf0vf2Worker1MacAddresseMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf2Worker1Labels) + pf0vf2Worker1MacAddresseMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf2Worker1Labels) Expect(pf0vf2Worker1MacAddresseMap).To(HaveLen(1)) pf0vf2Worker1MacAddress = pf0vf2Worker1MacAddresseMap[dpuNode1.Name] Expect(pf0vf2Worker1MacAddress).ToNot(BeEmpty()) - pf0vf2Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf2Worker2Labels) + pf0vf2Worker2MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf2Worker2Labels) Expect(pf0vf2Worker2MacAddressesMap).To(HaveLen(1)) pf0vf2Worker2MacAddress = pf0vf2Worker2MacAddressesMap[dpuNode2.Name] Expect(pf0vf2Worker2MacAddress).ToNot(BeEmpty()) @@ -960,18 +955,18 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} // Note: This is a workaround for testing to avoid rebooting the hosts. // MAC addresses will be set as part of the BFB, then when the host boots up, the mac address will be set. It("set host VF MAC addresses", func() { - hostWorkerNode1IP := GetNodeInternalIP(ctx, input.client, hostWorkerNode1) - hostWorkerNode2IP := GetNodeInternalIP(ctx, input.client, hostWorkerNode2) + hostWorkerNode1IP := GetNodeInternalIP(Ctx, input.Client, hostWorkerNode1) + hostWorkerNode2IP := GetNodeInternalIP(Ctx, input.Client, hostWorkerNode2) vpcutils.SetLinkMacAddress(hostWorkerNode1IP, hostPf0Vf2, pf0vf2Worker1MacAddress) vpcutils.SetLinkMacAddress(hostWorkerNode2IP, hostPf0Vf2, pf0vf2Worker2MacAddress) }) It("create netshoot pods and NetworkAttachmentDefinition", func() { - vpcutils.CreateTestNamespace(ctx, input.client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateTestNamespace(Ctx, input.Client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) nadName1 := nadNamePrefix + podName1 nadName2 := nadNamePrefix + podName2 - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName2, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName2, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) testPodConfigs = []*netshoot.TestPodConfig{ { Namespace: vpcTrafficTestNS, @@ -990,28 +985,28 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} CommandArgs: []string{deleteFlannelDefaultRouteCmd}, }, } - netshoot.CreatePods(ctx, input.client, testPodConfigs) + netshoot.CreatePods(Ctx, input.Client, testPodConfigs) }) It("verify netshoot pods are running", func() { - netshoot.WaitForPodsReady(ctx, input.client, testPodConfigs, vpcutils.LongTimeout) + netshoot.WaitForPodsReady(Ctx, input.Client, testPodConfigs, vpcutils.LongTimeout) }) It("create DPU NAD for br-int", func() { - ovnutils.CreateDPUIntergrationBridgeNetworkAttachmentDefinition(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, vpcOvnContextScope.CleanupLabels) + ovnutils.CreateDPUIntergrationBridgeNetworkAttachmentDefinition(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, vpcOvnContextScope.CleanupLabels) }) It("create dummy service consuming the SF", func() { - createDummyDPUService(ctx, input.client, dpfOperatorSystemNamespace, sfServiceName, vpcOvnContextScope.CleanupLabels, nil, serviceID, brIntNetwork, sfInterfaceName) + CreateDummyDPUService(Ctx, input.Client, DPFOperatorSystemNamespace, sfServiceName, vpcOvnContextScope.CleanupLabels, nil, serviceID, brIntNetwork, sfInterfaceName) }) It("verify SF DPUServiceInterface is ready", func() { // SF ServiceInterfaces will be ready only when we the service pods are deployed. - dpuservice.WaitForDPUServiceInterfacesReady(ctx, input.client, dpuClusterClient[0], []string{sfName}, dpfOperatorSystemNamespace) + dpuservice.WaitForDPUServiceInterfacesReady(Ctx, input.Client, DPUClusterClient[0], []string{sfName}, DPFOperatorSystemNamespace) }) It("verify dummy service is ready", func() { - dpuservice.WaitForDPUServices(ctx, input.client, dpfOperatorSystemNamespace, []string{sfServiceName}) + dpuservice.WaitForDPUServices(Ctx, input.Client, DPFOperatorSystemNamespace, []string{sfServiceName}) }) It("get SF pods ip addresses", func() { @@ -1020,14 +1015,14 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} } Eventually(func(g Gomega) { - sfPods = vpcutils.GetPodsMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, sfServiceLabels) + sfPods = vpcutils.GetPodsMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, sfServiceLabels) Expect(sfPods).To(HaveLen(2)) for _, pod := range sfPods { Expect(pod.Spec.NodeName).ToNot(BeEmpty()) } - pod1SFIP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, sfPods[0].Name, sfInterfaceName) - pod2SFIP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, sfPods[1].Name, sfInterfaceName) + pod1SFIP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, sfPods[0].Name, sfInterfaceName) + pod2SFIP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, sfPods[1].Name, sfInterfaceName) Expect(pod1SFIP).ToNot(BeEmpty()) Expect(pod2SFIP).ToNot(BeEmpty()) }, vpcutils.DefaultTimeout).Should(Succeed()) @@ -1036,16 +1031,16 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} It("verify netshoot vfs can ping sf pods on same and cross nodes", func() { By(fmt.Sprintf("Pinging from pod %s on %s node to Service pod %s on node %s", podName1, hostWorkerNode1, sfPods[0].Name, sfPods[0].Spec.NodeName)) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, pod1SFIP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, pod1SFIP) By(fmt.Sprintf("Pinging from pod %s on %s node to Service pod %s on node %s", podName1, hostWorkerNode1, sfPods[1].Name, sfPods[1].Spec.NodeName)) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, pod2SFIP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, pod2SFIP) By(fmt.Sprintf("Pinging from pod %s on %s node to Service pod %s on node %s", podName2, hostWorkerNode2, sfPods[0].Name, sfPods[0].Spec.NodeName)) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName2, pod1SFIP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName2, pod1SFIP) By(fmt.Sprintf("Pinging from pod %s on %s node to Service pod %s on node %s", podName2, hostWorkerNode2, sfPods[1].Name, sfPods[1].Spec.NodeName)) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName2, pod2SFIP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName2, pod2SFIP) }) }) @@ -1076,31 +1071,31 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} reportAfterEach(CurrentSpecReport()) } vpcOvnContextScope.CleanupAfter() - cleanupDPUClusterNodeLabels(ctx) + CleanupDPUClusterNodeLabels(Ctx) }) It("label DPU nodes with tenant and tenant-node labels", func() { - labelDPUNodesWithTenantAndTenantNode(ctx, dpuClusterClient[0], dpuNode1, dpuNode2, defaultTenant, "") + LabelDPUNodesWithTenantAndTenantNode(Ctx, DPUClusterClient[0], dpuNode1, dpuNode2, defaultTenant, "") }) It("create OVNIsolationClass object", func() { - createOVNIsolationClass(ctx, input.client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateOVNIsolationClass(Ctx, input.Client, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVPC object", func() { - createDPUVPC(ctx, input.client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) + CreateDPUVPC(Ctx, input.Client, vpcName, defaultTenant, ovnVPCProvisioner, vpcOvnContextScope.CleanupLabels) }) It("create DPUVirtualNetwork object", func() { - createDPUVirtualNetwork(ctx, input.client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) + CreateDPUVirtualNetwork(Ctx, input.Client, testnet1, vpcName, defaultTenant, vnet1DefaultSubnet, vpcOvnContextScope.CleanupLabels) }) It("verify DPUVirtualNetwork is ready", func() { - ovnutils.WaitForDPUServiceVirtualNetworkReady(ctx, input.client, testnet1, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUServiceVirtualNetworkReady(Ctx, input.Client, testnet1, DPFOperatorSystemNamespace) }) It("verify DPUVPC is ready", func() { - ovnutils.WaitForDPUVPCReady(ctx, input.client, vpcName, dpfOperatorSystemNamespace) + ovnutils.WaitForDPUVPCReady(Ctx, input.Client, vpcName, DPFOperatorSystemNamespace) }) It("create DPUServiceInterfaces on the nodes, same virtual network", func() { @@ -1111,9 +1106,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} ovnutils.InterfaceLabelKey: pf0vf7Worker2, } - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf2Worker1, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf2Worker1Labels), NodeName: &dpuNode1.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -1123,9 +1118,9 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} VirtualNetwork: &testnet1, }) // Creating pf0vf7 on second node that will not be part of the VPC that will simulated the endpoint for external network traffic - createVPCDPUServiceInterface(ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ + CreateVPCDPUServiceInterface(Ctx, input, dpuservice.TestDPUServiceInterfaceConfig{ Name: pf0vf7Worker2, - Namespace: dpfOperatorSystemNamespace, + Namespace: DPFOperatorSystemNamespace, Labels: cleanup.MergeMaps(vpcOvnContextScope.CleanupLabels, pf0vf7Worker2Labels), NodeName: &dpuNode2.Name, Type: dpuservicev1.InterfaceTypeVF, @@ -1137,26 +1132,26 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} It("verify DPUServiceInterfaces are ready", func() { dpuServiceInterfaceNames := []string{pf0vf2Worker1, pf0vf7Worker2} - dpuservice.WaitForDPUServiceInterfacesReady(ctx, input.client, dpuClusterClient[0], dpuServiceInterfaceNames, dpfOperatorSystemNamespace) + dpuservice.WaitForDPUServiceInterfacesReady(Ctx, input.Client, DPUClusterClient[0], dpuServiceInterfaceNames, DPFOperatorSystemNamespace) }) It("create DPU service chain on second worker node for external network traffic", func() { - createDPUServiceChainP0ToInterfaceMatchingLabels(ctx, input, p0ToPf0Vf7Gw, pf0vf7Worker2Labels, &dpuNode2.Name, vpcOvnContextScope.CleanupLabels) + CreateDPUServiceChainP0ToInterfaceMatchingLabels(Ctx, input, p0ToPf0Vf7Gw, pf0vf7Worker2Labels, &dpuNode2.Name, vpcOvnContextScope.CleanupLabels) }) It("reconfigure p0 to OVN VTEP external patch port DPU service chain to only exist on first node", func() { - createOrUpdateVPCDPUServiceChain(ctx, input, &dpuNode1.Name) + CreateOrUpdateVPCDPUServiceChain(Ctx, input, &dpuNode1.Name) }) It("wait for DPU service chains to be ready", func() { - dpuservice.WaitForDPUServiceChainsReady(ctx, input.client, dpuClusterClient[0], []string{p0ToPf0Vf7Gw, ovnutils.VpcOVNServiceChain}, dpfOperatorSystemNamespace, vpcutils.LongTimeout) + dpuservice.WaitForDPUServiceChainsReady(Ctx, input.Client, DPUClusterClient[0], []string{p0ToPf0Vf7Gw, ovnutils.VpcOVNServiceChain}, DPFOperatorSystemNamespace, vpcutils.LongTimeout) }) It("get the MAC addresses of the ServiceInterface objects", func() { pf0vf2Worker1Labels := map[string]string{ ovnutils.InterfaceLabelKey: pf0vf2Worker1, } - pf0vf2Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, pf0vf2Worker1Labels) + pf0vf2Worker1MacAddressesMap := ovnutils.GetServiceInterfaceMacAddressesMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, pf0vf2Worker1Labels) Expect(pf0vf2Worker1MacAddressesMap).To(HaveLen(1)) pf0vf2Worker1MacAddress = pf0vf2Worker1MacAddressesMap[dpuNode1.Name] @@ -1165,18 +1160,18 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} // Note: This is a workaround for testing to avoid rebooting the hosts. // MAC addresses will be set as part of the BFB, then when the host boots up, the mac address will be set. It("set host VF MAC addresses", func() { - workerNode1, _ := getTwoWorkerNodeNames(ctx, input.client) - workerNode1IP := GetNodeInternalIP(ctx, input.client, workerNode1) + workerNode1, _ := getTwoWorkerNodeNames(Ctx, input.Client) + workerNode1IP := GetNodeInternalIP(Ctx, input.Client, workerNode1) vpcutils.SetLinkMacAddress(workerNode1IP, hostPf0Vf2, pf0vf2Worker1MacAddress) }) It("create netshoot pods and NetworkAttachmentDefinitions", func() { - vpcutils.CreateTestNamespace(ctx, input.client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateTestNamespace(Ctx, input.Client, vpcTrafficTestNS, vpcOvnContextScope.CleanupLabels) nadName1 := nadNamePrefix + podName1 - vpcutils.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) + vpcutils.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, nadName1, hostPf0Vf2, ovnutils.VfsMTU, vpcOvnContextScope.CleanupLabels) gatewayIPCIDR := fmt.Sprintf("%s/%d", ovnutils.GatewayIPPoolGateway, ovnutils.GatewayMask) - nadName2 := ovnutils.CreateExternalEndpointPodNetworkAttachmentDefinition(ctx, input.client, vpcTrafficTestNS, podName2, 7, gatewayIPCIDR, vpcOvnContextScope.CleanupLabels) - workerNode1, workerNode2 := getTwoWorkerNodeNames(ctx, input.client) + nadName2 := ovnutils.CreateExternalEndpointPodNetworkAttachmentDefinition(Ctx, input.Client, vpcTrafficTestNS, podName2, 7, gatewayIPCIDR, vpcOvnContextScope.CleanupLabels) + workerNode1, workerNode2 := getTwoWorkerNodeNames(Ctx, input.Client) testPodConfigs = []*netshoot.TestPodConfig{ { Namespace: vpcTrafficTestNS, @@ -1195,31 +1190,31 @@ var _ = Describe("VPC OVN testcases", Labels{Domain.DPFSystem, Domain.DPFVPCOVN} CommandArgs: []string{deleteFlannelDefaultRouteCmd}, }, } - netshoot.CreatePods(ctx, input.client, testPodConfigs) + netshoot.CreatePods(Ctx, input.Client, testPodConfigs) }) It("verify netshoot pods are running", func() { - netshoot.WaitForPodsReady(ctx, input.client, testPodConfigs, vpcutils.LongTimeout) + netshoot.WaitForPodsReady(Ctx, input.Client, testPodConfigs, vpcutils.LongTimeout) }) It("get pod IP addresses", func() { - pod1IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName1, vfDefaultInterfaceName) - pod2IP = vpcutils.GetPodIPAddressFromNetworkStatus(ctx, input.client, vpcTrafficTestNS, podName2, vfDefaultInterfaceName) + pod1IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName1, vfDefaultInterfaceName) + pod2IP = vpcutils.GetPodIPAddressFromNetworkStatus(Ctx, input.Client, vpcTrafficTestNS, podName2, vfDefaultInterfaceName) Expect(pod1IP).ToNot(BeEmpty()) Expect(pod2IP).ToNot(BeEmpty()) }) It("verify netshoot vf pod can ping external network pod", func() { - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, pod2IP) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, pod2IP) }) It("verify performance with iperf to external network traffic", func() { - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, vpcTrafficTestNS, podName1, podName2, pod2IP) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, vpcTrafficTestNS, podName1, podName2, pod2IP) }) It("revert p0 to OVN VTEP external patch port DPU service chain to its original configuration", func() { - createOrUpdateVPCDPUServiceChain(ctx, input, nil) - dpuservice.WaitForDPUServiceChainsReady(ctx, input.client, dpuClusterClient[0], []string{ovnutils.VpcOVNServiceChain}, dpfOperatorSystemNamespace, vpcutils.LongTimeout) + CreateOrUpdateVPCDPUServiceChain(Ctx, input, nil) + dpuservice.WaitForDPUServiceChainsReady(Ctx, input.Client, DPUClusterClient[0], []string{ovnutils.VpcOVNServiceChain}, DPFOperatorSystemNamespace, vpcutils.LongTimeout) }) }) }) diff --git a/test/e2e/weave.go b/test/e2e/weave.go index 78d88150..2e4bc7e2 100644 --- a/test/e2e/weave.go +++ b/test/e2e/weave.go @@ -144,10 +144,10 @@ var ( weavePrerequisiteScope *cleanup.Scope ) -var weaveInput = &weaveTestInput{} +var WeaveInput = &WeaveTestInput{} -// vpcctlVNetResponse is used to parse create-vnet / get-vnet JSON responses. -type vpcctlVNetResponse struct { +// VpcctlVNetResponse is used to parse create-vnet / get-vnet JSON responses. +type VpcctlVNetResponse struct { VirtualNetwork struct { Spec struct { ID string `json:"id"` @@ -160,8 +160,8 @@ type vpcctlVNetResponse struct { } `json:"virtualNetwork"` } -// vpcctlAttachmentResponse is used to parse create-attachment / get-attachment JSON responses. -type vpcctlAttachmentResponse struct { +// VpcctlAttachmentResponse is used to parse create-attachment / get-attachment JSON responses. +type VpcctlAttachmentResponse struct { VirtualNetworkAttachment struct { Spec struct { ID string `json:"id"` @@ -175,8 +175,8 @@ type vpcctlAttachmentResponse struct { } `json:"virtualNetworkAttachment"` } -// vpcctlListAttachmentResponse is used to parse list-attachment JSON responses. -type vpcctlListAttachmentResponse struct { +// VpcctlListAttachmentResponse is used to parse list-attachment JSON responses. +type VpcctlListAttachmentResponse struct { VirtualNetworkAttachments []struct { Spec struct { ID string `json:"id"` @@ -184,37 +184,37 @@ type vpcctlListAttachmentResponse struct { } `json:"virtualNetworkAttachments"` } -// weaveTestInput holds objects loaded from config for Weave e2e (see applyWeaveConfig). -type weaveTestInput struct { - dhcpDaemonSet *appsv1.DaemonSet +// WeaveTestInput holds objects loaded from config for Weave e2e (see applyWeaveConfig). +type WeaveTestInput struct { + DHCPDaemonSet *appsv1.DaemonSet } -func (t *weaveTestInput) applyWeaveConfig(conf config) { +func (t *WeaveTestInput) ApplyWeaveConfig(conf Config) { dhcpDaemonSet := &appsv1.DaemonSet{} - dhcpObj := unstructuredFromFile(conf.DHCPDaemonSetPath) + dhcpObj := UnstructuredFromFile(conf.DHCPDaemonSetPath) Expect(machineryruntime.DefaultUnstructuredConverter.FromUnstructured(dhcpObj.Object, dhcpDaemonSet)).To(Succeed()) - t.dhcpDaemonSet = dhcpDaemonSet + t.DHCPDaemonSet = dhcpDaemonSet } // WeaveBeforeSuite is called from the e2e BeforeSuite to load Weave test artifacts from config. -func WeaveBeforeSuite(c config) { +func WeaveBeforeSuite(c Config) { By("Setting Weave configs for the test") - weaveInput.applyWeaveConfig(c) + WeaveInput.ApplyWeaveConfig(c) } -// getProvisionDPUClustersInputForWeave returns provision input for Weave tests. -func getProvisionDPUClustersInputForWeave(ctx context.Context, provisionInput ProvisionDPUClustersInput, cl client.Client) ProvisionDPUClustersInput { +// GetProvisionDPUClustersInputForWeave returns provision input for Weave tests. +func GetProvisionDPUClustersInputForWeave(ctx context.Context, provisionInput ProvisionDPUClustersInput, cl client.Client) ProvisionDPUClustersInput { if dpuClusterName != "" && dpuClusterNamespace != "" { name, ns := dpuClusterName, dpuClusterNamespace dc := &provisioningv1.DPUCluster{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}} if err := cl.Get(ctx, client.ObjectKeyFromObject(dc), dc); err == nil { - provisionInput.dpuClusters = []*provisioningv1.DPUCluster{dc} + provisionInput.DPUClusters = []*provisioningv1.DPUCluster{dc} return provisionInput } } - if len(provisionInput.dpuClusters) > 0 { - key := client.ObjectKeyFromObject(provisionInput.dpuClusters[0]) - if err := cl.Get(ctx, key, provisionInput.dpuClusters[0]); err == nil { + if len(provisionInput.DPUClusters) > 0 { + key := client.ObjectKeyFromObject(provisionInput.DPUClusters[0]) + if err := cl.Get(ctx, key, provisionInput.DPUClusters[0]); err == nil { return provisionInput } } @@ -223,31 +223,31 @@ func getProvisionDPUClustersInputForWeave(ctx context.Context, provisionInput Pr list := &provisioningv1.DPUClusterList{} Expect(cl.List(ctx, list)).To(Succeed()) if len(list.Items) > 0 { - provisionInput.dpuClusters = []*provisioningv1.DPUCluster{&list.Items[0]} + provisionInput.DPUClusters = []*provisioningv1.DPUCluster{&list.Items[0]} } return provisionInput } -// verifyOVSResponsive runs `ovs-vsctl show` on the given flow-controller pod and asserts +// VerifyOVSResponsive runs `ovs-vsctl show` on the given flow-controller pod and asserts // it returns non-empty output. Acts as an early sanity check that OVS is up. -func verifyOVSResponsive(pod *corev1.Pod) { +func VerifyOVSResponsive(pod *corev1.Pod) { By(fmt.Sprintf("Verifying OVS is responsive on pod %s (node %s)", pod.Name, pod.Spec.NodeName)) Eventually(func(g Gomega) { - out, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, + out, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, []string{"ovs-vsctl", "show"}) g.Expect(err).ToNot(HaveOccurred(), "ovs-vsctl show failed on pod %s: %s", pod.Name, out) g.Expect(strings.TrimSpace(out)).ToNot(BeEmpty(), "ovs-vsctl show returned empty output on pod %s", pod.Name) }).WithTimeout(30 * time.Second).WithPolling(weaveEventuallyPollInterval).Should(Succeed()) } -// getPFMACFromFlowControllerByPort reads the PF MAC for the given DPU-side port (e.g. "p0" or "p1") +// GetPFMACFromFlowControllerByPort reads the PF MAC for the given DPU-side port (e.g. "p0" or "p1") // from the smart_nic sysfs config inside the flow-controller pod. // NOTE: This will not work on BF4 ASTRA setup since ECPFs have different names. -func getPFMACFromFlowControllerByPort(pod *corev1.Pod, port string) string { +func GetPFMACFromFlowControllerByPort(pod *corev1.Pod, port string) string { cmd := []string{"sh", "-c", fmt.Sprintf(`grep -i '^MAC' /sys/class/net/%s/smart_nic/pf/config | head -1 | sed 's/^[^:]*:[[:space:]]*//'`, port)} var mac string Eventually(func(g Gomega) { - output, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, cmd) + output, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, cmd) g.Expect(err).ToNot(HaveOccurred(), "failed to read PF MAC (%s) from pod %s: %s", port, pod.Name, output) mac = strings.TrimSpace(output) g.Expect(mac).ToNot(BeEmpty(), "empty PF MAC (%s) from pod %s", port, pod.Name) @@ -257,29 +257,29 @@ func getPFMACFromFlowControllerByPort(pod *corev1.Pod, port string) string { return mac } -func assertVPCtlVNetPhaseReady(g Gomega, pod *corev1.Pod, vnetID string) { - out, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, +func AssertVPCtlVNetPhaseReady(g Gomega, pod *corev1.Pod, vnetID string) { + out, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, []string{"/vpcctl", "get-vnet", "--id", vnetID}) g.Expect(err).ToNot(HaveOccurred(), "vpcctl get-vnet %q on pod %s: %s", vnetID, pod.Name, out) - var resp vpcctlVNetResponse + var resp VpcctlVNetResponse g.Expect(json.Unmarshal([]byte(out), &resp)).To(Succeed(), "failed to parse get-vnet %q on pod %s: %s", vnetID, pod.Name, out) g.Expect(resp.VirtualNetwork.Status.State.Phase).To(Equal("PHASE_READY"), "virtual network %q on pod %s not PHASE_READY: %s", vnetID, pod.Name, out) } -func assertVPCtlAttachmentPhaseReady(g Gomega, pod *corev1.Pod, attachmentID string) { - out, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, +func AssertVPCtlAttachmentPhaseReady(g Gomega, pod *corev1.Pod, attachmentID string) { + out, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, []string{"/vpcctl", "get-attachment", "--id", attachmentID}) g.Expect(err).ToNot(HaveOccurred(), "vpcctl get-attachment %q on pod %s: %s", attachmentID, pod.Name, out) - var resp vpcctlAttachmentResponse + var resp VpcctlAttachmentResponse g.Expect(json.Unmarshal([]byte(out), &resp)).To(Succeed(), "failed to parse get-attachment %q on pod %s: %s", attachmentID, pod.Name, out) g.Expect(resp.VirtualNetworkAttachment.Status.State.Phase).To(Equal("PHASE_READY"), "attachment %q on pod %s not PHASE_READY: %s", attachmentID, pod.Name, out) } -// createVNetOnPod creates a virtual network on a flow-controller pod via vpcctl and asserts it reaches PHASE_READY. +// CreateVNetOnPod creates a virtual network on a flow-controller pod via vpcctl and asserts it reaches PHASE_READY. // The same vnetID and vni must be used on both flow-controller pods so that cross-node VXLAN traffic uses matching VNIs. -func createVNetOnPod(pod *corev1.Pod, vnetID string, vni uint32, subnet string) { +func CreateVNetOnPod(pod *corev1.Pod, vnetID string, vni uint32, subnet string) { By(fmt.Sprintf("Creating virtual network %q (vni=%d, subnet=%s) on pod %s", vnetID, vni, subnet, pod.Name)) cmd := []string{ "/vpcctl", "create-vnet", @@ -288,13 +288,13 @@ func createVNetOnPod(pod *corev1.Pod, vnetID string, vni uint32, subnet string) "--subnet-v4", subnet, } Eventually(func(g Gomega) { - output, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, cmd) + output, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, cmd) if err != nil && strings.Contains(output, "AlreadyExists") { - assertVPCtlVNetPhaseReady(g, pod, vnetID) + AssertVPCtlVNetPhaseReady(g, pod, vnetID) return } g.Expect(err).ToNot(HaveOccurred(), "vpcctl create-vnet failed on pod %s: %s", pod.Name, output) - var resp vpcctlVNetResponse + var resp VpcctlVNetResponse g.Expect(json.Unmarshal([]byte(output), &resp)).To(Succeed(), "failed to parse create-vnet response from pod %s: %s", pod.Name, output) g.Expect(resp.VirtualNetwork.Status.State.Phase).To(Equal("PHASE_READY"), "virtual network %q on pod %s not PHASE_READY: %s", vnetID, pod.Name, output) @@ -302,14 +302,14 @@ func createVNetOnPod(pod *corev1.Pod, vnetID string, vni uint32, subnet string) "failed to create virtual network %q on pod %s", vnetID, pod.Name) } -// listAttachmentIDs runs vpcctl list-attachment with the given filter flags (e.g. --nic-id, --vnet-id) +// ListAttachmentIDs runs vpcctl list-attachment with the given filter flags (e.g. --nic-id, --vnet-id) // and returns the attachment IDs from the JSON response. Used to discover stale attachments that // block create/delete operations without relying on parsing gRPC error strings. -func listAttachmentIDs(g Gomega, pod *corev1.Pod, filterFlags ...string) []string { +func ListAttachmentIDs(g Gomega, pod *corev1.Pod, filterFlags ...string) []string { args := append([]string{"/vpcctl", "list-attachment"}, filterFlags...) - out, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, args) + out, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, args) g.Expect(err).ToNot(HaveOccurred(), "vpcctl list-attachment %v failed on pod %s: %s", filterFlags, pod.Name, out) - var resp vpcctlListAttachmentResponse + var resp VpcctlListAttachmentResponse g.Expect(json.Unmarshal([]byte(out), &resp)).To(Succeed(), "failed to parse list-attachment response on pod %s: %s", pod.Name, out) ids := make([]string, 0, len(resp.VirtualNetworkAttachments)) @@ -319,10 +319,10 @@ func listAttachmentIDs(g Gomega, pod *corev1.Pod, filterFlags ...string) []strin return ids } -// createPFAttachmentAndWaitForHostIP creates a PF attachment on a flow-controller pod, waits until it is PHASE_READY, +// CreatePFAttachmentAndWaitForHostIP creates a PF attachment on a flow-controller pod, waits until it is PHASE_READY, // and returns the attachment ID together with the assigned host overlay IP (hostIpv4). // If the NIC already has a stale attachment from a previous test run or context, it is deleted and the create is retried. -func createPFAttachmentAndWaitForHostIP(pod *corev1.Pod, vnetID, pfMAC string) (attID, hostIP string) { +func CreatePFAttachmentAndWaitForHostIP(pod *corev1.Pod, vnetID, pfMAC string) (attID, hostIP string) { // Deterministic ID so the create is idempotent: if the tunnel drops after the server // processes the request, a retry returns AlreadyExists rather than FailedPrecondition // (the server enforces one attachment per NIC). @@ -342,16 +342,16 @@ func createPFAttachmentAndWaitForHostIP(pod *corev1.Pod, vnetID, pfMAC string) ( "--pf", pfMAC, } Eventually(func(g Gomega) { - output, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, cmd) + output, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, cmd) if err != nil && strings.Contains(output, "AlreadyExists") { - assertVPCtlAttachmentPhaseReady(g, pod, attID) + AssertVPCtlAttachmentPhaseReady(g, pod, attID) return } if err != nil && strings.Contains(output, "already attached") { - staleIDs := listAttachmentIDs(g, pod, "--nic-id", pfMAC) + staleIDs := ListAttachmentIDs(g, pod, "--nic-id", pfMAC) for _, staleID := range staleIDs { By(fmt.Sprintf("NIC %s has stale attachment %s — deleting before retry", pfMAC, staleID)) - delOut, delErr := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, + delOut, delErr := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, []string{"/vpcctl", "delete-attachment", "--id", staleID}) if delErr != nil && !strings.Contains(delOut, "NotFound") { g.Expect(delErr).ToNot(HaveOccurred(), "failed to delete stale attachment %s on pod %s: %s", staleID, pod.Name, delOut) @@ -360,16 +360,16 @@ func createPFAttachmentAndWaitForHostIP(pod *corev1.Pod, vnetID, pfMAC string) ( g.Expect(false).To(BeTrue(), "retry vpcctl create-attachment for nic %s after clearing %d stale attachment(s)", pfMAC, len(staleIDs)) } g.Expect(err).ToNot(HaveOccurred(), "vpcctl create-attachment failed on pod %s: %s", pod.Name, output) - var createResp vpcctlAttachmentResponse + var createResp VpcctlAttachmentResponse g.Expect(json.Unmarshal([]byte(output), &createResp)).To(Succeed(), "failed to parse create-attachment response from pod %s: %s", pod.Name, output) }).WithTimeout(weaveOperationTimeout).WithPolling(weaveEventuallyPollInterval).Should(Succeed(), "failed to create PF attachment for MAC %s on pod %s", pfMAC, pod.Name) By(fmt.Sprintf("Waiting for attachment %s on pod %s to reach PHASE_READY", attID, pod.Name)) Eventually(func(g Gomega) { - out, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, []string{"/vpcctl", "get-attachment", "--id", attID}) + out, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, []string{"/vpcctl", "get-attachment", "--id", attID}) g.Expect(err).ToNot(HaveOccurred()) - var resp vpcctlAttachmentResponse + var resp VpcctlAttachmentResponse g.Expect(json.Unmarshal([]byte(out), &resp)).To(Succeed()) g.Expect(resp.VirtualNetworkAttachment.Status.State.Phase).To(Equal("PHASE_READY")) hostIP = resp.VirtualNetworkAttachment.Status.HostIPv4 @@ -391,20 +391,20 @@ var dpuPortToDropNIC = map[string]string{ weaveDPUPortP1: "n1", } -// isolationBridgeName returns the OVS isolation bridge name for a VNI on a DPU port. -func isolationBridgeName(vni uint32, dpuPort string) string { +// IsolationBridgeName returns the OVS isolation bridge name for a VNI on a DPU port. +func IsolationBridgeName(vni uint32, dpuPort string) string { pci, ok := dpuPortToPCIUnderscored[dpuPort] Expect(ok).To(BeTrue(), "unknown DPU port %q", dpuPort) return fmt.Sprintf("br-isol-%d-%s", vni, pci) } -// verifyIsolationBridgeExists asserts that the OVS isolation bridge for the given VNI +// VerifyIsolationBridgeExists asserts that the OVS isolation bridge for the given VNI // on the given DPU port (br-isol--) is present on the flow-controller pod. -func verifyIsolationBridgeExists(pod *corev1.Pod, vni uint32, dpuPort string) { - bridge := isolationBridgeName(vni, dpuPort) +func VerifyIsolationBridgeExists(pod *corev1.Pod, vni uint32, dpuPort string) { + bridge := IsolationBridgeName(vni, dpuPort) By(fmt.Sprintf("Verifying bridge %s exists on pod %s", bridge, pod.Name)) Eventually(func(g Gomega) { - out, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, + out, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, []string{"ovs-vsctl", "list", "bridge", bridge}) g.Expect(err).ToNot(HaveOccurred(), "bridge %s not found on pod %s: %s", bridge, pod.Name, out) }).WithTimeout(weaveOperationTimeout).WithPolling(weaveEventuallyPollInterval).Should(Succeed()) @@ -414,15 +414,15 @@ func verifyIsolationBridgeExists(pod *corev1.Pod, vni uint32, dpuPort string) { // counters. Each sample is labeled with the bridge and the weave_* counter name. const weaveMetricFamily = "ovs_vswitchd_flow_packets_total" -// weaveMetrics holds weave packet counters from one scrape of a flow-controller pod, +// WeaveMetrics holds weave packet counters from one scrape of a flow-controller pod, // keyed by bridge then counter name (e.g. metrics["br-isol-1001-..."]["weave_host_tx"]). -type weaveMetrics map[string]map[string]uint64 +type WeaveMetrics map[string]map[string]uint64 -// scrapeWeaveMetrics performs a single weave-metrics scrape of a flow-controller pod, keyed [bridge][name]. -func scrapeWeaveMetrics(g Gomega, pod *corev1.Pod) weaveMetrics { +// ScrapeWeaveMetrics performs a single weave-metrics scrape of a flow-controller pod, keyed [bridge][name]. +func ScrapeWeaveMetrics(g Gomega, pod *corev1.Pod) WeaveMetrics { cmd := []string{"sh", "-c", `exec ovs-appctl -t /var/run/openvswitch/ovs-vswitchd.$(cat /var/run/openvswitch/ovs-vswitchd.pid).ctl metrics/show`} - out, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, cmd) + out, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, cmd) g.Expect(err).ToNot(HaveOccurred(), "ovs-appctl metrics/show failed on pod %s: %s", pod.Name, out) families, perr := (&expfmt.TextParser{}).TextToMetricFamilies(strings.NewReader(out)) @@ -430,7 +430,7 @@ func scrapeWeaveMetrics(g Gomega, pod *corev1.Pod) weaveMetrics { g.Expect(perr).ToNot(HaveOccurred(), "fatal parse error in metrics/show output on pod %s: %s", pod.Name, out) } - metrics := weaveMetrics{} + metrics := WeaveMetrics{} for _, m := range families[weaveMetricFamily].GetMetric() { var bridge, name string for _, l := range m.GetLabel() { @@ -454,19 +454,19 @@ func scrapeWeaveMetrics(g Gomega, pod *corev1.Pod) weaveMetrics { return metrics } -// readWeaveMetrics scrapes weave packet counters with retry, for standalone use outside an +// ReadWeaveMetrics scrapes weave packet counters with retry, for standalone use outside an // Eventually. Inside an outer Eventually, call scrapeWeaveMetrics instead. -func readWeaveMetrics(pod *corev1.Pod) weaveMetrics { - var metrics weaveMetrics +func ReadWeaveMetrics(pod *corev1.Pod) WeaveMetrics { + var metrics WeaveMetrics Eventually(func(g Gomega) { - metrics = scrapeWeaveMetrics(g, pod) + metrics = ScrapeWeaveMetrics(g, pod) }).WithTimeout(weaveOperationTimeout).WithPolling(weaveEventuallyPollInterval).Should(Succeed(), "failed to read weave metrics from pod %s", pod.Name) return metrics } -// metricDelta returns after-before for a counter on a bridge, asserting it did not go backwards -func metricDelta(g Gomega, before, after weaveMetrics, bridge, name string) uint64 { +// MetricDelta returns after-before for a counter on a bridge, asserting it did not go backwards +func MetricDelta(g Gomega, before, after WeaveMetrics, bridge, name string) uint64 { b, bOK := before[bridge][name] a, aOK := after[bridge][name] g.Expect(bOK).To(BeTrue(), "weave metric %s missing on bridge %s in before scrape", name, bridge) @@ -475,33 +475,33 @@ func metricDelta(g Gomega, before, after weaveMetrics, bridge, name string) uint return a - b } -// metricDeltaExpect describes how a set of weave counters should move between two scrapes. -type metricDeltaExpect struct { - // mustRiseBy maps a counter name to the minimum delta it must have gained. - mustRiseBy map[string]uint64 - // mustStayFlat lists counters whose delta must be exactly zero. - mustStayFlat []string +// MetricDeltaExpect describes how a set of weave counters should move between two scrapes. +type MetricDeltaExpect struct { + // MustRiseBy maps a counter name to the minimum delta it must have gained. + MustRiseBy map[string]uint64 + // MustStayFlat lists counters whose delta must be exactly zero. + MustStayFlat []string } -// assertMetricDeltas checks, on the given bridge, that every counter in expect.mustRiseBy advanced +// AssertMetricDeltas checks, on the given bridge, that every counter in expect.mustRiseBy advanced // by at least its minimum and every counter in expect.mustStayFlat did not move. -func assertMetricDeltas(g Gomega, before, after weaveMetrics, bridge string, expect metricDeltaExpect) { - for name, minDelta := range expect.mustRiseBy { - delta := metricDelta(g, before, after, bridge, name) +func AssertMetricDeltas(g Gomega, before, after WeaveMetrics, bridge string, expect MetricDeltaExpect) { + for name, minDelta := range expect.MustRiseBy { + delta := MetricDelta(g, before, after, bridge, name) g.Expect(delta).To(BeNumerically(">=", minDelta), "weave metric %s on bridge %s: delta %d < expected %d", name, bridge, delta, minDelta) } - for _, name := range expect.mustStayFlat { - delta := metricDelta(g, before, after, bridge, name) + for _, name := range expect.MustStayFlat { + delta := MetricDelta(g, before, after, bridge, name) g.Expect(delta).To(BeZero(), "weave metric %s on bridge %s: expected no change, got delta %d", name, bridge, delta) } } -// assertTxPacketsAccountedFor asserts every host_tx packet is accounted for as tx_sent or tx_dropped, +// AssertTxPacketsAccountedFor asserts every host_tx packet is accounted for as tx_sent or tx_dropped, // leaving only a small remainder (DHCP/ARP) under slack — i.e. no TX packets silently vanish. -func assertTxPacketsAccountedFor(g Gomega, before, after weaveMetrics, bridge string) { - hostTx := metricDelta(g, before, after, bridge, weaveMetricHostTx) - txSent := metricDelta(g, before, after, bridge, weaveMetricTxSent) - txDropped := metricDelta(g, before, after, bridge, weaveMetricTxDropped) +func AssertTxPacketsAccountedFor(g Gomega, before, after WeaveMetrics, bridge string) { + hostTx := MetricDelta(g, before, after, bridge, weaveMetricHostTx) + txSent := MetricDelta(g, before, after, bridge, weaveMetricTxSent) + txDropped := MetricDelta(g, before, after, bridge, weaveMetricTxDropped) accounted := txSent + txDropped g.Expect(hostTx).To(BeNumerically(">=", accounted), "tx accounting on %s: tx_sent(%d)+tx_dropped(%d)=%d exceeds host_tx delta %d", bridge, txSent, txDropped, accounted, hostTx) @@ -509,33 +509,33 @@ func assertTxPacketsAccountedFor(g Gomega, before, after weaveMetrics, bridge st "tx accounting on %s: host_tx delta %d exceeds tx_sent(%d)+tx_dropped(%d)=%d by >= slack %d", bridge, hostTx, txSent, txDropped, accounted, weaveTxAccountingSlack) } -// metricRef identifies one weave counter sampled before and after traffic: a counter name on a +// MetricRef identifies one weave counter sampled before and after traffic: a counter name on a // specific bridge, paired with its two scrapes. -type metricRef struct { - before, after weaveMetrics - bridge, name string +type MetricRef struct { + Before, After WeaveMetrics + Bridge, Name string } -// assertMetricDeltasMatch asserts the sender and receiver counters advanced by the same amount +// AssertMetricDeltasMatch asserts the sender and receiver counters advanced by the same amount // within tolerance — e.g. tx_sent on the sender DPU vs rx_decap on the receiver DPU track the same overlay // packets. -func assertMetricDeltasMatch(g Gomega, sender, receiver metricRef) { - src := metricDelta(g, sender.before, sender.after, sender.bridge, sender.name) - dst := metricDelta(g, receiver.before, receiver.after, receiver.bridge, receiver.name) +func AssertMetricDeltasMatch(g Gomega, sender, receiver MetricRef) { + src := MetricDelta(g, sender.Before, sender.After, sender.Bridge, sender.Name) + dst := MetricDelta(g, receiver.Before, receiver.After, receiver.Bridge, receiver.Name) // Absolute difference between the two counters. diff := max(src, dst) - min(src, dst) g.Expect(diff).To(BeNumerically("<=", weaveCrossNodePacketDriftTolerance), "%s on %s delta %d vs %s on %s delta %d differ by %d packets (> tolerance %d)", - sender.name, sender.bridge, src, receiver.name, receiver.bridge, dst, diff, weaveCrossNodePacketDriftTolerance) + sender.Name, sender.Bridge, src, receiver.Name, receiver.Bridge, dst, diff, weaveCrossNodePacketDriftTolerance) } -// ensureOverlayRoute ensures the route for subnet on a netshoot pod uses the DHCP +// EnsureOverlayRoute ensures the route for subnet on a netshoot pod uses the DHCP // gateway rather than being on-link. The CNI DHCP plugin sometimes fails to apply // option 121 classless static routes correctly. overlayIP is the pod's known overlay // address (from createPFAttachmentAndWaitForHostIP); for a /31 the gateway is the peer. // // restClient and restCfg must be for the host (management) cluster where netshoot pods run. -func ensureOverlayRoute(restClient *rest.RESTClient, restCfg *rest.Config, namespace, podName, overlayIP, subnet string) { +func EnsureOverlayRoute(restClient *rest.RESTClient, restCfg *rest.Config, namespace, podName, overlayIP, subnet string) { const overlayIface = "net1" ip := net.ParseIP(overlayIP).To4() Expect(ip).ToNot(BeNil(), "invalid overlay IP %s for pod %s", overlayIP, podName) @@ -553,10 +553,10 @@ func ensureOverlayRoute(restClient *rest.RESTClient, restCfg *rest.Config, names By(fmt.Sprintf("Overlay route on pod %s: %s via %s", podName, subnet, gateway)) } -// addRouteOnPodBetweenOverlayAndSubnet installs a route on a netshoot pod for the given (foreign) subnet +// AddRouteOnPodBetweenOverlayAndSubnet installs a route on a netshoot pod for the given (foreign) subnet // via the pod's /31 overlay peer. Used to force traffic destined for another VNet's subnet out the local // overlay interface so the isolation enforcement on the DPU is exercised. -func addRouteOnPodBetweenOverlayAndSubnet(restClient *rest.RESTClient, restCfg *rest.Config, namespace, podName, overlayIP, subnet string) { +func AddRouteOnPodBetweenOverlayAndSubnet(restClient *rest.RESTClient, restCfg *rest.Config, namespace, podName, overlayIP, subnet string) { const overlayIface = "net1" ip := net.ParseIP(overlayIP).To4() Expect(ip).ToNot(BeNil(), "invalid overlay IP %s for pod %s", overlayIP, podName) @@ -570,11 +570,11 @@ func addRouteOnPodBetweenOverlayAndSubnet(restClient *rest.RESTClient, restCfg * By(fmt.Sprintf("Cross-subnet route on pod %s: %s via %s", podName, subnet, gatewayIP)) } -// deleteAttachmentOnPod deletes a virtual network attachment via vpcctl on the given flow-controller pod. -func deleteAttachmentOnPod(pod *corev1.Pod, attID string) { +// DeleteAttachmentOnPod deletes a virtual network attachment via vpcctl on the given flow-controller pod. +func DeleteAttachmentOnPod(pod *corev1.Pod, attID string) { By(fmt.Sprintf("Deleting attachment %s on pod %s", attID, pod.Name)) Eventually(func(g Gomega) { - output, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, []string{"/vpcctl", "delete-attachment", "--id", attID}) + output, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, []string{"/vpcctl", "delete-attachment", "--id", attID}) if err != nil && strings.Contains(output, "NotFound") { return } @@ -583,20 +583,20 @@ func deleteAttachmentOnPod(pod *corev1.Pod, attID string) { "failed to delete attachment %s on pod %s", attID, pod.Name) } -// deleteVNetOnPod deletes a virtual network via vpcctl on the given flow-controller pod. +// DeleteVNetOnPod deletes a virtual network via vpcctl on the given flow-controller pod. // If an attachment is still attached (FailedPrecondition), it deletes the blocking attachment first. -func deleteVNetOnPod(pod *corev1.Pod, vnetID string) { +func DeleteVNetOnPod(pod *corev1.Pod, vnetID string) { By(fmt.Sprintf("Deleting virtual network %s on pod %s", vnetID, pod.Name)) Eventually(func(g Gomega) { - output, err := netshoot.ExecInPodOnce(dpuClusterRestClient[0], dpuClusterRestConfig[0], pod.Namespace, pod.Name, []string{"/vpcctl", "delete-vnet", "--id", vnetID}) + output, err := netshoot.ExecInPodOnce(DPUClusterRestClient[0], DPUClusterRestConfig[0], pod.Namespace, pod.Name, []string{"/vpcctl", "delete-vnet", "--id", vnetID}) if err != nil && strings.Contains(output, "NotFound") { return } if err != nil && strings.Contains(output, "still attached") { - staleIDs := listAttachmentIDs(g, pod, "--vnet-id", vnetID) + staleIDs := ListAttachmentIDs(g, pod, "--vnet-id", vnetID) for _, staleID := range staleIDs { By(fmt.Sprintf("VNet %s still has attachment %s — deleting before retry", vnetID, staleID)) - deleteAttachmentOnPod(pod, staleID) + DeleteAttachmentOnPod(pod, staleID) } g.Expect(fmt.Errorf("deleted %d blocking attachment(s) for vnet %s, retrying vnet delete", len(staleIDs), vnetID)).ToNot(HaveOccurred()) } @@ -605,8 +605,8 @@ func deleteVNetOnPod(pod *corev1.Pod, vnetID string) { "failed to delete virtual network %s on pod %s", vnetID, pod.Name) } -// createNetutilsHostPodOnNode creates a privileged hostNetwork netutils pod on nodeName and waits for Ready state. -func createNetutilsHostPodOnNode(ctx context.Context, c client.Client, namespace, podName, nodeName string) *corev1.Pod { +// CreateNetutilsHostPodOnNode creates a privileged hostNetwork netutils pod on nodeName and waits for Ready state. +func CreateNetutilsHostPodOnNode(ctx context.Context, c client.Client, namespace, podName, nodeName string) *corev1.Pod { By(fmt.Sprintf("Creating netutils host pod %s/%s on node %s", namespace, podName, nodeName)) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -619,7 +619,7 @@ func createNetutilsHostPodOnNode(ctx context.Context, c client.Client, namespace HostNetwork: true, DNSPolicy: corev1.DNSClusterFirstWithHostNet, RestartPolicy: corev1.RestartPolicyNever, - ImagePullSecrets: []corev1.LocalObjectReference{{Name: dpfPullSecretName}}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: DPFPullSecretName}}, Containers: []corev1.Container{{ Name: "netutils", Image: fmt.Sprintf("%s:%s", netutilsImage, tag), @@ -661,8 +661,8 @@ func createNetutilsHostPodOnNode(ctx context.Context, c client.Client, namespace return pod } -// releaseDHCPLeaseInPod runs weaveRDMAFlushCmdFmt inside the pod. Best-effort; preStop is the backstop. -func releaseDHCPLeaseInPod(restClient *rest.RESTClient, restCfg *rest.Config, pod *corev1.Pod, iface string) { +// ReleaseDHCPLeaseInPod runs weaveRDMAFlushCmdFmt inside the pod. Best-effort; preStop is the backstop. +func ReleaseDHCPLeaseInPod(restClient *rest.RESTClient, restCfg *rest.Config, pod *corev1.Pod, iface string) { By(fmt.Sprintf("Resetting dhcpcd state in pod %s/%s on %s", pod.Namespace, pod.Name, iface)) cmd := fmt.Sprintf(weaveRDMAFlushCmdFmt, iface) if out, err := netshoot.ExecInPodOnce(restClient, restCfg, pod.Namespace, pod.Name, @@ -671,9 +671,9 @@ func releaseDHCPLeaseInPod(restClient *rest.RESTClient, restCfg *rest.Config, po } } -// acquireDHCPLeaseInPod resets dhcpcd state, runs dhcpcd -L -1 -4, and verifies expectedIP is on iface. -func acquireDHCPLeaseInPod(restClient *rest.RESTClient, restCfg *rest.Config, pod *corev1.Pod, iface, expectedIP string) { - releaseDHCPLeaseInPod(restClient, restCfg, pod, iface) +// AcquireDHCPLeaseInPod resets dhcpcd state, runs dhcpcd -L -1 -4, and verifies expectedIP is on iface. +func AcquireDHCPLeaseInPod(restClient *rest.RESTClient, restCfg *rest.Config, pod *corev1.Pod, iface, expectedIP string) { + ReleaseDHCPLeaseInPod(restClient, restCfg, pod, iface) By(fmt.Sprintf("Running dhcpcd -L -1 -4 %s in pod %s/%s", iface, pod.Namespace, pod.Name)) out, err := netshoot.ExecInPodOnce(restClient, restCfg, pod.Namespace, pod.Name, @@ -688,8 +688,8 @@ func acquireDHCPLeaseInPod(restClient *rest.RESTClient, restCfg *rest.Config, po "expected IP %s not present in pod %s/%s on %s; got: %s", expectedIP, pod.Namespace, pod.Name, iface, addrOut) } -// runIBWriteBWPodToPod runs ib_write_bw between two pods and asserts the BW threshold. extraArgs forwards to both sides. -func runIBWriteBWPodToPod(restClient *rest.RESTClient, restCfg *rest.Config, serverPod, clientPod *corev1.Pod, dev, serverIP string, extraArgs ...string) { +// RunIBWriteBWPodToPod runs ib_write_bw between two pods and asserts the BW threshold. extraArgs forwards to both sides. +func RunIBWriteBWPodToPod(restClient *rest.RESTClient, restCfg *rest.Config, serverPod, clientPod *corev1.Pod, dev, serverIP string, extraArgs ...string) { // Joined as-is into the sh -c command. Safe only for plain shell flags (e.g. "--reversed"). extra := strings.Join(extraArgs, " ") durationSec := int(weaveIBWriteBWDuration / time.Second) diff --git a/test/e2e/weave_test.go b/test/e2e/weave_test.go index 162fe8f5..f1265b7c 100644 --- a/test/e2e/weave_test.go +++ b/test/e2e/weave_test.go @@ -39,66 +39,66 @@ var _ = Describe("Weave testcases", Labels{Domain.Weave}, Ordered, func() { ) BeforeAll(func() { - weavePrerequisiteScope = cleanupTracker.RegisterScope(cleanup.NamedScopeManual("weave-prerequisites")) - weaveContextScope = cleanupTracker.RegisterScope(cleanup.NamedScopeManual("weave-tests")) + weavePrerequisiteScope = CleanupTracker.RegisterScope(cleanup.NamedScopeManual("weave-prerequisites")) + weaveContextScope = CleanupTracker.RegisterScope(cleanup.NamedScopeManual("weave-tests")) for _, label := range CurrentSpecReport().Labels() { if label != Domain.RequiresNodes { continue } - if !input.hasDpuNodes() { + if !input.HasDpuNodes() { Skip("Skip test as there are no DPU nodes") } weavePrerequisiteScope.CleanupBefore() weaveContextScope.CleanupBefore() - provInput := getProvisionDPUClustersInputForWeave(ctx, getProvisionDPUClustersInput(), input.client) - Expect(provInput.dpuClusters).ToNot(BeEmpty(), "no DPU clusters found via config or discovery") + provInput := GetProvisionDPUClustersInputForWeave(Ctx, GetProvisionDPUClustersInput(), input.Client) + Expect(provInput.DPUClusters).ToNot(BeEmpty(), "no DPU clusters found via config or discovery") By("Creating DPU cluster client for verification") - getDPUClusterClients(ctx, provInput) - Expect(dpuClusterClient).ToNot(BeEmpty(), "no DPU cluster clients were created") + GetDPUClusterClients(Ctx, provInput) + Expect(DPUClusterClient).ToNot(BeEmpty(), "no DPU cluster clients were created") By("Verifying DPU cluster has ready nodes") - VerifyDPUClusterWithNodes(ctx, provInput) + VerifyDPUClusterWithNodes(Ctx, provInput) By("Waiting for DPU cluster pods to be ready") - VerifyClusterPods(ctx, dpuClusterClient[0], systemPodsToVerify) + VerifyClusterPods(Ctx, DPUClusterClient[0], systemPodsToVerify) By("Waiting for DPFOperatorConfig to be ready") - VerifyDPFOperatorConfigReady(ctx, input.client, 20*time.Minute) + VerifyDPFOperatorConfigReady(Ctx, input.Client, 20*time.Minute) By("Waiting for Weave pods on DPU cluster to be ready") - VerifyClusterPods(ctx, dpuClusterClient[0], weavePodsToVerify) + VerifyClusterPods(Ctx, DPUClusterClient[0], weavePodsToVerify) By("Getting ready flow controller pods") - flowControllerPods := netshoot.GetReadyPodsMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, + flowControllerPods := netshoot.GetReadyPodsMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, map[string]string{weaveDPUServiceLabelKey: weaveFlowControllerName}) Expect(flowControllerPods).To(HaveLen(2), "expected 2 ready %s pods", weaveFlowControllerName) By("Getting ready dhcp agent pods") - dhcpAgentPods := netshoot.GetReadyPodsMatchingLabels(ctx, dpuClusterClient[0], dpfOperatorSystemNamespace, + dhcpAgentPods := netshoot.GetReadyPodsMatchingLabels(Ctx, DPUClusterClient[0], DPFOperatorSystemNamespace, map[string]string{weaveDPUServiceLabelKey: weaveDHCPAgentName}) Expect(dhcpAgentPods).To(HaveLen(2), "expected 2 ready %s pods", weaveDHCPAgentName) - workerNode1, workerNode2 = getTwoWorkerNodeNames(ctx, input.client) + workerNode1, workerNode2 = getTwoWorkerNodeNames(Ctx, input.Client) By("Getting DPU cluster nodes in order") - dpuNode1, dpuNode2 := getDPUNodesInOrder(ctx, input.client, dpuClusterClient[0]) + dpuNode1, dpuNode2 := getDPUNodesInOrder(Ctx, input.Client, DPUClusterClient[0]) fcPod1 = netshoot.GetPodOnNode(flowControllerPods, dpuNode1.Name) fcPod2 = netshoot.GetPodOnNode(flowControllerPods, dpuNode2.Name) Expect(fcPod1).ToNot(BeNil(), "no flow-controller pod found on DPU node %s", dpuNode1.Name) Expect(fcPod2).ToNot(BeNil(), "no flow-controller pod found on DPU node %s", dpuNode2.Name) By("Verifying OVS is responsive on flow-controller pods") - verifyOVSResponsive(fcPod1) - verifyOVSResponsive(fcPod2) + VerifyOVSResponsive(fcPod1) + VerifyOVSResponsive(fcPod2) By("Getting PF MAC addresses for p0 and p1 from DPU flow-controller pods") - pfMACP0Node1 = getPFMACFromFlowControllerByPort(fcPod1, weaveDPUPortP0) - pfMACP0Node2 = getPFMACFromFlowControllerByPort(fcPod2, weaveDPUPortP0) - pfMACP1Node1 = getPFMACFromFlowControllerByPort(fcPod1, weaveDPUPortP1) - pfMACP1Node2 = getPFMACFromFlowControllerByPort(fcPod2, weaveDPUPortP1) + pfMACP0Node1 = GetPFMACFromFlowControllerByPort(fcPod1, weaveDPUPortP0) + pfMACP0Node2 = GetPFMACFromFlowControllerByPort(fcPod2, weaveDPUPortP0) + pfMACP1Node1 = GetPFMACFromFlowControllerByPort(fcPod1, weaveDPUPortP1) + pfMACP1Node2 = GetPFMACFromFlowControllerByPort(fcPod2, weaveDPUPortP1) } beforeAllSucceeded = true }) @@ -116,11 +116,11 @@ var _ = Describe("Weave testcases", Labels{Domain.Weave}, Ordered, func() { var dhcpDS *appsv1.DaemonSet It("should deploy host DHCP CNI daemon", func() { - dhcpDS = vpc.DeployDHCPDaemon(ctx, input.client, weaveInput.dhcpDaemonSet, weavePrerequisiteScope.CleanupLabels) + dhcpDS = vpc.DeployDHCPDaemon(Ctx, input.Client, WeaveInput.DHCPDaemonSet, weavePrerequisiteScope.CleanupLabels) }) It("should wait for DHCP daemon pods to be ready", func() { - vpc.WaitForDHCPDaemonReady(ctx, input.client, dhcpDS) + vpc.WaitForDHCPDaemonReady(Ctx, input.Client, dhcpDS) }) }) @@ -161,90 +161,90 @@ var _ = Describe("Weave testcases", Labels{Domain.Weave}, Ordered, func() { }) It("should create test namespace", func() { - vpc.CreateTestNamespace(ctx, input.client, trafficTestNS, weaveContextScope.CleanupLabels) + vpc.CreateTestNamespace(Ctx, input.Client, trafficTestNS, weaveContextScope.CleanupLabels) }) It("should create virtual network on both flow-controller pods", func() { - createVNetOnPod(fcPod1, trafficVNetID, trafficVNI, weaveVNetSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod1, trafficVNetID) }) - createVNetOnPod(fcPod2, trafficVNetID, trafficVNI, weaveVNetSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod2, trafficVNetID) }) + CreateVNetOnPod(fcPod1, trafficVNetID, trafficVNI, weaveVNetSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod1, trafficVNetID) }) + CreateVNetOnPod(fcPod2, trafficVNetID, trafficVNI, weaveVNetSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod2, trafficVNetID) }) }) It("should create PF attachments for p0 on both nodes", func() { var attIDP0Fc1, attIDP0Fc2 string - attIDP0Fc1, overlayIPP0Node1 = createPFAttachmentAndWaitForHostIP(fcPod1, trafficVNetID, pfMACP0Node1) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod1, attIDP0Fc1) }) - attIDP0Fc2, overlayIPP0Node2 = createPFAttachmentAndWaitForHostIP(fcPod2, trafficVNetID, pfMACP0Node2) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod2, attIDP0Fc2) }) + attIDP0Fc1, overlayIPP0Node1 = CreatePFAttachmentAndWaitForHostIP(fcPod1, trafficVNetID, pfMACP0Node1) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod1, attIDP0Fc1) }) + attIDP0Fc2, overlayIPP0Node2 = CreatePFAttachmentAndWaitForHostIP(fcPod2, trafficVNetID, pfMACP0Node2) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod2, attIDP0Fc2) }) }) It("should create PF attachments for p1 on both nodes", func() { var attIDP1Fc1, attIDP1Fc2 string - attIDP1Fc1, overlayIPP1Node1 = createPFAttachmentAndWaitForHostIP(fcPod1, trafficVNetID, pfMACP1Node1) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod1, attIDP1Fc1) }) - attIDP1Fc2, overlayIPP1Node2 = createPFAttachmentAndWaitForHostIP(fcPod2, trafficVNetID, pfMACP1Node2) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod2, attIDP1Fc2) }) + attIDP1Fc1, overlayIPP1Node1 = CreatePFAttachmentAndWaitForHostIP(fcPod1, trafficVNetID, pfMACP1Node1) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod1, attIDP1Fc1) }) + attIDP1Fc2, overlayIPP1Node2 = CreatePFAttachmentAndWaitForHostIP(fcPod2, trafficVNetID, pfMACP1Node2) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod2, attIDP1Fc2) }) }) It("should verify OVS isolation bridges exist on both DPU nodes", func() { - verifyIsolationBridgeExists(fcPod1, trafficVNI, weaveDPUPortP0) - verifyIsolationBridgeExists(fcPod1, trafficVNI, weaveDPUPortP1) - verifyIsolationBridgeExists(fcPod2, trafficVNI, weaveDPUPortP0) - verifyIsolationBridgeExists(fcPod2, trafficVNI, weaveDPUPortP1) + VerifyIsolationBridgeExists(fcPod1, trafficVNI, weaveDPUPortP0) + VerifyIsolationBridgeExists(fcPod1, trafficVNI, weaveDPUPortP1) + VerifyIsolationBridgeExists(fcPod2, trafficVNI, weaveDPUPortP0) + VerifyIsolationBridgeExists(fcPod2, trafficVNI, weaveDPUPortP1) }) It("should create DHCP NADs and netshoot pods on worker nodes", func() { nadP0 := weaveDHCPNADP0 nadP1 := weaveDHCPNADP1 - vpc.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, trafficTestNS, nadP0, weaveHostPFInterfaceP0, weavePFMTU, weaveContextScope.CleanupLabels) - vpc.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, trafficTestNS, nadP1, weaveHostPFInterfaceP1, weavePFMTU, weaveContextScope.CleanupLabels) + vpc.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, trafficTestNS, nadP0, weaveHostPFInterfaceP0, weavePFMTU, weaveContextScope.CleanupLabels) + vpc.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, trafficTestNS, nadP1, weaveHostPFInterfaceP1, weavePFMTU, weaveContextScope.CleanupLabels) testPodConfigs = []*netshoot.TestPodConfig{ {Namespace: trafficTestNS, Name: podP0Node1, NodeName: workerNode1, NADName: nadP0, Labels: weaveContextScope.CleanupLabels}, {Namespace: trafficTestNS, Name: podP0Node2, NodeName: workerNode2, NADName: nadP0, Labels: weaveContextScope.CleanupLabels}, {Namespace: trafficTestNS, Name: podP1Node1, NodeName: workerNode1, NADName: nadP1, Labels: weaveContextScope.CleanupLabels}, {Namespace: trafficTestNS, Name: podP1Node2, NodeName: workerNode2, NADName: nadP1, Labels: weaveContextScope.CleanupLabels}, } - netshoot.CreatePods(ctx, input.client, testPodConfigs) + netshoot.CreatePods(Ctx, input.Client, testPodConfigs) }) It("should verify netshoot pods are running", func() { - netshoot.WaitForPodsReady(ctx, input.client, testPodConfigs, vpc.LongTimeout) + netshoot.WaitForPodsReady(Ctx, input.Client, testPodConfigs, vpc.LongTimeout) }) It("should verify overlay routes on netshoot pods", func() { - ensureOverlayRoute(hostClusterRESTClient, input.restConfig, trafficTestNS, podP0Node1, overlayIPP0Node1, weaveVNetSubnet) - ensureOverlayRoute(hostClusterRESTClient, input.restConfig, trafficTestNS, podP0Node2, overlayIPP0Node2, weaveVNetSubnet) - ensureOverlayRoute(hostClusterRESTClient, input.restConfig, trafficTestNS, podP1Node1, overlayIPP1Node1, weaveVNetSubnet) - ensureOverlayRoute(hostClusterRESTClient, input.restConfig, trafficTestNS, podP1Node2, overlayIPP1Node2, weaveVNetSubnet) + EnsureOverlayRoute(HostClusterRESTClient, input.RestConfig, trafficTestNS, podP0Node1, overlayIPP0Node1, weaveVNetSubnet) + EnsureOverlayRoute(HostClusterRESTClient, input.RestConfig, trafficTestNS, podP0Node2, overlayIPP0Node2, weaveVNetSubnet) + EnsureOverlayRoute(HostClusterRESTClient, input.RestConfig, trafficTestNS, podP1Node1, overlayIPP1Node1, weaveVNetSubnet) + EnsureOverlayRoute(HostClusterRESTClient, input.RestConfig, trafficTestNS, podP1Node2, overlayIPP1Node2, weaveVNetSubnet) }) It("should verify cross-node ping succeeds on p0", func() { - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, trafficTestNS, podP0Node1, overlayIPP0Node2) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, trafficTestNS, podP0Node2, overlayIPP0Node1) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, trafficTestNS, podP0Node1, overlayIPP0Node2) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, trafficTestNS, podP0Node2, overlayIPP0Node1) }) It("should verify cross-node ping succeeds on p1", func() { - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, trafficTestNS, podP1Node1, overlayIPP1Node2) - netshoot.AssertPingSuccess(&hostClusterRESTClient, &input.restConfig, trafficTestNS, podP1Node2, overlayIPP1Node1) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, trafficTestNS, podP1Node1, overlayIPP1Node2) + netshoot.AssertPingSuccess(&HostClusterRESTClient, &input.RestConfig, trafficTestNS, podP1Node2, overlayIPP1Node1) }) It("should verify performance with iperf cross-node traffic on p0", func() { - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, trafficTestNS, podP0Node1, podP0Node2, overlayIPP0Node2) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, trafficTestNS, podP0Node1, podP0Node2, overlayIPP0Node2) }) It("should verify performance with iperf cross-node traffic on p1", func() { - netshoot.RunTrafficTest(&hostClusterRESTClient, &input.restConfig, trafficTestNS, podP1Node1, podP1Node2, overlayIPP1Node2) + netshoot.RunTrafficTest(&HostClusterRESTClient, &input.RestConfig, trafficTestNS, podP1Node1, podP1Node2, overlayIPP1Node2) }) It("should verify metrics across nodes under iperf load on p0", func() { - bridge := isolationBridgeName(trafficVNI, weaveDPUPortP0) + bridge := IsolationBridgeName(trafficVNI, weaveDPUPortP0) - baselineMetricsPod1 := readWeaveMetrics(fcPod1) - baselineMetricsPod2 := readWeaveMetrics(fcPod2) + baselineMetricsPod1 := ReadWeaveMetrics(fcPod1) + baselineMetricsPod2 := ReadWeaveMetrics(fcPod2) By("Running iperf cross-node on p0") - iperfResult := netshoot.RunTrafficTestWithResult(&hostClusterRESTClient, &input.restConfig, trafficTestNS, podP0Node1, podP0Node2, overlayIPP0Node2) + iperfResult := netshoot.RunTrafficTestWithResult(&HostClusterRESTClient, &input.RestConfig, trafficTestNS, podP0Node1, podP0Node2, overlayIPP0Node2) forwardBytes := iperfResult.Forward.End.SumSent.Bytes Expect(forwardBytes).To(BeNumerically(">", 0), "iperf reported zero forward bytes") // iperf3 exposes no packet counter, so derive it from bytes/MSS (segment payload). @@ -254,28 +254,28 @@ var _ = Describe("Weave testcases", Labels{Domain.Weave}, Ordered, func() { By("Verifying weave metrics across nodes") // Poll until the OVS scrape reflects the burst. - var currentMetricsPod1, currentMetricsPod2 weaveMetrics + var currentMetricsPod1, currentMetricsPod2 WeaveMetrics Eventually(func(g Gomega) { - currentMetricsPod1 = scrapeWeaveMetrics(g, fcPod1) - currentMetricsPod2 = scrapeWeaveMetrics(g, fcPod2) + currentMetricsPod1 = ScrapeWeaveMetrics(g, fcPod1) + currentMetricsPod2 = ScrapeWeaveMetrics(g, fcPod2) // Sender encaps (host_tx/tx_sent), receiver decaps (host_rx/rx_decap), neither drop. - assertMetricDeltas(g, baselineMetricsPod1, currentMetricsPod1, bridge, metricDeltaExpect{ - mustRiseBy: map[string]uint64{weaveMetricHostTx: minIperfPackets, weaveMetricTxSent: minIperfPackets}, - mustStayFlat: []string{weaveMetricTxDropped}, + AssertMetricDeltas(g, baselineMetricsPod1, currentMetricsPod1, bridge, MetricDeltaExpect{ + MustRiseBy: map[string]uint64{weaveMetricHostTx: minIperfPackets, weaveMetricTxSent: minIperfPackets}, + MustStayFlat: []string{weaveMetricTxDropped}, }) - assertMetricDeltas(g, baselineMetricsPod2, currentMetricsPod2, bridge, metricDeltaExpect{ - mustRiseBy: map[string]uint64{weaveMetricHostRx: minIperfPackets, weaveMetricRxDecap: minIperfPackets}, - mustStayFlat: []string{weaveMetricRxDropped}, + AssertMetricDeltas(g, baselineMetricsPod2, currentMetricsPod2, bridge, MetricDeltaExpect{ + MustRiseBy: map[string]uint64{weaveMetricHostRx: minIperfPackets, weaveMetricRxDecap: minIperfPackets}, + MustStayFlat: []string{weaveMetricRxDropped}, }) // Cross-DPU: packets encapped out of one DPU equal those decapped at the other, both directions. - assertMetricDeltasMatch(g, - metricRef{before: baselineMetricsPod1, after: currentMetricsPod1, bridge: bridge, name: weaveMetricTxSent}, - metricRef{before: baselineMetricsPod2, after: currentMetricsPod2, bridge: bridge, name: weaveMetricRxDecap}) - assertMetricDeltasMatch(g, - metricRef{before: baselineMetricsPod2, after: currentMetricsPod2, bridge: bridge, name: weaveMetricTxSent}, - metricRef{before: baselineMetricsPod1, after: currentMetricsPod1, bridge: bridge, name: weaveMetricRxDecap}) + AssertMetricDeltasMatch(g, + MetricRef{Before: baselineMetricsPod1, After: currentMetricsPod1, Bridge: bridge, Name: weaveMetricTxSent}, + MetricRef{Before: baselineMetricsPod2, After: currentMetricsPod2, Bridge: bridge, Name: weaveMetricRxDecap}) + AssertMetricDeltasMatch(g, + MetricRef{Before: baselineMetricsPod2, After: currentMetricsPod2, Bridge: bridge, Name: weaveMetricTxSent}, + MetricRef{Before: baselineMetricsPod1, After: currentMetricsPod1, Bridge: bridge, Name: weaveMetricRxDecap}) }).WithTimeout(weaveOperationTimeout).WithPolling(weaveEventuallyPollInterval).Should(Succeed()) }) }) @@ -316,87 +316,87 @@ var _ = Describe("Weave testcases", Labels{Domain.Weave}, Ordered, func() { }) It("should create test namespace", func() { - vpc.CreateTestNamespace(ctx, input.client, isolTestNS, weaveContextScope.CleanupLabels) + vpc.CreateTestNamespace(Ctx, input.Client, isolTestNS, weaveContextScope.CleanupLabels) }) It("should create both isolation virtual networks on both flow-controller pods", func() { - createVNetOnPod(fcPod1, isolVNet1ID, isolVNI1, weaveVNetSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod1, isolVNet1ID) }) - createVNetOnPod(fcPod2, isolVNet1ID, isolVNI1, weaveVNetSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod2, isolVNet1ID) }) - createVNetOnPod(fcPod1, isolVNet2ID, isolVNI2, weaveVNetSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod1, isolVNet2ID) }) - createVNetOnPod(fcPod2, isolVNet2ID, isolVNI2, weaveVNetSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod2, isolVNet2ID) }) + CreateVNetOnPod(fcPod1, isolVNet1ID, isolVNI1, weaveVNetSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod1, isolVNet1ID) }) + CreateVNetOnPod(fcPod2, isolVNet1ID, isolVNI1, weaveVNetSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod2, isolVNet1ID) }) + CreateVNetOnPod(fcPod1, isolVNet2ID, isolVNI2, weaveVNetSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod1, isolVNet2ID) }) + CreateVNetOnPod(fcPod2, isolVNet2ID, isolVNI2, weaveVNetSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod2, isolVNet2ID) }) }) It("should attach worker node 1 to vnet-1 and worker node 2 to vnet-2", func() { var attIDIsolFc1, attIDIsolFc2 string - attIDIsolFc1, overlayIP1 = createPFAttachmentAndWaitForHostIP(fcPod1, isolVNet1ID, pfMACP0Node1) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod1, attIDIsolFc1) }) - attIDIsolFc2, overlayIP2 = createPFAttachmentAndWaitForHostIP(fcPod2, isolVNet2ID, pfMACP0Node2) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod2, attIDIsolFc2) }) + attIDIsolFc1, overlayIP1 = CreatePFAttachmentAndWaitForHostIP(fcPod1, isolVNet1ID, pfMACP0Node1) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod1, attIDIsolFc1) }) + attIDIsolFc2, overlayIP2 = CreatePFAttachmentAndWaitForHostIP(fcPod2, isolVNet2ID, pfMACP0Node2) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod2, attIDIsolFc2) }) }) It("should verify OVS isolation bridges exist on each DPU for its PF-attached VNet", func() { // br-isol-- is created when that flow-controller has a PF attachment for the VNI // (bridgemanager), not only when CreateVirtualNetwork succeeded on that pod. - verifyIsolationBridgeExists(fcPod1, isolVNI1, weaveDPUPortP0) - verifyIsolationBridgeExists(fcPod2, isolVNI2, weaveDPUPortP0) + VerifyIsolationBridgeExists(fcPod1, isolVNI1, weaveDPUPortP0) + VerifyIsolationBridgeExists(fcPod2, isolVNI2, weaveDPUPortP0) }) It("should create DHCP NAD and netshoot pods on worker nodes", func() { nadName := weaveDHCPNADP0 - vpc.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, isolTestNS, nadName, weaveHostPFInterfaceP0, weavePFMTU, weaveContextScope.CleanupLabels) + vpc.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, isolTestNS, nadName, weaveHostPFInterfaceP0, weavePFMTU, weaveContextScope.CleanupLabels) testPodConfigs = []*netshoot.TestPodConfig{ {Namespace: isolTestNS, Name: isolPod1, NodeName: workerNode1, NADName: nadName, Labels: weaveContextScope.CleanupLabels}, {Namespace: isolTestNS, Name: isolPod2, NodeName: workerNode2, NADName: nadName, Labels: weaveContextScope.CleanupLabels}, } - netshoot.CreatePods(ctx, input.client, testPodConfigs) + netshoot.CreatePods(Ctx, input.Client, testPodConfigs) }) It("should verify netshoot pods are running", func() { - netshoot.WaitForPodsReady(ctx, input.client, testPodConfigs, vpc.LongTimeout) + netshoot.WaitForPodsReady(Ctx, input.Client, testPodConfigs, vpc.LongTimeout) }) It("should verify overlay routes on netshoot pods", func() { - ensureOverlayRoute(hostClusterRESTClient, input.restConfig, isolTestNS, isolPod1, overlayIP1, weaveVNetSubnet) - ensureOverlayRoute(hostClusterRESTClient, input.restConfig, isolTestNS, isolPod2, overlayIP2, weaveVNetSubnet) + EnsureOverlayRoute(HostClusterRESTClient, input.RestConfig, isolTestNS, isolPod1, overlayIP1, weaveVNetSubnet) + EnsureOverlayRoute(HostClusterRESTClient, input.RestConfig, isolTestNS, isolPod2, overlayIP2, weaveVNetSubnet) }) // Should run before the deny-ping below so the source ACL starts unlearned. It("should verify metrics for VNI-mismatch detection", func() { - srcBridge := isolationBridgeName(isolVNI1, weaveDPUPortP0) + srcBridge := IsolationBridgeName(isolVNI1, weaveDPUPortP0) dstBridge := fmt.Sprintf("br-drop-%s", dpuPortToDropNIC[weaveDPUPortP0]) - baselineMetricsPod1 := readWeaveMetrics(fcPod1) - baselineMetricsPod2 := readWeaveMetrics(fcPod2) + baselineMetricsPod1 := ReadWeaveMetrics(fcPod1) + baselineMetricsPod2 := ReadWeaveMetrics(fcPod2) By("Sending a ping burst across mismatched VNets") - _, _ = netshoot.PingBurst(hostClusterRESTClient, input.restConfig, isolTestNS, isolPod1, overlayIP2, weaveMetricBurstCount) + _, _ = netshoot.PingBurst(HostClusterRESTClient, input.RestConfig, isolTestNS, isolPod1, overlayIP2, weaveMetricBurstCount) By("Verifying weave metrics across mismatching VNets") - var currentMetricsPod1, currentMetricsPod2 weaveMetrics + var currentMetricsPod1, currentMetricsPod2 WeaveMetrics Eventually(func(g Gomega) { - currentMetricsPod1 = scrapeWeaveMetrics(g, fcPod1) - currentMetricsPod2 = scrapeWeaveMetrics(g, fcPod2) + currentMetricsPod1 = ScrapeWeaveMetrics(g, fcPod1) + currentMetricsPod2 = ScrapeWeaveMetrics(g, fcPod2) // Source: packets enter (host_tx); some leak before the ACL lands (tx_sent), the rest drop after (tx_dropped). // tx_dropped omitted because it rides the learned ACL flow and resets to 0 when that flow times out (~30s). - assertMetricDeltas(g, baselineMetricsPod1, currentMetricsPod1, srcBridge, metricDeltaExpect{ - mustRiseBy: map[string]uint64{weaveMetricHostTx: 1, weaveMetricTxSent: 1}, + AssertMetricDeltas(g, baselineMetricsPod1, currentMetricsPod1, srcBridge, MetricDeltaExpect{ + MustRiseBy: map[string]uint64{weaveMetricHostTx: 1, weaveMetricTxSent: 1}, }) // Destination: the leaked packets are counted as VNI mismatches on the p0 drop bridge. - assertMetricDeltas(g, baselineMetricsPod2, currentMetricsPod2, dstBridge, metricDeltaExpect{ - mustRiseBy: map[string]uint64{weaveMetricRxVNIMismatch: 1}, + AssertMetricDeltas(g, baselineMetricsPod2, currentMetricsPod2, dstBridge, MetricDeltaExpect{ + MustRiseBy: map[string]uint64{weaveMetricRxVNIMismatch: 1}, }) }).WithTimeout(weaveOperationTimeout).WithPolling(weaveEventuallyPollInterval).Should(Succeed()) }) It("should deny ping between worker nodes on different virtual networks", func() { - netshoot.AssertPingFailure(&hostClusterRESTClient, &input.restConfig, isolTestNS, isolPod1, overlayIP2) - netshoot.AssertPingFailure(&hostClusterRESTClient, &input.restConfig, isolTestNS, isolPod2, overlayIP1) + netshoot.AssertPingFailure(&HostClusterRESTClient, &input.RestConfig, isolTestNS, isolPod1, overlayIP2) + netshoot.AssertPingFailure(&HostClusterRESTClient, &input.RestConfig, isolTestNS, isolPod2, overlayIP1) }) }) @@ -417,10 +417,10 @@ var _ = Describe("Weave testcases", Labels{Domain.Weave}, Ordered, func() { ) BeforeAll(func() { - vpc.CreateTestNamespace(ctx, input.client, rdmaTestNS, weaveContextScope.CleanupLabels) - CopySecretToNamespace(ctx, input.client, dpfPullSecretName, dpfOperatorSystemNamespace, rdmaTestNS, weaveContextScope.CleanupLabels) - netutilsPod1 = createNetutilsHostPodOnNode(ctx, input.client, rdmaTestNS, rdmaPod1, workerNode1) - netutilsPod2 = createNetutilsHostPodOnNode(ctx, input.client, rdmaTestNS, rdmaPod2, workerNode2) + vpc.CreateTestNamespace(Ctx, input.Client, rdmaTestNS, weaveContextScope.CleanupLabels) + CopySecretToNamespace(Ctx, input.Client, DPFPullSecretName, DPFOperatorSystemNamespace, rdmaTestNS, weaveContextScope.CleanupLabels) + netutilsPod1 = CreateNetutilsHostPodOnNode(Ctx, input.Client, rdmaTestNS, rdmaPod1, workerNode1) + netutilsPod2 = CreateNetutilsHostPodOnNode(Ctx, input.Client, rdmaTestNS, rdmaPod2, workerNode2) }) AfterEach(func() { @@ -442,37 +442,37 @@ var _ = Describe("Weave testcases", Labels{Domain.Weave}, Ordered, func() { }) It("should create vnet on both flow-controller pods", func() { - createVNetOnPod(fcPod1, rdmaVNetID, rdmaVNI, weaveVNetSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod1, rdmaVNetID) }) - createVNetOnPod(fcPod2, rdmaVNetID, rdmaVNI, weaveVNetSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod2, rdmaVNetID) }) + CreateVNetOnPod(fcPod1, rdmaVNetID, rdmaVNI, weaveVNetSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod1, rdmaVNetID) }) + CreateVNetOnPod(fcPod2, rdmaVNetID, rdmaVNI, weaveVNetSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod2, rdmaVNetID) }) }) It("should create PF attachments for vnet on p0 of both nodes", func() { var attP0Fc1, attP0Fc2 string - attP0Fc1, overlayIPP0Node1 = createPFAttachmentAndWaitForHostIP(fcPod1, rdmaVNetID, pfMACP0Node1) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod1, attP0Fc1) }) - attP0Fc2, overlayIPP0Node2 = createPFAttachmentAndWaitForHostIP(fcPod2, rdmaVNetID, pfMACP0Node2) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod2, attP0Fc2) }) + attP0Fc1, overlayIPP0Node1 = CreatePFAttachmentAndWaitForHostIP(fcPod1, rdmaVNetID, pfMACP0Node1) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod1, attP0Fc1) }) + attP0Fc2, overlayIPP0Node2 = CreatePFAttachmentAndWaitForHostIP(fcPod2, rdmaVNetID, pfMACP0Node2) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod2, attP0Fc2) }) }) It("should verify OVS isolation bridges for vnet on p0 of both DPUs", func() { - verifyIsolationBridgeExists(fcPod1, rdmaVNI, weaveDPUPortP0) - verifyIsolationBridgeExists(fcPod2, rdmaVNI, weaveDPUPortP0) + VerifyIsolationBridgeExists(fcPod1, rdmaVNI, weaveDPUPortP0) + VerifyIsolationBridgeExists(fcPod2, rdmaVNI, weaveDPUPortP0) }) It("should plumb overlay IPs onto worker p0 PFs via dhcpcd", func() { - acquireDHCPLeaseInPod(hostClusterRESTClient, input.restConfig, netutilsPod1, weaveHostPFInterfaceP0, overlayIPP0Node1) - acquireDHCPLeaseInPod(hostClusterRESTClient, input.restConfig, netutilsPod2, weaveHostPFInterfaceP0, overlayIPP0Node2) + AcquireDHCPLeaseInPod(HostClusterRESTClient, input.RestConfig, netutilsPod1, weaveHostPFInterfaceP0, overlayIPP0Node1) + AcquireDHCPLeaseInPod(HostClusterRESTClient, input.RestConfig, netutilsPod2, weaveHostPFInterfaceP0, overlayIPP0Node2) }) It("should run ib_write_bw between the two hosts on p0 and meet the BW threshold", func() { - runIBWriteBWPodToPod(hostClusterRESTClient, input.restConfig, netutilsPod2, netutilsPod1, weaveHostPFRDMADeviceP0, overlayIPP0Node2) + RunIBWriteBWPodToPod(HostClusterRESTClient, input.RestConfig, netutilsPod2, netutilsPod1, weaveHostPFRDMADeviceP0, overlayIPP0Node2) }) It("should run ib_write_bw between the two hosts on p0 with --reversed and meet the BW threshold", func() { // Running with --reversed checks that the RDMA traffic also works in reverse direction for sanity purposes. - runIBWriteBWPodToPod(hostClusterRESTClient, input.restConfig, netutilsPod2, netutilsPod1, weaveHostPFRDMADeviceP0, overlayIPP0Node2, "--reversed") + RunIBWriteBWPodToPod(HostClusterRESTClient, input.RestConfig, netutilsPod2, netutilsPod1, weaveHostPFRDMADeviceP0, overlayIPP0Node2, "--reversed") }) }) @@ -513,83 +513,83 @@ var _ = Describe("Weave testcases", Labels{Domain.Weave}, Ordered, func() { }) It("should create test namespace", func() { - vpc.CreateTestNamespace(ctx, input.client, isolTestNS, weaveContextScope.CleanupLabels) + vpc.CreateTestNamespace(Ctx, input.Client, isolTestNS, weaveContextScope.CleanupLabels) }) It("should create both isolation virtual networks on flow-controller pod 1", func() { - createVNetOnPod(fcPod1, isolVNet1ID, isolVNI1, weaveVNetSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod1, isolVNet1ID) }) - createVNetOnPod(fcPod1, isolVNet2ID, isolVNI2, secondSubnet) - grpcCleanup = append(grpcCleanup, func() { deleteVNetOnPod(fcPod1, isolVNet2ID) }) + CreateVNetOnPod(fcPod1, isolVNet1ID, isolVNI1, weaveVNetSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod1, isolVNet1ID) }) + CreateVNetOnPod(fcPod1, isolVNet2ID, isolVNI2, secondSubnet) + grpcCleanup = append(grpcCleanup, func() { DeleteVNetOnPod(fcPod1, isolVNet2ID) }) }) It("should create two attachments on worker node 1", func() { var attIDIsolFc1, attIDIsolFc2 string - attIDIsolFc1, overlayIP1 = createPFAttachmentAndWaitForHostIP(fcPod1, isolVNet1ID, pfMACP0Node1) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod1, attIDIsolFc1) }) - attIDIsolFc2, overlayIP2 = createPFAttachmentAndWaitForHostIP(fcPod1, isolVNet2ID, pfMACP1Node1) - grpcCleanup = append(grpcCleanup, func() { deleteAttachmentOnPod(fcPod1, attIDIsolFc2) }) + attIDIsolFc1, overlayIP1 = CreatePFAttachmentAndWaitForHostIP(fcPod1, isolVNet1ID, pfMACP0Node1) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod1, attIDIsolFc1) }) + attIDIsolFc2, overlayIP2 = CreatePFAttachmentAndWaitForHostIP(fcPod1, isolVNet2ID, pfMACP1Node1) + grpcCleanup = append(grpcCleanup, func() { DeleteAttachmentOnPod(fcPod1, attIDIsolFc2) }) }) It("should verify OVS isolation bridges for both VNets on DPU node 1", func() { // br-isol-- is created when that flow-controller has a PF attachment for the VNI // (bridgemanager), not only when CreateVirtualNetwork succeeded on that pod. - verifyIsolationBridgeExists(fcPod1, isolVNI1, weaveDPUPortP0) - verifyIsolationBridgeExists(fcPod1, isolVNI2, weaveDPUPortP1) + VerifyIsolationBridgeExists(fcPod1, isolVNI1, weaveDPUPortP0) + VerifyIsolationBridgeExists(fcPod1, isolVNI2, weaveDPUPortP1) }) It("should create DHCP NADs and netshoot pods on worker node 1", func() { nadP0 := weaveDHCPNADP0 nadP1 := weaveDHCPNADP1 - vpc.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, isolTestNS, nadP0, weaveHostPFInterfaceP0, weavePFMTU, weaveContextScope.CleanupLabels) - vpc.CreateDHCPNetworkAttachmentDefinition(ctx, input.client, isolTestNS, nadP1, weaveHostPFInterfaceP1, weavePFMTU, weaveContextScope.CleanupLabels) + vpc.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, isolTestNS, nadP0, weaveHostPFInterfaceP0, weavePFMTU, weaveContextScope.CleanupLabels) + vpc.CreateDHCPNetworkAttachmentDefinition(Ctx, input.Client, isolTestNS, nadP1, weaveHostPFInterfaceP1, weavePFMTU, weaveContextScope.CleanupLabels) testPodConfigs = []*netshoot.TestPodConfig{ {Namespace: isolTestNS, Name: isolPod1, NodeName: workerNode1, NADName: nadP0, Labels: weaveContextScope.CleanupLabels}, {Namespace: isolTestNS, Name: isolPod2, NodeName: workerNode1, NADName: nadP1, Labels: weaveContextScope.CleanupLabels}, } - netshoot.CreatePods(ctx, input.client, testPodConfigs) + netshoot.CreatePods(Ctx, input.Client, testPodConfigs) }) It("should verify netshoot pods are running", func() { - netshoot.WaitForPodsReady(ctx, input.client, testPodConfigs, vpc.LongTimeout) + netshoot.WaitForPodsReady(Ctx, input.Client, testPodConfigs, vpc.LongTimeout) }) It("should verify overlay routes on netshoot pods", func() { - ensureOverlayRoute(hostClusterRESTClient, input.restConfig, isolTestNS, isolPod1, overlayIP1, weaveVNetSubnet) - ensureOverlayRoute(hostClusterRESTClient, input.restConfig, isolTestNS, isolPod2, overlayIP2, secondSubnet) + EnsureOverlayRoute(HostClusterRESTClient, input.RestConfig, isolTestNS, isolPod1, overlayIP1, weaveVNetSubnet) + EnsureOverlayRoute(HostClusterRESTClient, input.RestConfig, isolTestNS, isolPod2, overlayIP2, secondSubnet) }) It("should add route on netshoot pod 1", func() { - addRouteOnPodBetweenOverlayAndSubnet(hostClusterRESTClient, input.restConfig, isolTestNS, isolPod1, overlayIP1, secondSubnet) + AddRouteOnPodBetweenOverlayAndSubnet(HostClusterRESTClient, input.RestConfig, isolTestNS, isolPod1, overlayIP1, secondSubnet) }) It("should add route on netshoot pod 2", func() { - addRouteOnPodBetweenOverlayAndSubnet(hostClusterRESTClient, input.restConfig, isolTestNS, isolPod2, overlayIP2, weaveVNetSubnet) + AddRouteOnPodBetweenOverlayAndSubnet(HostClusterRESTClient, input.RestConfig, isolTestNS, isolPod2, overlayIP2, weaveVNetSubnet) }) It("should deny ping between pods on different virtual networks on the same node", func() { - netshoot.AssertPingFailure(&hostClusterRESTClient, &input.restConfig, isolTestNS, isolPod1, overlayIP2) - netshoot.AssertPingFailure(&hostClusterRESTClient, &input.restConfig, isolTestNS, isolPod2, overlayIP1) + netshoot.AssertPingFailure(&HostClusterRESTClient, &input.RestConfig, isolTestNS, isolPod1, overlayIP2) + netshoot.AssertPingFailure(&HostClusterRESTClient, &input.RestConfig, isolTestNS, isolPod2, overlayIP1) }) It("should verify metrics for an out-of-subnet destination", func() { - bridge := isolationBridgeName(isolVNI1, weaveDPUPortP0) - baselineMetrics := readWeaveMetrics(fcPod1) + bridge := IsolationBridgeName(isolVNI1, weaveDPUPortP0) + baselineMetrics := ReadWeaveMetrics(fcPod1) By("Sending a ping burst to an out-of-subnet destination") - _, _ = netshoot.PingBurst(hostClusterRESTClient, input.restConfig, isolTestNS, isolPod1, overlayIP2, weaveMetricBurstCount) + _, _ = netshoot.PingBurst(HostClusterRESTClient, input.RestConfig, isolTestNS, isolPod1, overlayIP2, weaveMetricBurstCount) By("Verifying weave metrics across out-of-subnet destination") // Poll until the OVS scrape reflects the burst. - var currentMetrics weaveMetrics + var currentMetrics WeaveMetrics Eventually(func(g Gomega) { - currentMetrics = scrapeWeaveMetrics(g, fcPod1) + currentMetrics = ScrapeWeaveMetrics(g, fcPod1) // tx_sent stays flat because an out-of-subnet destination is dropped before it is reached. - assertMetricDeltas(g, baselineMetrics, currentMetrics, bridge, metricDeltaExpect{ - mustRiseBy: map[string]uint64{weaveMetricHostTx: 1, weaveMetricTxDropped: 1}, - mustStayFlat: []string{weaveMetricTxSent}, + AssertMetricDeltas(g, baselineMetrics, currentMetrics, bridge, MetricDeltaExpect{ + MustRiseBy: map[string]uint64{weaveMetricHostTx: 1, weaveMetricTxDropped: 1}, + MustStayFlat: []string{weaveMetricTxSent}, }) - assertTxPacketsAccountedFor(g, baselineMetrics, currentMetrics, bridge) + AssertTxPacketsAccountedFor(g, baselineMetrics, currentMetrics, bridge) }).WithTimeout(weaveOperationTimeout).WithPolling(weaveEventuallyPollInterval).Should(Succeed()) }) })