devzero-inc · dray92 · Aug 20, 2026
diff --git a/api/v1/collectionpolicy_types.go b/api/v1/collectionpolicy_types.go
@@ -504,6 +504,13 @@ type Policies struct {
 	// +optional
 	KubeletFallbackTimeout string `json:"kubeletFallbackTimeout,omitempty"`
 
+	// SchedulerMetricsScrapeInterval bounds how often the SchedulerMetrics
+	// collector scrapes dz-scheduler's /metrics endpoint (defaults to 60s --
+	// see collector.DefaultSchedulerMetricsScrapeInterval's doc comment for
+	// why). A Go duration string (e.g. "60s", "30s").
+	// +optional
+	SchedulerMetricsScrapeInterval string `json:"schedulerMetricsScrapeInterval,omitempty"`
+
 	// ClusterSnapshotInterval is how often to take cluster snapshots (defaults to 3h)
 	ClusterSnapshotInterval string `json:"clusterSnapshotInterval,omitempty"`
 

diff --git a/config/crd/bases/devzero.io_collectionpolicies.yaml b/config/crd/bases/devzero.io_collectionpolicies.yaml
@@ -738,6 +738,13 @@ spec:
                       PATToken is the Personal Access Token used for automatic cluster token exchange
                       If ClusterToken is not provided but PATToken is, the system will exchange it for a cluster token
                     type: string
+                  schedulerMetricsScrapeInterval:
+                    description: |-
+                      SchedulerMetricsScrapeInterval bounds how often the SchedulerMetrics
+                      collector scrapes dz-scheduler's /metrics endpoint (defaults to 60s --
+                      see collector.DefaultSchedulerMetricsScrapeInterval's doc comment for
+                      why). A Go duration string (e.g. "60s", "30s").
+                    type: string
                   watchedCRDs:
                     description: WatchedCRDs is a list of custom resource definitions
                       to explicitly watch

diff --git a/config/manager/env_configmap.yaml b/config/manager/env_configmap.yaml
@@ -20,6 +20,7 @@ data:
   NODE_METRICS_CONCURRENCY: "20"
   NODEMON_REQUEST_TIMEOUT: "15s"
   KUBELET_FALLBACK_TIMEOUT: "15s"
+  SCHEDULER_METRICS_SCRAPE_INTERVAL: ""
   WATCHED_CRDS: ""
   DISABLED_COLLECTORS: ""
   # NVIDIA mig-parted ConfigMap location; empty = GPU operator defaults

diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml
@@ -22,6 +22,7 @@ resources:
 - metrics_auth_role.yaml
 - metrics_auth_role_binding.yaml
 - metrics_reader_role.yaml
+- metrics_reader_role_binding.yaml
 # For each CRD, "Editor" and "Viewer" roles are scaffolded by
 # default, aiding admins in cluster management. Those roles are
 # not used by the Project itself. You can comment the following lines

diff --git a/config/rbac/metrics_reader_role_binding.yaml b/config/rbac/metrics_reader_role_binding.yaml
@@ -0,0 +1,19 @@
+# Binds metrics-reader (nonResourceURLs: ["/metrics"], verb: get) to the
+# controller's own ServiceAccount so the SchedulerMetrics collector can scrape
+# dz-scheduler's /metrics endpoint. dz-scheduler runs kube-scheduler with
+# delegated authorization to the API server (SubjectAccessReview), and
+# "/metrics" is not in kube-scheduler's default AlwaysAllowPaths -- unlike
+# /healthz and /livez, an authenticated caller still needs an explicit RBAC
+# grant for this nonResourceURL, or every scrape 403s.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-reader-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -157,6 +157,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - coordination.k8s.io
+  resourceNames:
+  - dz-scheduler
+  resources:
+  - leases
+  verbs:
+  - get
 - apiGroups:
   - dakr.devzero.io
   resources:

diff --git a/dist/backend-install-gcp-lowpriv.yaml b/dist/backend-install-gcp-lowpriv.yaml
@@ -334,6 +334,14 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resourceNames:
+      - dz-scheduler
+    resources:
+      - leases
+    verbs:
+      - get
   - apiGroups:
       - dakr.devzero.io
     resources:
@@ -654,6 +662,19 @@ subjects:
     name: devzero-zxporter-controller-manager
     namespace: {{.zxporter_namespace}}
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: devzero-zxporter-metrics-reader-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: devzero-zxporter-metrics-reader
+subjects:
+  - kind: ServiceAccount
+    name: devzero-zxporter-controller-manager
+    namespace: {{.zxporter_namespace}}
+---
 apiVersion: v1
 data:
   BUFFER_SIZE: ""
@@ -701,6 +722,7 @@ data:
   NODE_METRICS_CONCURRENCY: "20"
   NODE_METRICS_INTERVAL: ""
   NODEMON_REQUEST_TIMEOUT: 15s
+  SCHEDULER_METRICS_SCRAPE_INTERVAL: ""
   TARGET_NAMESPACES: ""
   TOKEN_CONFIGMAP_NAME: devzero-zxporter-env-config
   TOKEN_CREDENTIALS_SECRET_NAME: devzero-zxporter-credentials
@@ -870,10 +892,10 @@ metadata:
   name: zxporter-nodemon
   namespace: {{.zxporter_namespace}}
   labels:
-    helm.sh/chart: zxporter-nodemon-0.1.6
+    helm.sh/chart: zxporter-nodemon-0.1.10
     app.kubernetes.io/name: zxporter-nodemon
     app.kubernetes.io/instance: zxporter-nodemon
-    app.kubernetes.io/version: "0.1.6"
+    app.kubernetes.io/version: "0.1.10"
     app.kubernetes.io/managed-by: Helm
 ---
 # Source: zxporter-nodemon/templates/dcgm-exporter-configmap.yaml
@@ -895,8 +917,12 @@ data:
     DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %).
     # DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned
     # DCGM_FI_PROF_SM_OCCUPANCY, gauge, The fraction of resident warps on a multiprocessor
-    # DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active (in %).
-    # DCGM_FI_PROF_DRAM_ACTIVE, gauge, The ratio of cycles the device memory interface is active sending or receiving data.
+    # Enabled: required for per-MIG-instance profiling data (GPU_I_PROFILE/GPU_I_ID
+    # labels only attach to DCGM_FI_PROF_* fields, never to plain DCGM_FI_DEV_*
+    # ones) and directly consumed by nodemon's GPUMigInstance parsing
+    # (GraphicsEngineActive/DRAMActive/TensorActive below).
+    DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active (in %).
+    DCGM_FI_PROF_DRAM_ACTIVE, gauge, The ratio of cycles the device memory interface is active sending or receiving data.
 
     # Memory usage,,
     DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB).
@@ -915,7 +941,9 @@ data:
     # DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipe is active.
     # DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipe is active.
     # DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipe is active.
-    # DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, The ratio of cycles the tensor (HMMA) pipe is active (off the peak sustained elapsed cycles)
+    # Enabled: see the GR_ENGINE_ACTIVE/DRAM_ACTIVE note above -- this is
+    # GPUMigInstance.TensorActive.
+    DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, The ratio of cycles the tensor (HMMA) pipe is active (off the peak sustained elapsed cycles)
 
     # Health,,
     # DCGM_FI_DEV_CLOCKS_EVENT_REASONS is not supported by DCGM 3.3.7
@@ -946,10 +974,10 @@ kind: ClusterRole
 metadata:
   name: zxporter-nodemon
   labels:
-    helm.sh/chart: zxporter-nodemon-0.1.6
+    helm.sh/chart: zxporter-nodemon-0.1.10
     app.kubernetes.io/name: zxporter-nodemon
     app.kubernetes.io/instance: zxporter-nodemon
-    app.kubernetes.io/version: "0.1.6"
+    app.kubernetes.io/version: "0.1.10"
     app.kubernetes.io/managed-by: Helm
 rules:
   - apiGroups:
@@ -996,10 +1024,10 @@ kind: ClusterRoleBinding
 metadata:
   name: zxporter-nodemon
   labels:
-    helm.sh/chart: zxporter-nodemon-0.1.6
+    helm.sh/chart: zxporter-nodemon-0.1.10
     app.kubernetes.io/name: zxporter-nodemon
     app.kubernetes.io/instance: zxporter-nodemon
-    app.kubernetes.io/version: "0.1.6"
+    app.kubernetes.io/version: "0.1.10"
     app.kubernetes.io/managed-by: Helm
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -1017,10 +1045,10 @@ metadata:
   name: zxporter-nodemon
   namespace: {{.zxporter_namespace}}
   labels:
-    helm.sh/chart: zxporter-nodemon-0.1.6
+    helm.sh/chart: zxporter-nodemon-0.1.10
     app.kubernetes.io/name: zxporter-nodemon
     app.kubernetes.io/instance: zxporter-nodemon
-    app.kubernetes.io/version: "0.1.6"
+    app.kubernetes.io/version: "0.1.10"
     app.kubernetes.io/managed-by: Helm
   annotations:
     ignore-check.kube-linter.io/privileged-container: "This daemon set needs to run DCGM Exporter as privileged to access the GPU metrics."
@@ -1127,6 +1155,14 @@ spec:
               memory: 256Mi
         - name: dcgm-exporter
           securityContext:
+            # SYS_ADMIN alone is not enough: enumerating MIG Compute Instance
+            # profiles via NVML needs broader host access than that one
+            # capability grants (confirmed live -- SYS_ADMIN-only reproduces
+            # "CacheManager Init Failed. Error: -17" the instant any MIG
+            # instance exists on the node; switching to privileged fixes it
+            # with zero other changes). This is a generic NVML/driver
+            # requirement, not provider-specific, so unlike the hostPath/
+            # LD_LIBRARY_PATH blocks below it must not be gated on gcp only.
             capabilities:
               add:
                 - SYS_ADMIN
@@ -1154,6 +1190,16 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            # Without this, nvidia-container-runtime's "auto" mode only exposes
+            # whole/non-MIG GPUs to a non-privileged container with no explicit
+            # device-plugin resource request (which this sidecar deliberately
+            # doesn't make, since it must see every GPU on the node, not just
+            # one). MIG-partitioned GPUs stay invisible without it -- confirmed
+            # by direct comparison against GPU Operator's own dcgm-exporter,
+            # which gets the same visibility via privileged:true instead; this
+            # is the minimal fix that doesn't need the broader grant.
+            - name: "NVIDIA_VISIBLE_DEVICES"
+              value: "all"
             - name: "LD_LIBRARY_PATH"
               value: "/usr/local/nvidia/lib64"
             - name: "DCGM_EXPORTER_KUBERNETES_GPU_ID_TYPE"
@@ -1174,10 +1220,10 @@ metadata:
   name: zxporter-nodemon-gpu
   namespace: {{.zxporter_namespace}}
   labels:
-    helm.sh/chart: zxporter-nodemon-0.1.6
+    helm.sh/chart: zxporter-nodemon-0.1.10
     app.kubernetes.io/name: zxporter-nodemon
     app.kubernetes.io/instance: zxporter-nodemon-gpu
-    app.kubernetes.io/version: "0.1.6"
+    app.kubernetes.io/version: "0.1.10"
     app.kubernetes.io/managed-by: Helm
   annotations:
     ignore-check.kube-linter.io/privileged-container: "This daemon set needs to run DCGM Exporter as privileged to access the GPU metrics."
@@ -1287,6 +1333,14 @@ spec:
               memory: 256Mi
         - name: dcgm-exporter
           securityContext:
+            # SYS_ADMIN alone is not enough: enumerating MIG Compute Instance
+            # profiles via NVML needs broader host access than that one
+            # capability grants (confirmed live -- SYS_ADMIN-only reproduces
+            # "CacheManager Init Failed. Error: -17" the instant any MIG
+            # instance exists on the node; switching to privileged fixes it
+            # with zero other changes). This is a generic NVML/driver
+            # requirement, not provider-specific, so unlike the hostPath/
+            # LD_LIBRARY_PATH blocks below it must not be gated on gcp only.
             capabilities:
               add:
                 - SYS_ADMIN
@@ -1314,6 +1368,16 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            # Without this, nvidia-container-runtime's "auto" mode only exposes
+            # whole/non-MIG GPUs to a non-privileged container with no explicit
+            # device-plugin resource request (which this sidecar deliberately
+            # doesn't make, since it must see every GPU on the node, not just
+            # one). MIG-partitioned GPUs stay invisible without it -- confirmed
+            # by direct comparison against GPU Operator's own dcgm-exporter,
+            # which gets the same visibility via privileged:true instead; this
+            # is the minimal fix that doesn't need the broader grant.
+            - name: "NVIDIA_VISIBLE_DEVICES"
+              value: "all"
             - name: "LD_LIBRARY_PATH"
               value: "/usr/local/nvidia/lib64"
             - name: "DCGM_EXPORTER_KUBERNETES_GPU_ID_TYPE"