From 0cc3160b516a5c9e49d514a32dd196f98c79de83 Mon Sep 17 00:00:00 2001 From: Rajath Agasthya Date: Thu, 6 Aug 2026 11:44:29 -0500 Subject: [PATCH] Mount ServiceAccount token when dcgm-exporter env needs API access The dcgm-exporter DaemonSet sets automountServiceAccountToken=false and re-enables it only when pod metadata enrichment is turned on through the typed enablePodLabels/enablePodUID fields. Users who configure the same features through raw dcgmExporter.env are missed by that condition, and setting DCGM_EXPORTER_CONFIGMAP_DATA crashes the exporter on startup since it cannot read the ServiceAccount token. This worked before the token was made opt-in. Mount the token when the user-provided env reads a ConfigMap or enables pod labels / pod UID. A DCGM_EXPORTER_CONFIGMAP_DATA value of "none" is the exporter default meaning no ConfigMap is read, so it does not count. Configurations that make no API calls keep the default. Apply the same env check to the cluster-scoped RBAC gate. Enrichment through raw env otherwise got a token but no matching permissions, leaving the pod informer with only the namespaced Role, which grants pod reads in the operator namespace without watch. The ConfigMap case keeps a separate condition since it needs no cluster-scoped RBAC: the namespaced Role already grants configmaps get/list and its RoleBinding is ungated. The daemonset test compared gpuv1.EnvVar against the container's corev1.EnvVar, which can never be equal. Both render as v1.EnvVar in failure output, so the mismatch stayed invisible until a test case set env on the ClusterPolicy. Signed-off-by: Rajath Agasthya --- controllers/object_controls.go | 56 ++++++++++++++++--- controllers/object_controls_test.go | 42 ++++++++++++-- controllers/transforms_test.go | 86 +++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+), 12 deletions(-) diff --git a/controllers/object_controls.go b/controllers/object_controls.go index 9130e60360..0113a43b66 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -116,6 +116,17 @@ const ( DCGMRemoteEngineEnvName = "DCGM_REMOTE_HOSTENGINE_INFO" // DCGMDefaultPort indicates default port bound to DCGM host engine DCGMDefaultPort = 5555 + // DCGMExporterConfigMapDataEnvName is the env name specifying the namespace:name + // ConfigMap with custom metrics + DCGMExporterConfigMapDataEnvName = "DCGM_EXPORTER_CONFIGMAP_DATA" + // DCGMExporterUndefinedConfigMapData is the exporter default meaning no ConfigMap is read + DCGMExporterUndefinedConfigMapData = "none" + // DCGMExporterPodLabelsEnvName is the env name enabling pod labels on metrics + DCGMExporterPodLabelsEnvName = "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS" + // DCGMExporterPodUIDEnvName is the env name enabling the pod UID label on metrics + DCGMExporterPodUIDEnvName = "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID" + // DCGMExporterPodLabelAllowlistEnvName is the env name holding the exported pod label allowlist + DCGMExporterPodLabelAllowlistEnvName = "DCGM_EXPORTER_KUBERNETES_POD_LABEL_ALLOWLIST_REGEX" // GPUDirectRDMAEnabledEnvName indicates if GPU direct RDMA is enabled through GPU operator GPUDirectRDMAEnabledEnvName = "GPU_DIRECT_RDMA_ENABLED" // UseHostMOFEDEnvName indicates if MOFED driver is pre-installed on the host @@ -446,7 +457,8 @@ func RoleBinding(n ClusterPolicyController) (gpuv1.State, error) { var rbacGates = map[string]func(*gpuv1.ClusterPolicySpec) bool{ "nvidia-dcgm-exporter-read-pods": func(config *gpuv1.ClusterPolicySpec) bool { - return config.DCGMExporter.IsKubernetesPodMetadataEnabled() + return config.DCGMExporter.IsKubernetesPodMetadataEnabled() || + dcgmExporterEnvEnablesPodMetadata(config.DCGMExporter.Env) }, } @@ -1868,20 +1880,20 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe // Inject pod-metadata enrichment env vars; RBAC is provisioned via the // 0210/0310 assets and the SA token is mounted below. if config.DCGMExporter.IsPodLabelsEnabled() { - setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", "true") + setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DCGMExporterPodLabelsEnvName, "true") } if config.DCGMExporter.IsPodUIDEnabled() { - setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID", "true") + setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DCGMExporterPodUIDEnvName, "true") } if len(config.DCGMExporter.PodLabelAllowlistRegex) > 0 { setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), - "DCGM_EXPORTER_KUBERNETES_POD_LABEL_ALLOWLIST_REGEX", + DCGMExporterPodLabelAllowlistEnvName, strings.Join(config.DCGMExporter.PodLabelAllowlistRegex, ",")) } - // Override the base asset's automountServiceAccountToken=false when - // enrichment is on so the pod informer has client-go credentials. - if config.DCGMExporter.IsKubernetesPodMetadataEnabled() { + // Override the base asset's automountServiceAccountToken=false when the + // configuration reads from the Kubernetes API. + if config.DCGMExporter.IsKubernetesPodMetadataEnabled() || dcgmExporterEnvRequiresAPIAccess(config.DCGMExporter.Env) { obj.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(true) } @@ -1928,6 +1940,36 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe return nil } +// dcgmExporterEnvRequiresAPIAccess returns true if a user-provided env var makes +// dcgm-exporter read from the Kubernetes API. +func dcgmExporterEnvRequiresAPIAccess(envs []gpuv1.EnvVar) bool { + if dcgmExporterEnvEnablesPodMetadata(envs) { + return true + } + for _, env := range envs { + if env.Name == DCGMExporterConfigMapDataEnvName { + // the exporter treats "none" and "" as no ConfigMap read + if env.Value != "" && env.Value != DCGMExporterUndefinedConfigMapData { + return true + } + } + } + return false +} + +// dcgmExporterEnvEnablesPodMetadata returns true if a user-provided env var turns +// on pod metadata enrichment. +func dcgmExporterEnvEnablesPodMetadata(envs []gpuv1.EnvVar) bool { + for _, env := range envs { + if env.Name == DCGMExporterPodLabelsEnvName || env.Name == DCGMExporterPodUIDEnvName { + if strings.EqualFold(env.Value, "true") { + return true + } + } + } + return false +} + func addExtraAnnotations(obj *appsv1.DaemonSet, annotations map[string]string) { if obj.Spec.Template.Annotations == nil { obj.Spec.Template.Annotations = make(map[string]string) diff --git a/controllers/object_controls_test.go b/controllers/object_controls_test.go index d84917c0b5..9c1bc9c7dd 100644 --- a/controllers/object_controls_test.go +++ b/controllers/object_controls_test.go @@ -863,7 +863,7 @@ func testDaemonsetCommon(t *testing.T, cp *gpuv1.ClusterPolicy, component string require.Equal(t, spec.args, mainCtr.Args, "unexpected Args") } for _, env := range spec.env { - require.Contains(t, mainCtr.Env, env, "env var not present") + require.Contains(t, mainCtr.Env, corev1.EnvVar{Name: env.Name, Value: env.Value}, "env var not present") } // TODO: implement checks for other common fields (i.e. Resources, securityContext, Tolerations, etc.) @@ -1536,6 +1536,14 @@ func getDCGMExporterTestInput(testCase string) *gpuv1.ClusterPolicy { cp.Spec.DCGMExporter.EnablePodLabels = ptr.To(true) case "pod-uid-enabled": cp.Spec.DCGMExporter.EnablePodUID = ptr.To(true) + case "pod-labels-env": + cp.Spec.DCGMExporter.Env = []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"}, + } + case "configmap-data-env": + cp.Spec.DCGMExporter.Env = []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "gpu-operator:custom-metrics"}, + } default: return nil } @@ -1552,6 +1560,7 @@ func getDCGMExporterTestOutput(testCase string) map[string]interface{} { "dcgmExporterImage": "nvcr.io/nvidia/k8s/dcgm-exporter:3.3.0-3.2.0-ubuntu22.04", "imagePullSecret": "ngc-secret", "clusterRoleExists": false, + "automountToken": false, } switch testCase { @@ -1566,11 +1575,24 @@ func getDCGMExporterTestOutput(testCase string) map[string]interface{} { "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS": "true", } output["clusterRoleExists"] = true + output["automountToken"] = true case "pod-uid-enabled": output["env"] = map[string]string{ "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID": "true", } output["clusterRoleExists"] = true + output["automountToken"] = true + case "pod-labels-env": + output["env"] = map[string]string{ + "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS": "true", + } + output["clusterRoleExists"] = true + output["automountToken"] = true + case "configmap-data-env": + output["env"] = map[string]string{ + "DCGM_EXPORTER_CONFIGMAP_DATA": "gpu-operator:custom-metrics", + } + output["automountToken"] = true default: return nil } @@ -1606,6 +1628,16 @@ func TestDCGMExporter(t *testing.T) { getDCGMExporterTestInput("pod-uid-enabled"), getDCGMExporterTestOutput("pod-uid-enabled"), }, + { + "PodLabelsViaEnv", + getDCGMExporterTestInput("pod-labels-env"), + getDCGMExporterTestOutput("pod-labels-env"), + }, + { + "ConfigMapDataViaEnv", + getDCGMExporterTestInput("configmap-data-env"), + getDCGMExporterTestOutput("configmap-data-env"), + }, } for _, tc := range testCases { @@ -1643,15 +1675,15 @@ func TestDCGMExporter(t *testing.T) { if tc.output["clusterRoleExists"].(bool) { require.NoError(t, clusterRoleGetErr, "ClusterRole should exist when pod metadata enrichment is enabled") require.NoError(t, clusterRoleBindingGetErr, "ClusterRoleBinding should exist when pod metadata enrichment is enabled") - require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be set when pod metadata enrichment is enabled") - require.True(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be true when pod metadata enrichment is enabled") } else { require.True(t, apierrors.IsNotFound(clusterRoleGetErr), "ClusterRole should not exist when pod metadata enrichment is disabled (got err=%v)", clusterRoleGetErr) require.True(t, apierrors.IsNotFound(clusterRoleBindingGetErr), "ClusterRoleBinding should not exist when pod metadata enrichment is disabled (got err=%v)", clusterRoleBindingGetErr) - require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be explicitly set false when pod metadata enrichment is disabled") - require.False(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be false when pod metadata enrichment is disabled") } + require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should always be set explicitly") + require.Equal(t, tc.output["automountToken"].(bool), *ds.Spec.Template.Spec.AutomountServiceAccountToken, + "unexpected AutomountServiceAccountToken on daemonset nvidia-dcgm-exporter") + require.Equal(t, tc.output["dcgmExporterImage"], dcgmExporterImage, "Unexpected configuration for dcgm-exporter image") // cleanup by deleting all kubernetes objects diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index fcbb6732a1..0ff56f8beb 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -1730,6 +1730,92 @@ func TestTransformDCGMExporter(t *testing.T) { WithRuntimeClassName("nvidia"). WithAutomountServiceAccountToken(true), }, + { + description: "transform dcgm exporter with custom metrics configmap env", + ds: NewDaemonset(). + WithContainer(corev1.Container{Name: "dcgm-exporter"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DCGMExporter: gpuv1.DCGMExporterSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "dcgm-exporter", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Env: []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "gpu-operator:exporter-metrics-config-map"}, + }, + }, + DCGM: gpuv1.DCGMSpec{ + Enabled: newBoolPtr(true), + }, + }, + expectedDs: NewDaemonset().WithContainer(corev1.Container{ + Name: "dcgm-exporter", + Image: "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"}, + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "gpu-operator:exporter-metrics-config-map"}, + }, + }).WithRuntimeClassName("nvidia"). + WithAutomountServiceAccountToken(true), + }, + { + description: "transform dcgm exporter with configmap data env set to none", + ds: NewDaemonset(). + WithContainer(corev1.Container{Name: "dcgm-exporter"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DCGMExporter: gpuv1.DCGMExporterSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "dcgm-exporter", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Env: []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "none"}, + }, + }, + DCGM: gpuv1.DCGMSpec{ + Enabled: newBoolPtr(true), + }, + }, + expectedDs: NewDaemonset().WithContainer(corev1.Container{ + Name: "dcgm-exporter", + Image: "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"}, + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "none"}, + }, + }).WithRuntimeClassName("nvidia"), + }, + { + description: "transform dcgm exporter with pod labels enabled via raw env", + ds: NewDaemonset(). + WithContainer(corev1.Container{Name: "dcgm-exporter"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DCGMExporter: gpuv1.DCGMExporterSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "dcgm-exporter", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Env: []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"}, + }, + }, + DCGM: gpuv1.DCGMSpec{ + Enabled: newBoolPtr(true), + }, + }, + expectedDs: NewDaemonset().WithContainer(corev1.Container{ + Name: "dcgm-exporter", + Image: "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"}, + {Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"}, + }, + }).WithRuntimeClassName("nvidia"). + WithAutomountServiceAccountToken(true), + }, } for _, tc := range testCases {