diff --git a/controllers/object_controls.go b/controllers/object_controls.go index 9130e6036..0113a43b6 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -116,6 +116,17 @@ const ( DCGMRemoteEngineEnvName = "DCGM_REMOTE_HOSTENGINE_INFO" // DCGMDefaultPort indicates default port bound to DCGM host engine DCGMDefaultPort = 5555 + // DCGMExporterConfigMapDataEnvName is the env name specifying the namespace:name + // ConfigMap with custom metrics + DCGMExporterConfigMapDataEnvName = "DCGM_EXPORTER_CONFIGMAP_DATA" + // DCGMExporterUndefinedConfigMapData is the exporter default meaning no ConfigMap is read + DCGMExporterUndefinedConfigMapData = "none" + // DCGMExporterPodLabelsEnvName is the env name enabling pod labels on metrics + DCGMExporterPodLabelsEnvName = "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS" + // DCGMExporterPodUIDEnvName is the env name enabling the pod UID label on metrics + DCGMExporterPodUIDEnvName = "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID" + // DCGMExporterPodLabelAllowlistEnvName is the env name holding the exported pod label allowlist + DCGMExporterPodLabelAllowlistEnvName = "DCGM_EXPORTER_KUBERNETES_POD_LABEL_ALLOWLIST_REGEX" // GPUDirectRDMAEnabledEnvName indicates if GPU direct RDMA is enabled through GPU operator GPUDirectRDMAEnabledEnvName = "GPU_DIRECT_RDMA_ENABLED" // UseHostMOFEDEnvName indicates if MOFED driver is pre-installed on the host @@ -446,7 +457,8 @@ func RoleBinding(n ClusterPolicyController) (gpuv1.State, error) { var rbacGates = map[string]func(*gpuv1.ClusterPolicySpec) bool{ "nvidia-dcgm-exporter-read-pods": func(config *gpuv1.ClusterPolicySpec) bool { - return config.DCGMExporter.IsKubernetesPodMetadataEnabled() + return config.DCGMExporter.IsKubernetesPodMetadataEnabled() || + dcgmExporterEnvEnablesPodMetadata(config.DCGMExporter.Env) }, } @@ -1868,20 +1880,20 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe // Inject pod-metadata enrichment env vars; RBAC is provisioned via the // 0210/0310 assets and the SA token is mounted below. if config.DCGMExporter.IsPodLabelsEnabled() { - setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", "true") + setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DCGMExporterPodLabelsEnvName, "true") } if config.DCGMExporter.IsPodUIDEnabled() { - setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID", "true") + setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DCGMExporterPodUIDEnvName, "true") } if len(config.DCGMExporter.PodLabelAllowlistRegex) > 0 { setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), - "DCGM_EXPORTER_KUBERNETES_POD_LABEL_ALLOWLIST_REGEX", + DCGMExporterPodLabelAllowlistEnvName, strings.Join(config.DCGMExporter.PodLabelAllowlistRegex, ",")) } - // Override the base asset's automountServiceAccountToken=false when - // enrichment is on so the pod informer has client-go credentials. - if config.DCGMExporter.IsKubernetesPodMetadataEnabled() { + // Override the base asset's automountServiceAccountToken=false when the + // configuration reads from the Kubernetes API. + if config.DCGMExporter.IsKubernetesPodMetadataEnabled() || dcgmExporterEnvRequiresAPIAccess(config.DCGMExporter.Env) { obj.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(true) } @@ -1928,6 +1940,36 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe return nil } +// dcgmExporterEnvRequiresAPIAccess returns true if a user-provided env var makes +// dcgm-exporter read from the Kubernetes API. +func dcgmExporterEnvRequiresAPIAccess(envs []gpuv1.EnvVar) bool { + if dcgmExporterEnvEnablesPodMetadata(envs) { + return true + } + for _, env := range envs { + if env.Name == DCGMExporterConfigMapDataEnvName { + // the exporter treats "none" and "" as no ConfigMap read + if env.Value != "" && env.Value != DCGMExporterUndefinedConfigMapData { + return true + } + } + } + return false +} + +// dcgmExporterEnvEnablesPodMetadata returns true if a user-provided env var turns +// on pod metadata enrichment. +func dcgmExporterEnvEnablesPodMetadata(envs []gpuv1.EnvVar) bool { + for _, env := range envs { + if env.Name == DCGMExporterPodLabelsEnvName || env.Name == DCGMExporterPodUIDEnvName { + if strings.EqualFold(env.Value, "true") { + return true + } + } + } + return false +} + func addExtraAnnotations(obj *appsv1.DaemonSet, annotations map[string]string) { if obj.Spec.Template.Annotations == nil { obj.Spec.Template.Annotations = make(map[string]string) diff --git a/controllers/object_controls_test.go b/controllers/object_controls_test.go index d84917c0b..9c1bc9c7d 100644 --- a/controllers/object_controls_test.go +++ b/controllers/object_controls_test.go @@ -863,7 +863,7 @@ func testDaemonsetCommon(t *testing.T, cp *gpuv1.ClusterPolicy, component string require.Equal(t, spec.args, mainCtr.Args, "unexpected Args") } for _, env := range spec.env { - require.Contains(t, mainCtr.Env, env, "env var not present") + require.Contains(t, mainCtr.Env, corev1.EnvVar{Name: env.Name, Value: env.Value}, "env var not present") } // TODO: implement checks for other common fields (i.e. Resources, securityContext, Tolerations, etc.) @@ -1536,6 +1536,14 @@ func getDCGMExporterTestInput(testCase string) *gpuv1.ClusterPolicy { cp.Spec.DCGMExporter.EnablePodLabels = ptr.To(true) case "pod-uid-enabled": cp.Spec.DCGMExporter.EnablePodUID = ptr.To(true) + case "pod-labels-env": + cp.Spec.DCGMExporter.Env = []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"}, + } + case "configmap-data-env": + cp.Spec.DCGMExporter.Env = []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "gpu-operator:custom-metrics"}, + } default: return nil } @@ -1552,6 +1560,7 @@ func getDCGMExporterTestOutput(testCase string) map[string]interface{} { "dcgmExporterImage": "nvcr.io/nvidia/k8s/dcgm-exporter:3.3.0-3.2.0-ubuntu22.04", "imagePullSecret": "ngc-secret", "clusterRoleExists": false, + "automountToken": false, } switch testCase { @@ -1566,11 +1575,24 @@ func getDCGMExporterTestOutput(testCase string) map[string]interface{} { "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS": "true", } output["clusterRoleExists"] = true + output["automountToken"] = true case "pod-uid-enabled": output["env"] = map[string]string{ "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID": "true", } output["clusterRoleExists"] = true + output["automountToken"] = true + case "pod-labels-env": + output["env"] = map[string]string{ + "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS": "true", + } + output["clusterRoleExists"] = true + output["automountToken"] = true + case "configmap-data-env": + output["env"] = map[string]string{ + "DCGM_EXPORTER_CONFIGMAP_DATA": "gpu-operator:custom-metrics", + } + output["automountToken"] = true default: return nil } @@ -1606,6 +1628,16 @@ func TestDCGMExporter(t *testing.T) { getDCGMExporterTestInput("pod-uid-enabled"), getDCGMExporterTestOutput("pod-uid-enabled"), }, + { + "PodLabelsViaEnv", + getDCGMExporterTestInput("pod-labels-env"), + getDCGMExporterTestOutput("pod-labels-env"), + }, + { + "ConfigMapDataViaEnv", + getDCGMExporterTestInput("configmap-data-env"), + getDCGMExporterTestOutput("configmap-data-env"), + }, } for _, tc := range testCases { @@ -1643,15 +1675,15 @@ func TestDCGMExporter(t *testing.T) { if tc.output["clusterRoleExists"].(bool) { require.NoError(t, clusterRoleGetErr, "ClusterRole should exist when pod metadata enrichment is enabled") require.NoError(t, clusterRoleBindingGetErr, "ClusterRoleBinding should exist when pod metadata enrichment is enabled") - require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be set when pod metadata enrichment is enabled") - require.True(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be true when pod metadata enrichment is enabled") } else { require.True(t, apierrors.IsNotFound(clusterRoleGetErr), "ClusterRole should not exist when pod metadata enrichment is disabled (got err=%v)", clusterRoleGetErr) require.True(t, apierrors.IsNotFound(clusterRoleBindingGetErr), "ClusterRoleBinding should not exist when pod metadata enrichment is disabled (got err=%v)", clusterRoleBindingGetErr) - require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be explicitly set false when pod metadata enrichment is disabled") - require.False(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be false when pod metadata enrichment is disabled") } + require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should always be set explicitly") + require.Equal(t, tc.output["automountToken"].(bool), *ds.Spec.Template.Spec.AutomountServiceAccountToken, + "unexpected AutomountServiceAccountToken on daemonset nvidia-dcgm-exporter") + require.Equal(t, tc.output["dcgmExporterImage"], dcgmExporterImage, "Unexpected configuration for dcgm-exporter image") // cleanup by deleting all kubernetes objects diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index fcbb6732a..0ff56f8be 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -1730,6 +1730,92 @@ func TestTransformDCGMExporter(t *testing.T) { WithRuntimeClassName("nvidia"). WithAutomountServiceAccountToken(true), }, + { + description: "transform dcgm exporter with custom metrics configmap env", + ds: NewDaemonset(). + WithContainer(corev1.Container{Name: "dcgm-exporter"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DCGMExporter: gpuv1.DCGMExporterSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "dcgm-exporter", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Env: []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "gpu-operator:exporter-metrics-config-map"}, + }, + }, + DCGM: gpuv1.DCGMSpec{ + Enabled: newBoolPtr(true), + }, + }, + expectedDs: NewDaemonset().WithContainer(corev1.Container{ + Name: "dcgm-exporter", + Image: "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"}, + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "gpu-operator:exporter-metrics-config-map"}, + }, + }).WithRuntimeClassName("nvidia"). + WithAutomountServiceAccountToken(true), + }, + { + description: "transform dcgm exporter with configmap data env set to none", + ds: NewDaemonset(). + WithContainer(corev1.Container{Name: "dcgm-exporter"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DCGMExporter: gpuv1.DCGMExporterSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "dcgm-exporter", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Env: []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "none"}, + }, + }, + DCGM: gpuv1.DCGMSpec{ + Enabled: newBoolPtr(true), + }, + }, + expectedDs: NewDaemonset().WithContainer(corev1.Container{ + Name: "dcgm-exporter", + Image: "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"}, + {Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "none"}, + }, + }).WithRuntimeClassName("nvidia"), + }, + { + description: "transform dcgm exporter with pod labels enabled via raw env", + ds: NewDaemonset(). + WithContainer(corev1.Container{Name: "dcgm-exporter"}), + cpSpec: &gpuv1.ClusterPolicySpec{ + DCGMExporter: gpuv1.DCGMExporterSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "dcgm-exporter", + Version: "v1.0.0", + ImagePullPolicy: "IfNotPresent", + Env: []gpuv1.EnvVar{ + {Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"}, + }, + }, + DCGM: gpuv1.DCGMSpec{ + Enabled: newBoolPtr(true), + }, + }, + expectedDs: NewDaemonset().WithContainer(corev1.Container{ + Name: "dcgm-exporter", + Image: "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0", + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + {Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"}, + {Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"}, + }, + }).WithRuntimeClassName("nvidia"). + WithAutomountServiceAccountToken(true), + }, } for _, tc := range testCases {