Skip to content

Mount ServiceAccount token when dcgm-exporter env needs API access #2721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rajathagasthya wants to merge 1 commit into
base: main
Choose a base branch
from
+172 −12
Open

Mount ServiceAccount token when dcgm-exporter env needs API access #2721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 49 additions & 7 deletions controllers/object_controls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,17 @@ const (
DCGMRemoteEngineEnvName = "DCGM_REMOTE_HOSTENGINE_INFO"
// DCGMDefaultPort indicates default port bound to DCGM host engine
DCGMDefaultPort = 5555
// DCGMExporterConfigMapDataEnvName is the env name specifying the namespace:name
// ConfigMap with custom metrics
DCGMExporterConfigMapDataEnvName = "DCGM_EXPORTER_CONFIGMAP_DATA"
// DCGMExporterUndefinedConfigMapData is the exporter default meaning no ConfigMap is read
DCGMExporterUndefinedConfigMapData = "none"
// DCGMExporterPodLabelsEnvName is the env name enabling pod labels on metrics
DCGMExporterPodLabelsEnvName = "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS"
// DCGMExporterPodUIDEnvName is the env name enabling the pod UID label on metrics
DCGMExporterPodUIDEnvName = "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID"
// DCGMExporterPodLabelAllowlistEnvName is the env name holding the exported pod label allowlist
DCGMExporterPodLabelAllowlistEnvName = "DCGM_EXPORTER_KUBERNETES_POD_LABEL_ALLOWLIST_REGEX"
// GPUDirectRDMAEnabledEnvName indicates if GPU direct RDMA is enabled through GPU operator
GPUDirectRDMAEnabledEnvName = "GPU_DIRECT_RDMA_ENABLED"
// UseHostMOFEDEnvName indicates if MOFED driver is pre-installed on the host
Expand Down Expand Up @@ -446,7 +457,8 @@ func RoleBinding(n ClusterPolicyController) (gpuv1.State, error) {

var rbacGates = map[string]func(*gpuv1.ClusterPolicySpec) bool{
"nvidia-dcgm-exporter-read-pods": func(config *gpuv1.ClusterPolicySpec) bool {
return config.DCGMExporter.IsKubernetesPodMetadataEnabled()
return config.DCGMExporter.IsKubernetesPodMetadataEnabled() ||
dcgmExporterEnvEnablesPodMetadata(config.DCGMExporter.Env)
},
}

Expand Down Expand Up @@ -1868,20 +1880,20 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
// Inject pod-metadata enrichment env vars; RBAC is provisioned via the
// 0210/0310 assets and the SA token is mounted below.
if config.DCGMExporter.IsPodLabelsEnabled() {
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", "true")
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DCGMExporterPodLabelsEnvName, "true")
}
if config.DCGMExporter.IsPodUIDEnabled() {
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID", "true")
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), DCGMExporterPodUIDEnvName, "true")
}
if len(config.DCGMExporter.PodLabelAllowlistRegex) > 0 {
setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]),
"DCGM_EXPORTER_KUBERNETES_POD_LABEL_ALLOWLIST_REGEX",
DCGMExporterPodLabelAllowlistEnvName,
strings.Join(config.DCGMExporter.PodLabelAllowlistRegex, ","))
}

// Override the base asset's automountServiceAccountToken=false when
// enrichment is on so the pod informer has client-go credentials.
if config.DCGMExporter.IsKubernetesPodMetadataEnabled() {
// Override the base asset's automountServiceAccountToken=false when the
// configuration reads from the Kubernetes API.
if config.DCGMExporter.IsKubernetesPodMetadataEnabled() || dcgmExporterEnvRequiresAPIAccess(config.DCGMExporter.Env) {
obj.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(true)
}

Expand Down Expand Up @@ -1928,6 +1940,36 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
return nil
}

// dcgmExporterEnvRequiresAPIAccess returns true if a user-provided env var makes
// dcgm-exporter read from the Kubernetes API.
func dcgmExporterEnvRequiresAPIAccess(envs []gpuv1.EnvVar) bool {
if dcgmExporterEnvEnablesPodMetadata(envs) {
return true
}
for _, env := range envs {
if env.Name == DCGMExporterConfigMapDataEnvName {
// the exporter treats "none" and "" as no ConfigMap read
if env.Value != "" && env.Value != DCGMExporterUndefinedConfigMapData {
return true
}
}
}
return false
}

// dcgmExporterEnvEnablesPodMetadata returns true if a user-provided env var turns
// on pod metadata enrichment.
func dcgmExporterEnvEnablesPodMetadata(envs []gpuv1.EnvVar) bool {
for _, env := range envs {
if env.Name == DCGMExporterPodLabelsEnvName || env.Name == DCGMExporterPodUIDEnvName {
if strings.EqualFold(env.Value, "true") {
return true
}
}
}
return false
}

func addExtraAnnotations(obj *appsv1.DaemonSet, annotations map[string]string) {
if obj.Spec.Template.Annotations == nil {
obj.Spec.Template.Annotations = make(map[string]string)
Expand Down
42 changes: 37 additions & 5 deletions controllers/object_controls_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -863,7 +863,7 @@ func testDaemonsetCommon(t *testing.T, cp *gpuv1.ClusterPolicy, component string
require.Equal(t, spec.args, mainCtr.Args, "unexpected Args")
}
for _, env := range spec.env {
require.Contains(t, mainCtr.Env, env, "env var not present")
require.Contains(t, mainCtr.Env, corev1.EnvVar{Name: env.Name, Value: env.Value}, "env var not present")
}
// TODO: implement checks for other common fields (i.e. Resources, securityContext, Tolerations, etc.)

Expand Down Expand Up @@ -1536,6 +1536,14 @@ func getDCGMExporterTestInput(testCase string) *gpuv1.ClusterPolicy {
cp.Spec.DCGMExporter.EnablePodLabels = ptr.To(true)
case "pod-uid-enabled":
cp.Spec.DCGMExporter.EnablePodUID = ptr.To(true)
case "pod-labels-env":
cp.Spec.DCGMExporter.Env = []gpuv1.EnvVar{
{Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"},
}
case "configmap-data-env":
cp.Spec.DCGMExporter.Env = []gpuv1.EnvVar{
{Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "gpu-operator:custom-metrics"},
}
default:
return nil
}
Expand All @@ -1552,6 +1560,7 @@ func getDCGMExporterTestOutput(testCase string) map[string]interface{} {
"dcgmExporterImage": "nvcr.io/nvidia/k8s/dcgm-exporter:3.3.0-3.2.0-ubuntu22.04",
"imagePullSecret": "ngc-secret",
"clusterRoleExists": false,
"automountToken": false,
}

switch testCase {
Expand All @@ -1566,11 +1575,24 @@ func getDCGMExporterTestOutput(testCase string) map[string]interface{} {
"DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS": "true",
}
output["clusterRoleExists"] = true
output["automountToken"] = true
case "pod-uid-enabled":
output["env"] = map[string]string{
"DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID": "true",
}
output["clusterRoleExists"] = true
output["automountToken"] = true
case "pod-labels-env":
output["env"] = map[string]string{
"DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS": "true",
}
output["clusterRoleExists"] = true
output["automountToken"] = true
case "configmap-data-env":
output["env"] = map[string]string{
"DCGM_EXPORTER_CONFIGMAP_DATA": "gpu-operator:custom-metrics",
}
output["automountToken"] = true
default:
return nil
}
Expand Down Expand Up @@ -1606,6 +1628,16 @@ func TestDCGMExporter(t *testing.T) {
getDCGMExporterTestInput("pod-uid-enabled"),
getDCGMExporterTestOutput("pod-uid-enabled"),
},
{
"PodLabelsViaEnv",
getDCGMExporterTestInput("pod-labels-env"),
getDCGMExporterTestOutput("pod-labels-env"),
},
{
"ConfigMapDataViaEnv",
getDCGMExporterTestInput("configmap-data-env"),
getDCGMExporterTestOutput("configmap-data-env"),
},
}

for _, tc := range testCases {
Expand Down Expand Up @@ -1643,15 +1675,15 @@ func TestDCGMExporter(t *testing.T) {
if tc.output["clusterRoleExists"].(bool) {
require.NoError(t, clusterRoleGetErr, "ClusterRole should exist when pod metadata enrichment is enabled")
require.NoError(t, clusterRoleBindingGetErr, "ClusterRoleBinding should exist when pod metadata enrichment is enabled")
require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be set when pod metadata enrichment is enabled")
require.True(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be true when pod metadata enrichment is enabled")
} else {
require.True(t, apierrors.IsNotFound(clusterRoleGetErr), "ClusterRole should not exist when pod metadata enrichment is disabled (got err=%v)", clusterRoleGetErr)
require.True(t, apierrors.IsNotFound(clusterRoleBindingGetErr), "ClusterRoleBinding should not exist when pod metadata enrichment is disabled (got err=%v)", clusterRoleBindingGetErr)
require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be explicitly set false when pod metadata enrichment is disabled")
require.False(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be false when pod metadata enrichment is disabled")
}

require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should always be set explicitly")
require.Equal(t, tc.output["automountToken"].(bool), *ds.Spec.Template.Spec.AutomountServiceAccountToken,
"unexpected AutomountServiceAccountToken on daemonset nvidia-dcgm-exporter")

require.Equal(t, tc.output["dcgmExporterImage"], dcgmExporterImage, "Unexpected configuration for dcgm-exporter image")

// cleanup by deleting all kubernetes objects
Expand Down
86 changes: 86 additions & 0 deletions controllers/transforms_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1730,6 +1730,92 @@ func TestTransformDCGMExporter(t *testing.T) {
WithRuntimeClassName("nvidia").
WithAutomountServiceAccountToken(true),
},
{
description: "transform dcgm exporter with custom metrics configmap env",
ds: NewDaemonset().
WithContainer(corev1.Container{Name: "dcgm-exporter"}),
cpSpec: &gpuv1.ClusterPolicySpec{
DCGMExporter: gpuv1.DCGMExporterSpec{
Repository: "nvcr.io/nvidia/cloud-native",
Image: "dcgm-exporter",
Version: "v1.0.0",
ImagePullPolicy: "IfNotPresent",
Env: []gpuv1.EnvVar{
{Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "gpu-operator:exporter-metrics-config-map"},
},
},
DCGM: gpuv1.DCGMSpec{
Enabled: newBoolPtr(true),
},
},
expectedDs: NewDaemonset().WithContainer(corev1.Container{
Name: "dcgm-exporter",
Image: "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0",
ImagePullPolicy: corev1.PullIfNotPresent,
Env: []corev1.EnvVar{
{Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"},
{Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "gpu-operator:exporter-metrics-config-map"},
},
}).WithRuntimeClassName("nvidia").
WithAutomountServiceAccountToken(true),
},
{
description: "transform dcgm exporter with configmap data env set to none",
ds: NewDaemonset().
WithContainer(corev1.Container{Name: "dcgm-exporter"}),
cpSpec: &gpuv1.ClusterPolicySpec{
DCGMExporter: gpuv1.DCGMExporterSpec{
Repository: "nvcr.io/nvidia/cloud-native",
Image: "dcgm-exporter",
Version: "v1.0.0",
ImagePullPolicy: "IfNotPresent",
Env: []gpuv1.EnvVar{
{Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "none"},
},
},
DCGM: gpuv1.DCGMSpec{
Enabled: newBoolPtr(true),
},
},
expectedDs: NewDaemonset().WithContainer(corev1.Container{
Name: "dcgm-exporter",
Image: "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0",
ImagePullPolicy: corev1.PullIfNotPresent,
Env: []corev1.EnvVar{
{Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"},
{Name: "DCGM_EXPORTER_CONFIGMAP_DATA", Value: "none"},
},
}).WithRuntimeClassName("nvidia"),
},
{
description: "transform dcgm exporter with pod labels enabled via raw env",
ds: NewDaemonset().
WithContainer(corev1.Container{Name: "dcgm-exporter"}),
cpSpec: &gpuv1.ClusterPolicySpec{
DCGMExporter: gpuv1.DCGMExporterSpec{
Repository: "nvcr.io/nvidia/cloud-native",
Image: "dcgm-exporter",
Version: "v1.0.0",
ImagePullPolicy: "IfNotPresent",
Env: []gpuv1.EnvVar{
{Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"},
},
},
DCGM: gpuv1.DCGMSpec{
Enabled: newBoolPtr(true),
},
},
expectedDs: NewDaemonset().WithContainer(corev1.Container{
Name: "dcgm-exporter",
Image: "nvcr.io/nvidia/cloud-native/dcgm-exporter:v1.0.0",
ImagePullPolicy: corev1.PullIfNotPresent,
Env: []corev1.EnvVar{
{Name: "DCGM_REMOTE_HOSTENGINE_INFO", Value: "nvidia-dcgm:5555"},
{Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"},
},
}).WithRuntimeClassName("nvidia").
WithAutomountServiceAccountToken(true),
},
}

for _, tc := range testCases {
Expand Down
Loading