diff --git a/.github/renovate.json b/.github/renovate.json index 8df57771cd..ed7231b40b 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -34,7 +34,8 @@ { "customType": "regex", "managerFilePatterns": [ - "deployments/gpu-operator/values.yaml" + "deployments/gpu-operator/values.yaml", + "config/samples/nvidia_v1alpha1_gpucluster.yaml" ], "matchStrings": [ "[-\\s]*repository:\\s*(?\\S+)\\s*\\n(?:\\s*#.*\\n|\\s*\\n)*[-\\s]*image:\\s*(?\\S+)\\s*\\n(?:\\s*#.*\\n|\\s*\\n)*[-\\s]*version:\\s*\"?(?[^\"\\s]+)\"?" @@ -113,6 +114,7 @@ { "matchFileNames": [ "deployments/gpu-operator/values.yaml", + "config/samples/nvidia_v1alpha1_gpucluster.yaml", "bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml" ], "matchPackageNames": [ @@ -121,6 +123,18 @@ "versioning": "regex:^v?(?\\d+)\\.(?\\d+)\\.(?\\d+)-\\d+\\.\\d+\\.\\d+-distroless$", "separateMajorMinor": false }, + { + "description": "Keep DRA driver version current in both the sample GPUCluster CR and the OLM bundle", + "matchFileNames": [ + "config/samples/nvidia_v1alpha1_gpucluster.yaml", + "bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml" + ], + "matchPackageNames": [ + "registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu" + ], + "versioning": "regex:^v?(?\\d+)\\.(?\\d+)\\.(?\\d+)$", + "separateMajorMinor": false + }, { "matchFileNames": ["bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml"], "matchPackageNames": [ diff --git a/cmd/cleanup-gpuclusters/main.go b/cmd/check-gpuclusters/main.go similarity index 52% rename from cmd/cleanup-gpuclusters/main.go rename to cmd/check-gpuclusters/main.go index 546af7e310..8b83ef8b8a 100644 --- a/cmd/cleanup-gpuclusters/main.go +++ b/cmd/check-gpuclusters/main.go @@ -20,12 +20,12 @@ import ( "context" "fmt" "os" + "strings" "time" log "github.com/sirupsen/logrus" "github.com/urfave/cli/v3" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,11 +39,11 @@ var logger = log.New() func main() { var debug bool - var crName string + var timeout time.Duration c := cli.Command{} - c.Name = "cleanup-gpuclusters" - c.Usage = "Delete the chart-managed GPUCluster CR and wait until it is gone" + c.Name = "check-gpuclusters" + c.Usage = "Fail while GPUCluster CRs exist so that helm uninstall aborts before the operator is removed" c.Version = info.GetVersionString() c.Flags = []cli.Flag{ &cli.BoolFlag{ @@ -53,12 +53,12 @@ func main() { Destination: &debug, Sources: cli.EnvVars("DEBUG"), }, - &cli.StringFlag{ - Name: "gpucluster-name", - Usage: "Name of the chart-managed GPUCluster CR to delete", - Required: true, - Destination: &crName, - Sources: cli.EnvVars("GPUCLUSTER_NAME"), + &cli.DurationFlag{ + Name: "timeout", + Usage: "How long to wait for already-terminating GPUCluster CRs to be deleted", + Value: 5 * time.Minute, + Destination: &timeout, + Sources: cli.EnvVars("TIMEOUT"), }, } c.Before = func(ctx context.Context, cli *cli.Command) (context.Context, error) { @@ -70,7 +70,7 @@ func main() { return ctx, nil } c.Action = func(ctx context.Context, _ *cli.Command) error { - return runDeleteGPUCluster(ctx, crName) + return checkGPUClusters(ctx, timeout) } err := c.Run(context.Background(), os.Args) @@ -80,12 +80,13 @@ func main() { } } -// runDeleteGPUCluster deletes the named GPUCluster CR and blocks until it is gone. The -// GPUCluster controller drains ResourceClaim-consuming operands under a finalizer -// before the CR disappears, so waiting here (from the chart's pre-delete hook) keeps -// the operator alive until that ordered teardown has completed. Scoped to the chart's -// own CR by name so CRs created outside the chart are never touched. -func runDeleteGPUCluster(ctx context.Context, name string) error { +// checkGPUClusters fails while any GPUCluster CR exists, so that the chart's pre-delete hook +// aborts helm uninstall before the operator is removed. A GPUCluster deleted after the +// operator is gone has no controller to process its finalizer: the CR stays stuck +// terminating and the DRA operands keep running. CRs that are already terminating are +// waited on instead of failed on, so an uninstall that follows a delete succeeds +// once the finalizer drain completes. The guard only lists CRs; it never deletes them. +func checkGPUClusters(ctx context.Context, timeout time.Duration) error { scheme := runtime.NewScheme() if err := nvidiav1alpha1.AddToScheme(scheme); err != nil { return fmt.Errorf("failed to add GPUCluster types to scheme: %w", err) @@ -99,34 +100,42 @@ func runDeleteGPUCluster(ctx context.Context, name string) error { return fmt.Errorf("failed to create client: %w", err) } - ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() for { - cr := &nvidiav1alpha1.GPUCluster{} - if err := k8sClient.Get(ctx, ctrlclient.ObjectKey{Name: name}, cr); err != nil { - // The GPUCluster CRD may not be installed (e.g. cleanup enabled without - // the DRA stack ever deployed); nothing to drain in that case. + list := &nvidiav1alpha1.GPUClusterList{} + if err := k8sClient.List(ctx, list); err != nil { + // The GPUCluster CRD may not be installed (e.g. the DRA stack was never + // deployed); nothing to check in that case. if meta.IsNoMatchError(err) { - logger.Info("GPUCluster CRD not installed, nothing to delete") + logger.Info("GPUCluster CRD not installed, nothing to check") return nil } - if apierrors.IsNotFound(err) { - logger.Infof("GPUCluster %s deleted", name) - return nil - } - return fmt.Errorf("failed to get GPUCluster %s: %w", name, err) + return fmt.Errorf("failed to list GPUCluster objects: %w", err) } - if cr.DeletionTimestamp.IsZero() { - logger.Infof("Deleting GPUCluster %s", name) - if err := k8sClient.Delete(ctx, cr); err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to delete GPUCluster %s: %w", name, err) + + var live []string + terminating := 0 + for _, cr := range list.Items { + if cr.DeletionTimestamp.IsZero() { + live = append(live, cr.Name) + } else { + terminating++ } } - logger.Infof("Waiting for GPUCluster %s to be deleted", name) + if len(live) > 0 { + return fmt.Errorf("GPUCluster CR(s) %s exist; delete them (kubectl delete gpuclusters %s) and wait for the deletion to complete, then retry the uninstall", + strings.Join(live, ", "), strings.Join(live, " ")) + } + if terminating == 0 { + logger.Info("No GPUCluster CRs found") + return nil + } + logger.Infof("Waiting for %d terminating GPUCluster CR(s) to be deleted", terminating) select { case <-ctx.Done(): - return fmt.Errorf("timed out waiting for GPUCluster %s to be deleted: %w", name, ctx.Err()) + return fmt.Errorf("timed out waiting for terminating GPUCluster CRs to be deleted: %w", ctx.Err()) case <-time.After(5 * time.Second): } } diff --git a/deployments/gpu-operator/templates/_helpers.tpl b/deployments/gpu-operator/templates/_helpers.tpl index 83ecd97cf0..088a8132c6 100644 --- a/deployments/gpu-operator/templates/_helpers.tpl +++ b/deployments/gpu-operator/templates/_helpers.tpl @@ -85,11 +85,3 @@ Full image name with tag {{- define "validator.fullimage" -}} {{- .Values.validator.repository -}}/{{- .Values.validator.image -}}:{{- .Values.validator.version | default .Chart.AppVersion -}} {{- end }} - -{{/* -Name of the chart-managed GPUCluster CR; the pre-delete cleanup hook deletes it by -this name. -*/}} -{{- define "gpu-operator.gpucluster-name" -}} -gpu-cluster -{{- end }} diff --git a/deployments/gpu-operator/templates/cleanup_gpucluster.yaml b/deployments/gpu-operator/templates/check_gpucluster.yaml similarity index 58% rename from deployments/gpu-operator/templates/cleanup_gpucluster.yaml rename to deployments/gpu-operator/templates/check_gpucluster.yaml index f5d0d83744..dcf51b493a 100644 --- a/deployments/gpu-operator/templates/cleanup_gpucluster.yaml +++ b/deployments/gpu-operator/templates/check_gpucluster.yaml @@ -1,15 +1,15 @@ apiVersion: batch/v1 kind: Job metadata: - name: gpu-operator-cleanup-gpucluster + name: gpu-operator-delete-gpuclusters-before-uninstall namespace: {{ .Release.Namespace }} annotations: - # Delete the chart-managed GPUCluster CR and wait for it to be gone before helm - # removes the operator (and, with operator.cleanupCRD, before the CRD cleanup - # hook runs). The operator drains ResourceClaim-consuming operands under a - # finalizer on CR deletion, so it must stay alive until the CR has disappeared. - # Unconditional because the CR outlives a gpuCluster.deployCR flip (resource - # policy keep); a no-op when the CRD or the CR does not exist. + # Fail the pre-delete phase while GPUCluster CRs exist, so helm does + # not remove the operator before it has processed their finalizers. + # A GPUCluster deleted after the operator is gone stays stuck terminating, + # with the DRA operands running. The guard never deletes CRs; it waits + # for CRs that are already terminating and is a no-op when the CRD or no + # CRs exist. "helm.sh/hook": pre-delete "helm.sh/hook-weight": "0" "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation @@ -17,9 +17,12 @@ metadata: {{- include "gpu-operator.labels" . | nindent 4 }} app.kubernetes.io/component: "gpu-operator" spec: + # Fail the uninstall on the first pod failure instead of retrying into + # helm's timeout. + backoffLimit: 0 template: metadata: - name: gpu-operator-cleanup-gpucluster + name: gpu-operator-delete-gpuclusters-before-uninstall labels: {{- include "gpu-operator.labels" . | nindent 8 }} app.kubernetes.io/component: "gpu-operator" @@ -38,11 +41,9 @@ spec: nodeSelector: {{- toYaml .Values.operator.nodeSelector | nindent 8 }} containers: - - name: cleanup-gpucluster + - name: check-gpuclusters image: {{ include "gpu-operator.fullimage" . }} imagePullPolicy: {{ .Values.operator.imagePullPolicy }} command: - - /usr/bin/cleanup-gpuclusters - - --gpucluster-name - - {{ include "gpu-operator.gpucluster-name" . }} - restartPolicy: OnFailure + - /usr/bin/check-gpuclusters + restartPolicy: Never diff --git a/deployments/gpu-operator/templates/gpucluster.yaml b/deployments/gpu-operator/templates/gpucluster.yaml deleted file mode 100644 index 51a96ab966..0000000000 --- a/deployments/gpu-operator/templates/gpucluster.yaml +++ /dev/null @@ -1,137 +0,0 @@ -{{- if .Values.gpuCluster.deployCR }} -apiVersion: nvidia.com/v1alpha1 -kind: GPUCluster -metadata: - name: {{ include "gpu-operator.gpucluster-name" . }} - labels: - {{- include "gpu-operator.labels" . | nindent 4 }} - app.kubernetes.io/component: "gpu-operator" - # CR deletion is owned by the pre-delete cleanup hook (finalizer drain); - # keep helm from also deleting it. - annotations: - "helm.sh/resource-policy": keep -spec: - draDriver: - {{- if .Values.draDriver.repository }} - repository: {{ .Values.draDriver.repository }} - {{- end }} - {{- if .Values.draDriver.image }} - image: {{ .Values.draDriver.image }} - {{- end }} - {{- if .Values.draDriver.version }} - version: {{ .Values.draDriver.version | quote }} - {{- end }} - {{- if .Values.draDriver.imagePullPolicy }} - imagePullPolicy: {{ .Values.draDriver.imagePullPolicy }} - {{- end }} - {{- if .Values.draDriver.imagePullSecrets }} - imagePullSecrets: {{ toYaml .Values.draDriver.imagePullSecrets | nindent 6 }} - {{- end }} - {{- if .Values.draDriver.featureGates }} - featureGates: {{ toYaml .Values.draDriver.featureGates | nindent 6 }} - {{- end }} - {{- if .Values.draDriver.gpus.kubeletPlugin }} - gpus: - kubeletPlugin: {{ toYaml .Values.draDriver.gpus.kubeletPlugin | nindent 8 }} - {{- end }} - computeDomains: - enabled: {{ .Values.draDriver.computeDomains.enabled }} - {{- if .Values.draDriver.computeDomains.controller }} - controller: {{ toYaml .Values.draDriver.computeDomains.controller | nindent 8 }} - {{- end }} - {{- if .Values.draDriver.computeDomains.kubeletPlugin }} - kubeletPlugin: {{ toYaml .Values.draDriver.computeDomains.kubeletPlugin | nindent 8 }} - {{- end }} - dcgm: - enabled: {{ .Values.dcgm.enabled }} - {{- if .Values.dcgm.repository }} - repository: {{ .Values.dcgm.repository }} - {{- end }} - {{- if .Values.dcgm.image }} - image: {{ .Values.dcgm.image }} - {{- end }} - {{- if .Values.dcgm.version }} - version: {{ .Values.dcgm.version | quote }} - {{- end }} - {{- if .Values.dcgm.imagePullPolicy }} - imagePullPolicy: {{ .Values.dcgm.imagePullPolicy }} - {{- end }} - {{- if .Values.dcgm.imagePullSecrets }} - imagePullSecrets: {{ toYaml .Values.dcgm.imagePullSecrets | nindent 6 }} - {{- end }} - {{- if .Values.dcgm.args }} - args: {{ toYaml .Values.dcgm.args | nindent 6 }} - {{- end }} - {{- if .Values.dcgm.env }} - env: {{ toYaml .Values.dcgm.env | nindent 6 }} - {{- end }} - {{- if .Values.dcgm.resources }} - resources: {{ toYaml .Values.dcgm.resources | nindent 6 }} - {{- end }} - {{- if .Values.dcgm.hostNetwork }} - hostNetwork: {{ .Values.dcgm.hostNetwork }} - {{- end }} - dcgmExporter: - enabled: {{ .Values.dcgmExporter.enabled }} - {{- if .Values.dcgmExporter.repository }} - repository: {{ .Values.dcgmExporter.repository }} - {{- end }} - {{- if .Values.dcgmExporter.image }} - image: {{ .Values.dcgmExporter.image }} - {{- end }} - {{- if .Values.dcgmExporter.version }} - version: {{ .Values.dcgmExporter.version | quote }} - {{- end }} - {{- if .Values.dcgmExporter.imagePullPolicy }} - imagePullPolicy: {{ .Values.dcgmExporter.imagePullPolicy }} - {{- end }} - {{- if .Values.dcgmExporter.imagePullSecrets }} - imagePullSecrets: {{ toYaml .Values.dcgmExporter.imagePullSecrets | nindent 6 }} - {{- end }} - {{- if .Values.dcgmExporter.annotations }} - annotations: {{ toYaml .Values.dcgmExporter.annotations | nindent 6 }} - {{- end }} - {{- if .Values.dcgmExporter.args }} - args: {{ toYaml .Values.dcgmExporter.args | nindent 6 }} - {{- end }} - {{- if .Values.dcgmExporter.env }} - env: {{ toYaml .Values.dcgmExporter.env | nindent 6 }} - {{- end }} - {{- if .Values.dcgmExporter.resources }} - resources: {{ toYaml .Values.dcgmExporter.resources | nindent 6 }} - {{- end }} - {{- if .Values.dcgmExporter.hostPID }} - hostPID: {{ .Values.dcgmExporter.hostPID }} - {{- end }} - {{- if .Values.dcgmExporter.hostNetwork }} - hostNetwork: {{ .Values.dcgmExporter.hostNetwork }} - {{- end }} - {{- if .Values.dcgmExporter.hpcJobMapping }} - hpcJobMapping: {{ toYaml .Values.dcgmExporter.hpcJobMapping | nindent 6 }} - {{- end }} - {{- if .Values.dcgmExporter.enablePodLabels }} - enablePodLabels: {{ .Values.dcgmExporter.enablePodLabels }} - {{- end }} - {{- if .Values.dcgmExporter.enablePodUID }} - enablePodUID: {{ .Values.dcgmExporter.enablePodUID }} - {{- end }} - {{- if .Values.dcgmExporter.podLabelAllowlistRegex }} - podLabelAllowlistRegex: {{ toYaml .Values.dcgmExporter.podLabelAllowlistRegex | nindent 6 }} - {{- end }} - {{- if .Values.dcgmExporter.service }} - service: {{ toYaml .Values.dcgmExporter.service | nindent 6 }} - {{- end }} - {{- if .Values.dcgmExporter.serviceMonitor }} - serviceMonitor: {{ toYaml .Values.dcgmExporter.serviceMonitor | nindent 6 }} - {{- end }} - {{- if and (.Values.dcgmExporter.config) (.Values.dcgmExporter.config.name) }} - config: - name: {{ .Values.dcgmExporter.config.name }} - {{- end }} - hostPaths: - driverInstallDir: {{ .Values.hostPaths.driverInstallDir }} - {{- if .Values.hostPaths.kubeletRootDir }} - kubeletRootDir: {{ .Values.hostPaths.kubeletRootDir }} - {{- end }} - daemonsets: {{ toYaml .Values.daemonsets | nindent 4 }} -{{- end }} diff --git a/deployments/gpu-operator/templates/validations.yaml b/deployments/gpu-operator/templates/validations.yaml index bb4677ebbe..c37e44e9d7 100644 --- a/deployments/gpu-operator/templates/validations.yaml +++ b/deployments/gpu-operator/templates/validations.yaml @@ -1,11 +1,3 @@ -{{- if and .Values.clusterPolicy.deployCR .Values.gpuCluster.deployCR }} -{{ fail "clusterPolicy.deployCR and gpuCluster.deployCR cannot both be true; only one CR can exist" }} -{{- end }} - -{{- if and .Values.gpuCluster.deployCR (and .Values.driver.enabled (not .Values.driver.nvidiaDriverCRD.enabled)) }} -{{ fail "the NVIDIADriver CRD must be enabled when deploying a GPUCluster CR, set driver.nvidiaDriverCRD.enabled=true" }} -{{- end }} - {{- if and (eq .Values.cdi.enabled false) (eq .Values.cdi.nriPluginEnabled true) }} {{ fail "the NRI Plugin cannot be enabled when CDI is disabled" }} {{- end }} @@ -21,10 +13,3 @@ {{- if and (.Values.devicePlugin.config.create) (empty .Values.devicePlugin.config.name) }} {{ fail "devicePlugin.config.name cannot be empty when devicePlugin.config.create is set to true" }} {{- end }} - -{{- if .Values.gpuCluster.deployCR }} -{{- $clusterSupportsDRA := or (.Capabilities.APIVersions.Has "resource.k8s.io/v1/DeviceClass") (.Capabilities.APIVersions.Has "resource.k8s.io/v1beta2/DeviceClass") (.Capabilities.APIVersions.Has "resource.k8s.io/v1beta1/DeviceClass") }} -{{- if not $clusterSupportsDRA }} -{{ fail "gpuCluster.deployCR=true requires a Kubernetes cluster that supports Dynamic Resource Allocation (no resource.k8s.io DeviceClass API is served). When rendering offline with 'helm template', pass --api-versions resource.k8s.io/v1/DeviceClass" }} -{{- end }} -{{- end }} diff --git a/deployments/gpu-operator/values.yaml b/deployments/gpu-operator/values.yaml index 84dd5a0517..126ee6f02f 100644 --- a/deployments/gpu-operator/values.yaml +++ b/deployments/gpu-operator/values.yaml @@ -565,52 +565,6 @@ ccManager: clusterPolicy: deployCR: true -# The GPUCluster CR stores the desired state of the GPU -# enablement stack based on Dynamic Resource Allocation (DRA). -# This is an experimental feature and is disabled by default. -# -# To enable the GPUCluster (DRA) enablement stack, set -# gpuCluster.deployCR=true and clusterPolicy.deployCR=false -# It is an invalid configuration for both CRs to exist. -gpuCluster: - deployCR: false - -# draDriver configures the NVIDIA DRA driver for GPUs which -# is managed by the GPUCluster CR (rendered only when -# gpuCluster.deployCR=true). -draDriver: - repository: registry.k8s.io/dra-driver-nvidia - image: dra-driver-nvidia-gpu - version: v0.4.1 - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - # featureGates toggles DRA driver feature gates; rendered as FEATURE_GATES. - # e.g. featureGates: {MPSSupport: true, NVMLDeviceHealthCheck: true} - featureGates: {} - # gpus configures the gpu.nvidia.com / mig.nvidia.com / vfio.gpu.nvidia.com - # capability (the gpus container of the kubelet-plugin DaemonSet). It is - # always deployed. - gpus: - # kubeletPlugin overrides env/resources for the gpus container. - # Scheduling is opinionated and not configurable here. All fields are optional: - kubeletPlugin: {} - # env: [] # list of {name, value} - # resources: {} # requests/limits - # healthcheck: {} # {enabled, port}; port defaults to 51516/51515 - # computeDomains configures the Multi-Node NVLink (MNNVL) compute-domain - # capability: a controller Deployment plus the compute-domains kubelet-plugin - # container. - computeDomains: - enabled: true - # controller overrides env/resources for the compute-domain controller Deployment. - # Scheduling is opinionated and not configurable here. All fields are optional: - controller: {} - # env: [] - # resources: {} - # kubeletPlugin overrides the compute-domains container (same fields as - # gpus.kubeletPlugin above). - kubeletPlugin: {} - # Array of extra K8s manifests to deploy # Supports use of custom Helm templates extraObjects: [] diff --git a/docker/Dockerfile b/docker/Dockerfile index 0b95563d31..90edeadafa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -108,7 +108,7 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/busybox COPY --from=builder /workspace/gpu-operator /usr/bin/ COPY --from=builder /workspace/manage-crds /usr/bin/ -COPY --from=builder /workspace/cleanup-gpuclusters /usr/bin/ +COPY --from=builder /workspace/check-gpuclusters /usr/bin/ COPY --from=builder /workspace/nvidia-validator /usr/bin/ COPY --from=sample-builder /build/vectorAdd /usr/bin/vectorAdd ARG CUDA_SAMPLES_VERSION