STOR-3090: Add CSI storage test for pod delete after host umount of mounted volume

pkg/clioptions/clusterdiscovery/csi.go

@@ -33,7 +33,7 @@ func InitCSITests() error {
 	}
 
 	// Load OCP specific tests first, because AddOpenShiftCSITests() modifies global list of
-	// testsuites.CSISuites used by AddDriverDefinition() below.
+	// testsuites.CSISuites and the OpenShift driver config registry used by those suites.
 	ocpManifestList := os.Getenv(OCPManifestEnvVar)
 	if ocpManifestList != "" {
 		manifests := strings.Split(ocpManifestList, ",")

test/extended/storage/csi/README.md

@@ -19,11 +19,15 @@ Example:
 
 ```yaml
 Driver: <CSI driver name>
+Capabilities:
+  podDeleteAfterUmount: true
 LUNStressTest:
   PodsTotal: 260
   Timeout: "40m"
 ```
 
+`Capabilities` lists OpenShift-only driver features from the OCP manifest. They are kept separate from upstream Kubernetes driver capabilities and are read by OpenShift CSI test suites when deciding whether to run or skip. `podDeleteAfterUmount` enables the suite that host-unmounts a CSI volume path and verifies pod deletion still succeeds.
+
 `LUNStressTest` is a test that stresses the CSI driver on a single node. The test picks a random scheudlable node and creates configured number of Pods + PVCs on it (260 by default).
 
 

test/extended/storage/csi/csi.go

@@ -13,19 +13,26 @@ import (
 
 var registerAlwaysOnCSISuites sync.Once
 
+var ocpCSIDriverConfig = map[string]*OpenShiftCSIDriverConfig{}
+
 const (
 	// The defaul timeout for the LUN stress test.
 	DefaultLUNStressTestTimeout = "40m"
 	// The default nr. of Pods to run in the LUN stress test.
 	DefaultLUNStressTestPodsTotal = 260
 )
 
+// OpenShiftCSICapability names an OpenShift-only CSI test feature.
+type OpenShiftCSICapability string
+
 // OpenShiftCSIDriverConfig holds definition test parameters of OpenShift specific CSI test
 type OpenShiftCSIDriverConfig struct {
 	// Name of the CSI driver.
 	Driver string
 	// Configuration of the LUN stress test. If nil, the test is skipped.
 	LUNStressTest *LUNStressTestConfig
+	// OpenShift-only capabilities read from the OCP manifest, keyed by capability name.
+	Capabilities map[OpenShiftCSICapability]bool
 }
 
 // Definition of the LUN stress test parameters.
@@ -57,6 +64,13 @@ func (d *OpenShiftCSIDriverConfig) GetObjectKind() schema.ObjectKind {
 	return nil
 }
 
+// OpenShiftCSIDriverConfigFor returns the OpenShift-specific CSI driver config loaded
+// from TEST_OCP_CSI_DRIVER_FILES, keyed by CSI driver name.
+func OpenShiftCSIDriverConfigFor(driverName string) (*OpenShiftCSIDriverConfig, bool) {
+	cfg, ok := ocpCSIDriverConfig[driverName]
+	return cfg, ok
+}
+
 // Register all OCP specific CSI tests into upstream testsuites.CSISuites.
 func AddOpenShiftCSITests(filename string) (string, error) {
 	bytes, err := os.ReadFile(filename)
@@ -75,10 +89,15 @@ func AddOpenShiftCSITests(filename string) (string, error) {
 	if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), bytes, cfg); err != nil {
 		return "", fmt.Errorf("%s: %w", filename, err)
 	}
+	if cfg.Driver == "" {
+		return "", fmt.Errorf("%s: missing Driver", filename)
+	}
+
+	ocpCSIDriverConfig[cfg.Driver] = cfg
 
 	// Register this OCP specific test suite in the upstream test framework.
 	// In the end, the test suite will be executed as any other upstream storage test.
-	// Note: this must be done before external.AddDriverDefinition which actually goes through
+	// Note: this must be done before  external.AddDriverDefinition which actually goes through
 	// the registered testsuites and generates ginkgo tests for them.
 	testsuites.CSISuites = append(testsuites.CSISuites, initSCSILUNOverflowCSISuite(cfg.LUNStressTest))
 	return cfg.Driver, nil
@@ -88,6 +107,6 @@ func AddOpenShiftCSITests(filename string) (string, error) {
 // driver manifest. Call before external.AddDriverDefinition. Safe to call once per process.
 func RegisterAlwaysOnCSISuites() {
 	registerAlwaysOnCSISuites.Do(func() {
-		testsuites.CSISuites = append(testsuites.CSISuites, initPVCCloneLargerCSISuite)
+		testsuites.CSISuites = append(testsuites.CSISuites, initPVCCloneLargerCSISuite, initPodDeleteAfterUmountCSISuite)
 	})
 }

test/extended/storage/csi/pod_delete_after_umount.go

@@ -0,0 +1,116 @@
+package csi
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	g "github.com/onsi/ginkgo/v2"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
+	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
+	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
+	storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
+	admissionapi "k8s.io/pod-security-admission/api"
+)
+
+// CapPodDeleteAfterUmount indicates the driver supports pod deletion after the volume
+// was force-unmounted on the node.
+const CapPodDeleteAfterUmount OpenShiftCSICapability = "podDeleteAfterUmount"
+
+func initPodDeleteAfterUmountCSISuite() storageframework.TestSuite {
+	return &podDeleteAfterUmountCSISuite{
+		tsInfo: storageframework.TestSuiteInfo{
+			Name: "OpenShift CSI extended - Pod delete after umount",
+			TestPatterns: []storageframework.TestPattern{
+				storageframework.DefaultFsDynamicPV,
+			},
+			SupportedSizeRange: e2evolume.SizeRange{
+				Min: "1Mi",
+			},
+		},
+	}
+}
+
+// podDeleteAfterUmountCSISuite verifies that a pod can be deleted after its CSI
+// volume mount has already been unmounted on the node.
+type podDeleteAfterUmountCSISuite struct {
+	tsInfo storageframework.TestSuiteInfo
+}
+
+var _ storageframework.TestSuite = &podDeleteAfterUmountCSISuite{}
+
+func (s *podDeleteAfterUmountCSISuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
+	return s.tsInfo
+}
+
+func (s *podDeleteAfterUmountCSISuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
+	cfg, ok := OpenShiftCSIDriverConfigFor(driver.GetDriverInfo().Name)
+	if !ok || !cfg.Capabilities[CapPodDeleteAfterUmount] {
+		e2eskipper.Skipf("Driver %q does not support pod delete after umount - skipping", driver.GetDriverInfo().Name)
+	}
+}
+
+func (s *podDeleteAfterUmountCSISuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
+	f := e2e.NewFrameworkWithCustomTimeouts("csi-pod-delete-umount", storageframework.GetDriverTimeouts(driver))
+	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
+
+	g.It("should delete pod after volume directory was umounted on the node", func(ctx context.Context) {
+		config := driver.PrepareTest(ctx, f)
+		hostExec := storageutils.NewHostExec(f)
+		g.DeferCleanup(hostExec.Cleanup)
+
+		g.By("Creating a dynamically provisioned volume")
+		resource := storageframework.CreateVolumeResource(ctx, driver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange)
+		g.DeferCleanup(resource.CleanupResource)
+
+		g.By("Creating a pod that mounts the volume")
+		podConfig := e2epod.Config{
+			NS:            f.Namespace.Name,
+			PVCs:          []*v1.PersistentVolumeClaim{resource.Pvc},
+			SeLinuxLabel:  e2epod.GetLinuxLabel(),
+			NodeSelection: config.ClientNodeSelection,
+			ImageID:       e2epod.GetDefaultTestImageID(),
+		}
+		pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart)
+		e2e.ExpectNoError(err, "creating pod with PVC")
+		g.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod)
+
+		pvc, err := f.ClientSet.CoreV1().PersistentVolumeClaims(resource.Pvc.Namespace).Get(ctx, resource.Pvc.Name, metav1.GetOptions{})
+		e2e.ExpectNoError(err, "re-fetching PVC after pod is running")
+		pvName := pvc.Spec.VolumeName
+		if pvName == "" {
+			e2e.Failf("PVC %s has empty Spec.VolumeName after pod is running", pvc.Name)
+		}
+
+		node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, pod.Spec.NodeName, metav1.GetOptions{})
+		e2e.ExpectNoError(err, "getting pod node %s", pod.Spec.NodeName)
+
+		mountPath := csiPodVolumeMountPath(string(pod.UID), pvName)
+		g.By("Verifying volume is mounted on the pod node")
+		err = hostExec.IssueCommand(ctx, fmt.Sprintf("mountpoint -q %q", mountPath), node)
+		e2e.ExpectNoError(err, "expected %s to be a mountpoint before umount", mountPath)
+
+		g.By("Unmounting and removing the volume directory on the node")
+		err = hostExec.IssueCommand(ctx, fmt.Sprintf("umount -f %q && rmdir %q", mountPath, mountPath), node)
+		e2e.ExpectNoError(err, "umount and rmdir of volume mount path %s", mountPath)
+
+		g.By("Verifying the path is no longer a mountpoint")
+		err = hostExec.IssueCommand(ctx, fmt.Sprintf("mountpoint -q %q", mountPath), node)
+		if err == nil {
+			e2e.Failf("expected %s to not be a mountpoint after umount", mountPath)
+		}
+
+		g.By("Deleting the pod; TearDown must succeed despite the missing mount [OCPBUGS-10816]")
+		err = e2epod.DeletePodWithWait(ctx, f.ClientSet, pod)
+		e2e.ExpectNoError(err, "deleting pod after volume directory was umounted")
+	})
+}
+
+// csiPodVolumeMountPath returns the kubelet CSI NodePublish mount path for a pod volume.
+func csiPodVolumeMountPath(podUID, pvName string) string {
+	return filepath.Join("/var/lib/kubelet/pods", podUID, "volumes", "kubernetes.io~csi", pvName, "mount")
+}