From da4359cf8205da8240be22fbcc99e1db56bf6a56 Mon Sep 17 00:00:00 2001 From: Gagan163264 Date: Sat, 16 May 2026 02:25:46 +0530 Subject: [PATCH 1/6] Trigger node deletion when machine is deleted and update machine descriptions --- pkg/util/provider/machinecontroller/machine.go | 10 +++++++++- pkg/util/provider/machinecontroller/machine_test.go | 4 ++-- pkg/util/provider/machinecontroller/node.go | 8 ++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pkg/util/provider/machinecontroller/machine.go b/pkg/util/provider/machinecontroller/machine.go index fd88fc9db1..6ef35575be 100644 --- a/pkg/util/provider/machinecontroller/machine.go +++ b/pkg/util/provider/machinecontroller/machine.go @@ -100,6 +100,14 @@ func (c *controller) deleteMachine(obj any) { } } c.enqueueMachineTermination(machine, "handling terminating machine object DELETE event") + + if c.targetCoreClient != nil { + if nodeName := machine.Labels[v1alpha1.NodeLabelKey]; nodeName != "" { + if err := c.targetCoreClient.CoreV1().Nodes().Delete(context.Background(), nodeName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + klog.Errorf("Machine %q: failed to delete node %q: %v", machine.Name, nodeName, err) + } + } + } } // getKeyForObj returns key for object, else returns false @@ -530,7 +538,7 @@ func (c *controller) triggerCreationFlow(ctx context.Context, createMachineReque if machine.Status.CurrentStatus.Phase == "" || machine.Status.CurrentStatus.Phase == v1alpha1.MachineCrashLoopBackOff { clone := clone.DeepCopy() clone.Status.LastOperation = v1alpha1.LastOperation{ - Description: "Creating machine on cloud provider", + Description: "Creating machine on cloud provider. Waiting for node object to register", State: v1alpha1.MachineStateProcessing, Type: v1alpha1.MachineOperationCreate, LastUpdateTime: metav1.Now(), diff --git a/pkg/util/provider/machinecontroller/machine_test.go b/pkg/util/provider/machinecontroller/machine_test.go index 6fb97ca0fd..952eaf3df7 100644 --- a/pkg/util/provider/machinecontroller/machine_test.go +++ b/pkg/util/provider/machinecontroller/machine_test.go @@ -677,7 +677,7 @@ var _ = Describe("machine", func() { LastUpdateTime: metav1.Now(), }, LastOperation: v1alpha1.LastOperation{ - Description: "Creating machine on cloud provider", + Description: "Creating machine on cloud provider. Waiting for node object to register", State: v1alpha1.MachineStateProcessing, Type: v1alpha1.MachineOperationCreate, LastUpdateTime: metav1.Now(), @@ -722,7 +722,7 @@ var _ = Describe("machine", func() { LastUpdateTime: metav1.Now(), }, LastOperation: v1alpha1.LastOperation{ - Description: "Creating machine on cloud provider", + Description: "Creating machine on cloud provider. Waiting for node object to register", State: v1alpha1.MachineStateProcessing, Type: v1alpha1.MachineOperationCreate, LastUpdateTime: metav1.Now(), diff --git a/pkg/util/provider/machinecontroller/node.go b/pkg/util/provider/machinecontroller/node.go index 174805c7c1..1c2bf3f4fc 100644 --- a/pkg/util/provider/machinecontroller/node.go +++ b/pkg/util/provider/machinecontroller/node.go @@ -68,6 +68,10 @@ func (c *controller) updateNode(oldObj, newObj any) { // as the update handler will be triggered again due to kubelet updates. machine, err := c.getMachineFromNode(node.Name) if err != nil { + if errors.Is(err, errNoMachineMatch) && node.DeletionTimestamp != nil { + c.enqueueNode(node, fmt.Sprintf("handling node UPDATE event. Node %q is being deleted with no backing machine", node.Name)) + return + } klog.Errorf("unable to handle update event for node %q, couldn't fetch associated machine. Error: %v", node.Name, err) return } @@ -153,6 +157,10 @@ func (c *controller) reconcileClusterNodeKey(key string) error { machine, err := c.getMachineFromNode(node.Name) if err != nil { if errors.Is(err, errNoMachineMatch) { + if node.DeletionTimestamp != nil { + klog.V(2).Infof("ClusterNode %q: Node is being deleted with no backing machine, removing finalizer", key) + return c.removeNodeFinalizers(ctx, node) + } klog.Errorf("ClusterNode %q: No machine found matching node, skipping adding finalizers", key) return nil } From 9864805cd389694842e6dd14e73519c755759e66 Mon Sep 17 00:00:00 2001 From: Gagan163264 Date: Fri, 7 Aug 2026 13:12:26 +0530 Subject: [PATCH 2/6] Address review comments --- pkg/util/provider/machinecontroller/machine.go | 4 ++-- pkg/util/provider/machinecontroller/node.go | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/util/provider/machinecontroller/machine.go b/pkg/util/provider/machinecontroller/machine.go index 6ef35575be..df74f43b0e 100644 --- a/pkg/util/provider/machinecontroller/machine.go +++ b/pkg/util/provider/machinecontroller/machine.go @@ -104,7 +104,7 @@ func (c *controller) deleteMachine(obj any) { if c.targetCoreClient != nil { if nodeName := machine.Labels[v1alpha1.NodeLabelKey]; nodeName != "" { if err := c.targetCoreClient.CoreV1().Nodes().Delete(context.Background(), nodeName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - klog.Errorf("Machine %q: failed to delete node %q: %v", machine.Name, nodeName, err) + klog.Errorf("failed to delete backing node %q of machine %q queued for deletion: %v", nodeName, machine.Name, err) } } } @@ -538,7 +538,7 @@ func (c *controller) triggerCreationFlow(ctx context.Context, createMachineReque if machine.Status.CurrentStatus.Phase == "" || machine.Status.CurrentStatus.Phase == v1alpha1.MachineCrashLoopBackOff { clone := clone.DeepCopy() clone.Status.LastOperation = v1alpha1.LastOperation{ - Description: "Creating machine on cloud provider. Waiting for node object to register", + Description: "VM created on cloud provider. Waiting for node registration", State: v1alpha1.MachineStateProcessing, Type: v1alpha1.MachineOperationCreate, LastUpdateTime: metav1.Now(), diff --git a/pkg/util/provider/machinecontroller/node.go b/pkg/util/provider/machinecontroller/node.go index 1c2bf3f4fc..2069ba1698 100644 --- a/pkg/util/provider/machinecontroller/node.go +++ b/pkg/util/provider/machinecontroller/node.go @@ -63,9 +63,10 @@ func (c *controller) updateNode(oldObj, newObj any) { return } - // Do not process node updates if there is no associated machine - // In case of transient errors while fetching machine, do not retry - // as the update handler will be triggered again due to kubelet updates. + // Do not process node updates if there is no associated machine, except + // when the node is being deleted, in that case enqueue it for finalizer removal. + // For transient fetch errors, do not retry; the update handler will be triggered + // again due to kubelet updates. machine, err := c.getMachineFromNode(node.Name) if err != nil { if errors.Is(err, errNoMachineMatch) && node.DeletionTimestamp != nil { @@ -152,8 +153,10 @@ func (c *controller) reconcileClusterNodeKey(key string) error { return nil } - // Ignore node updates without an associated machine. Retry only for errors other than errNoMachineMatch; - // transient fetch errors will be eventually requeued by the update handler due to kubelet updates. + // Ignore node updates without an associated machine, except when the node is being + // deleted, then remove the MCM finalizer to unblock deletion. Retry only for + // errors other than errNoMachineMatch; transient fetch errors will be eventually + // requeued by the update handler due to kubelet updates. machine, err := c.getMachineFromNode(node.Name) if err != nil { if errors.Is(err, errNoMachineMatch) { @@ -163,9 +166,10 @@ func (c *controller) reconcileClusterNodeKey(key string) error { } klog.Errorf("ClusterNode %q: No machine found matching node, skipping adding finalizers", key) return nil + } else { + klog.Errorf("ClusterNode %q: error fetching machine for node: %v", key, err) + return err } - klog.Errorf("ClusterNode %q: error fetching machine for node: %v", key, err) - return err } if node.DeletionTimestamp != nil { From e02a450bf23b6821c6ec2e418a1d1463570cac65 Mon Sep 17 00:00:00 2001 From: Gagan163264 Date: Fri, 7 Aug 2026 13:37:38 +0530 Subject: [PATCH 3/6] Fix test --- pkg/util/provider/machinecontroller/machine_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/provider/machinecontroller/machine_test.go b/pkg/util/provider/machinecontroller/machine_test.go index 952eaf3df7..766a1ee9fe 100644 --- a/pkg/util/provider/machinecontroller/machine_test.go +++ b/pkg/util/provider/machinecontroller/machine_test.go @@ -677,7 +677,7 @@ var _ = Describe("machine", func() { LastUpdateTime: metav1.Now(), }, LastOperation: v1alpha1.LastOperation{ - Description: "Creating machine on cloud provider. Waiting for node object to register", + Description: "VM created on cloud provider. Waiting for node registration", State: v1alpha1.MachineStateProcessing, Type: v1alpha1.MachineOperationCreate, LastUpdateTime: metav1.Now(), @@ -722,7 +722,7 @@ var _ = Describe("machine", func() { LastUpdateTime: metav1.Now(), }, LastOperation: v1alpha1.LastOperation{ - Description: "Creating machine on cloud provider. Waiting for node object to register", + Description: "VM created on cloud provider. Waiting for node registration", State: v1alpha1.MachineStateProcessing, Type: v1alpha1.MachineOperationCreate, LastUpdateTime: metav1.Now(), From 70001bf31303a02cc4f52535c9c9fb83c77b6591 Mon Sep 17 00:00:00 2001 From: Gagan163264 Date: Wed, 12 Aug 2026 11:17:46 +0530 Subject: [PATCH 4/6] Remove redundant else --- pkg/util/provider/machinecontroller/node.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/util/provider/machinecontroller/node.go b/pkg/util/provider/machinecontroller/node.go index 2069ba1698..57b473d93f 100644 --- a/pkg/util/provider/machinecontroller/node.go +++ b/pkg/util/provider/machinecontroller/node.go @@ -166,10 +166,9 @@ func (c *controller) reconcileClusterNodeKey(key string) error { } klog.Errorf("ClusterNode %q: No machine found matching node, skipping adding finalizers", key) return nil - } else { - klog.Errorf("ClusterNode %q: error fetching machine for node: %v", key, err) - return err } + klog.Errorf("ClusterNode %q: error fetching machine for node: %v", key, err) + return err } if node.DeletionTimestamp != nil { From 92ba83b3bc184784e8f9cbb426fd90073c79ae12 Mon Sep 17 00:00:00 2001 From: Gagan163264 Date: Wed, 12 Aug 2026 11:33:02 +0530 Subject: [PATCH 5/6] Change log message --- pkg/util/provider/machinecontroller/machine.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/util/provider/machinecontroller/machine.go b/pkg/util/provider/machinecontroller/machine.go index df74f43b0e..16803110cb 100644 --- a/pkg/util/provider/machinecontroller/machine.go +++ b/pkg/util/provider/machinecontroller/machine.go @@ -101,10 +101,11 @@ func (c *controller) deleteMachine(obj any) { } c.enqueueMachineTermination(machine, "handling terminating machine object DELETE event") + // Node retains finalizer here, removed by reconcileClusterNodeKey once DeletionTimestamp is set if c.targetCoreClient != nil { if nodeName := machine.Labels[v1alpha1.NodeLabelKey]; nodeName != "" { if err := c.targetCoreClient.CoreV1().Nodes().Delete(context.Background(), nodeName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - klog.Errorf("failed to delete backing node %q of machine %q queued for deletion: %v", nodeName, machine.Name, err) + klog.Errorf("failed to delete backing node %q of deleted machine %q: %v", nodeName, machine.Name, err) } } } From 2cf292ef3d95e504616bebf1d70b9cfa4512a704 Mon Sep 17 00:00:00 2001 From: Gagan163264 Date: Wed, 12 Aug 2026 12:14:06 +0530 Subject: [PATCH 6/6] Improve error handling and logging for node deletion --- pkg/util/provider/machinecontroller/machine.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/util/provider/machinecontroller/machine.go b/pkg/util/provider/machinecontroller/machine.go index 16803110cb..f75ced4ae0 100644 --- a/pkg/util/provider/machinecontroller/machine.go +++ b/pkg/util/provider/machinecontroller/machine.go @@ -104,8 +104,15 @@ func (c *controller) deleteMachine(obj any) { // Node retains finalizer here, removed by reconcileClusterNodeKey once DeletionTimestamp is set if c.targetCoreClient != nil { if nodeName := machine.Labels[v1alpha1.NodeLabelKey]; nodeName != "" { - if err := c.targetCoreClient.CoreV1().Nodes().Delete(context.Background(), nodeName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - klog.Errorf("failed to delete backing node %q of deleted machine %q: %v", nodeName, machine.Name, err) + err := c.targetCoreClient.CoreV1().Nodes().Delete(context.Background(), nodeName, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + klog.Errorf("backing node %q does not exist/already deleted for deleted machine %q: %v", nodeName, machine.Name, err) + } else { + klog.Errorf("failed to delete backing node %q of deleted machine %q: %v", nodeName, machine.Name, err) + } + } else { + klog.Infof("Successfully triggered deletion of backing node %q for deleted machine %q", nodeName, machine.Name) } } }