Use finalizers to avoid losing delete events and to ensure full resource cleanup (#941)

* Add Finalizer functions to Cluster; add/remove finalizer on Create/Delete events
* Check if clusters have a deletion timestamp and we missed that event. Run Delete() and remove finalizer when done.
* Fix nil handling when using Service from map; Remove Service, Endpoint entries from their maps - just like with Secrets
* Add handling of ResourceNotFound to all delete functions (Service, Endpoint, LogicalBackup CronJob, PDB and Secret) - this is not a real error when deleting things
* Emit events when there are issues deleting resources to the user is informed
* Depend the removal of the Finalizer on all resources being deleted successfully first. Otherwise the next sync run should let us try again
* Add config option to enable finalizers
* Removed dangling whitespace at EOL
* config.EnableFinalizers is a bool pointer

---------

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>
This commit is contained in:
Christian Rohmann
2024-01-04 16:22:53 +01:00
committed by GitHub
co-authored by Felix Kunde
parent 9581ba969b
commit 743aade45f
8 changed files with 201 additions and 30 deletions
+1
View File
@@ -91,6 +91,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate
result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName
result.EnableCrossNamespaceSecret = fromCRD.Kubernetes.EnableCrossNamespaceSecret
result.EnableFinalizers = util.CoalesceBool(fromCRD.Kubernetes.EnableFinalizers, util.False())
result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName
if fromCRD.Kubernetes.InfrastructureRolesDefs != nil {
+23 -11
View File
@@ -290,9 +290,10 @@ func (c *Controller) processEvent(event ClusterEvent) {
teamName := strings.ToLower(cl.Spec.TeamID)
c.curWorkerCluster.Store(event.WorkerID, cl)
cl.Delete()
// Fixme - no error handling for delete ?
// c.eventRecorder.Eventf(cl.GetReference, v1.EventTypeWarning, "Delete", "%v", cl.Error)
if err := cl.Delete(); err != nil {
cl.Error = fmt.Sprintf("Could not delete cluster: %v", err)
c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Delete", "%v", cl.Error)
}
func() {
defer c.clustersMu.Unlock()
@@ -325,16 +326,27 @@ func (c *Controller) processEvent(event ClusterEvent) {
}
c.curWorkerCluster.Store(event.WorkerID, cl)
err = cl.Sync(event.NewSpec)
if err != nil {
cl.Error = fmt.Sprintf("could not sync cluster: %v", err)
c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Sync", "%v", cl.Error)
lg.Error(cl.Error)
return
// has this cluster been marked as deleted already, then we shall start cleaning up
if !cl.ObjectMeta.DeletionTimestamp.IsZero() {
lg.Infof("Cluster has a DeletionTimestamp of %s, starting deletion now.", cl.ObjectMeta.DeletionTimestamp.Format(time.RFC3339))
if err = cl.Delete(); err != nil {
cl.Error = fmt.Sprintf("Error deleting cluster and its resources: %v", err)
c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Delete", "%v", cl.Error)
lg.Error(cl.Error)
return
}
lg.Infof("cluster has been deleted")
} else {
if err = cl.Sync(event.NewSpec); err != nil {
cl.Error = fmt.Sprintf("could not sync cluster: %v", err)
c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Sync", "%v", cl.Error)
lg.Error(cl.Error)
return
}
lg.Infof("cluster has been synced")
}
cl.Error = ""
lg.Infof("cluster has been synced")
}
}