feat (operator): add support for IRSA (aws resources access) (#3128)

* fix data to POSIX and sed working on macos
* add ServiceAccountGetter to the newFakeK8sAnnotationsClient for unit tests
* try to update the service account
* use irsa_role_arn since we need the full arn, and remove enable_irsa
* move sa sync code to existing sync.go file to be all together
* change all Irsa to IRSA to follow go idiomatic that capitalize initialisms or acronyms
* using Update instead of Patch for the service account syn
* document the new option and add the key in the values/configs
* add the new option to the administrator docs
* trying to increase the timeout for the flaky test after sync

---------

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>

pkg/cluster/sync.go

@@ -107,6 +107,10 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		}
 	}
 
+	if err = c.syncPodServiceAccount(); err != nil {
+		c.logger.Errorf("could not sync pod service account: %v", err)
+	}
+
 	if err = c.syncStatefulSet(); err != nil {
 		if !k8sutil.ResourceAlreadyExists(err) {
 			err = fmt.Errorf("could not sync statefulsets: %v", err)
@@ -630,6 +634,10 @@ func (c *Cluster) syncStatefulSet() error {
 		if !cmp.rollingUpdate {
 			updatedPodAnnotations := map[string]*string{}
 			for _, anno := range cmp.deletedPodAnnotations {
+				// during IRSA migration let kube2iam annotation drain naturally via pod rotation
+				if c.OpConfig.IRSARoleARN != "" && anno == constants.KubeIAmAnnotation {
+					continue
+				}
 				updatedPodAnnotations[anno] = nil
 			}
 			for anno, val := range desiredSts.Spec.Template.Annotations {
@@ -1803,6 +1811,7 @@ func (c *Cluster) syncLogicalBackupJob() error {
 	// no existing logical backup job, create new one
 	c.logger.Info("could not find the cluster's logical backup job")
 
+
 	if err = c.createLogicalBackupJob(); err == nil {
 		c.logger.Infof("created missing logical backup job %s", jobName)
 	} else {
@@ -1817,3 +1826,62 @@ func (c *Cluster) syncLogicalBackupJob() error {
 
 	return nil
 }
+
+func (c *Cluster) syncPodServiceAccount() error {
+	sa, err := c.KubeClient.ServiceAccounts(c.Namespace).Get(context.TODO(), c.OpConfig.PodServiceAccountName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("could not get pod service account %q: %v", c.OpConfig.PodServiceAccountName, err)
+	}
+
+	changed := false
+
+	if c.OpConfig.IRSARoleARN != "" {
+		if val, ok := sa.Annotations[constants.IRSAAnnotation]; !ok || val != c.OpConfig.IRSARoleARN {
+			if sa.Annotations == nil {
+				sa.Annotations = make(map[string]string)
+			}
+			sa.Annotations[constants.IRSAAnnotation] = c.OpConfig.IRSARoleARN
+			changed = true
+		}
+	} else {
+		if _, ok := sa.Annotations[constants.IRSAAnnotation]; ok {
+			delete(sa.Annotations, constants.IRSAAnnotation)
+			changed = true
+		}
+	}
+
+	if changed {
+		if _, err = c.KubeClient.ServiceAccounts(c.Namespace).Update(context.TODO(), sa, metav1.UpdateOptions{}); err != nil {
+			return fmt.Errorf("could not update pod service account %q: %v", sa.Name, err)
+		}
+		c.logger.Infof("synced annotations on pod service account %q", sa.Name)
+	}
+
+	if c.OpConfig.IRSARoleARN != "" {
+		c.logIRSAMigrationProgress()
+	}
+
+	return nil
+}
+
+func (c *Cluster) logIRSAMigrationProgress() {
+	pods, err := c.listPods()
+	if err != nil {
+		c.logger.Warnf("IRSA migration: could not list pods: %v", err)
+		return
+	}
+
+	total := len(pods)
+	remaining := 0
+	for _, pod := range pods {
+		if _, ok := pod.Annotations[constants.KubeIAmAnnotation]; ok {
+			remaining++
+		}
+	}
+
+	if remaining > 0 {
+		c.logger.Infof("IRSA migration in progress: %d/%d pods still carry kube2iam annotation, will be removed on next rotation", remaining, total)
+	} else {
+		c.logger.Infof("IRSA migration complete: all %d pods have rotated, kube2iam annotation fully drained", total)
+	}
+}