feat (operator): add support for IRSA (aws resources access) (#3128)

* fix data to POSIX and sed working on macos
* add ServiceAccountGetter to the newFakeK8sAnnotationsClient for unit tests
* try to update the service account
* use irsa_role_arn since we need the full arn, and remove enable_irsa
* move sa sync code to existing sync.go file to be all together
* change all Irsa to IRSA to follow go idiomatic that capitalize initialisms or acronyms
* using Update instead of Patch for the service account syn
* document the new option and add the key in the values/configs
* add the new option to the administrator docs
* trying to increase the timeout for the flaky test after sync

---------

Co-authored-by: Felix Kunde <felix-kunde@gmx.de>

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -73,6 +73,8 @@ spec:
                     type: integer
                   gcp_credentials:
                     type: string
+                  irsa_role_arn:
+                    type: string
                   kube_iam_role:
                     type: string
                   log_s3_bucket:

charts/postgres-operator/values.yaml

@@ -360,6 +360,8 @@ configAwsOrGcp:
 
   # AWS IAM role to supply in the iam.amazonaws.com/role annotation of Postgres pods
   # kube_iam_role: ""
+  # Full ARN for IRSA (IAM Roles for Service Accounts) on EKS
+  # irsa_role_arn: ""
 
   # S3 bucket to use for shipping postgres daily logs
   # log_s3_bucket: ""

docs/administrator.md

@@ -1094,6 +1094,32 @@ configuration:
     wal_s3_bucket: your-backup-path
 ```
 
+Alternatively, if your cluster uses EKS with OIDC, you can use
+[IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html)
+(IAM Roles for Service Accounts) instead of kube2iam. Set `irsa_role_arn` to
+the full ARN of the IAM role:
+
+**OperatorConfiguration**
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: OperatorConfiguration
+metadata:
+  name: postgresql-operator-configuration
+configuration:
+  aws_or_gcp:
+    aws_region: eu-central-1
+    irsa_role_arn: arn:aws:iam::123456789012:role/postgres-pod-role
+    wal_s3_bucket: your-backup-path
+```
+
+When `irsa_role_arn` is set the operator annotates the pod service account with
+`eks.amazonaws.com/role-arn` on every reconcile. The EKS OIDC webhook then
+injects an AWS web identity token into each pod, which takes precedence over
+the EC2 metadata credentials used by kube2iam. Both `kube_iam_role` and
+`irsa_role_arn` can coexist during a migration — existing pods retain the
+kube2iam annotation until they are rotated, at which point only IRSA is used.
+
 The referenced IAM role should contain the following privileges to make sure
 Postgres can send compressed WAL files to the given S3 bucket:
 
@@ -1204,6 +1230,7 @@ aws_or_gcp:
   # additional_secret_mount_path: ""
   # aws_region: eu-central-1
   # kube_iam_role: ""
+  # irsa_role_arn: ""
   # log_s3_bucket: ""
   # wal_s3_bucket: ""
   wal_gs_bucket: "postgres-backups-bucket-28302F2"  # name of bucket on where to save the WAL-E logs
@@ -1253,6 +1280,7 @@ aws_or_gcp:
   additional_secret_mount_path: "/var/secrets/google"  # or where ever you want to mount the file
   # aws_region: eu-central-1
   # kube_iam_role: ""
+  # irsa_role_arn: ""
   # log_s3_bucket: ""
   # wal_s3_bucket: ""
   wal_gs_bucket: "postgres-backups-bucket-28302F2"  # name of bucket on where to save the WAL-E logs

docs/reference/operator_parameters.md

@@ -798,6 +798,15 @@ yet officially supported.
   [kube2iam](https://github.com/jtblin/kube2iam) project on AWS. The default is
   empty.
 
+* **irsa_role_arn**
+  Full AWS IAM role ARN to supply in the `eks.amazonaws.com/role-arn` annotation
+  of the Postgres pod service account, enabling
+  [IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html)
+  (IAM Roles for Service Accounts) on EKS. When set, the operator annotates the
+  pod service account on every sync so that the EKS OIDC webhook can inject AWS
+  credentials directly into pods. Must be a full ARN, e.g.
+  `arn:aws:iam::123456789012:role/my-postgres-role`. The default is empty.
+
 * **aws_region**
   AWS region used to store EBS volumes. The default is `eu-central-1`. Note,
   this option is not meant for specifying the AWS region for backups and

e2e/tests/test_e2e.py

@@ -1430,7 +1430,7 @@ class EndToEndTestCase(unittest.TestCase):
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
             "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources)
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
-                             "Operator does not get in sync")
+                             "Operator does not get in sync", retries=120)
 
         # wait for switched over
         k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label)

manifests/configmap.yaml

@@ -79,6 +79,7 @@ data:
   # inherited_annotations: owned-by
   # inherited_labels: application,environment
   # kube_iam_role: ""
+  # irsa_role_arn: ""
   kubernetes_use_configmaps: "false"
   # log_s3_bucket: ""
   # logical_backup_azure_storage_account_name: ""

manifests/operatorconfiguration.crd.yaml

@@ -73,6 +73,8 @@ spec:
                     type: integer
                   gcp_credentials:
                     type: string
+                  irsa_role_arn:
+                    type: string
                   kube_iam_role:
                     type: string
                   log_s3_bucket:

manifests/postgresql-operator-default-configuration.yaml

@@ -171,6 +171,7 @@ configuration:
     # enable_ebs_gp3_migration_max_size: 1000
     # gcp_credentials: ""
     # kube_iam_role: ""
+    # irsa_role_arn: ""
     # log_s3_bucket: ""
     # wal_az_storage_account: ""
     # wal_gs_bucket: ""

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -252,6 +252,7 @@ type AWSGCPConfiguration struct {
 	WALAZStorageAccount          string `json:"wal_az_storage_account,omitempty"`
 	LogS3Bucket                  string `json:"log_s3_bucket,omitempty"`
 	KubeIAMRole                  string `json:"kube_iam_role,omitempty"`
+	IRSARoleARN                  string `json:"irsa_role_arn,omitempty"`
 	AdditionalSecretMount        string `json:"additional_secret_mount,omitempty"`
 	AdditionalSecretMountPath    string `json:"additional_secret_mount_path,omitempty"`
 	EnableEBSGp3Migration        bool   `json:"enable_ebs_gp3_migration,omitempty"`

pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml

@@ -73,6 +73,8 @@ spec:
                     type: integer
                   gcp_credentials:
                     type: string
+                  irsa_role_arn:
+                    type: string
                   kube_iam_role:
                     type: string
                   log_s3_bucket:

pkg/cluster/cluster.go

@@ -1140,6 +1140,12 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 		c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume size sync.")
 	}
 
+	// Pod service account (IRSA annotation sync)
+	if err := c.syncPodServiceAccount(); err != nil {
+		c.logger.Errorf("could not sync pod service account: %v", err)
+		updateFailed = true
+	}
+
 	// Statefulset
 	func() {
 		if err := c.syncStatefulSet(); err != nil {

pkg/cluster/sync.go

@@ -107,6 +107,10 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		}
 	}
 
+	if err = c.syncPodServiceAccount(); err != nil {
+		c.logger.Errorf("could not sync pod service account: %v", err)
+	}
+
 	if err = c.syncStatefulSet(); err != nil {
 		if !k8sutil.ResourceAlreadyExists(err) {
 			err = fmt.Errorf("could not sync statefulsets: %v", err)
@@ -630,6 +634,10 @@ func (c *Cluster) syncStatefulSet() error {
 		if !cmp.rollingUpdate {
 			updatedPodAnnotations := map[string]*string{}
 			for _, anno := range cmp.deletedPodAnnotations {
+				// during IRSA migration let kube2iam annotation drain naturally via pod rotation
+				if c.OpConfig.IRSARoleARN != "" && anno == constants.KubeIAmAnnotation {
+					continue
+				}
 				updatedPodAnnotations[anno] = nil
 			}
 			for anno, val := range desiredSts.Spec.Template.Annotations {
@@ -1803,6 +1811,7 @@ func (c *Cluster) syncLogicalBackupJob() error {
 	// no existing logical backup job, create new one
 	c.logger.Info("could not find the cluster's logical backup job")
 
+
 	if err = c.createLogicalBackupJob(); err == nil {
 		c.logger.Infof("created missing logical backup job %s", jobName)
 	} else {
@@ -1817,3 +1826,62 @@ func (c *Cluster) syncLogicalBackupJob() error {
 
 	return nil
 }
+
+func (c *Cluster) syncPodServiceAccount() error {
+	sa, err := c.KubeClient.ServiceAccounts(c.Namespace).Get(context.TODO(), c.OpConfig.PodServiceAccountName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("could not get pod service account %q: %v", c.OpConfig.PodServiceAccountName, err)
+	}
+
+	changed := false
+
+	if c.OpConfig.IRSARoleARN != "" {
+		if val, ok := sa.Annotations[constants.IRSAAnnotation]; !ok || val != c.OpConfig.IRSARoleARN {
+			if sa.Annotations == nil {
+				sa.Annotations = make(map[string]string)
+			}
+			sa.Annotations[constants.IRSAAnnotation] = c.OpConfig.IRSARoleARN
+			changed = true
+		}
+	} else {
+		if _, ok := sa.Annotations[constants.IRSAAnnotation]; ok {
+			delete(sa.Annotations, constants.IRSAAnnotation)
+			changed = true
+		}
+	}
+
+	if changed {
+		if _, err = c.KubeClient.ServiceAccounts(c.Namespace).Update(context.TODO(), sa, metav1.UpdateOptions{}); err != nil {
+			return fmt.Errorf("could not update pod service account %q: %v", sa.Name, err)
+		}
+		c.logger.Infof("synced annotations on pod service account %q", sa.Name)
+	}
+
+	if c.OpConfig.IRSARoleARN != "" {
+		c.logIRSAMigrationProgress()
+	}
+
+	return nil
+}
+
+func (c *Cluster) logIRSAMigrationProgress() {
+	pods, err := c.listPods()
+	if err != nil {
+		c.logger.Warnf("IRSA migration: could not list pods: %v", err)
+		return
+	}
+
+	total := len(pods)
+	remaining := 0
+	for _, pod := range pods {
+		if _, ok := pod.Annotations[constants.KubeIAmAnnotation]; ok {
+			remaining++
+		}
+	}
+
+	if remaining > 0 {
+		c.logger.Infof("IRSA migration in progress: %d/%d pods still carry kube2iam annotation, will be removed on next rotation", remaining, total)
+	} else {
+		c.logger.Infof("IRSA migration complete: all %d pods have rotated, kube2iam annotation fully drained", total)
+	}
+}

pkg/cluster/util_test.go

@@ -53,6 +53,7 @@ func newFakeK8sAnnotationsClient() (k8sutil.KubernetesClient, *k8sFake.Clientset
 		EndpointsGetter:              clientSet.CoreV1(),
 		ConfigMapsGetter:             clientSet.CoreV1(),
 		PodsGetter:                   clientSet.CoreV1(),
+		ServiceAccountsGetter:        clientSet.CoreV1(),
 		DeploymentsGetter:            clientSet.AppsV1(),
 		CronJobsGetter:               clientSet.BatchV1(),
 	}, clientSet

pkg/controller/controller.go

@@ -247,6 +247,12 @@ func (c *Controller) initPodServiceAccount() {
 			c.PodServiceAccount.Name = c.opConfig.PodServiceAccountName
 		}
 		c.PodServiceAccount.Namespace = ""
+		if c.opConfig.IRSARoleARN != "" {
+			if c.PodServiceAccount.Annotations == nil {
+				c.PodServiceAccount.Annotations = make(map[string]string)
+			}
+			c.PodServiceAccount.Annotations[constants.IRSAAnnotation] = c.opConfig.IRSARoleARN
+		}
 	}
 
 	// actual service accounts are deployed at the time of Postgres/Spilo cluster creation

pkg/controller/operator_config.go

@@ -175,6 +175,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.AWSRegion = fromCRD.AWSGCP.AWSRegion
 	result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket
 	result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole
+	result.IRSARoleARN = fromCRD.AWSGCP.IRSARoleARN
 	result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket
 	result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials
 	result.WALAZStorageAccount = fromCRD.AWSGCP.WALAZStorageAccount

pkg/util/config/config.go

@@ -194,6 +194,7 @@ type Config struct {
 	WALES3Bucket                             string            `name:"wal_s3_bucket"`
 	LogS3Bucket                              string            `name:"log_s3_bucket"`
 	KubeIAMRole                              string            `name:"kube_iam_role"`
+	IRSARoleARN                              string            `name:"irsa_role_arn"`
 	WALGSBucket                              string            `name:"wal_gs_bucket"`
 	GCPCredentials                           string            `name:"gcp_credentials"`
 	WALAZStorageAccount                      string            `name:"wal_az_storage_account"`

pkg/util/constants/annotations.go

@@ -4,6 +4,7 @@ package constants
 const (
 	ZalandoDNSNameAnnotation           = "external-dns.alpha.kubernetes.io/hostname"
 	KubeIAmAnnotation                  = "iam.amazonaws.com/role"
+	IRSAAnnotation                     = "eks.amazonaws.com/role-arn"
 	VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by"
 	PostgresqlControllerAnnotationKey  = "acid.zalan.do/controller"
 )