From 7578f9d2c0a0c5f70ed8dc374ce4693ffaf20116 Mon Sep 17 00:00:00 2001 From: Tiago Condeixa Date: Wed, 15 Jul 2026 18:19:10 +0200 Subject: [PATCH] feat (operator): add support for IRSA (aws resources access) (#3128) * fix data to POSIX and sed working on macos * add ServiceAccountGetter to the newFakeK8sAnnotationsClient for unit tests * try to update the service account * use irsa_role_arn since we need the full arn, and remove enable_irsa * move sa sync code to existing sync.go file to be all together * change all Irsa to IRSA to follow go idiomatic that capitalize initialisms or acronyms * using Update instead of Patch for the service account syn * document the new option and add the key in the values/configs * add the new option to the administrator docs * trying to increase the timeout for the flaky test after sync --------- Co-authored-by: Felix Kunde --- .../crds/operatorconfigurations.yaml | 2 + charts/postgres-operator/values.yaml | 2 + docs/administrator.md | 28 ++++++++ docs/reference/operator_parameters.md | 9 +++ e2e/tests/test_e2e.py | 2 +- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 + ...gresql-operator-default-configuration.yaml | 1 + .../v1/operator_configuration_type.go | 1 + .../v1/operatorconfiguration.crd.yaml | 2 + pkg/cluster/cluster.go | 6 ++ pkg/cluster/sync.go | 68 +++++++++++++++++++ pkg/cluster/util_test.go | 1 + pkg/controller/controller.go | 6 ++ pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + pkg/util/constants/annotations.go | 1 + 17 files changed, 133 insertions(+), 1 deletion(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 9cd6b27b7..ebe4e7089 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -73,6 +73,8 @@ spec: type: integer gcp_credentials: type: string + irsa_role_arn: + type: string kube_iam_role: type: string log_s3_bucket: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 3a2194ef0..4ea5b6319 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -360,6 +360,8 @@ configAwsOrGcp: # AWS IAM role to supply in the iam.amazonaws.com/role annotation of Postgres pods # kube_iam_role: "" + # Full ARN for IRSA (IAM Roles for Service Accounts) on EKS + # irsa_role_arn: "" # S3 bucket to use for shipping postgres daily logs # log_s3_bucket: "" diff --git a/docs/administrator.md b/docs/administrator.md index ec0bc28dc..49718d1ea 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -1094,6 +1094,32 @@ configuration: wal_s3_bucket: your-backup-path ``` +Alternatively, if your cluster uses EKS with OIDC, you can use +[IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) +(IAM Roles for Service Accounts) instead of kube2iam. Set `irsa_role_arn` to +the full ARN of the IAM role: + +**OperatorConfiguration** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + aws_or_gcp: + aws_region: eu-central-1 + irsa_role_arn: arn:aws:iam::123456789012:role/postgres-pod-role + wal_s3_bucket: your-backup-path +``` + +When `irsa_role_arn` is set the operator annotates the pod service account with +`eks.amazonaws.com/role-arn` on every reconcile. The EKS OIDC webhook then +injects an AWS web identity token into each pod, which takes precedence over +the EC2 metadata credentials used by kube2iam. Both `kube_iam_role` and +`irsa_role_arn` can coexist during a migration — existing pods retain the +kube2iam annotation until they are rotated, at which point only IRSA is used. + The referenced IAM role should contain the following privileges to make sure Postgres can send compressed WAL files to the given S3 bucket: @@ -1204,6 +1230,7 @@ aws_or_gcp: # additional_secret_mount_path: "" # aws_region: eu-central-1 # kube_iam_role: "" + # irsa_role_arn: "" # log_s3_bucket: "" # wal_s3_bucket: "" wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs @@ -1253,6 +1280,7 @@ aws_or_gcp: additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file # aws_region: eu-central-1 # kube_iam_role: "" + # irsa_role_arn: "" # log_s3_bucket: "" # wal_s3_bucket: "" wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index dd7487352..cf1d18c14 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -798,6 +798,15 @@ yet officially supported. [kube2iam](https://github.com/jtblin/kube2iam) project on AWS. The default is empty. +* **irsa_role_arn** + Full AWS IAM role ARN to supply in the `eks.amazonaws.com/role-arn` annotation + of the Postgres pod service account, enabling + [IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) + (IAM Roles for Service Accounts) on EKS. When set, the operator annotates the + pod service account on every sync so that the EKS OIDC webhook can inject AWS + credentials directly into pods. Must be a full ARN, e.g. + `arn:aws:iam::123456789012:role/my-postgres-role`. The default is empty. + * **aws_region** AWS region used to store EBS volumes. The default is `eu-central-1`. Note, this option is not meant for specifying the AWS region for backups and diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 7ce80a6c9..8b64185e9 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1430,7 +1430,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") + "Operator does not get in sync", retries=120) # wait for switched over k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 1c663c757..c7f4d2655 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -79,6 +79,7 @@ data: # inherited_annotations: owned-by # inherited_labels: application,environment # kube_iam_role: "" + # irsa_role_arn: "" kubernetes_use_configmaps: "false" # log_s3_bucket: "" # logical_backup_azure_storage_account_name: "" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 9cd6b27b7..ebe4e7089 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -73,6 +73,8 @@ spec: type: integer gcp_credentials: type: string + irsa_role_arn: + type: string kube_iam_role: type: string log_s3_bucket: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index daa7f7b13..e25a7b339 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -171,6 +171,7 @@ configuration: # enable_ebs_gp3_migration_max_size: 1000 # gcp_credentials: "" # kube_iam_role: "" + # irsa_role_arn: "" # log_s3_bucket: "" # wal_az_storage_account: "" # wal_gs_bucket: "" diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index b84bce03a..046dd0761 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -252,6 +252,7 @@ type AWSGCPConfiguration struct { WALAZStorageAccount string `json:"wal_az_storage_account,omitempty"` LogS3Bucket string `json:"log_s3_bucket,omitempty"` KubeIAMRole string `json:"kube_iam_role,omitempty"` + IRSARoleARN string `json:"irsa_role_arn,omitempty"` AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` AdditionalSecretMountPath string `json:"additional_secret_mount_path,omitempty"` EnableEBSGp3Migration bool `json:"enable_ebs_gp3_migration,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml b/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml index 9cd6b27b7..ebe4e7089 100644 --- a/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml +++ b/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml @@ -73,6 +73,8 @@ spec: type: integer gcp_credentials: type: string + irsa_role_arn: + type: string kube_iam_role: type: string log_s3_bucket: diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 06c76b279..836bd086e 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1140,6 +1140,12 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume size sync.") } + // Pod service account (IRSA annotation sync) + if err := c.syncPodServiceAccount(); err != nil { + c.logger.Errorf("could not sync pod service account: %v", err) + updateFailed = true + } + // Statefulset func() { if err := c.syncStatefulSet(); err != nil { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index af3f786df..5ada7ea9a 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -107,6 +107,10 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } + if err = c.syncPodServiceAccount(); err != nil { + c.logger.Errorf("could not sync pod service account: %v", err) + } + if err = c.syncStatefulSet(); err != nil { if !k8sutil.ResourceAlreadyExists(err) { err = fmt.Errorf("could not sync statefulsets: %v", err) @@ -630,6 +634,10 @@ func (c *Cluster) syncStatefulSet() error { if !cmp.rollingUpdate { updatedPodAnnotations := map[string]*string{} for _, anno := range cmp.deletedPodAnnotations { + // during IRSA migration let kube2iam annotation drain naturally via pod rotation + if c.OpConfig.IRSARoleARN != "" && anno == constants.KubeIAmAnnotation { + continue + } updatedPodAnnotations[anno] = nil } for anno, val := range desiredSts.Spec.Template.Annotations { @@ -1803,6 +1811,7 @@ func (c *Cluster) syncLogicalBackupJob() error { // no existing logical backup job, create new one c.logger.Info("could not find the cluster's logical backup job") + if err = c.createLogicalBackupJob(); err == nil { c.logger.Infof("created missing logical backup job %s", jobName) } else { @@ -1817,3 +1826,62 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } + +func (c *Cluster) syncPodServiceAccount() error { + sa, err := c.KubeClient.ServiceAccounts(c.Namespace).Get(context.TODO(), c.OpConfig.PodServiceAccountName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("could not get pod service account %q: %v", c.OpConfig.PodServiceAccountName, err) + } + + changed := false + + if c.OpConfig.IRSARoleARN != "" { + if val, ok := sa.Annotations[constants.IRSAAnnotation]; !ok || val != c.OpConfig.IRSARoleARN { + if sa.Annotations == nil { + sa.Annotations = make(map[string]string) + } + sa.Annotations[constants.IRSAAnnotation] = c.OpConfig.IRSARoleARN + changed = true + } + } else { + if _, ok := sa.Annotations[constants.IRSAAnnotation]; ok { + delete(sa.Annotations, constants.IRSAAnnotation) + changed = true + } + } + + if changed { + if _, err = c.KubeClient.ServiceAccounts(c.Namespace).Update(context.TODO(), sa, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("could not update pod service account %q: %v", sa.Name, err) + } + c.logger.Infof("synced annotations on pod service account %q", sa.Name) + } + + if c.OpConfig.IRSARoleARN != "" { + c.logIRSAMigrationProgress() + } + + return nil +} + +func (c *Cluster) logIRSAMigrationProgress() { + pods, err := c.listPods() + if err != nil { + c.logger.Warnf("IRSA migration: could not list pods: %v", err) + return + } + + total := len(pods) + remaining := 0 + for _, pod := range pods { + if _, ok := pod.Annotations[constants.KubeIAmAnnotation]; ok { + remaining++ + } + } + + if remaining > 0 { + c.logger.Infof("IRSA migration in progress: %d/%d pods still carry kube2iam annotation, will be removed on next rotation", remaining, total) + } else { + c.logger.Infof("IRSA migration complete: all %d pods have rotated, kube2iam annotation fully drained", total) + } +} diff --git a/pkg/cluster/util_test.go b/pkg/cluster/util_test.go index d97668ee8..dfae6237a 100644 --- a/pkg/cluster/util_test.go +++ b/pkg/cluster/util_test.go @@ -53,6 +53,7 @@ func newFakeK8sAnnotationsClient() (k8sutil.KubernetesClient, *k8sFake.Clientset EndpointsGetter: clientSet.CoreV1(), ConfigMapsGetter: clientSet.CoreV1(), PodsGetter: clientSet.CoreV1(), + ServiceAccountsGetter: clientSet.CoreV1(), DeploymentsGetter: clientSet.AppsV1(), CronJobsGetter: clientSet.BatchV1(), }, clientSet diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 879e44ad4..63434efed 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -247,6 +247,12 @@ func (c *Controller) initPodServiceAccount() { c.PodServiceAccount.Name = c.opConfig.PodServiceAccountName } c.PodServiceAccount.Namespace = "" + if c.opConfig.IRSARoleARN != "" { + if c.PodServiceAccount.Annotations == nil { + c.PodServiceAccount.Annotations = make(map[string]string) + } + c.PodServiceAccount.Annotations[constants.IRSAAnnotation] = c.opConfig.IRSARoleARN + } } // actual service accounts are deployed at the time of Postgres/Spilo cluster creation diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index b3765e387..667384e30 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -175,6 +175,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AWSRegion = fromCRD.AWSGCP.AWSRegion result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole + result.IRSARoleARN = fromCRD.AWSGCP.IRSARoleARN result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials result.WALAZStorageAccount = fromCRD.AWSGCP.WALAZStorageAccount diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 14ae537cc..95324a747 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -194,6 +194,7 @@ type Config struct { WALES3Bucket string `name:"wal_s3_bucket"` LogS3Bucket string `name:"log_s3_bucket"` KubeIAMRole string `name:"kube_iam_role"` + IRSARoleARN string `name:"irsa_role_arn"` WALGSBucket string `name:"wal_gs_bucket"` GCPCredentials string `name:"gcp_credentials"` WALAZStorageAccount string `name:"wal_az_storage_account"` diff --git a/pkg/util/constants/annotations.go b/pkg/util/constants/annotations.go index 0330ddcb8..d5d925e2e 100644 --- a/pkg/util/constants/annotations.go +++ b/pkg/util/constants/annotations.go @@ -4,6 +4,7 @@ package constants const ( ZalandoDNSNameAnnotation = "external-dns.alpha.kubernetes.io/hostname" KubeIAmAnnotation = "iam.amazonaws.com/role" + IRSAAnnotation = "eks.amazonaws.com/role-arn" VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by" PostgresqlControllerAnnotationKey = "acid.zalan.do/controller" )