Merge branch 'master' into update-ebs-tags-inherit-labels

Makefile

@@ -29,16 +29,19 @@ PKG := `go list ./... | grep -v /vendor/`
 
 ifeq ($(DEBUG),1)
 	DOCKERFILE = DebugDockerfile
-	DEBUG_POSTFIX := -debug-$(shell date hhmmss)
+	DEBUG_POSTFIX := -debug-$(shell date +"%H%M%S")
 	BUILD_FLAGS += -gcflags "-N -l"
 else
 	DOCKERFILE = Dockerfile
 endif
 
+
 ifeq ($(FRESH),1)
   DEBUG_FRESH=$(shell date +"%H-%M-%S")
 endif
 
+SED := $(shell command -v gsed 2>/dev/null || command -v sed)
+
 ifdef CDP_PULL_REQUEST_NUMBER
 	CDP_TAG := -${CDP_BUILD_VERSION}
 endif
@@ -69,8 +72,8 @@ $(GENERATED_CRDS): $(GENERATED)
 	go tool controller-gen crd:crdVersions=v1,allowDangerousTypes=true paths=./pkg/apis/acid.zalan.do/... output:crd:dir=manifests
 	@mv manifests/acid.zalan.do_postgresqls.yaml manifests/postgresql.crd.yaml
 	@# hack to use lowercase kind and listKind
-	@sed -i -e 's/kind: Postgresql/kind: postgresql/' manifests/postgresql.crd.yaml
+	@$(SED) -i -e 's/kind: Postgresql/kind: postgresql/' manifests/postgresql.crd.yaml
-	@sed -i -e 's/listKind: PostgresqlList/listKind: postgresqlList/' manifests/postgresql.crd.yaml
+	@$(SED) -i -e 's/listKind: PostgresqlList/listKind: postgresqlList/' manifests/postgresql.crd.yaml
 	@hack/adjust_postgresql_crd.sh
 	@mv manifests/acid.zalan.do_operatorconfigurations.yaml manifests/operatorconfiguration.crd.yaml
 	@mv manifests/acid.zalan.do_postgresteams.yaml manifests/postgresteam.crd.yaml

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -77,6 +77,8 @@ spec:
                     type: integer
                   gcp_credentials:
                     type: string
+                  irsa_role_arn:
+                    type: string
                   kube_iam_role:
                     type: string
                   log_s3_bucket:

charts/postgres-operator/templates/clusterrole.yaml

@@ -234,6 +234,7 @@ rules:
   verbs:
   - get
   - create
+  - update
 # to create role bindings to the postgres-pod service account
 - apiGroups:
   - rbac.authorization.k8s.io

charts/postgres-operator/values.yaml

@@ -360,6 +360,8 @@ configAwsOrGcp:
 
   # AWS IAM role to supply in the iam.amazonaws.com/role annotation of Postgres pods
   # kube_iam_role: ""
+  # Full ARN for IRSA (IAM Roles for Service Accounts) on EKS
+  # irsa_role_arn: ""
 
   # S3 bucket to use for shipping postgres daily logs
   # log_s3_bucket: ""

docs/administrator.md

@@ -1094,6 +1094,32 @@ configuration:
     wal_s3_bucket: your-backup-path
 ```
 
+Alternatively, if your cluster uses EKS with OIDC, you can use
+[IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html)
+(IAM Roles for Service Accounts) instead of kube2iam. Set `irsa_role_arn` to
+the full ARN of the IAM role:
+
+**OperatorConfiguration**
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: OperatorConfiguration
+metadata:
+  name: postgresql-operator-configuration
+configuration:
+  aws_or_gcp:
+    aws_region: eu-central-1
+    irsa_role_arn: arn:aws:iam::123456789012:role/postgres-pod-role
+    wal_s3_bucket: your-backup-path
+```
+
+When `irsa_role_arn` is set the operator annotates the pod service account with
+`eks.amazonaws.com/role-arn` on every reconcile. The EKS OIDC webhook then
+injects an AWS web identity token into each pod, which takes precedence over
+the EC2 metadata credentials used by kube2iam. Both `kube_iam_role` and
+`irsa_role_arn` can coexist during a migration — existing pods retain the
+kube2iam annotation until they are rotated, at which point only IRSA is used.
+
 The referenced IAM role should contain the following privileges to make sure
 Postgres can send compressed WAL files to the given S3 bucket:
 
@@ -1204,6 +1230,7 @@ aws_or_gcp:
   # additional_secret_mount_path: ""
   # aws_region: eu-central-1
   # kube_iam_role: ""
+  # irsa_role_arn: ""
   # log_s3_bucket: ""
   # wal_s3_bucket: ""
   wal_gs_bucket: "postgres-backups-bucket-28302F2"  # name of bucket on where to save the WAL-E logs
@@ -1253,6 +1280,7 @@ aws_or_gcp:
   additional_secret_mount_path: "/var/secrets/google"  # or where ever you want to mount the file
   # aws_region: eu-central-1
   # kube_iam_role: ""
+  # irsa_role_arn: ""
   # log_s3_bucket: ""
   # wal_s3_bucket: ""
   wal_gs_bucket: "postgres-backups-bucket-28302F2"  # name of bucket on where to save the WAL-E logs

docs/reference/operator_parameters.md

@@ -42,14 +42,14 @@ and change it.
 
 To test the CRD-based configuration locally, use the following
 
-```bash
+```
-  kubectl create -f manifests/operatorconfiguration.crd.yaml # registers the CRD
+kubectl create -f manifests/operatorconfiguration.crd.yaml # registers the CRD
-  kubectl create -f manifests/postgresql-operator-default-configuration.yaml
+kubectl create -f manifests/postgresql-operator-default-configuration.yaml
 
-  kubectl create -f manifests/operator-service-account-rbac.yaml
+kubectl create -f manifests/operator-service-account-rbac.yaml
-  kubectl create -f manifests/postgres-operator.yaml # set the env var as mentioned above
+kubectl create -f manifests/postgres-operator.yaml # set the env var as mentioned above
 
-  kubectl get operatorconfigurations postgresql-operator-default-configuration -o yaml
+kubectl get operatorconfigurations postgresql-operator-default-configuration -o yaml
 ```
 
 The CRD-based configuration is more powerful than the one based on ConfigMaps
@@ -798,6 +798,15 @@ yet officially supported.
   [kube2iam](https://github.com/jtblin/kube2iam) project on AWS. The default is
   empty.
 
+* **irsa_role_arn**
+  Full AWS IAM role ARN to supply in the `eks.amazonaws.com/role-arn` annotation
+  of the Postgres pod service account, enabling
+  [IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html)
+  (IAM Roles for Service Accounts) on EKS. When set, the operator annotates the
+  pod service account on every sync so that the EKS OIDC webhook can inject AWS
+  credentials directly into pods. Must be a full ARN, e.g.
+  `arn:aws:iam::123456789012:role/my-postgres-role`. The default is empty.
+
 * **aws_region**
   AWS region used to store EBS volumes. The default is `eu-central-1`. Note,
   this option is not meant for specifying the AWS region for backups and

e2e/tests/test_e2e.py

@@ -1430,7 +1430,7 @@ class EndToEndTestCase(unittest.TestCase):
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
             "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources)
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},
-                             "Operator does not get in sync")
+                             "Operator does not get in sync", retries=120)
 
         # wait for switched over
         k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label)

hack/adjust_postgresql_crd.sh

@@ -13,12 +13,14 @@
 
 file="${1:-"manifests/postgresql.crd.yaml"}"
 
-sed -i '/^[[:space:]]*standby:$/{
+SED=$(command -v gsed 2>/dev/null || command -v sed)
+
+$SED -i '/^[[:space:]]*standby:$/{
     # Capture the indentation
     s/^\([[:space:]]*\)standby:$/\1standby:\n\1  anyOf:\n\1  - required:\n\1    - s3_wal_path\n\1  - required:\n\1    - gs_wal_path\n\1  - required:\n\1    - standby_host\n\1  not:\n\1    required:\n\1    - s3_wal_path\n\1    - gs_wal_path/
 }' "$file"
 
-sed -i '/^[[:space:]]*maintenanceWindows:$/{
+$SED -i '/^[[:space:]]*maintenanceWindows:$/{
     # Capture the indentation
     s/^\([[:space:]]*\)maintenanceWindows:$/\1maintenanceWindows:\n\1  items:\n\1    pattern: '\''^\\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))-((2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))\\ *$'\''\n\1    type: string/
 }' "$file"

manifests/configmap.yaml

@@ -80,6 +80,7 @@ data:
   # inherited_annotations: owned-by
   # inherited_labels: application,environment
   # kube_iam_role: ""
+  # irsa_role_arn: ""
   kubernetes_use_configmaps: "false"
   # log_s3_bucket: ""
   # logical_backup_azure_storage_account_name: ""

manifests/operatorconfiguration.crd.yaml

@@ -77,6 +77,8 @@ spec:
                     type: integer
                   gcp_credentials:
                     type: string
+                  irsa_role_arn:
+                    type: string
                   kube_iam_role:
                     type: string
                   log_s3_bucket:

manifests/postgresql-operator-default-configuration.yaml

@@ -174,6 +174,7 @@ configuration:
     # enable_ebs_gp3_migration_max_size: 1000
     # gcp_credentials: ""
     # kube_iam_role: ""
+    # irsa_role_arn: ""
     # log_s3_bucket: ""
     # wal_az_storage_account: ""
     # wal_gs_bucket: ""

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -252,6 +252,7 @@ type AWSGCPConfiguration struct {
 	WALAZStorageAccount          string   `json:"wal_az_storage_account,omitempty"`
 	LogS3Bucket                  string   `json:"log_s3_bucket,omitempty"`
 	KubeIAMRole                  string   `json:"kube_iam_role,omitempty"`
+  IRSARoleARN                  string   `json:"irsa_role_arn,omitempty"`
 	AdditionalSecretMount        string   `json:"additional_secret_mount,omitempty"`
 	AdditionalSecretMountPath    string   `json:"additional_secret_mount_path,omitempty"`
 	EnableEBSGp3Migration        bool     `json:"enable_ebs_gp3_migration,omitempty"`

pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml

@@ -77,6 +77,8 @@ spec:
                     type: integer
                   gcp_credentials:
                     type: string
+                  irsa_role_arn:
+                    type: string
                   kube_iam_role:
                     type: string
                   log_s3_bucket:

pkg/cluster/cluster.go

@@ -1140,6 +1140,12 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error {
 		c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume size sync.")
 	}
 
+	// Pod service account (IRSA annotation sync)
+	if err := c.syncPodServiceAccount(); err != nil {
+		c.logger.Errorf("could not sync pod service account: %v", err)
+		updateFailed = true
+	}
+
 	// Statefulset
 	func() {
 		if err := c.syncStatefulSet(); err != nil {

pkg/cluster/k8sres.go

@@ -2278,6 +2278,12 @@ func (c *Cluster) generatePrimaryPodDisruptionBudget() *policyv1.PodDisruptionBu
 		labels[c.OpConfig.PodRoleLabel] = string(Master)
 	}
 
+	// When master selector is disabled and synchronous_mode_strict is on, require
+	// master + synchronous_node_count (default 1) healthy pods for write quorum.
+	if pdbMasterLabelSelector != nil && !*pdbMasterLabelSelector && minAvailable.IntVal > 0 && c.Spec.SynchronousModeStrict {
+		minAvailable = intstr.FromInt32(int32(c.Spec.SynchronousNodeCount + 1))
+	}
+
 	return &policyv1.PodDisruptionBudget{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:            c.PrimaryPodDisruptionBudgetName(),

pkg/cluster/k8sres_test.go

@@ -2691,13 +2691,13 @@ func TestGeneratePodDisruptionBudget(t *testing.T) {
 				k8sutil.KubernetesClient{},
 				acidv1.Postgresql{
 					ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"},
-					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}},
+					Spec:       acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3, Patroni: acidv1.Patroni{SynchronousModeStrict: true, SynchronousNodeCount: 1}}},
 				logger,
 				eventRecorder),
 			check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{
 				testPodDisruptionBudgetOwnerReference,
 				hasName("postgres-myapp-database-pdb"),
-				hasMinAvailable(1),
+				hasMinAvailable(2),
 				testLabelsAndSelectors(true),
 			},
 		},

pkg/cluster/sync.go

@@ -107,6 +107,10 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error {
 		}
 	}
 
+	if err = c.syncPodServiceAccount(); err != nil {
+		c.logger.Errorf("could not sync pod service account: %v", err)
+	}
+
 	if err = c.syncStatefulSet(); err != nil {
 		if !k8sutil.ResourceAlreadyExists(err) {
 			err = fmt.Errorf("could not sync statefulsets: %v", err)
@@ -630,6 +634,10 @@ func (c *Cluster) syncStatefulSet() error {
 		if !cmp.rollingUpdate {
 			updatedPodAnnotations := map[string]*string{}
 			for _, anno := range cmp.deletedPodAnnotations {
+				// during IRSA migration let kube2iam annotation drain naturally via pod rotation
+				if c.OpConfig.IRSARoleARN != "" && anno == constants.KubeIAmAnnotation {
+					continue
+				}
 				updatedPodAnnotations[anno] = nil
 			}
 			for anno, val := range desiredSts.Spec.Template.Annotations {
@@ -1803,6 +1811,7 @@ func (c *Cluster) syncLogicalBackupJob() error {
 	// no existing logical backup job, create new one
 	c.logger.Info("could not find the cluster's logical backup job")
 
+
 	if err = c.createLogicalBackupJob(); err == nil {
 		c.logger.Infof("created missing logical backup job %s", jobName)
 	} else {
@@ -1817,3 +1826,62 @@ func (c *Cluster) syncLogicalBackupJob() error {
 
 	return nil
 }
+
+func (c *Cluster) syncPodServiceAccount() error {
+	sa, err := c.KubeClient.ServiceAccounts(c.Namespace).Get(context.TODO(), c.OpConfig.PodServiceAccountName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("could not get pod service account %q: %v", c.OpConfig.PodServiceAccountName, err)
+	}
+
+	changed := false
+
+	if c.OpConfig.IRSARoleARN != "" {
+		if val, ok := sa.Annotations[constants.IRSAAnnotation]; !ok || val != c.OpConfig.IRSARoleARN {
+			if sa.Annotations == nil {
+				sa.Annotations = make(map[string]string)
+			}
+			sa.Annotations[constants.IRSAAnnotation] = c.OpConfig.IRSARoleARN
+			changed = true
+		}
+	} else {
+		if _, ok := sa.Annotations[constants.IRSAAnnotation]; ok {
+			delete(sa.Annotations, constants.IRSAAnnotation)
+			changed = true
+		}
+	}
+
+	if changed {
+		if _, err = c.KubeClient.ServiceAccounts(c.Namespace).Update(context.TODO(), sa, metav1.UpdateOptions{}); err != nil {
+			return fmt.Errorf("could not update pod service account %q: %v", sa.Name, err)
+		}
+		c.logger.Infof("synced annotations on pod service account %q", sa.Name)
+	}
+
+	if c.OpConfig.IRSARoleARN != "" {
+		c.logIRSAMigrationProgress()
+	}
+
+	return nil
+}
+
+func (c *Cluster) logIRSAMigrationProgress() {
+	pods, err := c.listPods()
+	if err != nil {
+		c.logger.Warnf("IRSA migration: could not list pods: %v", err)
+		return
+	}
+
+	total := len(pods)
+	remaining := 0
+	for _, pod := range pods {
+		if _, ok := pod.Annotations[constants.KubeIAmAnnotation]; ok {
+			remaining++
+		}
+	}
+
+	if remaining > 0 {
+		c.logger.Infof("IRSA migration in progress: %d/%d pods still carry kube2iam annotation, will be removed on next rotation", remaining, total)
+	} else {
+		c.logger.Infof("IRSA migration complete: all %d pods have rotated, kube2iam annotation fully drained", total)
+	}
+}

pkg/cluster/util_test.go

@@ -53,6 +53,7 @@ func newFakeK8sAnnotationsClient() (k8sutil.KubernetesClient, *k8sFake.Clientset
 		EndpointsGetter:              clientSet.CoreV1(),
 		ConfigMapsGetter:             clientSet.CoreV1(),
 		PodsGetter:                   clientSet.CoreV1(),
+		ServiceAccountsGetter:        clientSet.CoreV1(),
 		DeploymentsGetter:            clientSet.AppsV1(),
 		CronJobsGetter:               clientSet.BatchV1(),
 	}, clientSet

pkg/controller/controller.go

@@ -247,6 +247,12 @@ func (c *Controller) initPodServiceAccount() {
 			c.PodServiceAccount.Name = c.opConfig.PodServiceAccountName
 		}
 		c.PodServiceAccount.Namespace = ""
+		if c.opConfig.IRSARoleARN != "" {
+			if c.PodServiceAccount.Annotations == nil {
+				c.PodServiceAccount.Annotations = make(map[string]string)
+			}
+			c.PodServiceAccount.Annotations[constants.IRSAAnnotation] = c.opConfig.IRSARoleARN
+		}
 	}
 
 	// actual service accounts are deployed at the time of Postgres/Spilo cluster creation

pkg/controller/operator_config.go

@@ -175,6 +175,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	result.AWSRegion = fromCRD.AWSGCP.AWSRegion
 	result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket
 	result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole
+	result.IRSARoleARN = fromCRD.AWSGCP.IRSARoleARN
 	result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket
 	result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials
 	result.WALAZStorageAccount = fromCRD.AWSGCP.WALAZStorageAccount

pkg/util/config/config.go

@@ -194,6 +194,7 @@ type Config struct {
 	WALES3Bucket                             string            `name:"wal_s3_bucket"`
 	LogS3Bucket                              string            `name:"log_s3_bucket"`
 	KubeIAMRole                              string            `name:"kube_iam_role"`
+	IRSARoleARN                              string            `name:"irsa_role_arn"`
 	WALGSBucket                              string            `name:"wal_gs_bucket"`
 	GCPCredentials                           string            `name:"gcp_credentials"`
 	WALAZStorageAccount                      string            `name:"wal_az_storage_account"`

pkg/util/constants/annotations.go

@@ -4,6 +4,7 @@ package constants
 const (
 	ZalandoDNSNameAnnotation           = "external-dns.alpha.kubernetes.io/hostname"
 	KubeIAmAnnotation                  = "iam.amazonaws.com/role"
+	IRSAAnnotation                     = "eks.amazonaws.com/role-arn"
 	VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by"
 	PostgresqlControllerAnnotationKey  = "acid.zalan.do/controller"
 )

ui/requirements.txt

@@ -2,7 +2,7 @@ backoff==2.2.1
 boto3==1.34.110
 boto==2.49.0
 click==8.1.7
-Flask==3.0.3
+Flask==3.1.3
 furl==2.1.3
 gevent==24.2.1
 jq==1.7.0