diff --git a/Makefile b/Makefile index d85b74ed3..e6bf4d88f 100644 --- a/Makefile +++ b/Makefile @@ -29,16 +29,19 @@ PKG := `go list ./... | grep -v /vendor/` ifeq ($(DEBUG),1) DOCKERFILE = DebugDockerfile - DEBUG_POSTFIX := -debug-$(shell date hhmmss) + DEBUG_POSTFIX := -debug-$(shell date +"%H%M%S") BUILD_FLAGS += -gcflags "-N -l" else DOCKERFILE = Dockerfile endif + ifeq ($(FRESH),1) DEBUG_FRESH=$(shell date +"%H-%M-%S") endif +SED := $(shell command -v gsed 2>/dev/null || command -v sed) + ifdef CDP_PULL_REQUEST_NUMBER CDP_TAG := -${CDP_BUILD_VERSION} endif @@ -69,8 +72,8 @@ $(GENERATED_CRDS): $(GENERATED) go tool controller-gen crd:crdVersions=v1,allowDangerousTypes=true paths=./pkg/apis/acid.zalan.do/... output:crd:dir=manifests @mv manifests/acid.zalan.do_postgresqls.yaml manifests/postgresql.crd.yaml @# hack to use lowercase kind and listKind - @sed -i -e 's/kind: Postgresql/kind: postgresql/' manifests/postgresql.crd.yaml - @sed -i -e 's/listKind: PostgresqlList/listKind: postgresqlList/' manifests/postgresql.crd.yaml + @$(SED) -i -e 's/kind: Postgresql/kind: postgresql/' manifests/postgresql.crd.yaml + @$(SED) -i -e 's/listKind: PostgresqlList/listKind: postgresqlList/' manifests/postgresql.crd.yaml @hack/adjust_postgresql_crd.sh @mv manifests/acid.zalan.do_operatorconfigurations.yaml manifests/operatorconfiguration.crd.yaml @mv manifests/acid.zalan.do_postgresteams.yaml manifests/postgresteam.crd.yaml diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 983a3ed5a..528a74d58 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -77,6 +77,8 @@ spec: type: integer gcp_credentials: type: string + irsa_role_arn: + type: string kube_iam_role: type: string log_s3_bucket: diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index ad3b46064..6846a4bdf 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -234,6 +234,7 @@ rules: verbs: - get - create + - update # to create role bindings to the postgres-pod service account - apiGroups: - rbac.authorization.k8s.io diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 3a2194ef0..4ea5b6319 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -360,6 +360,8 @@ configAwsOrGcp: # AWS IAM role to supply in the iam.amazonaws.com/role annotation of Postgres pods # kube_iam_role: "" + # Full ARN for IRSA (IAM Roles for Service Accounts) on EKS + # irsa_role_arn: "" # S3 bucket to use for shipping postgres daily logs # log_s3_bucket: "" diff --git a/docs/administrator.md b/docs/administrator.md index ec0bc28dc..49718d1ea 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -1094,6 +1094,32 @@ configuration: wal_s3_bucket: your-backup-path ``` +Alternatively, if your cluster uses EKS with OIDC, you can use +[IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) +(IAM Roles for Service Accounts) instead of kube2iam. Set `irsa_role_arn` to +the full ARN of the IAM role: + +**OperatorConfiguration** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + aws_or_gcp: + aws_region: eu-central-1 + irsa_role_arn: arn:aws:iam::123456789012:role/postgres-pod-role + wal_s3_bucket: your-backup-path +``` + +When `irsa_role_arn` is set the operator annotates the pod service account with +`eks.amazonaws.com/role-arn` on every reconcile. The EKS OIDC webhook then +injects an AWS web identity token into each pod, which takes precedence over +the EC2 metadata credentials used by kube2iam. Both `kube_iam_role` and +`irsa_role_arn` can coexist during a migration — existing pods retain the +kube2iam annotation until they are rotated, at which point only IRSA is used. + The referenced IAM role should contain the following privileges to make sure Postgres can send compressed WAL files to the given S3 bucket: @@ -1204,6 +1230,7 @@ aws_or_gcp: # additional_secret_mount_path: "" # aws_region: eu-central-1 # kube_iam_role: "" + # irsa_role_arn: "" # log_s3_bucket: "" # wal_s3_bucket: "" wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs @@ -1253,6 +1280,7 @@ aws_or_gcp: additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file # aws_region: eu-central-1 # kube_iam_role: "" + # irsa_role_arn: "" # log_s3_bucket: "" # wal_s3_bucket: "" wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index dd7487352..02c6db8d0 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -42,14 +42,14 @@ and change it. To test the CRD-based configuration locally, use the following -```bash - kubectl create -f manifests/operatorconfiguration.crd.yaml # registers the CRD - kubectl create -f manifests/postgresql-operator-default-configuration.yaml +``` +kubectl create -f manifests/operatorconfiguration.crd.yaml # registers the CRD +kubectl create -f manifests/postgresql-operator-default-configuration.yaml - kubectl create -f manifests/operator-service-account-rbac.yaml - kubectl create -f manifests/postgres-operator.yaml # set the env var as mentioned above +kubectl create -f manifests/operator-service-account-rbac.yaml +kubectl create -f manifests/postgres-operator.yaml # set the env var as mentioned above - kubectl get operatorconfigurations postgresql-operator-default-configuration -o yaml +kubectl get operatorconfigurations postgresql-operator-default-configuration -o yaml ``` The CRD-based configuration is more powerful than the one based on ConfigMaps @@ -798,6 +798,15 @@ yet officially supported. [kube2iam](https://github.com/jtblin/kube2iam) project on AWS. The default is empty. +* **irsa_role_arn** + Full AWS IAM role ARN to supply in the `eks.amazonaws.com/role-arn` annotation + of the Postgres pod service account, enabling + [IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) + (IAM Roles for Service Accounts) on EKS. When set, the operator annotates the + pod service account on every sync so that the EKS OIDC webhook can inject AWS + credentials directly into pods. Must be a full ARN, e.g. + `arn:aws:iam::123456789012:role/my-postgres-role`. The default is empty. + * **aws_region** AWS region used to store EBS volumes. The default is `eu-central-1`. Note, this option is not meant for specifying the AWS region for backups and diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 7ce80a6c9..8b64185e9 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1430,7 +1430,7 @@ class EndToEndTestCase(unittest.TestCase): k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") + "Operator does not get in sync", retries=120) # wait for switched over k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) diff --git a/hack/adjust_postgresql_crd.sh b/hack/adjust_postgresql_crd.sh index d06b74a2d..29d2788a7 100755 --- a/hack/adjust_postgresql_crd.sh +++ b/hack/adjust_postgresql_crd.sh @@ -13,12 +13,14 @@ file="${1:-"manifests/postgresql.crd.yaml"}" -sed -i '/^[[:space:]]*standby:$/{ +SED=$(command -v gsed 2>/dev/null || command -v sed) + +$SED -i '/^[[:space:]]*standby:$/{ # Capture the indentation s/^\([[:space:]]*\)standby:$/\1standby:\n\1 anyOf:\n\1 - required:\n\1 - s3_wal_path\n\1 - required:\n\1 - gs_wal_path\n\1 - required:\n\1 - standby_host\n\1 not:\n\1 required:\n\1 - s3_wal_path\n\1 - gs_wal_path/ }' "$file" -sed -i '/^[[:space:]]*maintenanceWindows:$/{ +$SED -i '/^[[:space:]]*maintenanceWindows:$/{ # Capture the indentation s/^\([[:space:]]*\)maintenanceWindows:$/\1maintenanceWindows:\n\1 items:\n\1 pattern: '\''^\\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))-((2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))\\ *$'\''\n\1 type: string/ }' "$file" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 5806fe7b2..c069df237 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -80,6 +80,7 @@ data: # inherited_annotations: owned-by # inherited_labels: application,environment # kube_iam_role: "" + # irsa_role_arn: "" kubernetes_use_configmaps: "false" # log_s3_bucket: "" # logical_backup_azure_storage_account_name: "" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 983a3ed5a..528a74d58 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -77,6 +77,8 @@ spec: type: integer gcp_credentials: type: string + irsa_role_arn: + type: string kube_iam_role: type: string log_s3_bucket: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 2091e102f..d511f45d2 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -174,6 +174,7 @@ configuration: # enable_ebs_gp3_migration_max_size: 1000 # gcp_credentials: "" # kube_iam_role: "" + # irsa_role_arn: "" # log_s3_bucket: "" # wal_az_storage_account: "" # wal_gs_bucket: "" diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 4f04121ab..35f703505 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -252,6 +252,7 @@ type AWSGCPConfiguration struct { WALAZStorageAccount string `json:"wal_az_storage_account,omitempty"` LogS3Bucket string `json:"log_s3_bucket,omitempty"` KubeIAMRole string `json:"kube_iam_role,omitempty"` + IRSARoleARN string `json:"irsa_role_arn,omitempty"` AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` AdditionalSecretMountPath string `json:"additional_secret_mount_path,omitempty"` EnableEBSGp3Migration bool `json:"enable_ebs_gp3_migration,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml b/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml index 983a3ed5a..528a74d58 100644 --- a/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml +++ b/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml @@ -77,6 +77,8 @@ spec: type: integer gcp_credentials: type: string + irsa_role_arn: + type: string kube_iam_role: type: string log_s3_bucket: diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 06c76b279..836bd086e 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1140,6 +1140,12 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume size sync.") } + // Pod service account (IRSA annotation sync) + if err := c.syncPodServiceAccount(); err != nil { + c.logger.Errorf("could not sync pod service account: %v", err) + updateFailed = true + } + // Statefulset func() { if err := c.syncStatefulSet(); err != nil { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 09cffea36..505977ed9 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2278,6 +2278,12 @@ func (c *Cluster) generatePrimaryPodDisruptionBudget() *policyv1.PodDisruptionBu labels[c.OpConfig.PodRoleLabel] = string(Master) } + // When master selector is disabled and synchronous_mode_strict is on, require + // master + synchronous_node_count (default 1) healthy pods for write quorum. + if pdbMasterLabelSelector != nil && !*pdbMasterLabelSelector && minAvailable.IntVal > 0 && c.Spec.SynchronousModeStrict { + minAvailable = intstr.FromInt32(int32(c.Spec.SynchronousNodeCount + 1)) + } + return &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: c.PrimaryPodDisruptionBudgetName(), diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index d51efb085..cb2655777 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -2691,13 +2691,13 @@ func TestGeneratePodDisruptionBudget(t *testing.T) { k8sutil.KubernetesClient{}, acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, - Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3, Patroni: acidv1.Patroni{SynchronousModeStrict: true, SynchronousNodeCount: 1}}}, logger, eventRecorder), check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ testPodDisruptionBudgetOwnerReference, hasName("postgres-myapp-database-pdb"), - hasMinAvailable(1), + hasMinAvailable(2), testLabelsAndSelectors(true), }, }, diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index af3f786df..5ada7ea9a 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -107,6 +107,10 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } + if err = c.syncPodServiceAccount(); err != nil { + c.logger.Errorf("could not sync pod service account: %v", err) + } + if err = c.syncStatefulSet(); err != nil { if !k8sutil.ResourceAlreadyExists(err) { err = fmt.Errorf("could not sync statefulsets: %v", err) @@ -630,6 +634,10 @@ func (c *Cluster) syncStatefulSet() error { if !cmp.rollingUpdate { updatedPodAnnotations := map[string]*string{} for _, anno := range cmp.deletedPodAnnotations { + // during IRSA migration let kube2iam annotation drain naturally via pod rotation + if c.OpConfig.IRSARoleARN != "" && anno == constants.KubeIAmAnnotation { + continue + } updatedPodAnnotations[anno] = nil } for anno, val := range desiredSts.Spec.Template.Annotations { @@ -1803,6 +1811,7 @@ func (c *Cluster) syncLogicalBackupJob() error { // no existing logical backup job, create new one c.logger.Info("could not find the cluster's logical backup job") + if err = c.createLogicalBackupJob(); err == nil { c.logger.Infof("created missing logical backup job %s", jobName) } else { @@ -1817,3 +1826,62 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } + +func (c *Cluster) syncPodServiceAccount() error { + sa, err := c.KubeClient.ServiceAccounts(c.Namespace).Get(context.TODO(), c.OpConfig.PodServiceAccountName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("could not get pod service account %q: %v", c.OpConfig.PodServiceAccountName, err) + } + + changed := false + + if c.OpConfig.IRSARoleARN != "" { + if val, ok := sa.Annotations[constants.IRSAAnnotation]; !ok || val != c.OpConfig.IRSARoleARN { + if sa.Annotations == nil { + sa.Annotations = make(map[string]string) + } + sa.Annotations[constants.IRSAAnnotation] = c.OpConfig.IRSARoleARN + changed = true + } + } else { + if _, ok := sa.Annotations[constants.IRSAAnnotation]; ok { + delete(sa.Annotations, constants.IRSAAnnotation) + changed = true + } + } + + if changed { + if _, err = c.KubeClient.ServiceAccounts(c.Namespace).Update(context.TODO(), sa, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("could not update pod service account %q: %v", sa.Name, err) + } + c.logger.Infof("synced annotations on pod service account %q", sa.Name) + } + + if c.OpConfig.IRSARoleARN != "" { + c.logIRSAMigrationProgress() + } + + return nil +} + +func (c *Cluster) logIRSAMigrationProgress() { + pods, err := c.listPods() + if err != nil { + c.logger.Warnf("IRSA migration: could not list pods: %v", err) + return + } + + total := len(pods) + remaining := 0 + for _, pod := range pods { + if _, ok := pod.Annotations[constants.KubeIAmAnnotation]; ok { + remaining++ + } + } + + if remaining > 0 { + c.logger.Infof("IRSA migration in progress: %d/%d pods still carry kube2iam annotation, will be removed on next rotation", remaining, total) + } else { + c.logger.Infof("IRSA migration complete: all %d pods have rotated, kube2iam annotation fully drained", total) + } +} diff --git a/pkg/cluster/util_test.go b/pkg/cluster/util_test.go index d97668ee8..dfae6237a 100644 --- a/pkg/cluster/util_test.go +++ b/pkg/cluster/util_test.go @@ -53,6 +53,7 @@ func newFakeK8sAnnotationsClient() (k8sutil.KubernetesClient, *k8sFake.Clientset EndpointsGetter: clientSet.CoreV1(), ConfigMapsGetter: clientSet.CoreV1(), PodsGetter: clientSet.CoreV1(), + ServiceAccountsGetter: clientSet.CoreV1(), DeploymentsGetter: clientSet.AppsV1(), CronJobsGetter: clientSet.BatchV1(), }, clientSet diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 879e44ad4..63434efed 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -247,6 +247,12 @@ func (c *Controller) initPodServiceAccount() { c.PodServiceAccount.Name = c.opConfig.PodServiceAccountName } c.PodServiceAccount.Namespace = "" + if c.opConfig.IRSARoleARN != "" { + if c.PodServiceAccount.Annotations == nil { + c.PodServiceAccount.Annotations = make(map[string]string) + } + c.PodServiceAccount.Annotations[constants.IRSAAnnotation] = c.opConfig.IRSARoleARN + } } // actual service accounts are deployed at the time of Postgres/Spilo cluster creation diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 1b2c521ae..9d73c5892 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -175,6 +175,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.AWSRegion = fromCRD.AWSGCP.AWSRegion result.LogS3Bucket = fromCRD.AWSGCP.LogS3Bucket result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole + result.IRSARoleARN = fromCRD.AWSGCP.IRSARoleARN result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials result.WALAZStorageAccount = fromCRD.AWSGCP.WALAZStorageAccount diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index e234028e1..1fdef223b 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -194,6 +194,7 @@ type Config struct { WALES3Bucket string `name:"wal_s3_bucket"` LogS3Bucket string `name:"log_s3_bucket"` KubeIAMRole string `name:"kube_iam_role"` + IRSARoleARN string `name:"irsa_role_arn"` WALGSBucket string `name:"wal_gs_bucket"` GCPCredentials string `name:"gcp_credentials"` WALAZStorageAccount string `name:"wal_az_storage_account"` diff --git a/pkg/util/constants/annotations.go b/pkg/util/constants/annotations.go index 0330ddcb8..d5d925e2e 100644 --- a/pkg/util/constants/annotations.go +++ b/pkg/util/constants/annotations.go @@ -4,6 +4,7 @@ package constants const ( ZalandoDNSNameAnnotation = "external-dns.alpha.kubernetes.io/hostname" KubeIAmAnnotation = "iam.amazonaws.com/role" + IRSAAnnotation = "eks.amazonaws.com/role-arn" VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by" PostgresqlControllerAnnotationKey = "acid.zalan.do/controller" ) diff --git a/ui/requirements.txt b/ui/requirements.txt index ace18641d..4a8874dfe 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -2,7 +2,7 @@ backoff==2.2.1 boto3==1.34.110 boto==2.49.0 click==8.1.7 -Flask==3.0.3 +Flask==3.1.3 furl==2.1.3 gevent==24.2.1 jq==1.7.0