diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index c801346e4..231f60a84 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -227,6 +227,10 @@ spec: items: type: string pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' + masterPoolerServiceAnnotations: + type: object + additionalProperties: + type: string masterServiceAnnotations: type: object additionalProperties: @@ -408,6 +412,10 @@ spec: replicaLoadBalancer: type: boolean description: deprecated + replicaPoolerServiceAnnotations: + type: object + additionalProperties: + type: string replicaServiceAnnotations: type: object additionalProperties: diff --git a/docs/administrator.md b/docs/administrator.md index 60db0f3f1..b1fb07642 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -905,6 +905,7 @@ precedence): 2. Globally configured `custom_service_annotations` 3. `serviceAnnotations` specified in the cluster manifest 4. `masterServiceAnnotations` and `replicaServiceAnnotations` specified in the cluster manifest +5. `masterPoolerServiceAnnotations` and `replicaPoolerServiceAnnotations` specified in the cluster manifest (only for connection pooler services) To limit the range of IP addresses that can reach a load balancer, specify the desired ranges in the `allowedSourceRanges` field (applies to both master and diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index ae23dabb9..4b91705be 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -203,6 +203,18 @@ These parameters are grouped directly under the `spec` key in the manifest. This field overrides `serviceAnnotations` with the same key for the replica service if not empty. +* **masterPoolerServiceAnnotations** + A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) + to the master connection pooler service created for the database cluster. + This field overrides `serviceAnnotations` and `masterServiceAnnotations` + with the same key for the master pooler service if not empty. + +* **replicaPoolerServiceAnnotations** + A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) + to the replica connection pooler service created for the database cluster. + This field overrides `serviceAnnotations` and `replicaServiceAnnotations` + with the same key for the replica pooler service if not empty. + * **enableShmVolume** Start a database pod without limitations on shm memory. By default Docker limit `/dev/shm` to `64M` (see e.g. the [docker diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 39811824e..19e1334c4 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -3256,6 +3256,12 @@ spec: pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' type: string type: array + masterPoolerServiceAnnotations: + additionalProperties: + type: string + description: MasterPoolerServiceAnnotations takes precedence over other + annotations for master pooler service if not empty + type: object masterServiceAnnotations: additionalProperties: type: string @@ -3560,6 +3566,12 @@ spec: replicaLoadBalancer: description: deprecated type: boolean + replicaPoolerServiceAnnotations: + additionalProperties: + type: string + description: ReplicaPoolerServiceAnnotations takes precedence over other + annotations for replica pooler service if not empty + type: object replicaServiceAnnotations: additionalProperties: type: string diff --git a/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml b/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml index 39811824e..19e1334c4 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml +++ b/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml @@ -3256,6 +3256,12 @@ spec: pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' type: string type: array + masterPoolerServiceAnnotations: + additionalProperties: + type: string + description: MasterPoolerServiceAnnotations takes precedence over other + annotations for master pooler service if not empty + type: object masterServiceAnnotations: additionalProperties: type: string @@ -3560,6 +3566,12 @@ spec: replicaLoadBalancer: description: deprecated type: boolean + replicaPoolerServiceAnnotations: + additionalProperties: + type: string + description: ReplicaPoolerServiceAnnotations takes precedence over other + annotations for replica pooler service if not empty + type: object replicaServiceAnnotations: additionalProperties: type: string diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 1dadfd06c..56cd80461 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -111,8 +111,12 @@ type PostgresSpec struct { // MasterServiceAnnotations takes precedence over ServiceAnnotations for master role if not empty MasterServiceAnnotations map[string]string `json:"masterServiceAnnotations,omitempty"` // ReplicaServiceAnnotations takes precedence over ServiceAnnotations for replica role if not empty - ReplicaServiceAnnotations map[string]string `json:"replicaServiceAnnotations,omitempty"` - TLS *TLSDescription `json:"tls,omitempty"` + ReplicaServiceAnnotations map[string]string `json:"replicaServiceAnnotations,omitempty"` + // MasterPoolerServiceAnnotations takes precedence over other annotations for master pooler service if not empty + MasterPoolerServiceAnnotations map[string]string `json:"masterPoolerServiceAnnotations,omitempty"` + // ReplicaPoolerServiceAnnotations takes precedence over other annotations for replica pooler service if not empty + ReplicaPoolerServiceAnnotations map[string]string `json:"replicaPoolerServiceAnnotations,omitempty"` + TLS *TLSDescription `json:"tls,omitempty"` AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` Streams []Stream `json:"streams,omitempty"` Env []v1.EnvVar `json:"env,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 159a87f35..56ce5b498 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -855,6 +855,20 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = val } } + if in.MasterPoolerServiceAnnotations != nil { + in, out := &in.MasterPoolerServiceAnnotations, &out.MasterPoolerServiceAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ReplicaPoolerServiceAnnotations != nil { + in, out := &in.ReplicaPoolerServiceAnnotations, &out.ReplicaPoolerServiceAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.TLS != nil { in, out := &in.TLS, &out.TLS *out = new(TLSDescription) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index d78d4c92e..8220f1798 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -1026,6 +1026,136 @@ func TestServiceAnnotations(t *testing.T) { } } +func TestPoolerServiceAnnotations(t *testing.T) { + enabled := true + tests := []struct { + about string + role PostgresRole + enableMasterPoolerLoadBalancer *bool + enableReplicaPoolerLoadBalancer *bool + operatorAnnotations map[string]string + serviceAnnotations map[string]string + masterServiceAnnotations map[string]string + replicaServiceAnnotations map[string]string + masterPoolerServiceAnnotations map[string]string + replicaPoolerServiceAnnotations map[string]string + expect map[string]string + }{ + { + about: "Master pooler annotations override service annotations", + role: "master", + operatorAnnotations: make(map[string]string), + serviceAnnotations: map[string]string{ + "foo": "bar", + }, + masterServiceAnnotations: map[string]string{ + "baz": "qux", + }, + masterPoolerServiceAnnotations: map[string]string{ + "foo": "pooler-bar", + "pooler": "master", + }, + expect: map[string]string{ + "foo": "pooler-bar", + "baz": "qux", + "pooler": "master", + }, + }, + { + about: "Replica pooler annotations override service annotations", + role: "replica", + operatorAnnotations: make(map[string]string), + serviceAnnotations: map[string]string{ + "foo": "bar", + }, + replicaServiceAnnotations: map[string]string{ + "baz": "qux", + }, + replicaPoolerServiceAnnotations: map[string]string{ + "foo": "pooler-bar", + "pooler": "replica", + }, + expect: map[string]string{ + "foo": "pooler-bar", + "baz": "qux", + "pooler": "replica", + }, + }, + { + about: "Master pooler with load balancer and pooler annotations", + role: "master", + enableMasterPoolerLoadBalancer: &enabled, + operatorAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), + masterPoolerServiceAnnotations: map[string]string{ + "pooler-key": "pooler-value", + }, + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "acid-test-pooler-stg.test.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "pooler-key": "pooler-value", + }, + }, + { + about: "Master pooler annotations not applied to replica pooler", + role: "replica", + operatorAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), + masterPoolerServiceAnnotations: map[string]string{ + "master-only": "value", + }, + expect: make(map[string]string), + }, + { + about: "Replica pooler annotations not applied to master pooler", + role: "master", + operatorAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), + replicaPoolerServiceAnnotations: map[string]string{ + "replica-only": "value", + }, + expect: make(map[string]string), + }, + } + + for _, tt := range tests { + t.Run(tt.about, func(t *testing.T) { + cl.OpConfig.CustomServiceAnnotations = tt.operatorAnnotations + cl.OpConfig.EnableMasterPoolerLoadBalancer = false + cl.OpConfig.EnableReplicaPoolerLoadBalancer = false + cl.OpConfig.MasterDNSNameFormat = "{cluster}-stg.{namespace}.{hostedzone}" + cl.OpConfig.MasterLegacyDNSNameFormat = "{cluster}-stg.{team}.{hostedzone}" + cl.OpConfig.ReplicaDNSNameFormat = "{cluster}-stg-repl.{namespace}.{hostedzone}" + cl.OpConfig.ReplicaLegacyDNSNameFormat = "{cluster}-stg-repl.{team}.{hostedzone}" + cl.OpConfig.DbHostedZone = "db.example.com" + + cl.Postgresql.Spec.ClusterName = "" + cl.Postgresql.Spec.TeamID = "acid" + cl.Postgresql.Spec.ServiceAnnotations = tt.serviceAnnotations + cl.Postgresql.Spec.MasterServiceAnnotations = tt.masterServiceAnnotations + cl.Postgresql.Spec.ReplicaServiceAnnotations = tt.replicaServiceAnnotations + cl.Postgresql.Spec.MasterPoolerServiceAnnotations = tt.masterPoolerServiceAnnotations + cl.Postgresql.Spec.ReplicaPoolerServiceAnnotations = tt.replicaPoolerServiceAnnotations + cl.Postgresql.Spec.EnableMasterPoolerLoadBalancer = tt.enableMasterPoolerLoadBalancer + cl.Postgresql.Spec.EnableReplicaPoolerLoadBalancer = tt.enableReplicaPoolerLoadBalancer + + got := cl.generatePoolerServiceAnnotations(tt.role, &cl.Postgresql.Spec) + if got == nil { + got = make(map[string]string) + } + if len(tt.expect) != len(got) { + t.Errorf("expected %d annotation(s), got %d: %v", len(tt.expect), len(got), got) + return + } + for k, v := range got { + if tt.expect[k] != v { + t.Errorf("expected annotation '%v' with value '%v', got value '%v'", k, tt.expect[k], v) + } + } + }) + } +} + func TestInitSystemUsers(t *testing.T) { // reset system users, pooler and stream section cl.systemUsers = make(map[string]spec.PgUser) diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index ac4ce67d8..5a33e902f 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -24,6 +24,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/retryutil" + "maps" ) var poolerRunAsUser = int64(100) @@ -532,6 +533,15 @@ func (c *Cluster) generatePoolerServiceAnnotations(role PostgresRole, spec *acid var dnsString string annotations := c.getCustomServiceAnnotations(role, spec) + if spec != nil { + switch role { + case Master: + maps.Copy(annotations, spec.MasterPoolerServiceAnnotations) + case Replica: + maps.Copy(annotations, spec.ReplicaPoolerServiceAnnotations) + } + } + if c.shouldCreateLoadBalancerForPoolerService(role, spec) { // set ELB Timeout annotation with default value if _, ok := annotations[constants.ElbTimeoutAnnotationName]; !ok {