Merge branch 'master' into minikube

docs/reference/operator_parameters.md

@@ -107,8 +107,13 @@ Those are top-level keys, containing both leaf keys and groups.
 * **kubernetes_use_configmaps**
   Select if setup uses endpoints (default), or configmaps to manage leader when
   DCS is kubernetes (not etcd or similar). In OpenShift it is not possible to
-  use endpoints option, and configmaps is required. By default,
+  use endpoints option, and configmaps is required. Starting with K8s 1.33,
-  `kubernetes_use_configmaps: false`, meaning endpoints will be used.
+  endpoints are marked as deprecated. It's recommended to switch to config maps
+  instead. But, to do so make sure you scale the Postgres cluster down to just
+  one primary pod (e.g. using `max_instances` option). Otherwise, you risk
+  running into a split-brain scenario.
+  By default, `kubernetes_use_configmaps: false`, meaning endpoints will be used.
+  Starting from v1.16.0 the default will be changed to `true`.
 
 * **docker_image**
   Spilo Docker image for Postgres instances. For production, don't rely on the

go.mod

@@ -3,6 +3,7 @@ module github.com/zalando/postgres-operator
 go 1.25.0
 
 require (
+	github.com/Masterminds/semver v1.5.0
 	github.com/aws/aws-sdk-go v1.53.8
 	github.com/golang/mock v1.6.0
 	github.com/lib/pq v1.10.9
@@ -12,7 +13,6 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/crypto v0.41.0
-	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.30.4
 	k8s.io/apiextensions-apiserver v0.25.9
@@ -21,10 +21,7 @@ require (
 	k8s.io/code-generator v0.25.9
 )
 
-require golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect
-
 require (
-	github.com/Masterminds/semver v1.5.0
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
@@ -62,6 +59,7 @@ require (
 	golang.org/x/text v0.28.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.36.0 // indirect
+	golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

go.sum

@@ -121,8 +121,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
 golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
-golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o=
-golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=

logical-backup/dump.sh

@@ -122,7 +122,21 @@ function aws_upload {
 function gcs_upload {
     PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
 
-    gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP"
+    #Set local LOGICAL_GOOGLE_APPLICATION_CREDENTIALS to nothing or
+    #value of LOGICAL_GOOGLE_APPLICATION_CREDENTIALS env var. Needed
+    #because `set -o nounset` is globally set
+    local LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS=${LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS:-}
+
+    GSUTIL_OPTIONS=("-o" "Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS")
+
+    #If GOOGLE_APPLICATION_CREDENTIALS is not set try to get
+    #creds from metadata
+    if [[ -z $LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS ]]
+    then
+	    GSUTIL_OPTIONS[1]="GoogleCompute:service_account=default"
+    fi
+
+    gsutil ${GSUTIL_OPTIONS[@]} cp - "$PATH_TO_BACKUP"
 }
 
 function upload {

manifests/operator-service-account-rbac.yaml

@@ -59,13 +59,20 @@ rules:
   - get
   - patch
   - update
-# to read configuration from ConfigMaps
+# to read configuration from ConfigMaps and help Patroni manage the cluster if endpoints are not used
 - apiGroups:
   - ""
   resources:
   - configmaps
   verbs:
+  - create
+  - delete
+  - deletecollection
   - get
+  - list
+  - patch
+  - update
+  - watch
 # to send events to the CRs
 - apiGroups:
   - ""
@@ -78,7 +85,7 @@ rules:
   - patch
   - update
   - watch
-# to manage endpoints which are also used by Patroni
+# to manage endpoints which are also used by Patroni (if it is using config maps)
 - apiGroups:
   - ""
   resources:
@@ -249,7 +256,21 @@ kind: ClusterRole
 metadata:
   name: postgres-pod
 rules:
-# Patroni needs to watch and manage endpoints
+# Patroni needs to watch and manage config maps (or endpoints)
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# Patroni needs to watch and manage endpoints (or config maps)
 - apiGroups:
   - ""
   resources:

pkg/cluster/cluster.go

@@ -841,6 +841,14 @@ func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) {
 		return false, "new service's owner references do not match the current ones"
 	}
 
+	if !reflect.DeepEqual(old.Spec.Selector, new.Spec.Selector) {
+		return false, "new service's selector does not match the current one"
+	}
+
+	if old.Spec.ExternalTrafficPolicy != new.Spec.ExternalTrafficPolicy {
+		return false, "new service's ExternalTrafficPolicy does not match the current one"
+	}
+
 	return true, ""
 }
 

pkg/cluster/cluster_test.go

@@ -1341,14 +1341,21 @@ func TestCompareEnv(t *testing.T) {
 	}
 }
 
-func newService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service {
+func newService(
+	annotations map[string]string,
+	svcType v1.ServiceType,
+	sourceRanges []string,
+	selector map[string]string,
+	policy v1.ServiceExternalTrafficPolicyType) *v1.Service {
 	svc := &v1.Service{
 		Spec: v1.ServiceSpec{
-			Type:                     svcT,
+			Selector:                 selector,
-			LoadBalancerSourceRanges: lbSr,
+			Type:                     svcType,
+			LoadBalancerSourceRanges: sourceRanges,
+			ExternalTrafficPolicy:    policy,
 		},
 	}
-	svc.Annotations = ann
+	svc.Annotations = annotations
 	return svc
 }
 
@@ -1365,13 +1372,18 @@ func TestCompareServices(t *testing.T) {
 		},
 	}
 
+	defaultPolicy := v1.ServiceExternalTrafficPolicyTypeCluster
+
 	serviceWithOwnerReference := newService(
 		map[string]string{
 			constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
 			constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 		},
 		v1.ServiceTypeClusterIP,
-		[]string{"128.141.0.0/16", "137.138.0.0/16"})
+		[]string{"128.141.0.0/16", "137.138.0.0/16"},
+		nil,
+		defaultPolicy,
+	)
 
 	ownerRef := metav1.OwnerReference{
 		APIVersion: "acid.zalan.do/v1",
@@ -1397,14 +1409,16 @@ func TestCompareServices(t *testing.T) {
 					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 				},
 				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+				[]string{"128.141.0.0/16", "137.138.0.0/16"},
+				nil, defaultPolicy),
 			new: newService(
 				map[string]string{
 					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
 					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 				},
 				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+				[]string{"128.141.0.0/16", "137.138.0.0/16"},
+				nil, defaultPolicy),
 			match: true,
 		},
 		{
@@ -1415,14 +1429,16 @@ func TestCompareServices(t *testing.T) {
 					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 				},
 				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+				[]string{"128.141.0.0/16", "137.138.0.0/16"},
+				nil, defaultPolicy),
 			new: newService(
 				map[string]string{
 					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
 					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 				},
 				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+				[]string{"128.141.0.0/16", "137.138.0.0/16"},
+				nil, defaultPolicy),
 			match:  false,
 			reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`,
 		},
@@ -1434,14 +1450,16 @@ func TestCompareServices(t *testing.T) {
 					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 				},
 				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+				[]string{"128.141.0.0/16", "137.138.0.0/16"},
+				nil, defaultPolicy),
 			new: newService(
 				map[string]string{
 					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
 					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 				},
 				v1.ServiceTypeLoadBalancer,
-				[]string{"185.249.56.0/22"}),
+				[]string{"185.249.56.0/22"},
+				nil, defaultPolicy),
 			match:  false,
 			reason: `new service's LoadBalancerSourceRange does not match the current one`,
 		},
@@ -1453,14 +1471,16 @@ func TestCompareServices(t *testing.T) {
 					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 				},
 				v1.ServiceTypeLoadBalancer,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+				[]string{"128.141.0.0/16", "137.138.0.0/16"},
+				nil, defaultPolicy),
 			new: newService(
 				map[string]string{
 					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
 					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 				},
 				v1.ServiceTypeLoadBalancer,
-				[]string{}),
+				[]string{},
+				nil, defaultPolicy),
 			match:  false,
 			reason: `new service's LoadBalancerSourceRange does not match the current one`,
 		},
@@ -1472,10 +1492,39 @@ func TestCompareServices(t *testing.T) {
 					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
 				},
 				v1.ServiceTypeClusterIP,
-				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
+				[]string{"128.141.0.0/16", "137.138.0.0/16"},
+				nil, defaultPolicy),
 			new:   serviceWithOwnerReference,
 			match: false,
 		},
+		{
+			about: "new service has a label selector",
+			current: newService(
+				map[string]string{},
+				v1.ServiceTypeClusterIP,
+				[]string{},
+				nil, defaultPolicy),
+			new: newService(
+				map[string]string{},
+				v1.ServiceTypeClusterIP,
+				[]string{},
+				map[string]string{"cluster-name": "clstr", "spilo-role": "master"}, defaultPolicy),
+			match: false,
+		},
+		{
+			about: "services differ on external traffic policy",
+			current: newService(
+				map[string]string{},
+				v1.ServiceTypeClusterIP,
+				[]string{},
+				nil, defaultPolicy),
+			new: newService(
+				map[string]string{},
+				v1.ServiceTypeClusterIP,
+				[]string{},
+				nil, v1.ServiceExternalTrafficPolicyTypeLocal),
+			match: false,
+		},
 	}
 
 	for _, tt := range tests {

pkg/cluster/k8sres.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"path"
+	"slices"
 	"sort"
 	"strings"
 
@@ -12,19 +14,16 @@ import (
 	"github.com/sirupsen/logrus"
 
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
-	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
-	batchv1 "k8s.io/api/batch/v1"
-	"k8s.io/apimachinery/pkg/labels"
-
 	acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1"
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util"
@@ -1928,7 +1927,7 @@ func (c *Cluster) generateSingleUserSecret(pgUser spec.PgUser) *v1.Secret {
 
 	// if secret lives in another namespace we cannot set ownerReferences
 	var ownerReferences []metav1.OwnerReference
-	if c.Config.OpConfig.EnableCrossNamespaceSecret && strings.Contains(username, ".") {
+	if c.Config.OpConfig.EnableCrossNamespaceSecret && c.Postgresql.ObjectMeta.Namespace != pgUser.Namespace {
 		ownerReferences = nil
 	} else {
 		ownerReferences = c.ownerReferences()

pkg/cluster/pod.go

@@ -3,12 +3,11 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"slices"
 	"sort"
 	"strconv"
 	"time"
 
-	"golang.org/x/exp/slices"
-
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

pkg/cluster/sync.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"reflect"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -15,8 +17,6 @@ import (
 	"github.com/zalando/postgres-operator/pkg/util"
 	"github.com/zalando/postgres-operator/pkg/util/constants"
 	"github.com/zalando/postgres-operator/pkg/util/k8sutil"
-	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1"

pkg/cluster/sync_test.go

@@ -2,15 +2,14 @@ package cluster
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
+	"slices"
 	"testing"
 	"time"
 
-	"context"
-
-	"golang.org/x/exp/slices"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"

ui/requirements.txt

@@ -9,6 +9,6 @@ jq==1.7.0
 json_delta>=2.0.2
 kubernetes==11.0.0
 python-json-logger==2.0.7
-requests==2.32.2
+requests==2.32.4
 stups-tokens>=1.1.19
 werkzeug==3.0.6