diff --git a/docs/administrator.md b/docs/administrator.md index f394b70ab..f28bd93f0 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -195,12 +195,14 @@ from numerous escape characters in the latter log entry, view it in CLI with used internally in K8s. The StatefulSet is replaced if the following properties change: + - annotations - volumeClaimTemplates - template volumes The StatefulSet is replaced and a rolling updates is triggered if the following properties differ between the old and new state: + - container name, ports, image, resources, env, envFrom, securityContext and volumeMounts - template labels, annotations, service account, securityContext, affinity, priority class and termination grace period @@ -898,6 +900,7 @@ services: There are multiple options to specify service annotations that will be merged with each other and override in the following order (where latter take precedence): + 1. Default annotations if LoadBalancer is enabled 2. Globally configured `custom_service_annotations` 3. `serviceAnnotations` specified in the cluster manifest diff --git a/docs/developer.md b/docs/developer.md index c006aded0..0429f951c 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -16,7 +16,7 @@ under the ~/go/src sub directories. Given the schema above, the Postgres Operator source code located at `github.com/zalando/postgres-operator` should be put at --`~/go/src/github.com/zalando/postgres-operator`. +`~/go/src/github.com/zalando/postgres-operator`. ```bash export GOPATH=~/go @@ -105,6 +105,7 @@ and K8s-like APIs for its custom resource definitions, namely the Postgres CRD and the operator CRD. The usage of the code generation follows conventions from the K8s community. Relevant scripts live in the `hack` directory: + * `update-codegen.sh` triggers code generation for the APIs defined in `pkg/apis/acid.zalan.do/`, * `verify-codegen.sh` checks if the generated code is up-to-date (to be used within CI). @@ -112,6 +113,7 @@ The `/pkg/generated/` contains the resultant code. To make these scripts work, you may need to `export GOPATH=$(go env GOPATH)` References for code generation are: + * [Relevant pull request](https://github.com/zalando/postgres-operator/pull/369) See comments there for minor issues that can sometimes broke the generation process. * [Code generator source code](https://github.com/kubernetes/code-generator) @@ -315,6 +317,7 @@ precedence. Update the following Go files that obtain the configuration parameter from the manifest files: + * [operator_configuration_type.go](https://github.com/zalando/postgres-operator/blob/master/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go) * [operator_config.go](https://github.com/zalando/postgres-operator/blob/master/pkg/controller/operator_config.go) * [config.go](https://github.com/zalando/postgres-operator/blob/master/pkg/util/config/config.go) @@ -323,6 +326,7 @@ Postgres manifest parameters are defined in the [api package](https://github.com The operator behavior has to be implemented at least in [k8sres.go](https://github.com/zalando/postgres-operator/blob/master/pkg/cluster/k8sres.go). Validation of CRD parameters is controlled in [crds.go](https://github.com/zalando/postgres-operator/blob/master/pkg/apis/acid.zalan.do/v1/crds.go). Please, reflect your changes in tests, for example in: + * [config_test.go](https://github.com/zalando/postgres-operator/blob/master/pkg/util/config/config_test.go) * [k8sres_test.go](https://github.com/zalando/postgres-operator/blob/master/pkg/cluster/k8sres_test.go) * [util_test.go](https://github.com/zalando/postgres-operator/blob/master/pkg/apis/acid.zalan.do/v1/util_test.go) @@ -330,6 +334,7 @@ Please, reflect your changes in tests, for example in: ### Updating manifest files For the CRD-based configuration, please update the following files: + * the default [OperatorConfiguration](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql-operator-default-configuration.yaml) * the CRD's [validation](https://github.com/zalando/postgres-operator/blob/master/manifests/operatorconfiguration.crd.yaml) * the CRD's validation in the [Helm chart](https://github.com/zalando/postgres-operator/blob/master/charts/postgres-operator/crds/operatorconfigurations.yaml) @@ -342,6 +347,7 @@ Last but no least, update the [ConfigMap](https://github.com/zalando/postgres-op Finally, add a section for each new configuration option and/or cluster manifest parameter in the reference documents: + * [config reference](reference/operator_parameters.md) * [manifest reference](reference/cluster_manifest.md) diff --git a/docs/quickstart.md b/docs/quickstart.md index 2d6742354..fa16d1813 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -10,7 +10,7 @@ hence set it up first. For local tests we recommend to use one of the following solutions: * [minikube](https://github.com/kubernetes/minikube/releases), which creates a - single-node K8s cluster inside a VM (requires KVM or VirtualBox), + K8s cluster inside a container or VM (requires Docker, KVM, Hyper-V, HyperKit, VirtualBox, or similar), * [kind](https://kind.sigs.k8s.io/) and [k3d](https://k3d.io), which allows creating multi-nodes K8s clusters running on Docker (requires Docker) @@ -20,7 +20,7 @@ This quickstart assumes that you have started minikube or created a local kind cluster. Note that you can also use built-in K8s support in the Docker Desktop for Mac to follow the steps of this tutorial. You would have to replace `minikube start` and `minikube delete` with your launch actions for the Docker -built-in K8s support. +Desktop built-in K8s support. ## Configuration Options diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 95bfb4cf3..7e7cbeaf0 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -107,8 +107,13 @@ Those are top-level keys, containing both leaf keys and groups. * **kubernetes_use_configmaps** Select if setup uses endpoints (default), or configmaps to manage leader when DCS is kubernetes (not etcd or similar). In OpenShift it is not possible to - use endpoints option, and configmaps is required. By default, - `kubernetes_use_configmaps: false`, meaning endpoints will be used. + use endpoints option, and configmaps is required. Starting with K8s 1.33, + endpoints are marked as deprecated. It's recommended to switch to config maps + instead. But, to do so make sure you scale the Postgres cluster down to just + one primary pod (e.g. using `max_instances` option). Otherwise, you risk + running into a split-brain scenario. + By default, `kubernetes_use_configmaps: false`, meaning endpoints will be used. + Starting from v1.16.0 the default will be changed to `true`. * **docker_image** Spilo Docker image for Postgres instances. For production, don't rely on the diff --git a/go.mod b/go.mod index d047b2453..150074cf8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/zalando/postgres-operator go 1.25.0 require ( + github.com/Masterminds/semver v1.5.0 github.com/aws/aws-sdk-go v1.53.8 github.com/golang/mock v1.6.0 github.com/lib/pq v1.10.9 @@ -12,7 +13,6 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 golang.org/x/crypto v0.41.0 - golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.30.4 k8s.io/apiextensions-apiserver v0.25.9 @@ -21,10 +21,7 @@ require ( k8s.io/code-generator v0.25.9 ) -require golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect - require ( - github.com/Masterminds/semver v1.5.0 github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect @@ -62,6 +59,7 @@ require ( golang.org/x/text v0.28.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.36.0 // indirect + golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 2d66aef46..cab88ce55 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= -golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= -golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= diff --git a/logical-backup/dump.sh b/logical-backup/dump.sh index 25641c3b5..a250670a6 100755 --- a/logical-backup/dump.sh +++ b/logical-backup/dump.sh @@ -122,7 +122,21 @@ function aws_upload { function gcs_upload { PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz - gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP" + #Set local LOGICAL_GOOGLE_APPLICATION_CREDENTIALS to nothing or + #value of LOGICAL_GOOGLE_APPLICATION_CREDENTIALS env var. Needed + #because `set -o nounset` is globally set + local LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS=${LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS:-} + + GSUTIL_OPTIONS=("-o" "Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS") + + #If GOOGLE_APPLICATION_CREDENTIALS is not set try to get + #creds from metadata + if [[ -z $LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS ]] + then + GSUTIL_OPTIONS[1]="GoogleCompute:service_account=default" + fi + + gsutil ${GSUTIL_OPTIONS[@]} cp - "$PATH_TO_BACKUP" } function upload { diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index bf27f99f1..2cc1edcd1 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -59,13 +59,20 @@ rules: - get - patch - update -# to read configuration from ConfigMaps +# to read configuration from ConfigMaps and help Patroni manage the cluster if endpoints are not used - apiGroups: - "" resources: - configmaps verbs: + - create + - delete + - deletecollection - get + - list + - patch + - update + - watch # to send events to the CRs - apiGroups: - "" @@ -78,7 +85,7 @@ rules: - patch - update - watch -# to manage endpoints which are also used by Patroni +# to manage endpoints which are also used by Patroni (if it is using config maps) - apiGroups: - "" resources: @@ -249,7 +256,21 @@ kind: ClusterRole metadata: name: postgres-pod rules: -# Patroni needs to watch and manage endpoints +# Patroni needs to watch and manage config maps (or endpoints) +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# Patroni needs to watch and manage endpoints (or config maps) - apiGroups: - "" resources: diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index a13e0a587..4903e5bc8 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -846,6 +846,14 @@ func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) { return false, "new service's owner references do not match the current ones" } + if !reflect.DeepEqual(old.Spec.Selector, new.Spec.Selector) { + return false, "new service's selector does not match the current one" + } + + if old.Spec.ExternalTrafficPolicy != new.Spec.ExternalTrafficPolicy { + return false, "new service's ExternalTrafficPolicy does not match the current one" + } + return true, "" } diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 09d9df972..25f61db98 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -1341,14 +1341,21 @@ func TestCompareEnv(t *testing.T) { } } -func newService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service { +func newService( + annotations map[string]string, + svcType v1.ServiceType, + sourceRanges []string, + selector map[string]string, + policy v1.ServiceExternalTrafficPolicyType) *v1.Service { svc := &v1.Service{ Spec: v1.ServiceSpec{ - Type: svcT, - LoadBalancerSourceRanges: lbSr, + Selector: selector, + Type: svcType, + LoadBalancerSourceRanges: sourceRanges, + ExternalTrafficPolicy: policy, }, } - svc.Annotations = ann + svc.Annotations = annotations return svc } @@ -1365,13 +1372,18 @@ func TestCompareServices(t *testing.T) { }, } + defaultPolicy := v1.ServiceExternalTrafficPolicyTypeCluster + serviceWithOwnerReference := newService( map[string]string{ constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}) + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, + defaultPolicy, + ) ownerRef := metav1.OwnerReference{ APIVersion: "acid.zalan.do/v1", @@ -1397,14 +1409,16 @@ func TestCompareServices(t *testing.T) { constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy), new: newService( map[string]string{ constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy), match: true, }, { @@ -1415,14 +1429,16 @@ func TestCompareServices(t *testing.T) { constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy), new: newService( map[string]string{ constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy), match: false, reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`, }, @@ -1434,14 +1450,16 @@ func TestCompareServices(t *testing.T) { constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy), new: newService( map[string]string{ constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeLoadBalancer, - []string{"185.249.56.0/22"}), + []string{"185.249.56.0/22"}, + nil, defaultPolicy), match: false, reason: `new service's LoadBalancerSourceRange does not match the current one`, }, @@ -1453,14 +1471,16 @@ func TestCompareServices(t *testing.T) { constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy), new: newService( map[string]string{ constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeLoadBalancer, - []string{}), + []string{}, + nil, defaultPolicy), match: false, reason: `new service's LoadBalancerSourceRange does not match the current one`, }, @@ -1472,10 +1492,39 @@ func TestCompareServices(t *testing.T) { constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, }, v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy), new: serviceWithOwnerReference, match: false, }, + { + about: "new service has a label selector", + current: newService( + map[string]string{}, + v1.ServiceTypeClusterIP, + []string{}, + nil, defaultPolicy), + new: newService( + map[string]string{}, + v1.ServiceTypeClusterIP, + []string{}, + map[string]string{"cluster-name": "clstr", "spilo-role": "master"}, defaultPolicy), + match: false, + }, + { + about: "services differ on external traffic policy", + current: newService( + map[string]string{}, + v1.ServiceTypeClusterIP, + []string{}, + nil, defaultPolicy), + new: newService( + map[string]string{}, + v1.ServiceTypeClusterIP, + []string{}, + nil, v1.ServiceExternalTrafficPolicyTypeLocal), + match: false, + }, } for _, tt := range tests { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 02da1f339..c49ecb980 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "fmt" + "maps" "path" + "slices" "sort" "strings" @@ -12,19 +14,16 @@ import ( "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "golang.org/x/exp/maps" - "golang.org/x/exp/slices" - batchv1 "k8s.io/api/batch/v1" - "k8s.io/apimachinery/pkg/labels" - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" @@ -1941,7 +1940,7 @@ func (c *Cluster) generateSingleUserSecret(pgUser spec.PgUser) *v1.Secret { // if secret lives in another namespace we cannot set ownerReferences var ownerReferences []metav1.OwnerReference - if c.Config.OpConfig.EnableCrossNamespaceSecret && strings.Contains(username, ".") { + if c.Config.OpConfig.EnableCrossNamespaceSecret && c.Postgresql.ObjectMeta.Namespace != pgUser.Namespace { ownerReferences = nil } else { ownerReferences = c.ownerReferences() diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 7fc95090e..b28e03078 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -3,12 +3,11 @@ package cluster import ( "context" "fmt" + "slices" "sort" "strconv" "time" - "golang.org/x/exp/slices" - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 9067c8c46..c632e598c 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -4,8 +4,10 @@ import ( "context" "encoding/json" "fmt" + "maps" "reflect" "regexp" + "slices" "strconv" "strings" "time" @@ -15,8 +17,6 @@ import ( "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" - "golang.org/x/exp/maps" - "golang.org/x/exp/slices" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index f9d1d7873..e670daa70 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -2,15 +2,14 @@ package cluster import ( "bytes" + "context" "fmt" "io" "net/http" + "slices" "testing" "time" - "context" - - "golang.org/x/exp/slices" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" diff --git a/ui/requirements.txt b/ui/requirements.txt index 783c0aac3..75bcc1952 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -9,6 +9,6 @@ jq==1.7.0 json_delta>=2.0.2 kubernetes==11.0.0 python-json-logger==2.0.7 -requests==2.32.2 +requests==2.32.4 stups-tokens>=1.1.19 werkzeug==3.0.6