From a5663da64f7a94af8dde395852cb2cf309ae27bc Mon Sep 17 00:00:00 2001 From: Jociele Padilha <45459238+jopadi@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:43:25 +0100 Subject: [PATCH] add the pg version 16 (#2557) * add the pg version 16 * add comma after pg16 in crds api * change minimal_major_version to 12 * add new spilo image for pg16 * edit the registry from current and lazy spilo * Update e2e/run.sh Co-authored-by: Felix Kunde * Update README.md Co-authored-by: Felix Kunde * add pg 11 to be compatible for the existing DBs * update pq, pyyaml,k8s and kind version * skip test_infrastructure_roles * skip another test * remove the skipping * adjust the verification of new Patroni version states --------- Co-authored-by: Felix Kunde --- README.md | 6 ++-- .../templates/deployment.yaml | 3 +- .../crds/operatorconfigurations.yaml | 6 ++-- .../postgres-operator/crds/postgresqls.yaml | 2 +- charts/postgres-operator/values.yaml | 6 ++-- docker/logical-backup/Dockerfile | 1 - docs/administrator.md | 2 +- docs/reference/cluster_manifest.md | 4 +-- docs/reference/operator_parameters.md | 4 +-- docs/user.md | 14 ++++----- e2e/Makefile | 2 +- e2e/requirements.txt | 4 +-- e2e/run.sh | 2 +- e2e/tests/k8s_api.py | 4 +-- e2e/tests/test_e2e.py | 4 +-- go.mod | 2 +- go.sum | 4 +-- manifests/complete-postgres-manifest.yaml | 4 +-- manifests/configmap.yaml | 6 ++-- manifests/minimal-postgres-manifest.yaml | 2 +- manifests/operatorconfiguration.crd.yaml | 6 ++-- ...gresql-operator-default-configuration.yaml | 6 ++-- manifests/postgresql.crd.yaml | 1 - manifests/standby-manifest.yaml | 2 +- pkg/apis/acid.zalan.do/v1/crds.go | 3 -- .../v1/operator_configuration_type.go | 4 +-- pkg/apis/acid.zalan.do/v1/util_test.go | 10 +++---- pkg/cluster/k8sres_test.go | 30 +++++++++---------- pkg/cluster/majorversionupgrade.go | 3 +- pkg/controller/operator_config.go | 6 ++-- pkg/util/config/config.go | 6 ++-- ui/manifests/deployment.yaml | 3 +- ui/operator_ui/main.py | 2 +- ui/run_local.sh | 3 +- 34 files changed, 79 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 810f4d198..6b127310a 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,13 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as ### PostgreSQL features -* Supports PostgreSQL 15, starting from 10+ +* Supports PostgreSQL 16, starting from 11+ * Streaming replication cluster via Patroni * Point-In-Time-Recovery with -[pg_basebackup](https://www.postgresql.org/docs/11/app-pgbasebackup.html) / +[pg_basebackup](https://www.postgresql.org/docs/16/app-pgbasebackup.html) / [WAL-E](https://github.com/wal-e/wal-e) via [Spilo](https://github.com/zalando/spilo) * Preload libraries: [bg_mon](https://github.com/CyberDem0n/bg_mon), -[pg_stat_statements](https://www.postgresql.org/docs/15/pgstatstatements.html), +[pg_stat_statements](https://www.postgresql.org/docs/16/pgstatstatements.html), [pgextwlist](https://github.com/dimitri/pgextwlist), [pg_auth_mon](https://github.com/RafiaSabih/pg_auth_mon) * Incl. popular Postgres extensions such as diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index 6a545576c..cf3d60752 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -88,8 +88,7 @@ spec: "15", "14", "13", - "12", - "11" + "12" ] } {{- if .Values.extraEnvs }} diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index b0d3966f6..bfe17d07b 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -68,7 +68,7 @@ spec: type: string docker_image: type: string - default: "ghcr.io/zalando/spilo-15:3.0-p1" + default: "ghcr.io/zalando/spilo-16:3.2-p2" enable_crd_registration: type: boolean default: true @@ -167,10 +167,10 @@ spec: type: string minimal_major_version: type: string - default: "11" + default: "12" target_major_version: type: string - default: "15" + default: "16" kubernetes: type: object properties: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 496267ecf..fcb278cbe 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -371,12 +371,12 @@ spec: version: type: string enum: - - "10" - "11" - "12" - "13" - "14" - "15" + - "16" parameters: type: object additionalProperties: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 3449b47af..a7bc7310c 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -38,7 +38,7 @@ configGeneral: # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" # Spilo docker image - docker_image: ghcr.io/zalando/spilo-15:3.0-p1 + docker_image: ghcr.io/zalando/spilo-16:3.2-p2 # key name for annotation to ignore globally configured instance limits # ignore_instance_limits_annotation_key: "" @@ -89,9 +89,9 @@ configMajorVersionUpgrade: # - acid # minimal Postgres major version that will not automatically be upgraded - minimal_major_version: "11" + minimal_major_version: "12" # target Postgres major version when upgrading clusters automatically - target_major_version: "15" + target_major_version: "16" configKubernetes: # list of additional capabilities for postgres container diff --git a/docker/logical-backup/Dockerfile b/docker/logical-backup/Dockerfile index 8465c3012..8770e5e1a 100644 --- a/docker/logical-backup/Dockerfile +++ b/docker/logical-backup/Dockerfile @@ -30,7 +30,6 @@ RUN apt-get update \ postgresql-client-14 \ postgresql-client-13 \ postgresql-client-12 \ - postgresql-client-11 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/docs/administrator.md b/docs/administrator.md index b3efe5193..1e5caa6a4 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -1217,7 +1217,7 @@ aws_or_gcp: If cluster members have to be (re)initialized restoring physical backups happens automatically either from the backup location or by running -[pg_basebackup](https://www.postgresql.org/docs/15/app-pgbasebackup.html) +[pg_basebackup](https://www.postgresql.org/docs/16/app-pgbasebackup.html) on one of the other running instances (preferably replicas if they do not lag behind). You can test restoring backups by [cloning](user.md#how-to-clone-an-existing-postgresql-cluster) clusters. diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 2b25a28a8..4adf58668 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -621,7 +621,7 @@ the global configuration before adding the `tls` section'. ## Change data capture streams This sections enables change data capture (CDC) streams via Postgres' -[logical decoding](https://www.postgresql.org/docs/15/logicaldecoding.html) +[logical decoding](https://www.postgresql.org/docs/16/logicaldecoding.html) feature and `pgoutput` plugin. While the Postgres operator takes responsibility for providing the setup to publish change events, it relies on external tools to consume them. At Zalando, we are using a workflow based on @@ -653,7 +653,7 @@ can have the following properties: and `payloadColumn`). The CDC operator is following the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/). The application is responsible for putting events into a (JSON/B or VARCHAR) payload column of the outbox table in the structure of the specified target - event type. The operator will create a [PUBLICATION](https://www.postgresql.org/docs/15/logical-replication-publication.html) + event type. The operator will create a [PUBLICATION](https://www.postgresql.org/docs/16/logical-replication-publication.html) in Postgres for all tables specified for one `database` and `applicationId`. The CDC operator will consume from it shortly after transactions are committed to the outbox table. The `idColumn` will be used in telemetry for diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 00e648bb5..341570a55 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -250,12 +250,12 @@ CRD-configuration, they are grouped under the `major_version_upgrade` key. * **minimal_major_version** The minimal Postgres major version that will not automatically be upgraded - when `major_version_upgrade_mode` is set to `"full"`. The default is `"11"`. + when `major_version_upgrade_mode` is set to `"full"`. The default is `"12"`. * **target_major_version** The target Postgres major version when upgrading clusters automatically which violate the configured allowed `minimal_major_version` when - `major_version_upgrade_mode` is set to `"full"`. The default is `"15"`. + `major_version_upgrade_mode` is set to `"full"`. The default is `"16"`. ## Kubernetes resources diff --git a/docs/user.md b/docs/user.md index 2c5fea995..056df7065 100644 --- a/docs/user.md +++ b/docs/user.md @@ -30,7 +30,7 @@ spec: databases: foo: zalando postgresql: - version: "15" + version: "16" ``` Once you cloned the Postgres Operator [repository](https://github.com/zalando/postgres-operator) @@ -109,7 +109,7 @@ metadata: spec: [...] postgresql: - version: "15" + version: "16" parameters: password_encryption: scram-sha-256 ``` @@ -517,7 +517,7 @@ Postgres Operator will create the following NOLOGIN roles: The `_owner` role is the database owner and should be used when creating new database objects. All members of the `admin` role, e.g. teams API roles, can -become the owner with the `SET ROLE` command. [Default privileges](https://www.postgresql.org/docs/15/sql-alterdefaultprivileges.html) +become the owner with the `SET ROLE` command. [Default privileges](https://www.postgresql.org/docs/16/sql-alterdefaultprivileges.html) are configured for the owner role so that the `_reader` role automatically gets read-access (SELECT) to new tables and sequences and the `_writer` receives write-access (INSERT, UPDATE, DELETE on tables, @@ -594,7 +594,7 @@ spec: ### Schema `search_path` for default roles -The schema [`search_path`](https://www.postgresql.org/docs/15/ddl-schemas.html#DDL-SCHEMAS-PATH) +The schema [`search_path`](https://www.postgresql.org/docs/16/ddl-schemas.html#DDL-SCHEMAS-PATH) for each role will include the role name and the schemas, this role should have access to. So `foo_bar_writer` does not have to schema-qualify tables from schemas `foo_bar_writer, bar`, while `foo_writer` can look up `foo_writer` and @@ -695,7 +695,7 @@ handle it. ### HugePages support -The operator supports [HugePages](https://www.postgresql.org/docs/15/kernel-resources.html#LINUX-HUGEPAGES). +The operator supports [HugePages](https://www.postgresql.org/docs/16/kernel-resources.html#LINUX-HUGEPAGES). To enable HugePages, set the matching resource requests and/or limits in the manifest: ```yaml @@ -758,7 +758,7 @@ If you need to define a `nodeAffinity` for all your Postgres clusters use the ## In-place major version upgrade Starting with Spilo 13, operator supports in-place major version upgrade to a -higher major version (e.g. from PG 10 to PG 13). To trigger the upgrade, +higher major version (e.g. from PG 11 to PG 13). To trigger the upgrade, simply increase the version in the manifest. It is your responsibility to test your applications against the new version before the upgrade; downgrading is not supported. The easiest way to do so is to try the upgrade on the cloned @@ -838,7 +838,7 @@ spec: ### Clone directly Another way to get a fresh copy of your source DB cluster is via -[pg_basebackup](https://www.postgresql.org/docs/15/app-pgbasebackup.html). To +[pg_basebackup](https://www.postgresql.org/docs/16/app-pgbasebackup.html). To use this feature simply leave out the timestamp field from the clone section. The operator will connect to the service of the source cluster by name. If the cluster is called test, then the connection string will look like host=test diff --git a/e2e/Makefile b/e2e/Makefile index 25688b920..9d3681ce5 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -46,7 +46,7 @@ tools: # install pinned version of 'kind' # go install must run outside of a dir with a (module-based) Go project ! # otherwise go install updates project's dependencies and/or behaves differently - cd "/tmp" && GO111MODULE=on go install sigs.k8s.io/kind@v0.14.0 + cd "/tmp" && GO111MODULE=on go install sigs.k8s.io/kind@v0.22.0 e2etest: tools copy clean ./run.sh main diff --git a/e2e/requirements.txt b/e2e/requirements.txt index ea5405b56..d904585be 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -1,3 +1,3 @@ -kubernetes==24.2.0 +kubernetes==29.2.0 timeout_decorator==0.5.0 -pyyaml==6.0 +pyyaml==6.0.1 diff --git a/e2e/run.sh b/e2e/run.sh index ecef56d08..1adca479d 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -8,7 +8,7 @@ IFS=$'\n\t' readonly cluster_name="postgres-operator-e2e-tests" readonly kubeconfig_path="/tmp/kind-config-${cluster_name}" -readonly spilo_image="registry.opensource.zalan.do/acid/spilo-15-e2e:0.1" +readonly spilo_image="registry.opensource.zalan.do/acid/spilo-16-e2e:0.1" readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.4" export GOPATH=${GOPATH-~/go} diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py index 34cf3659c..12e45f4b0 100644 --- a/e2e/tests/k8s_api.py +++ b/e2e/tests/k8s_api.py @@ -314,7 +314,7 @@ class K8s: def get_patroni_running_members(self, pod="acid-minimal-cluster-0"): result = self.get_patroni_state(pod) - return list(filter(lambda x: "State" in x and x["State"] == "running", result)) + return list(filter(lambda x: "State" in x and x["State"] in ["running", "streaming"], result)) def get_deployment_replica_count(self, name="acid-minimal-cluster-pooler", namespace="default"): try: @@ -583,7 +583,7 @@ class K8sBase: def get_patroni_running_members(self, pod): result = self.get_patroni_state(pod) - return list(filter(lambda x: x["State"] == "running", result)) + return list(filter(lambda x: x["State"] in ["running", "streaming"], result)) def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'): ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 2f36c010c..e0854f1f7 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -12,8 +12,8 @@ from kubernetes import client from tests.k8s_api import K8s from kubernetes.client.rest import ApiException -SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-15-e2e:0.1" -SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-15-e2e:0.2" +SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-16-e2e:0.1" +SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-16-e2e:0.2" def to_selector(labels): diff --git a/go.mod b/go.mod index f6255ed09..d2efea8a8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/aws/aws-sdk-go v1.42.18 github.com/golang/mock v1.6.0 - github.com/lib/pq v1.10.4 + github.com/lib/pq v1.10.9 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/pkg/errors v0.9.1 github.com/r3labs/diff v1.1.0 diff --git a/go.sum b/go.sum index 79f271819..fdf6ad8b4 100644 --- a/go.sum +++ b/go.sum @@ -63,8 +63,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= -github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index a074007ad..9a6dca4d8 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -10,7 +10,7 @@ metadata: # "delete-date": "2020-08-31" # can only be deleted on that day if "delete-date "key is configured # "delete-clustername": "acid-test-cluster" # can only be deleted when name matches if "delete-clustername" key is configured spec: - dockerImage: ghcr.io/zalando/spilo-15:3.0-p1 + dockerImage: ghcr.io/zalando/spilo-16:3.2-p2 teamId: "acid" numberOfInstances: 2 users: # Application/Robot users @@ -48,7 +48,7 @@ spec: defaultRoles: true defaultUsers: false postgresql: - version: "15" + version: "16" parameters: # Expert section shared_buffers: "32MB" max_connections: "10" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 6a10ade2c..a4d91e63d 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -34,7 +34,7 @@ data: default_memory_request: 100Mi # delete_annotation_date_key: delete-date # delete_annotation_name_key: delete-clustername - docker_image: ghcr.io/zalando/spilo-15:3.0-p1 + docker_image: ghcr.io/zalando/spilo-16:3.2-p2 # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" # enable_crd_registration: "true" @@ -107,7 +107,7 @@ data: # max_memory_request: 4Gi # min_cpu_limit: 250m # min_memory_limit: 250Mi - # minimal_major_version: "11" + # minimal_major_version: "12" # node_readiness_label: "status:ready" # node_readiness_label_merge: "OR" # oauth_token_secret_name: postgresql-operator @@ -157,7 +157,7 @@ data: spilo_privileged: "false" storage_resize_mode: "pvc" super_username: postgres - # target_major_version: "15" + # target_major_version: "16" # team_admin_role: "admin" # team_api_role_configuration: "log_statement:all" # teams_api_url: http://fake-teams-api.default.svc.cluster.local diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index 00f11ebf7..d22327905 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -17,4 +17,4 @@ spec: preparedDatabases: bar: {} postgresql: - version: "15" + version: "16" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 73e667821..51ffb24a4 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -66,7 +66,7 @@ spec: type: string docker_image: type: string - default: "ghcr.io/zalando/spilo-15:3.0-p1" + default: "ghcr.io/zalando/spilo-16:3.2-p2" enable_crd_registration: type: boolean default: true @@ -165,10 +165,10 @@ spec: type: string minimal_major_version: type: string - default: "11" + default: "12" target_major_version: type: string - default: "15" + default: "16" kubernetes: type: object properties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 044841338..aff79ef71 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -3,7 +3,7 @@ kind: OperatorConfiguration metadata: name: postgresql-operator-default-configuration configuration: - docker_image: ghcr.io/zalando/spilo-15:3.0-p1 + docker_image: ghcr.io/zalando/spilo-16:3.2-p2 # enable_crd_registration: true # crd_categories: # - all @@ -39,8 +39,8 @@ configuration: major_version_upgrade_mode: "off" # major_version_upgrade_team_allow_list: # - acid - minimal_major_version: "11" - target_major_version: "15" + minimal_major_version: "12" + target_major_version: "16" kubernetes: # additional_pod_capabilities: # - "SYS_NICE" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index da2e4dd16..4371bf561 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -369,7 +369,6 @@ spec: version: type: string enum: - - "10" - "11" - "12" - "13" diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml index 2db4d489b..aece29dae 100644 --- a/manifests/standby-manifest.yaml +++ b/manifests/standby-manifest.yaml @@ -8,7 +8,7 @@ spec: size: 1Gi numberOfInstances: 1 postgresql: - version: "15" + version: "16" # Make this a standby cluster and provide either the s3 bucket path of source cluster or the remote primary host for continuous streaming. standby: # s3_wal_path: "s3://mybucket/spilo/acid-minimal-cluster/abcd1234-2a4b-4b2a-8c9c-c1234defg567/wal/14/" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 65499d9ce..ae0a180ac 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -589,9 +589,6 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "version": { Type: "string", Enum: []apiextv1.JSON{ - { - Raw: []byte(`"10"`), - }, { Raw: []byte(`"11"`), }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 69d591e42..fb099c746 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -49,8 +49,8 @@ type PostgresUsersConfiguration struct { type MajorVersionUpgradeConfiguration struct { MajorVersionUpgradeMode string `json:"major_version_upgrade_mode" default:"off"` // off - no actions, manual - manifest triggers action, full - manifest and minimal version violation trigger upgrade MajorVersionUpgradeTeamAllowList []string `json:"major_version_upgrade_team_allow_list,omitempty"` - MinimalMajorVersion string `json:"minimal_major_version" default:"11"` - TargetMajorVersion string `json:"target_major_version" default:"15"` + MinimalMajorVersion string `json:"minimal_major_version" default:"12"` + TargetMajorVersion string `json:"target_major_version" default:"16"` } // KubernetesMetaConfiguration defines k8s conf required for all Postgres clusters and the operator itself diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 2b1e6a6b2..bb01816c0 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -217,7 +217,7 @@ var unmarshalCluster = []struct { "127.0.0.1/32" ], "postgresql": { - "version": "15", + "version": "16", "parameters": { "shared_buffers": "32MB", "max_connections": "10", @@ -277,7 +277,7 @@ var unmarshalCluster = []struct { }, Spec: PostgresSpec{ PostgresqlParam: PostgresqlParam{ - PgVersion: "15", + PgVersion: "16", Parameters: map[string]string{ "shared_buffers": "32MB", "max_connections": "10", @@ -337,7 +337,7 @@ var unmarshalCluster = []struct { }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"15","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"16","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, { about: "example with clone", @@ -402,7 +402,7 @@ var postgresqlList = []struct { out PostgresqlList err error }{ - {"expect success", []byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace":"default","resourceVersion":"30446957","selfLink":"/apis/acid.zalan.do/v1/namespaces/default/postgresqls/acid-testcluster42","uid":"857cd208-33dc-11e7-b20a-0699041e4b03"},"spec":{"allowedSourceRanges":["185.85.220.0/22"],"numberOfInstances":1,"postgresql":{"version":"15"},"teamId":"acid","volume":{"size":"10Gi"}},"status":{"PostgresClusterStatus":"Running"}}],"kind":"List","metadata":{},"resourceVersion":"","selfLink":""}`), + {"expect success", []byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace":"default","resourceVersion":"30446957","selfLink":"/apis/acid.zalan.do/v1/namespaces/default/postgresqls/acid-testcluster42","uid":"857cd208-33dc-11e7-b20a-0699041e4b03"},"spec":{"allowedSourceRanges":["185.85.220.0/22"],"numberOfInstances":1,"postgresql":{"version":"16"},"teamId":"acid","volume":{"size":"10Gi"}},"status":{"PostgresClusterStatus":"Running"}}],"kind":"List","metadata":{},"resourceVersion":"","selfLink":""}`), PostgresqlList{ TypeMeta: metav1.TypeMeta{ Kind: "List", @@ -423,7 +423,7 @@ var postgresqlList = []struct { }, Spec: PostgresSpec{ ClusterName: "testcluster42", - PostgresqlParam: PostgresqlParam{PgVersion: "15"}, + PostgresqlParam: PostgresqlParam{PgVersion: "16"}, Volume: Volume{Size: "10Gi"}, TeamID: "acid", AllowedSourceRanges: []string{"185.85.220.0/22"}, diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index ff75cd6d4..6e5a669ad 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -72,18 +72,18 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { }{ { subtest: "Patroni default configuration", - pgParam: &acidv1.PostgresqlParam{PgVersion: "15"}, + pgParam: &acidv1.PostgresqlParam{PgVersion: "16"}, patroni: &acidv1.Patroni{}, opConfig: &config.Config{ Auth: config.Auth{ PamRoleName: "zalandos", }, }, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/15/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{}}}`, }, { subtest: "Patroni configured", - pgParam: &acidv1.PostgresqlParam{PgVersion: "15"}, + pgParam: &acidv1.PostgresqlParam{PgVersion: "16"}, patroni: &acidv1.Patroni{ InitDB: map[string]string{ "encoding": "UTF8", @@ -102,38 +102,38 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { FailsafeMode: util.True(), }, opConfig: &config.Config{}, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/15/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"synchronous_node_count":1,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}},"failsafe_mode":true}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"synchronous_node_count":1,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}},"failsafe_mode":true}}}`, }, { subtest: "Patroni failsafe_mode configured globally", - pgParam: &acidv1.PostgresqlParam{PgVersion: "15"}, + pgParam: &acidv1.PostgresqlParam{PgVersion: "16"}, patroni: &acidv1.Patroni{}, opConfig: &config.Config{ EnablePatroniFailsafeMode: util.True(), }, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/15/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`, }, { subtest: "Patroni failsafe_mode configured globally, disabled for cluster", - pgParam: &acidv1.PostgresqlParam{PgVersion: "15"}, + pgParam: &acidv1.PostgresqlParam{PgVersion: "16"}, patroni: &acidv1.Patroni{ FailsafeMode: util.False(), }, opConfig: &config.Config{ EnablePatroniFailsafeMode: util.True(), }, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/15/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":false}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":false}}}`, }, { subtest: "Patroni failsafe_mode disabled globally, configured for cluster", - pgParam: &acidv1.PostgresqlParam{PgVersion: "15"}, + pgParam: &acidv1.PostgresqlParam{PgVersion: "16"}, patroni: &acidv1.Patroni{ FailsafeMode: util.True(), }, opConfig: &config.Config{ EnablePatroniFailsafeMode: util.False(), }, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/15/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/16/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`, }, } for _, tt := range tests { @@ -164,15 +164,15 @@ func TestExtractPgVersionFromBinPath(t *testing.T) { }, { subTest: "test current bin path against hard coded template", - binPath: "/usr/lib/postgresql/15/bin", + binPath: "/usr/lib/postgresql/16/bin", template: pgBinariesLocationTemplate, - expected: "15", + expected: "16", }, { subTest: "test alternative bin path against a matching template", - binPath: "/usr/pgsql-15/bin", + binPath: "/usr/pgsql-16/bin", template: "/usr/pgsql-%v/bin", - expected: "15", + expected: "16", }, } @@ -2100,7 +2100,7 @@ func TestSidecars(t *testing.T) { spec = acidv1.PostgresSpec{ PostgresqlParam: acidv1.PostgresqlParam{ - PgVersion: "15", + PgVersion: "16", Parameters: map[string]string{ "max_connections": "100", }, diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index ae926d85e..5a1599cda 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -11,7 +11,6 @@ import ( // VersionMap Map of version numbers var VersionMap = map[string]int{ - "10": 100000, "11": 110000, "12": 120000, "13": 130000, @@ -36,7 +35,7 @@ func (c *Cluster) GetDesiredMajorVersionAsInt() int { func (c *Cluster) GetDesiredMajorVersion() string { if c.Config.OpConfig.MajorVersionUpgradeMode == "full" { - // e.g. current is 10, minimal is 11 allowing 11 to 15 clusters, everything below is upgraded + // e.g. current is 12, minimal is 12 allowing 12 to 16 clusters, everything below is upgraded if IsBiggerPostgresVersion(c.Spec.PgVersion, c.Config.OpConfig.MinimalMajorVersion) { c.logger.Infof("overwriting configured major version %s to %s", c.Spec.PgVersion, c.Config.OpConfig.TargetMajorVersion) return c.Config.OpConfig.TargetMajorVersion diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 328307d97..eaad11dff 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -39,7 +39,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.EnableTeamIdClusternamePrefix = fromCRD.EnableTeamIdClusternamePrefix result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps - result.DockerImage = util.Coalesce(fromCRD.DockerImage, "ghcr.io/zalando/spilo-15:3.0-p1") + result.DockerImage = util.Coalesce(fromCRD.DockerImage, "ghcr.io/zalando/spilo-16:3.2-p2") result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8) result.MinInstances = fromCRD.MinInstances result.MaxInstances = fromCRD.MaxInstances @@ -62,8 +62,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // major version upgrade config result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "off") result.MajorVersionUpgradeTeamAllowList = fromCRD.MajorVersionUpgrade.MajorVersionUpgradeTeamAllowList - result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "11") - result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "15") + result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "12") + result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "16") // kubernetes config result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 7ea9f9e29..4c6575218 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -173,7 +173,7 @@ type Config struct { WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"ghcr.io/zalando/spilo-15:3.0-p1"` + DockerImage string `name:"docker_image" default:"ghcr.io/zalando/spilo-16:3.2-p2"` SidecarImages map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers SidecarContainers []v1.Container `name:"sidecars"` PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` @@ -244,8 +244,8 @@ type Config struct { EnableTeamIdClusternamePrefix bool `name:"enable_team_id_clustername_prefix" default:"false"` MajorVersionUpgradeMode string `name:"major_version_upgrade_mode" default:"off"` MajorVersionUpgradeTeamAllowList []string `name:"major_version_upgrade_team_allow_list" default:""` - MinimalMajorVersion string `name:"minimal_major_version" default:"11"` - TargetMajorVersion string `name:"target_major_version" default:"15"` + MinimalMajorVersion string `name:"minimal_major_version" default:"12"` + TargetMajorVersion string `name:"target_major_version" default:"16"` PatroniAPICheckInterval time.Duration `name:"patroni_api_check_interval" default:"1s"` PatroniAPICheckTimeout time.Duration `name:"patroni_api_check_timeout" default:"5s"` EnablePatroniFailsafeMode *bool `name:"enable_patroni_failsafe_mode" default:"false"` diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 960f4c77d..a4b52f4d3 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -77,8 +77,7 @@ spec: "15", "14", "13", - "12", - "11" + "12" ] } # Exemple of settings to make snapshot view working in the ui when using AWS diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index 5eb54d962..d0f1d3911 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -321,7 +321,7 @@ DEFAULT_UI_CONFIG = { 'users_visible': True, 'databases_visible': True, 'resources_visible': RESOURCES_VISIBLE, - 'postgresql_versions': ['11', '12', '13', '14', '15', '16'], + 'postgresql_versions': ['12', '13', '14', '15', '16'], 'dns_format_string': '{0}.{1}', 'pgui_link': '', 'static_network_whitelist': {}, diff --git a/ui/run_local.sh b/ui/run_local.sh index 7af9b17ee..77f4da760 100755 --- a/ui/run_local.sh +++ b/ui/run_local.sh @@ -35,8 +35,7 @@ default_operator_ui_config='{ "15", "14", "13", - "12", - "11" + "12" ], "static_network_whitelist": { "localhost": ["172.0.0.1/32"]