diff --git a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md index 93392a68e..3b731eb72 100644 --- a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md +++ b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md @@ -9,7 +9,7 @@ assignees: '' Please, answer some short questions which should help us to understand your problem / question better? -- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.6.3 +- **Which image of the operator are you using?** e.g. ghcr.io/zalando/postgres-operator:v1.15.1 - **Where do you run it - cloud or metal? Kubernetes or OpenShift?** [AWS K8s | GCP ... | Bare Metal K8s] - **Are you running Postgres Operator in production?** [yes | no] - **Type of issue?** [Bug report, question, feature request, etc.] diff --git a/.github/workflows/publish_ghcr_image.yaml b/.github/workflows/publish_ghcr_image.yaml new file mode 100644 index 000000000..2fc22d0c9 --- /dev/null +++ b/.github/workflows/publish_ghcr_image.yaml @@ -0,0 +1,115 @@ +name: Publish multiarch postgres-operator images on ghcr.io + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + IMAGE_NAME_UI: ${{ github.repository }}-ui + +on: + push: + tags: + - '*' + +jobs: + publish: + name: Build, test and push image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - uses: actions/setup-go@v6 + with: + go-version: "^1.26.4" + + - name: Run unit tests + run: make test + + - name: Define image name + id: image + run: | + OPERATOR_IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${GITHUB_REF/refs\/tags\//}" + echo "OPERATOR_IMAGE=$OPERATOR_IMAGE" >> $GITHUB_OUTPUT + + - name: Define pooler image name + id: image_pooler + run: | + POOLER_IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/pgbouncer:${GITHUB_REF/refs\/tags\//}" + echo "POOLER_IMAGE=$POOLER_IMAGE" >> $GITHUB_OUTPUT + + - name: Define UI image name + id: image_ui + run: | + UI_IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_UI }}:${GITHUB_REF/refs\/tags\//}" + echo "UI_IMAGE=$UI_IMAGE" >> $GITHUB_OUTPUT + + - name: Define logical backup image name + id: image_lb + run: | + BACKUP_IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/logical-backup:${GITHUB_REF_NAME}" + echo "BACKUP_IMAGE=$BACKUP_IMAGE" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GHCR + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push multiarch operator image to ghcr + uses: docker/build-push-action@v7 + with: + context: . + file: docker/Dockerfile + push: true + build-args: BASE_IMAGE=alpine:3 + tags: "${{ steps.image.outputs.OPERATOR_IMAGE }}" + platforms: linux/amd64,linux/arm64 + + - name: Build and push multiarch pooler image to ghcr + uses: docker/build-push-action@v7 + with: + context: pooler + push: true + build-args: BASE_IMAGE=alpine:3.22 + tags: "${{ steps.image_pooler.outputs.POOLER_IMAGE }}" + platforms: linux/amd64,linux/arm64 + + - name: Build and push multiarch ui image to ghcr + uses: docker/build-push-action@v7 + with: + context: ui + push: true + build-args: BASE_IMAGE=python:3.11-slim + tags: "${{ steps.image_ui.outputs.UI_IMAGE }}" + platforms: linux/amd64,linux/arm64 + + - name: Build and push multiarch logical-backup image to ghcr + uses: docker/build-push-action@v7 + with: + context: logical-backup + push: true + build-args: BASE_IMAGE=ubuntu:22.04 + tags: "${{ steps.image_lb.outputs.BACKUP_IMAGE }}" + platforms: linux/amd64,linux/arm64 + + - name: Build and push postgres-operator chart to ghcr + run: | + helm package charts/postgres-operator + helm push postgres-operator-*.tgz oci://${{ env.REGISTRY }}/zalando/charts + rm -rf postgres-operator-*.tgz + + - name: Build and push postgres-operator-ui chart to ghcr + run: | + helm package charts/postgres-operator-ui + helm push postgres-operator-ui-*.tgz oci://${{ env.REGISTRY }}/zalando/charts + rm -rf postgres-operator-ui-*.tgz diff --git a/.github/workflows/run_e2e.yaml b/.github/workflows/run_e2e.yaml index cff0d49ef..393109b00 100644 --- a/.github/workflows/run_e2e.yaml +++ b/.github/workflows/run_e2e.yaml @@ -11,15 +11,15 @@ jobs: name: End-2-End tests runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 with: - go-version: "^1.15.6" + go-version: "^1.26.4" - name: Make dependencies - run: make deps mocks - - name: Compile - run: make linux + run: make mocks + - name: Code generation + run: make codegen - name: Run unit tests - run: go test ./... + run: make test - name: Run end-2-end tests run: make e2e diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index de14267b4..c511e8fa1 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -11,18 +11,18 @@ jobs: name: Unit tests and coverage runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 with: - go-version: "^1.15.6" + go-version: "^1.26.4" - name: Make dependencies - run: make deps mocks + run: make mocks - name: Compile run: make linux - name: Run unit tests run: go test -race -covermode atomic -coverprofile=coverage.out ./... - name: Convert coverage to lcov - uses: jandelgado/gcov2lcov-action@v1.0.8 + uses: jandelgado/gcov2lcov-action@v1.1.1 - name: Coveralls uses: coverallsapp/github-action@master with: diff --git a/.gitignore b/.gitignore index e062f8479..65aad49c5 100644 --- a/.gitignore +++ b/.gitignore @@ -94,9 +94,14 @@ coverage.xml # e2e tests e2e/manifests +e2e/tls # Translations *.mo *.pot mocks + +ui/.npm/ + +.DS_Store diff --git a/CODEOWNERS b/CODEOWNERS index 398856c66..96213da15 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1,2 @@ # global owners -* @erthalion @sdudoladov @Jan-M @CyberDem0n @avaczi @FxKu @RafiaSabih +* @sdudoladov @Jan-M @FxKu @jopadi @idanovinda @hughcapet @mikkeloscar diff --git a/LICENSE b/LICENSE index 7c0f459a5..2f94edc31 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2021 Zalando SE +Copyright (c) 2026 Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MAINTAINERS b/MAINTAINERS index 572e6d971..6ff2aa62e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1,5 +1,7 @@ -Dmitrii Dolgov Sergey Dudoladov Felix Kunde Jan Mussler -Rafia Sabih \ No newline at end of file +Jociele Padilha +Ida Novindasari +Polina Bungina +Mikkel Larsen diff --git a/Makefile b/Makefile index 5d0d960f0..3613c1044 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ -.PHONY: clean local test linux macos mocks docker push scm-source.json e2e +.PHONY: clean local test linux macos mocks docker pooler push e2e BINARY ?= postgres-operator BUILD_FLAGS ?= -v +GOARCH ?= amd64 CGO_ENABLED ?= 0 ifeq ($(RACE),1) BUILD_FLAGS += -race -a @@ -12,13 +13,17 @@ LOCAL_BUILD_FLAGS ?= $(BUILD_FLAGS) LDFLAGS ?= -X=main.version=$(VERSION) DOCKERDIR = docker -IMAGE ?= registry.opensource.zalan.do/acid/$(BINARY) +BASE_IMAGE ?= alpine:latest +IMAGE ?= ghcr.io/zalando/$(BINARY) TAG ?= $(VERSION) GITHEAD = $(shell git rev-parse --short HEAD) GITURL = $(shell git config --get remote.origin.url) GITSTATUS = $(shell git status --porcelain || echo "no changes") SOURCES = cmd/main.go VERSION ?= $(shell git describe --tags --always --dirty) +CRD_SOURCES = $(shell find pkg/apis/zalando.org pkg/apis/acid.zalan.do -name '*.go' -not -name '*.deepcopy.go') +GENERATED_CRDS = manifests/postgresteam.crd.yaml manifests/postgresql.crd.yaml pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml +GENERATED = pkg/apis/zalando.org/v1/zz_generated.deepcopy.go pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go DIRS := cmd pkg PKG := `go list ./... | grep -v /vendor/` @@ -42,52 +47,64 @@ ifndef GOPATH GOPATH := $(HOME)/go endif -PATH := $(GOPATH)/bin:$(PATH) -SHELL := env PATH=$(PATH) $(SHELL) +PATH := $(GOPATH)/bin:$(PATH) +SHELL := env PATH="$(PATH)" $(SHELL) +IMAGE_TAG := $(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_FRESH)$(DEBUG_POSTFIX) +POOLER_TAG := $(IMAGE)/pgbouncer:$(TAG)$(CDP_TAG)$(DEBUG_FRESH)$(DEBUG_POSTFIX) default: local clean: - rm -rf build scm-source.json + rm -rf build + rm $(GENERATED) + rm $(GENERATED_CRDS) -local: ${SOURCES} +verify: hack/verify-codegen.sh - CGO_ENABLED=${CGO_ENABLED} go build -o build/${BINARY} $(LOCAL_BUILD_FLAGS) -ldflags "$(LDFLAGS)" $^ -linux: ${SOURCES} - GOOS=linux GOARCH=amd64 CGO_ENABLED=${CGO_ENABLED} go build -o build/linux/${BINARY} ${BUILD_FLAGS} -ldflags "$(LDFLAGS)" $^ +$(GENERATED): go.mod $(CRD_SOURCES) + hack/update-codegen.sh -macos: ${SOURCES} - GOOS=darwin GOARCH=amd64 CGO_ENABLED=${CGO_ENABLED} go build -o build/macos/${BINARY} ${BUILD_FLAGS} -ldflags "$(LDFLAGS)" $^ +$(GENERATED_CRDS): $(GENERATED) + go tool controller-gen crd:crdVersions=v1,allowDangerousTypes=true paths=./pkg/apis/acid.zalan.do/... output:crd:dir=manifests + @mv manifests/acid.zalan.do_postgresqls.yaml manifests/postgresql.crd.yaml + @# hack to use lowercase kind and listKind + @sed -i -e 's/kind: Postgresql/kind: postgresql/' manifests/postgresql.crd.yaml + @sed -i -e 's/listKind: PostgresqlList/listKind: postgresqlList/' manifests/postgresql.crd.yaml + @hack/adjust_postgresql_crd.sh + @mv manifests/acid.zalan.do_operatorconfigurations.yaml manifests/operatorconfiguration.crd.yaml + @mv manifests/acid.zalan.do_postgresteams.yaml manifests/postgresteam.crd.yaml + @cp manifests/postgresql.crd.yaml pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml + @cp manifests/operatorconfiguration.crd.yaml pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml -docker-context: scm-source.json linux - mkdir -p docker/build/ - cp build/linux/${BINARY} scm-source.json docker/build/ +local: ${SOURCES} $(GENERATED_CRDS) + CGO_ENABLED=${CGO_ENABLED} go build -o build/${BINARY} $(LOCAL_BUILD_FLAGS) -ldflags "$(LDFLAGS)" $(SOURCES) -docker: ${DOCKERDIR}/${DOCKERFILE} docker-context +wasm: ${SOURCES} $(GENERATED_CRDS) + GOOS=wasip1 GOARCH=wasm CGO_ENABLED=${CGO_ENABLED} go build -o build/${BINARY}.wasm ${BUILD_FLAGS} -ldflags "$(LDFLAGS)" $(SOURCES) + +linux: ${SOURCES} $(GENERATED_CRDS) + GOOS=linux GOARCH=${GOARCH} CGO_ENABLED=${CGO_ENABLED} go build -o build/linux/${BINARY} ${BUILD_FLAGS} -ldflags "$(LDFLAGS)" $(SOURCES) + +macos: ${SOURCES} $(GENERATED_CRDS) + GOOS=darwin GOARCH=${GOARCH} CGO_ENABLED=${CGO_ENABLED} go build -o build/macos/${BINARY} ${BUILD_FLAGS} -ldflags "$(LDFLAGS)" $(SOURCES) + +docker: $(GENERATED_CRDS) ${DOCKERDIR}/${DOCKERFILE} echo `(env)` echo "Tag ${TAG}" echo "Version ${VERSION}" echo "CDP tag ${CDP_TAG}" echo "git describe $(shell git describe --tags --always --dirty)" - cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_FRESH)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" . + docker build --rm -t "$(IMAGE_TAG)" -f "${DOCKERDIR}/${DOCKERFILE}" --build-arg VERSION="${VERSION}" --build-arg BASE_IMAGE="${BASE_IMAGE}" . + +pooler: + cd pooler; docker build --rm -t "$(POOLER_TAG)" --build-arg VERSION="${VERSION}" --build-arg BASE_IMAGE="${BASE_IMAGE}" . indocker-race: - docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.8.1 bash -c "make linux" - -push: - docker push "$(IMAGE):$(TAG)$(CDP_TAG)" - -scm-source.json: .git - echo '{\n "url": "git:$(GITURL)",\n "revision": "$(GITHEAD)",\n "author": "$(USER)",\n "status": "$(GITSTATUS)"\n}' > scm-source.json + docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.26.4 bash -c "make linux" mocks: - GO111MODULE=on go generate ./... - -tools: - GO111MODULE=on go get k8s.io/client-go@kubernetes-1.20.6 - GO111MODULE=on go get github.com/golang/mock/mockgen@v1.4.4 - GO111MODULE=on go mod tidy + go generate ./... fmt: @gofmt -l -w -s $(DIRS) @@ -96,12 +113,10 @@ vet: @go vet $(PKG) @staticcheck $(PKG) -deps: tools - GO111MODULE=on go mod vendor +test: mocks $(GENERATED) $(GENERATED_CRDS) + go test ./... -test: - hack/verify-codegen.sh - GO111MODULE=on go test ./... +codegen: $(GENERATED) -e2e: docker # build operator image to be tested +e2e: docker pooler # build operator and pooler images to be tested cd e2e; make e2etest diff --git a/README.md b/README.md index ebe84bf5f..31e9c1748 100644 --- a/README.md +++ b/README.md @@ -16,60 +16,60 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as * Rolling updates on Postgres cluster changes, incl. quick minor version updates * Live volume resize without pod restarts (AWS EBS, PVC) * Database connection pooling with PGBouncer -* Support fast in place major version upgrade to PG13. Supports global upgrade of all clusters. -* Restore and cloning Postgres clusters (incl. major version upgrade) -* Additionally logical backups to S3 bucket can be configured -* Standby cluster from S3 WAL archive +* Support fast in place major version upgrade. Supports global upgrade of all clusters. +* Pod protection during bootstrap phase and configurable maintenance windows +* Restore and cloning Postgres clusters on AWS, GCS and Azure +* Additionally logical backups to S3 or GCS bucket can be configured +* Standby cluster from S3 or GCS WAL archive * Configurable for non-cloud environments * Basic credential and user management on K8s, eases application deployments * Support for custom TLS certificates * UI to create and edit Postgres cluster manifests -* Works well on Amazon AWS, Google Cloud, OpenShift and locally on Kind -* Support for AWS EBS gp2 to gp3 migration, supporting iops and throughput configuration +* Compatible with OpenShift ### PostgreSQL features -* Supports PostgreSQL 13, starting from 9.6+ +* Supports PostgreSQL 18, starting from 14+ * Streaming replication cluster via Patroni * Point-In-Time-Recovery with -[pg_basebackup](https://www.postgresql.org/docs/11/app-pgbasebackup.html) / -[WAL-E](https://github.com/wal-e/wal-e) via [Spilo](https://github.com/zalando/spilo) +[pg_basebackup](https://www.postgresql.org/docs/18/app-pgbasebackup.html) / +[WAL-G](https://github.com/wal-g/wal-g) or [WAL-E](https://github.com/wal-e/wal-e) via [Spilo](https://github.com/zalando/spilo) * Preload libraries: [bg_mon](https://github.com/CyberDem0n/bg_mon), -[pg_stat_statements](https://www.postgresql.org/docs/9.4/pgstatstatements.html), +[pg_stat_statements](https://www.postgresql.org/docs/18/pgstatstatements.html), [pgextwlist](https://github.com/dimitri/pgextwlist), [pg_auth_mon](https://github.com/RafiaSabih/pg_auth_mon) * Incl. popular Postgres extensions such as [decoderbufs](https://github.com/debezium/postgres-decoderbufs), [hypopg](https://github.com/HypoPG/hypopg), [pg_cron](https://github.com/citusdata/pg_cron), +[pg_repack](https://github.com/reorg/pg_repack), [pg_partman](https://github.com/pgpartman/pg_partman), [pg_stat_kcache](https://github.com/powa-team/pg_stat_kcache), +[pg_audit](https://github.com/pgaudit/pgaudit), +[pgfaceting](https://github.com/cybertec-postgresql/pgfaceting), [pgq](https://github.com/pgq/pgq), +[pgvector](https://github.com/pgvector/pgvector), [plpgsql_check](https://github.com/okbob/plpgsql_check), +[plproxy](https://github.com/plproxy/plproxy), [postgis](https://postgis.net/), +[roaringbitmap](https://github.com/ChenHuajun/pg_roaringbitmap), [set_user](https://github.com/pgaudit/set_user) and [timescaledb](https://github.com/timescale/timescaledb) The Postgres Operator has been developed at Zalando and is being used in -production for over three years. +production for over five years. -## Notes on Postgres 13 support - -If you are new to the operator, you can skip this and just start using the Postgres operator as is, Postgres 13 is ready to go. - -The Postgres operator supports Postgres 13 with the new Spilo Image that includes also the recent Patroni version to support PG13 settings. -More work on optimizing restarts and rolling upgrades is pending. - -If you are already using the Postgres operator in older version with a Spilo 12 Docker image you need to be aware of the changes for the backup path. -We introduce the major version into the backup path to smoothen the [major version upgrade](docs/administrator.md#minor-and-major-version-upgrade) that is now supported manually. - -The new operator configuration can set a compatibility flag *enable_spilo_wal_path_compat* to make Spilo look for wal segments in the current path but also old format paths. -This comes at potential performance costs and should be disabled after a few days. - -The newest Spilo 13 image is: `registry.opensource.zalan.do/acid/spilo-13:2.0-p7` - -The last Spilo 12 image is: `registry.opensource.zalan.do/acid/spilo-12:1.6-p5` +## Supported Postgres & K8s versions +| Release | Postgres versions | K8s versions | Golang | +| :-------- | :---------------: | :---------------: | :-----: | +| next | 14 → 18 | 1.27+ | 1.26.4 | +| v1.15.1 | 13 → 17 | 1.27+ | 1.25.3 | +| v1.14.0 | 13 → 17 | 1.27+ | 1.23.4 | +| v1.13.0 | 12 → 16 | 1.27+ | 1.22.5 | +| v1.12.0 | 11 → 16 | 1.27+ | 1.22.3 | +| v1.11.0 | 11 → 16 | 1.27+ | 1.21.7 | +| v1.10.1 | 10 → 15 | 1.21+ | 1.19.8 | ## Getting started @@ -78,7 +78,8 @@ For a quick first impression follow the instructions of this ## Supported setups of Postgres and Applications -![Features](docs/diagrams/neutral_operator.png) +![Features](docs/diagrams/neutral_operator_dark.png#gh-dark-mode-only) +![Features](docs/diagrams/neutral_operator_light.png#gh-light-mode-only) ## Documentation @@ -94,9 +95,3 @@ There is a browser-friendly version of this documentation at * [Configuration options](docs/reference/operator_parameters.md) * [Postgres manifest reference](docs/reference/cluster_manifest.md) * [Command-line options and environment variables](docs/reference/command_line_and_environment.md) - -## Community - -There are two places to get in touch with the community: -1. The [GitHub issue tracker](https://github.com/zalando/postgres-operator/issues) -2. The **#postgres-operator** [slack channel](https://postgres-slack.herokuapp.com) diff --git a/build-ci.sh b/build-ci.sh index a4805ad28..16ef3d245 100755 --- a/build-ci.sh +++ b/build-ci.sh @@ -9,4 +9,4 @@ mkdir -p "$team_repo" ln -s "$PWD" "$project_dir" cd "$project_dir" -make deps clean docker push +make clean docker push diff --git a/charts/postgres-operator-ui/Chart.yaml b/charts/postgres-operator-ui/Chart.yaml index e9c10868c..871640467 100644 --- a/charts/postgres-operator-ui/Chart.yaml +++ b/charts/postgres-operator-ui/Chart.yaml @@ -1,7 +1,7 @@ -apiVersion: v1 +apiVersion: v2 name: postgres-operator-ui -version: 1.6.3 -appVersion: 1.6.3 +version: 1.15.1 +appVersion: 1.15.1 home: https://github.com/zalando/postgres-operator description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience keywords: diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml index f76bfcbb4..20408aeaf 100644 --- a/charts/postgres-operator-ui/index.yaml +++ b/charts/postgres-operator-ui/index.yaml @@ -1,11 +1,12 @@ apiVersion: v1 entries: postgres-operator-ui: - - apiVersion: v1 - appVersion: 1.6.3 - created: "2021-05-27T19:04:33.425637932+02:00" - description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience - digest: 08b810aa632dcc719e4785ef184e391267f7c460caa99677f2d00719075aac78 + - apiVersion: v2 + appVersion: 1.15.1 + created: "2025-12-11T12:44:25.470723322+01:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: 4bbb750934366038d692711f924151182b7be131b6822d011f5a4e51cf609482 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -21,13 +22,14 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-ui-1.6.3.tgz - version: 1.6.3 - - apiVersion: v1 - appVersion: 1.6.2 - created: "2021-05-27T19:04:33.422124263+02:00" - description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience - digest: 14d1559bb0bd1e1e828f2daaaa6f6ac9ffc268d79824592c3589b55dd39241f6 + - postgres-operator-ui-1.15.1.tgz + version: 1.15.1 + - apiVersion: v2 + appVersion: 1.14.0 + created: "2025-12-11T12:44:25.468680645+01:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: e87ed898079a852957a67a4caf3fbd27b9098e413f5d961b7a771a6ae8b3e17c home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -43,13 +45,14 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-ui-1.6.2.tgz - version: 1.6.2 - - apiVersion: v1 - appVersion: 1.6.1 - created: "2021-05-27T19:04:33.419640902+02:00" - description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience - digest: 3d321352f2f1e7bb7450aa8876e3d818aa9f9da9bd4250507386f0490f2c1969 + - postgres-operator-ui-1.14.0.tgz + version: 1.14.0 + - apiVersion: v2 + appVersion: 1.13.0 + created: "2025-12-11T12:44:25.466716836+01:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: e0444e516b50f82002d1a733527813c51759a627cefdd1005cea73659f824ea8 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -65,13 +68,14 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-ui-1.6.1.tgz - version: 1.6.1 - - apiVersion: v1 - appVersion: 1.6.0 - created: "2021-05-27T19:04:33.41788193+02:00" - description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience - digest: 1e0aa1e7db3c1daa96927ffbf6fdbcdb434562f961833cb5241ddbe132220ee4 + - postgres-operator-ui-1.13.0.tgz + version: 1.13.0 + - apiVersion: v2 + appVersion: 1.12.2 + created: "2025-12-11T12:44:25.464739895+01:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: cbcef400c23ccece27d97369ad629278265c013e0a45c0b7f33e7568a082fedd home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -87,13 +91,14 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-ui-1.6.0.tgz - version: 1.6.0 - - apiVersion: v1 - appVersion: 1.5.0 - created: "2021-05-27T19:04:33.416056821+02:00" - description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience - digest: c91ea39e6d51d57f4048fb1b6ec53b40823f2690eb88e4e4f1a036367b9fdd61 + - postgres-operator-ui-1.12.2.tgz + version: 1.12.2 + - apiVersion: v2 + appVersion: 1.11.0 + created: "2025-12-11T12:44:25.462698399+01:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: a45f2284045c2a9a79750a36997386444f39b01ac722b17c84b431457577a3a2 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -109,6 +114,29 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-ui-1.5.0.tgz - version: 1.5.0 -generated: "2021-05-27T19:04:33.41380858+02:00" + - postgres-operator-ui-1.11.0.tgz + version: 1.11.0 + - apiVersion: v2 + appVersion: 1.10.1 + created: "2025-12-11T12:44:25.460357063+01:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: 2e5e7a82aebee519ec57c6243eb8735124aa4585a3a19c66ffd69638fbeb11ce + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - ui + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator-ui + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-ui-1.10.1.tgz + version: 1.10.1 +generated: "2025-12-11T12:44:25.45732896+01:00" diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.10.1.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.10.1.tgz new file mode 100644 index 000000000..c0719f7bf Binary files /dev/null and b/charts/postgres-operator-ui/postgres-operator-ui-1.10.1.tgz differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.11.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.11.0.tgz new file mode 100644 index 000000000..7612a159b Binary files /dev/null and b/charts/postgres-operator-ui/postgres-operator-ui-1.11.0.tgz differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.12.2.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.12.2.tgz new file mode 100644 index 000000000..f34fd8f11 Binary files /dev/null and b/charts/postgres-operator-ui/postgres-operator-ui-1.12.2.tgz differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.13.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.13.0.tgz new file mode 100644 index 000000000..21aadc076 Binary files /dev/null and b/charts/postgres-operator-ui/postgres-operator-ui-1.13.0.tgz differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.14.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.14.0.tgz new file mode 100644 index 000000000..8e229d0f5 Binary files /dev/null and b/charts/postgres-operator-ui/postgres-operator-ui-1.14.0.tgz differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.15.1.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.15.1.tgz new file mode 100644 index 000000000..2eae35f92 Binary files /dev/null and b/charts/postgres-operator-ui/postgres-operator-ui-1.15.1.tgz differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz deleted file mode 100644 index d8527f293..000000000 Binary files a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz and /dev/null differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.6.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.6.0.tgz deleted file mode 100644 index 7d3b2b738..000000000 Binary files a/charts/postgres-operator-ui/postgres-operator-ui-1.6.0.tgz and /dev/null differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.6.1.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.6.1.tgz deleted file mode 100644 index c59d20b2f..000000000 Binary files a/charts/postgres-operator-ui/postgres-operator-ui-1.6.1.tgz and /dev/null differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.6.2.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.6.2.tgz deleted file mode 100644 index 2e5298164..000000000 Binary files a/charts/postgres-operator-ui/postgres-operator-ui-1.6.2.tgz and /dev/null differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.6.3.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.6.3.tgz deleted file mode 100644 index 45e3d00b2..000000000 Binary files a/charts/postgres-operator-ui/postgres-operator-ui-1.6.3.tgz and /dev/null differ diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index b72b108e0..c8797e42e 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -7,8 +7,9 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} name: {{ template "postgres-operator-ui.fullname" . }} + namespace: {{ .Release.Namespace }} spec: - replicas: 1 + replicas: {{ .Values.replicaCount }} selector: matchLabels: app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} @@ -18,6 +19,10 @@ spec: labels: app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "postgres-operator-ui.serviceAccountName" . }} {{- if .Values.imagePullSecrets }} @@ -41,7 +46,7 @@ spec: {{- toYaml .Values.resources | nindent 12 }} env: - name: "APP_URL" - value: "http://localhost:8081" + value: {{ .Values.envs.appUrl }} - name: "OPERATOR_API_URL" value: {{ .Values.envs.operatorApiUrl | quote }} - name: "OPERATOR_CLUSTER_NAME_LABEL" @@ -62,16 +67,39 @@ spec: value: |- { "docs_link":"https://postgres-operator.readthedocs.io/en/latest/", - "dns_format_string": "{1}-{0}.{2}", + "dns_format_string": "{0}.{1}", "databases_visible": true, "master_load_balancer_visible": true, "nat_gateways_visible": false, "replica_load_balancer_visible": true, "resources_visible": true, "users_visible": true, + "cost_ebs": 0.0952, + "cost_iops": 0.006, + "cost_throughput": 0.0476, + "cost_core": 0.0575, + "cost_memory": 0.014375, + "free_iops": 3000, + "free_throughput": 125, + "limit_iops": 16000, + "limit_throughput": 1000, "postgresql_versions": [ - "13", - "12", - "11" + "18", + "17", + "16", + "15", + "14", ] } + {{- if .Values.extraEnvs }} + {{- .Values.extraEnvs | toYaml | nindent 12 }} + {{- end }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} diff --git a/charts/postgres-operator-ui/templates/ingress.yaml b/charts/postgres-operator-ui/templates/ingress.yaml index 73fa2e817..75bf79090 100644 --- a/charts/postgres-operator-ui/templates/ingress.yaml +++ b/charts/postgres-operator-ui/templates/ingress.yaml @@ -1,7 +1,10 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "postgres-operator-ui.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} + +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1beta1 {{- else -}} apiVersion: extensions/v1beta1 @@ -9,6 +12,7 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: name: {{ $fullName }} + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} @@ -19,6 +23,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: +{{- if .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} +{{- end }} {{- if .Values.ingress.tls }} tls: {{- range .Values.ingress.tls }} @@ -36,9 +43,18 @@ spec: paths: {{- range .paths }} - path: {{ . }} + {{ if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion -}} + pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else -}} backend: serviceName: {{ $fullName }} servicePort: {{ $svcPort }} + {{- end -}} {{- end }} {{- end }} {{- end }} diff --git a/charts/postgres-operator-ui/templates/service.yaml b/charts/postgres-operator-ui/templates/service.yaml index bc40fbbb1..c93e076ed 100644 --- a/charts/postgres-operator-ui/templates/service.yaml +++ b/charts/postgres-operator-ui/templates/service.yaml @@ -6,7 +6,12 @@ metadata: helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} name: {{ template "postgres-operator-ui.fullname" . }} + namespace: {{ .Release.Namespace }} spec: ports: - port: {{ .Values.service.port }} diff --git a/charts/postgres-operator-ui/templates/serviceaccount.yaml b/charts/postgres-operator-ui/templates/serviceaccount.yaml index 4c5a25543..94a9ca52e 100644 --- a/charts/postgres-operator-ui/templates/serviceaccount.yaml +++ b/charts/postgres-operator-ui/templates/serviceaccount.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "postgres-operator-ui.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ template "postgres-operator-ui.name" . }} helm.sh/chart: {{ template "postgres-operator-ui.chart" . }} diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index c9f521464..c308335b2 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -6,9 +6,9 @@ replicaCount: 1 # configure ui image image: - registry: registry.opensource.zalan.do - repository: acid/postgres-operator-ui - tag: v1.6.3 + registry: ghcr.io + repository: zalando/postgres-operator-ui + tag: v1.15.1 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -39,15 +39,49 @@ resources: # configure UI ENVs envs: - # IMPORTANT: While operator chart and UI chart are idendependent, this is the interface between + # IMPORTANT: While operator chart and UI chart are independent, this is the interface between # UI and operator API. Insert the service name of the operator API here! + appUrl: "http://localhost:8081" operatorApiUrl: "http://postgres-operator:8080" operatorClusterNameLabel: "cluster-name" resourcesVisible: "False" + # Set to "*" to allow viewing/creation of clusters in all namespaces targetNamespace: "default" teams: - "acid" +# Extra pod annotations +podAnnotations: + {} + +# configure extra UI ENVs +# Extra ENVs are writen in kubenertes format and added "as is" to the pod's env variables +# https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables +# UI specific env variables can be found here: https://github.com/zalando/postgres-operator/blob/master/ui/operator_ui/main.py +extraEnvs: + [] + # Exemple of settings to make snapshot view working in the ui when using AWS + # - name: SPILO_S3_BACKUP_PREFIX + # value: spilo/ + # - name: AWS_ACCESS_KEY_ID + # valueFrom: + # secretKeyRef: + # name: + # key: AWS_ACCESS_KEY_ID + # - name: AWS_SECRET_ACCESS_KEY + # valueFrom: + # secretKeyRef: + # name: + # key: AWS_SECRET_ACCESS_KEY + # - name: AWS_DEFAULT_REGION + # valueFrom: + # secretKeyRef: + # name: + # key: AWS_DEFAULT_REGION + # - name: SPILO_S3_BACKUP_BUCKET + # value: + # configure UI service service: type: "ClusterIP" @@ -55,17 +89,36 @@ service: # If the type of the service is NodePort a port can be specified using the nodePort field # If the nodePort field is not specified, or if it has no value, then a random port is used # nodePort: 32521 + annotations: + {} # configure UI ingress. If needed: "enabled: true" ingress: enabled: false - annotations: {} + annotations: + {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" + ingressClassName: "" hosts: - host: ui.example.org - paths: [""] + paths: ["/"] tls: [] # - secretName: ui-tls # hosts: # - ui.exmaple.org + +# priority class for operator-ui pod +priorityClassName: "" + +# Affinity for pod assignment +# Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} + +# Node labels for pod assignment +# Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +# Tolerations for pod assignment +# Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] diff --git a/charts/postgres-operator/Chart.yaml b/charts/postgres-operator/Chart.yaml index 28d0f811d..6f0d2e762 100644 --- a/charts/postgres-operator/Chart.yaml +++ b/charts/postgres-operator/Chart.yaml @@ -1,7 +1,7 @@ -apiVersion: v1 +apiVersion: v2 name: postgres-operator -version: 1.6.3 -appVersion: 1.6.3 +version: 1.15.1 +appVersion: 1.15.1 home: https://github.com/zalando/postgres-operator description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes keywords: diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 82a737ae6..5875b5808 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -4,565 +4,967 @@ metadata: name: operatorconfigurations.acid.zalan.do labels: app.kubernetes.io/name: postgres-operator - annotations: - "helm.sh/hook": crd-install spec: group: acid.zalan.do names: + categories: + - all kind: OperatorConfiguration listKind: OperatorConfigurationList plural: operatorconfigurations - singular: operatorconfiguration shortNames: - opconfig - categories: - - all + singular: operatorconfiguration scope: Namespaced versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - name: Image - type: string - description: Spilo image to be used for Pods + - additionalPrinterColumns: + - description: Spilo image to be used for Pods jsonPath: .configuration.docker_image - - name: Cluster-Label + name: Image type: string - description: Label for K8s resources created by operator + - description: Label for K8s resources created by operator jsonPath: .configuration.kubernetes.cluster_name_label - - name: Service-Account + name: Cluster-Label type: string - description: Name of service account to be used + - description: Name of service account to be used jsonPath: .configuration.kubernetes.pod_service_account_name - - name: Min-Instances - type: integer - description: Minimum number of instances per Postgres cluster + name: Service-Account + type: string + - description: Minimum number of instances per Postgres cluster jsonPath: .configuration.min_instances - - name: Age - type: date + name: Min-Instances + type: integer + - description: Age of the OperatorConfiguration resource jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - configuration + description: OperatorConfiguration defines the specification for the OperatorConfiguration. properties: - kind: - type: string - enum: - - OperatorConfiguration apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string - enum: - - acid.zalan.do/v1 configuration: - type: object + description: OperatorConfigurationData defines the operation config properties: - docker_image: - type: string - default: "registry.opensource.zalan.do/acid/spilo-13:2.0-p7" - enable_crd_validation: - type: boolean - default: true - enable_lazy_spilo_upgrade: - type: boolean - default: false - enable_pgversion_env_var: - type: boolean - default: true - enable_shm_volume: - type: boolean - default: true - enable_spilo_wal_path_compat: - type: boolean - default: false - etcd_host: - type: string - default: "" - kubernetes_use_configmaps: - type: boolean - default: false - max_instances: - type: integer - minimum: -1 # -1 = disabled - default: -1 - min_instances: - type: integer - minimum: -1 # -1 = disabled - default: -1 - resync_period: - type: string - default: "30m" - repair_period: - type: string - default: "5m" - set_memory_request_to_limit: - type: boolean - default: false - sidecar_docker_images: - type: object - additionalProperties: - type: string - sidecars: - type: array - nullable: true - items: - type: object - x-kubernetes-preserve-unknown-fields: true - workers: - type: integer - minimum: 1 - default: 8 - users: - type: object - properties: - replication_username: - type: string - default: standby - super_username: - type: string - default: postgres - major_version_upgrade: - type: object - properties: - major_version_upgrade_mode: - type: string - default: "off" - minimal_major_version: - type: string - default: "9.5" - target_major_version: - type: string - default: "13" - kubernetes: - type: object - properties: - additional_pod_capabilities: - type: array - items: - type: string - cluster_domain: - type: string - default: "cluster.local" - cluster_labels: - type: object - additionalProperties: - type: string - default: - application: spilo - cluster_name_label: - type: string - default: "cluster-name" - custom_pod_annotations: - type: object - additionalProperties: - type: string - delete_annotation_date_key: - type: string - delete_annotation_name_key: - type: string - downscaler_annotations: - type: array - items: - type: string - enable_init_containers: - type: boolean - default: true - enable_pod_antiaffinity: - type: boolean - default: false - enable_pod_disruption_budget: - type: boolean - default: true - enable_sidecars: - type: boolean - default: true - infrastructure_roles_secret_name: - type: string - infrastructure_roles_secrets: - type: array - nullable: true - items: - type: object - required: - - secretname - - userkey - - passwordkey - properties: - secretname: - type: string - userkey: - type: string - passwordkey: - type: string - rolekey: - type: string - defaultuservalue: - type: string - defaultrolevalue: - type: string - details: - type: string - template: - type: boolean - inherited_annotations: - type: array - items: - type: string - inherited_labels: - type: array - items: - type: string - master_pod_move_timeout: - type: string - default: "20m" - node_readiness_label: - type: object - additionalProperties: - type: string - oauth_token_secret_name: - type: string - default: "postgresql-operator" - pdb_name_format: - type: string - default: "postgres-{cluster}-pdb" - pod_antiaffinity_topology_key: - type: string - default: "kubernetes.io/hostname" - pod_environment_configmap: - type: string - pod_environment_secret: - type: string - pod_management_policy: - type: string - enum: - - "ordered_ready" - - "parallel" - default: "ordered_ready" - pod_priority_class_name: - type: string - pod_role_label: - type: string - default: "spilo-role" - pod_service_account_definition: - type: string - default: "" - pod_service_account_name: - type: string - default: "postgres-pod" - pod_service_account_role_binding_definition: - type: string - default: "" - pod_terminate_grace_period: - type: string - default: "5m" - secret_name_template: - type: string - default: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" - spilo_allow_privilege_escalation: - type: boolean - default: true - spilo_runasuser: - type: integer - spilo_runasgroup: - type: integer - spilo_fsgroup: - type: integer - spilo_privileged: - type: boolean - default: false - storage_resize_mode: - type: string - enum: - - "ebs" - - "pvc" - - "off" - default: "pvc" - toleration: - type: object - additionalProperties: - type: string - watched_namespace: - type: string - postgres_pod_resources: - type: object - properties: - default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "1" - default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "100m" - default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "500Mi" - default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "100Mi" - min_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "250m" - min_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "250Mi" - timeouts: - type: object - properties: - pod_label_wait_timeout: - type: string - default: "10m" - pod_deletion_wait_timeout: - type: string - default: "10m" - ready_wait_interval: - type: string - default: "4s" - ready_wait_timeout: - type: string - default: "30s" - resource_check_interval: - type: string - default: "3s" - resource_check_timeout: - type: string - default: "10m" - load_balancer: - type: object - properties: - custom_service_annotations: - type: object - additionalProperties: - type: string - db_hosted_zone: - type: string - default: "db.example.com" - enable_master_load_balancer: - type: boolean - default: true - enable_replica_load_balancer: - type: boolean - default: false - external_traffic_policy: - type: string - enum: - - "Cluster" - - "Local" - default: "Cluster" - master_dns_name_format: - type: string - default: "{cluster}.{team}.{hostedzone}" - replica_dns_name_format: - type: string - default: "{cluster}-repl.{team}.{hostedzone}" aws_or_gcp: - type: object + description: AWSGCPConfiguration defines the configuration for AWS properties: additional_secret_mount: type: string additional_secret_mount_path: type: string - default: "/meta/credentials" aws_region: + default: eu-central-1 type: string - default: "eu-central-1" enable_ebs_gp3_migration: type: boolean - default: false enable_ebs_gp3_migration_max_size: + format: int64 type: integer - default: 1000 gcp_credentials: type: string kube_iam_role: type: string log_s3_bucket: type: string + wal_az_storage_account: + type: string wal_gs_bucket: type: string wal_s3_bucket: type: string - logical_backup: type: object + connection_pooler: + description: ConnectionPoolerConfiguration defines default configuration + for connection pooler properties: - logical_backup_docker_image: + connection_pooler_default_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ type: string - default: "registry.opensource.zalan.do/acid/logical-backup:v1.6.3" + connection_pooler_default_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + connection_pooler_default_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + connection_pooler_default_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + connection_pooler_image: + default: ghcr.io/zalando/postgres-operator/pgbouncer:latest + type: string + connection_pooler_max_db_connections: + format: int32 + type: integer + connection_pooler_mode: + default: transaction + enum: + - session + - transaction + type: string + connection_pooler_number_of_instances: + default: 2 + format: int32 + minimum: 1 + type: integer + connection_pooler_schema: + default: pooler + type: string + connection_pooler_user: + default: pooler + type: string + type: object + crd_categories: + items: + type: string + type: array + debug: + description: OperatorDebugConfiguration defines options for the debug + mode + properties: + debug_logging: + default: true + type: boolean + enable_database_access: + default: true + type: boolean + type: object + docker_image: + default: ghcr.io/zalando/spilo-18:4.1-p1 + type: string + enable_crd_registration: + default: true + type: boolean + enable_lazy_spilo_upgrade: + type: boolean + enable_maintenance_windows: + default: true + type: boolean + enable_pgversion_env_var: + default: true + type: boolean + enable_shm_volume: + default: true + type: boolean + enable_spilo_wal_path_compat: + type: boolean + enable_team_id_clustername_prefix: + type: boolean + etcd_host: + default: "" + type: string + ignore_instance_limits_annotation_key: + type: string + ignore_resources_limits_annotation_key: + type: string + kubernetes: + description: KubernetesMetaConfiguration defines k8s conf required + for all Postgres clusters and the operator itself + properties: + additional_pod_capabilities: + items: + type: string + type: array + cluster_domain: + default: cluster.local + type: string + cluster_labels: + additionalProperties: + type: string + default: + application: spilo + type: object + cluster_name_label: + default: cluster-name + type: string + custom_pod_annotations: + additionalProperties: + type: string + type: object + delete_annotation_date_key: + type: string + delete_annotation_name_key: + type: string + downscaler_annotations: + items: + type: string + type: array + enable_cross_namespace_secret: + type: boolean + enable_finalizers: + type: boolean + enable_init_containers: + default: true + type: boolean + enable_owner_references: + type: boolean + enable_persistent_volume_claim_deletion: + default: true + type: boolean + enable_pod_antiaffinity: + type: boolean + enable_pod_disruption_budget: + default: true + type: boolean + enable_readiness_probe: + type: boolean + enable_secrets_deletion: + default: true + type: boolean + enable_sidecars: + default: true + type: boolean + ignored_annotations: + items: + type: string + type: array + infrastructure_roles_secret_name: + description: |- + NamespacedName comprises a resource name, with a mandatory namespace, + rendered as "/". Being a type captures intent and + helps make sure that UIDs, namespaced names and non-namespaced names + do not get conflated in code. For most use cases, namespace and name + will already have been format validated at the API entry point, so we + don't do that here. Where that's not the case (e.g. in testing), + consider using NamespacedNameOrDie() in testing.go in this package. + + from: https://github.com/kubernetes/apimachinery/blob/master/pkg/types/namespacedname.go + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + infrastructure_roles_secrets: + description: namespaced name of the secret containing infrastructure + roles names and passwords + items: + properties: + defaultrolevalue: + type: string + defaultuservalue: + type: string + details: + description: This field point out the detailed yaml definition + of the role, if exists + type: string + passwordkey: + type: string + rolekey: + type: string + secretname: + description: |- + Name of a secret which describes the role, and optionally name of a + configmap with an extra information + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + template: + type: boolean + userkey: + type: string + type: object + type: array + inherited_annotations: + items: + type: string + type: array + inherited_labels: + items: + type: string + type: array + liveness_probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + master_pod_move_timeout: + default: 20m + description: timeout for successful migration of master pods from + unschedulable node + format: int64 + type: integer + node_readiness_label: + additionalProperties: + type: string + type: object + node_readiness_label_merge: + enum: + - AND + - OR + type: string + oauth_token_secret_name: + default: postgres-operator + description: namespaced name of the secret containing the OAuth2 + token to pass to the teams API + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + pdb_master_label_selector: + default: true + type: boolean + pdb_name_format: + default: postgres-{cluster}-pdb + description: defines the template for PDB names + type: string + persistent_volume_claim_retention_policy: + additionalProperties: + type: string + type: object + pod_antiaffinity_preferred_during_scheduling: + type: boolean + pod_antiaffinity_topology_key: + default: kubernetes.io/hostname + type: string + pod_environment_configmap: + description: namespaced name of the ConfigMap with environment + variables to populate on every pod + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + pod_environment_secret: + type: string + pod_management_policy: + default: ordered_ready + enum: + - ordered_ready + - parallel + type: string + pod_priority_class_name: + type: string + pod_role_label: + default: spilo-role + type: string + pod_service_account_definition: + type: string + pod_service_account_name: + default: postgres-pod + type: string + pod_service_account_role_binding_definition: + type: string + pod_terminate_grace_period: + default: 5m + description: Postgres pods are terminated forcefully after this + timeout + format: int64 + type: integer + secret_name_template: + default: '{username}.{cluster}.credentials.{tprkind}.{tprgroup}' + description: |- + template for database user secrets generated by the operator, + here username contains the namespace in the format namespace.username + if the user is in different namespace than cluster and cross namespace secrets + are enabled via `enable_cross_namespace_secret` flag in the configuration. + type: string + share_pgsocket_with_sidecars: + type: boolean + spilo_allow_privilege_escalation: + default: true + type: boolean + spilo_fsgroup: + format: int64 + type: integer + spilo_privileged: + type: boolean + spilo_runasgroup: + format: int64 + type: integer + spilo_runasuser: + format: int64 + type: integer + storage_resize_mode: + default: pvc + enum: + - ebs + - mixed + - pvc + - "off" + type: string + toleration: + additionalProperties: + type: string + type: object + watched_namespace: + type: string + type: object + kubernetes_use_configmaps: + default: true + type: boolean + load_balancer: + description: LoadBalancerConfiguration defines the LB configuration + properties: + custom_service_annotations: + additionalProperties: + type: string + type: object + db_hosted_zone: + type: string + enable_master_load_balancer: + type: boolean + enable_master_node_port: + type: boolean + enable_master_pooler_load_balancer: + type: boolean + enable_master_pooler_node_port: + type: boolean + enable_replica_load_balancer: + type: boolean + enable_replica_node_port: + type: boolean + enable_replica_pooler_load_balancer: + type: boolean + enable_replica_pooler_node_port: + type: boolean + external_traffic_policy: + default: Cluster + enum: + - Cluster + - Local + type: string + master_dns_name_format: + default: '{cluster}.{namespace}.{hostedzone}' + description: defines the DNS name string template for the master + load balancer cluster + type: string + master_legacy_dns_name_format: + default: '{cluster}.{team}.{hostedzone}' + description: deprecated DNS template for master load balancer + using team name + type: string + replica_dns_name_format: + default: '{cluster}-repl.{namespace}.{hostedzone}' + description: defines the DNS name string template for the replica + load balancer cluster + type: string + replica_legacy_dns_name_format: + default: '{cluster}-repl.{team}.{hostedzone}' + description: deprecated DNS template for replica load balancer + using team name + type: string + type: object + logging_rest_api: + description: LoggingRESTAPIConfiguration defines Logging API conf + properties: + api_port: + default: 8080 + type: integer + cluster_history_entries: + default: 1000 + type: integer + ring_log_lines: + default: 100 + type: integer + type: object + logical_backup: + description: OperatorLogicalBackupConfiguration defines configuration + for logical backup + properties: + logical_backup_azure_storage_account_key: + type: string + logical_backup_azure_storage_account_name: + type: string + logical_backup_azure_storage_container: + type: string + logical_backup_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + logical_backup_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + logical_backup_cronjob_environment_secret: + type: string + logical_backup_docker_image: + default: ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1 + type: string + logical_backup_failed_jobs_history_limit: + default: 3 + format: int32 + minimum: 0 + type: integer logical_backup_google_application_credentials: type: string logical_backup_job_prefix: + default: logical-backup- + type: string + logical_backup_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + logical_backup_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - default: "logical-backup-" logical_backup_provider: + default: s3 + enum: + - az + - gcs + - s3 type: string - default: "s3" logical_backup_s3_access_key_id: type: string logical_backup_s3_bucket: type: string + logical_backup_s3_bucket_prefix: + type: string logical_backup_s3_endpoint: type: string logical_backup_s3_region: type: string + logical_backup_s3_retention_time: + type: string logical_backup_s3_secret_access_key: type: string logical_backup_s3_sse: type: string logical_backup_schedule: + default: 30 00 * * * + pattern: ^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$ type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - default: "30 00 * * *" - debug: + logical_backup_successful_jobs_history_limit: + default: 3 + format: int32 + minimum: 0 + type: integer + logical_backup_ttl_seconds_after_finished: + default: 86400 + format: int32 + minimum: 0 + type: integer type: object + maintenance_windows: + type: array + major_version_upgrade: + description: MajorVersionUpgradeConfiguration defines how to execute + major version upgrades of Postgres. properties: - debug_logging: - type: boolean - default: true - enable_database_access: - type: boolean - default: true - teams_api: - type: object - properties: - enable_admin_role_for_users: - type: boolean - default: true - enable_postgres_team_crd: - type: boolean - default: true - enable_postgres_team_crd_superusers: - type: boolean - default: false - enable_team_member_deprecation: - type: boolean - default: false - enable_team_superuser: - type: boolean - default: false - enable_teams_api: - type: boolean - default: true - pam_configuration: + major_version_upgrade_mode: + default: manual + enum: + - "off" + - manual + - full type: string - default: "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees" - pam_role_name: - type: string - default: "zalandos" - postgres_superuser_teams: - type: array + major_version_upgrade_team_allow_list: items: type: string - protected_role_names: type: array - items: - type: string - default: - - admin - role_deletion_suffix: + minimal_major_version: + default: "14" type: string - default: "_deleted" - team_admin_role: + target_major_version: + default: "18" type: string - default: "admin" - team_api_role_configuration: - type: object - additionalProperties: - type: string - default: - log_statement: all - teams_api_url: - type: string - default: "https://teams.example.com/api/" - logging_rest_api: type: object + max_instances: + default: -1 + description: -1 = disabled + format: int32 + minimum: -1 + type: integer + min_instances: + default: -1 + description: -1 = disabled + format: int32 + minimum: -1 + type: integer + patroni: + description: PatroniConfiguration defines configuration for Patroni properties: - api_port: - type: integer - default: 8080 - cluster_history_entries: - type: integer - default: 1000 - ring_log_lines: - type: integer - default: 100 - scalyr: # deprecated + enable_patroni_failsafe_mode: + type: boolean type: object + postgres_pod_resources: + description: PostgresPodResourcesDefaults defines the spec of default + resources + properties: + default_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + default_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + default_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + default_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + max_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + max_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + min_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + min_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + repair_period: + default: 5m + description: period between consecutive repair requests + format: int64 + type: integer + resync_period: + default: 30m + description: period between consecutive sync requests + format: int64 + type: integer + scalyr: + description: ScalyrConfiguration defines the configuration for ScalyrAPI properties: scalyr_api_key: type: string scalyr_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' default: "1" - scalyr_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + scalyr_cpu_request: + default: 100m + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "100m" scalyr_image: type: string scalyr_memory_limit: + default: 500Mi + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "500Mi" scalyr_memory_request: + default: 50Mi + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "50Mi" scalyr_server_url: + default: https://upload.eu.scalyr.com type: string - default: "https://upload.eu.scalyr.com" - connection_pooler: type: object + set_memory_request_to_limit: + type: boolean + sidecar_docker_images: + additionalProperties: + type: string + type: object + sidecars: + type: object + x-kubernetes-preserve-unknown-fields: true + teams_api: + description: TeamsAPIConfiguration defines the configuration of TeamsAPI properties: - connection_pooler_schema: + enable_admin_role_for_users: + default: true + type: boolean + enable_postgres_team_crd: + default: true + type: boolean + enable_postgres_team_crd_superusers: + type: boolean + enable_team_member_deprecation: + type: boolean + enable_team_superuser: + type: boolean + enable_teams_api: + type: boolean + pam_configuration: + default: https://info.example.com/oauth2/tokeninfo?access_token= + uid realm=/employees type: string - default: "pooler" - connection_pooler_user: + pam_role_name: + default: zalandos type: string - default: "pooler" - connection_pooler_image: + postgres_superuser_teams: + items: + type: string + type: array + protected_role_names: + default: '["admin", "cron_admin"]' + items: + type: string + type: array + role_deletion_suffix: + default: _deleted type: string - default: "registry.opensource.zalan.do/acid/pgbouncer:master-16" - connection_pooler_max_db_connections: + team_admin_role: + default: admin + type: string + team_api_role_configuration: + additionalProperties: + type: string + default: + log_statement: all + type: object + teams_api_url: + default: https://teams.example.com/api/ + type: string + type: object + timeouts: + description: OperatorTimeouts defines the timeout of ResourceCheck, + PodWait, ReadyWait + properties: + patroni_api_check_interval: + default: 1s + description: interval between consecutive attempts of operator + calling the Patroni API + format: int64 type: integer - default: 60 - connection_pooler_mode: - type: string - enum: - - "session" - - "transaction" - default: "transaction" - connection_pooler_number_of_instances: + patroni_api_check_timeout: + default: 5s + description: timeout when waiting for successful response from + Patroni API + format: int64 type: integer - minimum: 1 - default: 2 - connection_pooler_default_cpu_limit: + pod_deletion_wait_timeout: + default: 10m + description: timeout when waiting for the Postgres pods to be + deleted + format: int64 + type: integer + pod_label_wait_timeout: + default: 10m + description: timeout when waiting for pod role and cluster labels + format: int64 + type: integer + ready_wait_interval: + default: 4s + description: interval between consecutive attempts waiting for + postgresql CRD to be created + format: int64 + type: integer + ready_wait_timeout: + default: 30s + description: timeout for the complete postgres CRD creation + format: int64 + type: integer + resource_check_interval: + default: 3s + description: interval to wait between consecutive attempts to + check for some K8s resources + format: int64 + type: integer + resource_check_timeout: + default: 10m + description: timeout when waiting for the presence of a certain + K8s resource + format: int64 + type: integer + type: object + users: + description: PostgresUsersConfiguration defines the system users of + Postgres. + properties: + additional_owner_roles: + items: + type: string + type: array + enable_password_rotation: + type: boolean + password_rotation_interval: + default: 90 + format: int32 + type: integer + password_rotation_user_retention: + default: 120 + format: int32 + type: integer + replication_username: + default: standby type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "1" - connection_pooler_default_cpu_request: + super_username: + default: postgres type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "500m" - connection_pooler_default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "100Mi" - connection_pooler_default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "100Mi" - status: + type: object + workers: + default: 8 + format: int32 + minimum: 1 + type: integer type: object - additionalProperties: - type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + required: + - configuration + - metadata + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index aead7fe69..2bbf5ee49 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -4,525 +4,4419 @@ metadata: name: postgresqls.acid.zalan.do labels: app.kubernetes.io/name: postgres-operator - annotations: - "helm.sh/hook": crd-install spec: group: acid.zalan.do names: + categories: + - all kind: postgresql listKind: postgresqlList plural: postgresqls - singular: postgresql shortNames: - pg - categories: - - all + singular: postgresql scope: Namespaced versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - name: Team - type: string - description: Team responsible for Postgres cluster + - additionalPrinterColumns: + - description: Team responsible for Postgres cluster jsonPath: .spec.teamId - - name: Version + name: Team type: string - description: PostgreSQL version + - description: PostgreSQL version jsonPath: .spec.postgresql.version - - name: Pods - type: integer - description: Number of Pods per Postgres cluster + name: Version + type: string + - description: Number of Pods per Postgres cluster jsonPath: .spec.numberOfInstances - - name: Volume - type: string - description: Size of the bound volume + name: Pods + type: integer + - description: Size of the bound volume jsonPath: .spec.volume.size - - name: CPU-Request + name: Volume type: string - description: Requested CPU for Postgres containers + - description: Requested CPU for Postgres containers jsonPath: .spec.resources.requests.cpu - - name: Memory-Request + name: CPU-Request type: string - description: Requested memory for Postgres containers + - description: Requested memory for Postgres containers jsonPath: .spec.resources.requests.memory - - name: Age - type: date - jsonPath: .metadata.creationTimestamp - - name: Status + name: Memory-Request type: string - description: Current sync status of postgresql resource + - description: Age of the PostgreSQL cluster + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Current sync status of postgresql resource jsonPath: .status.PostgresClusterStatus + name: Status + type: string + name: v1 schema: openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - spec + description: Postgresql defines PostgreSQL Custom Resource Definition Object. properties: - kind: - type: string - enum: - - postgresql apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string - enum: - - acid.zalan.do/v1 - spec: + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: type: object - required: - - numberOfInstances - - teamId - - postgresql - - volume + spec: + description: PostgresSpec defines the specification for the PostgreSQL + TPR. properties: additionalVolumes: - type: array items: - type: object - required: - - name - - mountPath - - volumeSource + description: AdditionalVolume specs additional optional volumes + for statefulset properties: - name: - type: string + isSubPathExpr: + type: boolean mountPath: type: string + name: + type: string + subPath: + type: string targetContainers: - type: array - nullable: true items: type: string + nullable: true + type: array volumeSource: type: object x-kubernetes-preserve-unknown-fields: true - subPath: - type: string - allowedSourceRanges: + required: + - mountPath + - name + - volumeSource + type: object type: array - nullable: true + allowedSourceRanges: + description: load balancers' source ranges are the same for master + and replica services items: + pattern: ^((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:))\/(12[0-8]|1[01][0-9]|[1-9]?[0-9]))$ type: string - pattern: '^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])$' + nullable: true + type: array clone: - type: object - required: - - cluster + description: CloneDescription describes which cluster the new should + clone and up to which point in time properties: cluster: type: string - s3_endpoint: - type: string s3_access_key_id: type: string - s3_secret_access_key: + s3_endpoint: type: string s3_force_path_style: type: boolean + s3_secret_access_key: + type: string s3_wal_path: type: string timestamp: + description: |- + The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + Example: 1996-12-19T16:39:57-08:00 + Note: this field requires a timezone + pattern: ^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$ type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' - # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC - # Example: 1996-12-19T16:39:57-08:00 - # Note: this field requires a timezone uid: format: uuid type: string - connectionPooler: + required: + - cluster type: object + connectionPooler: + description: |- + ConnectionPooler Options for connection pooler + + pgbouncer-large (with higher resources) or odyssey-small (with smaller + resources) + Type string `json:"type,omitempty"` + + makes sense to expose. E.g. pool size (min/max boundaries), max client + connections etc. properties: dockerImage: type: string maxDBConnections: + format: int32 type: integer mode: - type: string enum: - - "session" - - "transaction" + - session + - transaction + type: string numberOfInstances: + format: int32 + minimum: 1 type: integer - minimum: 2 resources: - type: object - required: - - requests - - limits + description: Resources describes requests and limits for the cluster + resouces. properties: limits: - type: object - required: - - cpu - - memory + description: ResourceDescription describes CPU and memory + resources defined for a cluster. properties: cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + type: object requests: - type: object - required: - - cpu - - memory + description: ResourceDescription describes CPU and memory + resources defined for a cluster. properties: cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + type: object + type: object schema: type: string user: type: string - databases: type: object + databases: additionalProperties: type: string - # Note: usernames specified here as database owners must be declared in the users key of the spec key. + description: |- + Note: usernames specified here as database owners must be declared + in the users key of the spec key. + type: object dockerImage: type: string enableConnectionPooler: type: boolean - enableReplicaConnectionPooler: - type: boolean enableLogicalBackup: type: boolean enableMasterLoadBalancer: + description: |- + vars that enable load balancers are pointers because it is important to know if any of them is omitted from the Postgres manifest + in that case the var evaluates to nil and the value is taken from the operator config + type: boolean + enableMasterNodePort: + description: |- + vars to enable and configure nodeport services + set ports to 0 or nil to let kubernetes decide which port to use + overrides loadbalancer configuration + type: boolean + enableMasterPoolerLoadBalancer: + type: boolean + enableMasterPoolerNodePort: + type: boolean + enableReplicaConnectionPooler: type: boolean enableReplicaLoadBalancer: type: boolean + enableReplicaNodePort: + type: boolean + enableReplicaPoolerLoadBalancer: + type: boolean + enableReplicaPoolerNodePort: + type: boolean enableShmVolume: type: boolean - init_containers: # deprecated - type: array - nullable: true + env: items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name type: object - x-kubernetes-preserve-unknown-fields: true + type: array + init_containers: + description: deprecated + items: + description: A single application container that you want to run + within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be + a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a + single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize + policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array initContainers: - type: array - nullable: true items: + description: A single application container that you want to run + within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be + a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a + single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize + policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name type: object - x-kubernetes-preserve-unknown-fields: true - logicalBackupSchedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - maintenanceWindows: type: array - items: - type: string - pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' - numberOfInstances: - type: integer - minimum: 0 - patroni: - type: object + livenessProbe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. properties: - initdb: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes a custom header to be + used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + logicalBackupRetention: + type: string + logicalBackupSchedule: + pattern: ^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$ + type: string + maintenanceWindows: + items: + pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' + type: string + type: array + masterNodePort: + format: int32 + type: integer + masterPoolerNodePort: + format: int32 + type: integer + masterServiceAnnotations: + additionalProperties: + type: string + description: MasterServiceAnnotations takes precedence over ServiceAnnotations + for master role if not empty + type: object + nodeAffinity: + description: Node affinity is a group of node affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + numberOfInstances: + format: int32 + minimum: 0 + type: integer + patroni: + description: Patroni contains Patroni-specific configuration + properties: + failsafe_mode: + type: boolean + initdb: additionalProperties: type: string + type: object loop_wait: + format: int32 type: integer maximum_lag_on_failover: + format: int64 type: integer pg_hba: - type: array items: type: string + type: array retry_timeout: + format: int32 type: integer slots: - type: object additionalProperties: - type: object additionalProperties: type: string + type: object + type: object synchronous_mode: type: boolean synchronous_mode_strict: type: boolean - ttl: + synchronous_node_count: + format: int32 + type: integer + ttl: + format: int32 type: integer - podAnnotations: type: object + pod_priority_class_name: + description: deprecated + type: string + podAnnotations: additionalProperties: type: string - pod_priority_class_name: # deprecated - type: string + type: object podPriorityClassName: type: string postgresql: - type: object - required: - - version + description: PostgresqlParam describes PostgreSQL version and pairs + of configuration parameter name - values. properties: - version: - type: string - enum: - - "9.3" - - "9.4" - - "9.5" - - "9.6" - - "10" - - "11" - - "12" - - "13" parameters: - type: object additionalProperties: type: string - preparedDatabases: + type: object + version: + enum: + - "14" + - "15" + - "16" + - "17" + - "18" + type: string + required: + - version type: object + preparedDatabases: additionalProperties: - type: object + description: PreparedDatabase describes elements to be bootstrapped properties: defaultUsers: type: boolean extensions: - type: object additionalProperties: type: string - schemas: type: object + schemas: additionalProperties: - type: object + description: PreparedSchema describes elements to be bootstrapped + per schema properties: - defaultUsers: - type: boolean defaultRoles: type: boolean - replicaLoadBalancer: # deprecated - type: boolean - resources: + defaultUsers: + type: boolean + type: object + type: object + secretNamespace: + type: string + type: object type: object - required: - - requests - - limits + replicaLoadBalancer: + description: deprecated + type: boolean + replicaNodePort: + format: int32 + type: integer + replicaPoolerNodePort: + format: int32 + type: integer + replicaServiceAnnotations: + additionalProperties: + type: string + description: ReplicaServiceAnnotations takes precedence over ServiceAnnotations + for replica role if not empty + type: object + resources: + description: Resources describes requests and limits for the cluster + resouces. properties: limits: - type: object - required: - - cpu - - memory + description: ResourceDescription describes CPU and memory resources + defined for a cluster. properties: cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. + type: object requests: - type: object - required: - - cpu - - memory + description: ResourceDescription describes CPU and memory resources + defined for a cluster. properties: cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + type: object + type: object schedulerName: type: string serviceAnnotations: - type: object additionalProperties: type: string + type: object sidecars: - type: array - nullable: true items: + description: Sidecar defines a container to be run in the same pod + as the Postgres container. + properties: + command: + items: + type: string + type: array + env: + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be + a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + type: string + name: + type: string + ports: + items: + description: ContainerPort represents a network port in a + single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + resources: + description: Resources describes requests and limits for the + cluster resouces. + properties: + limits: + description: ResourceDescription describes CPU and memory + resources defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + requests: + description: ResourceDescription describes CPU and memory + resources defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + type: object type: object - x-kubernetes-preserve-unknown-fields: true - spiloRunAsUser: + type: array + spiloFSGroup: + format: int64 type: integer spiloRunAsGroup: + format: int64 type: integer - spiloFSGroup: + spiloRunAsUser: + format: int64 type: integer standby: - type: object - required: + anyOf: + - required: - s3_wal_path + - required: + - gs_wal_path + - required: + - standby_host + not: + required: + - s3_wal_path + - gs_wal_path + description: |- + StandbyDescription contains remote primary config and/or s3/gs wal path. + standby_host can be specified alone or together with either s3_wal_path OR gs_wal_path (mutually exclusive). + At least one field must be specified. s3_wal_path and gs_wal_path are mutually exclusive. properties: + gs_wal_path: + type: string s3_wal_path: type: string + standby_host: + type: string + standby_port: + type: string + standby_primary_slot_name: + type: string + type: object + streams: + items: + description: Stream defines properties for creating FabricEventStream + resources + properties: + applicationId: + type: string + batchSize: + format: int32 + type: integer + cpu: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + database: + type: string + enableRecovery: + type: boolean + filter: + additionalProperties: + type: string + type: object + memory: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + tables: + additionalProperties: + description: StreamTable defines properties of outbox tables + for FabricEventStreams + properties: + eventType: + type: string + idColumn: + type: string + ignoreRecovery: + type: boolean + payloadColumn: + type: string + recoveryEventType: + type: string + required: + - eventType + type: object + type: object + required: + - applicationId + - database + - tables + type: object + type: array teamId: type: string tls: - type: object - required: - - secretName + description: TLSDescription specs TLS properties properties: - secretName: + caFile: + type: string + caSecretName: type: string certificateFile: type: string privateKeyFile: type: string - caFile: + secretName: type: string - caSecretName: - type: string - nodeAffinity: + required: + - secretName type: object - properties: - preferredDuringSchedulingIgnoredDuringExecution: - type: array - items: - type: object - required: - - weight - - preference - properties: - preference: - type: object - properties: - matchExpressions: - type: array - items: - type: object - required: - - key - - operator - properties: - key: - type: string - operator: - type: string - values: - type: array - items: - type: string - matchFields: - type: array - items: - type: object - required: - - key - - operator - properties: - key: - type: string - operator: - type: string - values: - type: array - items: - type: string - weight: - format: int32 - type: integer - requiredDuringSchedulingIgnoredDuringExecution: - type: object - required: - - nodeSelectorTerms - properties: - nodeSelectorTerms: - type: array - items: - type: object - properties: - matchExpressions: - type: array - items: - type: object - required: - - key - - operator - properties: - key: - type: string - operator: - type: string - values: - type: array - items: - type: string - matchFields: - type: array - items: - type: object - required: - - key - - operator - properties: - key: - type: string - operator: - type: string - values: - type: array - items: - type: string tolerations: - type: array items: - type: object - required: - - key - - operator - - effect + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. type: string - enum: - - Equal - - Exists - value: - type: string - effect: - type: string - enum: - - NoExecute - - NoSchedule - - PreferNoSchedule tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 type: integer - useLoadBalancer: # deprecated + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + useLoadBalancer: + description: |- + deprecated load balancer settings maintained for backward compatibility + see "Load balancers" operator docs type: boolean users: - type: object additionalProperties: - type: array - nullable: true - description: "Role flags specified here must not contradict each other" + description: UserFlags defines flags (such as superuser, nologin) + that could be assigned to individual users items: - type: string enum: - bypassrls - BYPASSRLS @@ -552,24 +4446,116 @@ spec: - SUPERUSER - nosuperuser - NOSUPERUSER - volume: + type: string + type: array type: object - required: - - size + usersIgnoringSecretRotation: + items: + type: string + nullable: true + type: array + usersWithInPlaceSecretRotation: + items: + type: string + nullable: true + type: array + usersWithSecretRotation: + items: + type: string + nullable: true + type: array + volume: + description: Volume describes a single volume in the manifest. properties: iops: + format: int64 type: integer + isSubPathExpr: + type: boolean + selector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic size: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero. storageClass: type: string subPath: type: string throughput: + format: int64 type: integer - status: + type: + type: string + required: + - size + type: object + required: + - numberOfInstances + - postgresql + - teamId + - volume type: object - additionalProperties: - type: string + status: + description: PostgresStatus contains status of the PostgreSQL cluster + (running, creation failed etc.) + properties: + PostgresClusterStatus: + type: string + required: + - PostgresClusterStatus + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/postgres-operator/crds/postgresteams.yaml b/charts/postgres-operator/crds/postgresteams.yaml index fbf873b84..b7a36848d 100644 --- a/charts/postgres-operator/crds/postgresteams.yaml +++ b/charts/postgres-operator/crds/postgresteams.yaml @@ -4,8 +4,6 @@ metadata: name: postgresteams.acid.zalan.do labels: app.kubernetes.io/name: postgres-operator - annotations: - "helm.sh/hook": crd-install spec: group: acid.zalan.do names: diff --git a/charts/postgres-operator/index.yaml b/charts/postgres-operator/index.yaml index bbd762104..7128b8eb9 100644 --- a/charts/postgres-operator/index.yaml +++ b/charts/postgres-operator/index.yaml @@ -1,11 +1,12 @@ apiVersion: v1 entries: postgres-operator: - - apiVersion: v1 - appVersion: 1.6.3 - created: "2021-05-27T19:04:25.199523943+02:00" - description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes - digest: ea08f991bf23c9ad114bca98ebcbe3e2fa15beab163061399394905eaee89b35 + - apiVersion: v2 + appVersion: 1.15.1 + created: "2025-12-17T14:48:33.832345061+01:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: 9f3edc3d796105c02c04eaae28a78e58fb08c1847a9de012245fd6ac2c0d2c00 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -20,13 +21,14 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-1.6.3.tgz - version: 1.6.3 - - apiVersion: v1 - appVersion: 1.6.2 - created: "2021-05-27T19:04:25.198182197+02:00" - description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes - digest: d886f8a0879ca07d1e5246ee7bc55710e1c872f3977280fe495db6fc2057a7f4 + - postgres-operator-1.15.1.tgz + version: 1.15.1 + - apiVersion: v2 + appVersion: 1.15.0 + created: "2025-12-17T14:48:33.826117296+01:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: 002dd47647bf51fbba023bd1762d807be478cf37de7a44b80cd01ac1f20bd94a home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -41,13 +43,14 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-1.6.2.tgz - version: 1.6.2 - - apiVersion: v1 - appVersion: 1.6.1 - created: "2021-05-27T19:04:25.19687586+02:00" - description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes - digest: 4ba5972cd486dcaa2d11c5613a6f97f6b7b831822e610fe9e10a57ea1db23556 + - postgres-operator-1.15.0.tgz + version: 1.15.0 + - apiVersion: v2 + appVersion: 1.14.0 + created: "2025-12-17T14:48:33.819729144+01:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: 36e1571f3f455b213f16cdda7b1158648e8e84deb804ba47ed6b9b6d19263ba8 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -62,13 +65,14 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-1.6.1.tgz - version: 1.6.1 - - apiVersion: v1 - appVersion: 1.6.0 - created: "2021-05-27T19:04:25.195600766+02:00" - description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes - digest: f52149718ea364f46b4b9eec9a65f6253ad182bb78df541d14cd5277b9c8a8c3 + - postgres-operator-1.14.0.tgz + version: 1.14.0 + - apiVersion: v2 + appVersion: 1.13.0 + created: "2025-12-17T14:48:33.81038602+01:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: a839601689aea0a7e6bc0712a5244d435683cf3314c95794097ff08540e1dfef home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -83,13 +87,14 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-1.6.0.tgz - version: 1.6.0 - - apiVersion: v1 - appVersion: 1.5.0 - created: "2021-05-27T19:04:25.193985032+02:00" - description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes - digest: 198351d5db52e65cdf383d6f3e1745d91ac1e2a01121f8476f8b1be728b09531 + - postgres-operator-1.13.0.tgz + version: 1.13.0 + - apiVersion: v2 + appVersion: 1.12.2 + created: "2025-12-17T14:48:33.803256825+01:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: 65858d14a40d7fd90c32bd9fc60021acc9555c161079f43a365c70171eaf21d8 home: https://github.com/zalando/postgres-operator keywords: - postgres @@ -104,6 +109,50 @@ entries: sources: - https://github.com/zalando/postgres-operator urls: - - postgres-operator-1.5.0.tgz - version: 1.5.0 -generated: "2021-05-27T19:04:25.191897769+02:00" + - postgres-operator-1.12.2.tgz + version: 1.12.2 + - apiVersion: v2 + appVersion: 1.11.0 + created: "2025-12-17T14:48:33.797369053+01:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: 3914b5e117bda0834f05c9207f007e2ac372864cf6e86dcc2e1362bbe46c14d9 + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-1.11.0.tgz + version: 1.11.0 + - apiVersion: v2 + appVersion: 1.10.1 + created: "2025-12-17T14:48:33.791368349+01:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: cc3baa41753da92466223d0b334df27e79c882296577b404a8e9071411fcf19c + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-1.10.1.tgz + version: 1.10.1 +generated: "2025-12-17T14:48:33.785159183+01:00" diff --git a/charts/postgres-operator/postgres-operator-1.10.1.tgz b/charts/postgres-operator/postgres-operator-1.10.1.tgz new file mode 100644 index 000000000..5ecb27a5b Binary files /dev/null and b/charts/postgres-operator/postgres-operator-1.10.1.tgz differ diff --git a/charts/postgres-operator/postgres-operator-1.11.0.tgz b/charts/postgres-operator/postgres-operator-1.11.0.tgz new file mode 100644 index 000000000..61c2eadb0 Binary files /dev/null and b/charts/postgres-operator/postgres-operator-1.11.0.tgz differ diff --git a/charts/postgres-operator/postgres-operator-1.12.2.tgz b/charts/postgres-operator/postgres-operator-1.12.2.tgz new file mode 100644 index 000000000..a74c25c47 Binary files /dev/null and b/charts/postgres-operator/postgres-operator-1.12.2.tgz differ diff --git a/charts/postgres-operator/postgres-operator-1.13.0.tgz b/charts/postgres-operator/postgres-operator-1.13.0.tgz new file mode 100644 index 000000000..3d7ca4ce6 Binary files /dev/null and b/charts/postgres-operator/postgres-operator-1.13.0.tgz differ diff --git a/charts/postgres-operator/postgres-operator-1.14.0.tgz b/charts/postgres-operator/postgres-operator-1.14.0.tgz new file mode 100644 index 000000000..df95fd01d Binary files /dev/null and b/charts/postgres-operator/postgres-operator-1.14.0.tgz differ diff --git a/charts/postgres-operator/postgres-operator-1.15.0.tgz b/charts/postgres-operator/postgres-operator-1.15.0.tgz new file mode 100644 index 000000000..e029732ae Binary files /dev/null and b/charts/postgres-operator/postgres-operator-1.15.0.tgz differ diff --git a/charts/postgres-operator/postgres-operator-1.15.1.tgz b/charts/postgres-operator/postgres-operator-1.15.1.tgz new file mode 100644 index 000000000..7dce6f87d Binary files /dev/null and b/charts/postgres-operator/postgres-operator-1.15.1.tgz differ diff --git a/charts/postgres-operator/postgres-operator-1.5.0.tgz b/charts/postgres-operator/postgres-operator-1.5.0.tgz deleted file mode 100644 index 6e1a48ab7..000000000 Binary files a/charts/postgres-operator/postgres-operator-1.5.0.tgz and /dev/null differ diff --git a/charts/postgres-operator/postgres-operator-1.6.0.tgz b/charts/postgres-operator/postgres-operator-1.6.0.tgz deleted file mode 100644 index d52c405eb..000000000 Binary files a/charts/postgres-operator/postgres-operator-1.6.0.tgz and /dev/null differ diff --git a/charts/postgres-operator/postgres-operator-1.6.1.tgz b/charts/postgres-operator/postgres-operator-1.6.1.tgz deleted file mode 100644 index 48ffb9014..000000000 Binary files a/charts/postgres-operator/postgres-operator-1.6.1.tgz and /dev/null differ diff --git a/charts/postgres-operator/postgres-operator-1.6.2.tgz b/charts/postgres-operator/postgres-operator-1.6.2.tgz deleted file mode 100644 index 4daf847e1..000000000 Binary files a/charts/postgres-operator/postgres-operator-1.6.2.tgz and /dev/null differ diff --git a/charts/postgres-operator/postgres-operator-1.6.3.tgz b/charts/postgres-operator/postgres-operator-1.6.3.tgz deleted file mode 100644 index af84bf57b..000000000 Binary files a/charts/postgres-operator/postgres-operator-1.6.3.tgz and /dev/null differ diff --git a/charts/postgres-operator/templates/_helpers.tpl b/charts/postgres-operator/templates/_helpers.tpl index c958a99a6..cb8c69c2b 100644 --- a/charts/postgres-operator/templates/_helpers.tpl +++ b/charts/postgres-operator/templates/_helpers.tpl @@ -38,6 +38,13 @@ Create a pod service account name. {{ default (printf "%s-%v" (include "postgres-operator.fullname" .) "pod") .Values.podServiceAccount.name }} {{- end -}} +{{/* +Create a pod priority class name. +*/}} +{{- define "postgres-pod.priorityClassName" -}} +{{ default (printf "%s-%v" (include "postgres-operator.fullname" .) "pod") .Values.podPriorityClassName.name }} +{{- end -}} + {{/* Create a controller ID. */}} @@ -57,18 +64,16 @@ Flatten nested config options when ConfigMap is used as ConfigTarget */}} {{- define "flattenValuesForConfigMap" }} {{- range $key, $value := . }} - {{- if or (kindIs "string" $value) (kindIs "int" $value) }} -{{ $key }}: {{ $value | quote }} - {{- end }} {{- if kindIs "slice" $value }} {{ $key }}: {{ join "," $value | quote }} - {{- end }} - {{- if kindIs "map" $value }} + {{- else if kindIs "map" $value }} {{- $list := list }} {{- range $subKey, $subValue := $value }} {{- $list = append $list (printf "%s:%s" $subKey $subValue) }} -{{ $key }}: {{ join "," $list | quote }} {{- end }} +{{ $key }}: {{ join "," $list | quote }} + {{- else }} +{{ $key }}: {{ $value | quote }} {{- end }} {{- end }} {{- end }} diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml index 33c43822f..fdccf16d3 100644 --- a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml +++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} rules: -# Patroni needs to watch and manage endpoints +# Patroni needs to watch and manage config maps or endpoints {{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }} - apiGroups: - "" @@ -24,12 +24,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get {{- else }} - apiGroups: - "" diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 885bad3f7..ad3b46064 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -34,16 +34,34 @@ rules: - get - list - watch +# all verbs allowed for event streams +{{- if .Values.enableStreams }} +- apiGroups: + - zalando.org + resources: + - fabriceventstreams + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +{{- end }} # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - - create - get +{{- if toString .Values.configGeneral.enable_crd_registration | eq "true" }} + - create - patch - update +{{- end }} # to send events to the CRs - apiGroups: - "" @@ -71,12 +89,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get {{- else }} # to read configuration from ConfigMaps - apiGroups: @@ -108,6 +120,7 @@ rules: - create - delete - get + - patch - update # to check nodes for node readiness label - apiGroups: @@ -127,8 +140,8 @@ rules: - delete - get - list -{{- if toString .Values.configKubernetes.storage_resize_mode | eq "pvc" }} - patch +{{- if or (toString .Values.configKubernetes.storage_resize_mode | eq "pvc") (toString .Values.configKubernetes.storage_resize_mode | eq "mixed") }} - update {{- end }} # to read existing PVs. Creation should be done via dynamic provisioning @@ -184,6 +197,7 @@ rules: - get - list - patch + - update # to CRUD cron jobs for logical backups - apiGroups: - batch diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 836babe1d..9ea574172 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -3,15 +3,16 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ template "postgres-operator.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ template "postgres-operator.name" . }} helm.sh/chart: {{ template "postgres-operator.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} data: - {{- if .Values.podPriorityClassName }} - pod_priority_class_name: {{ .Values.podPriorityClassName }} - {{- end }} +{{- if or .Values.podPriorityClassName.create .Values.podPriorityClassName.name }} + pod_priority_class_name: {{ include "postgres-pod.priorityClassName" . }} +{{- end }} pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} {{- include "flattenValuesForConfigMap" .Values.configGeneral | indent 2 }} {{- include "flattenValuesForConfigMap" .Values.configUsers | indent 2 }} @@ -25,4 +26,5 @@ data: {{- include "flattenValuesForConfigMap" .Values.configLoggingRestApi | indent 2 }} {{- include "flattenValuesForConfigMap" .Values.configTeamsApi | indent 2 }} {{- include "flattenValuesForConfigMap" .Values.configConnectionPooler | indent 2 }} +{{- include "flattenValuesForConfigMap" .Values.configPatroni | indent 2 }} {{- end }} diff --git a/charts/postgres-operator/templates/crds.yaml b/charts/postgres-operator/templates/crds.yaml deleted file mode 100644 index 733830014..000000000 --- a/charts/postgres-operator/templates/crds.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{{ if .Values.crd.create }} -{{- range $path, $bytes := .Files.Glob "crds/*.yaml" }} -{{ $.Files.Get $path }} ---- -{{- end }} -{{- end }} diff --git a/charts/postgres-operator/templates/deployment.yaml b/charts/postgres-operator/templates/deployment.yaml index 89500ae94..c2eb0ba23 100644 --- a/charts/postgres-operator/templates/deployment.yaml +++ b/charts/postgres-operator/templates/deployment.yaml @@ -7,6 +7,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} name: {{ template "postgres-operator.fullname" . }} + namespace: {{ .Release.Namespace }} spec: replicas: 1 selector: @@ -36,6 +37,10 @@ spec: - name: {{ .Chart.Name }} image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.extraArgs }} + args: +{{ toYaml .Values.extraArgs | indent 8 }} + {{- end }} env: {{- if .Values.enableJsonLogging }} - name: ENABLE_JSON_LOGGING @@ -51,11 +56,22 @@ spec: {{- if .Values.controllerID.create }} - name: CONTROLLER_ID value: {{ template "postgres-operator.controllerID" . }} + {{- end }} + {{- if .Values.extraEnvs }} +{{ toYaml .Values.extraEnvs | indent 8 }} {{- end }} resources: {{ toYaml .Values.resources | indent 10 }} securityContext: {{ toYaml .Values.securityContext | indent 10 }} + {{- if .Values.readinessProbe }} + readinessProbe: + httpGet: + path: /readyz + port: {{ .Values.configLoggingRestApi.api_port }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | indent 8 }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index be1608297..b72bfb899 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -3,40 +3,43 @@ apiVersion: "acid.zalan.do/v1" kind: OperatorConfiguration metadata: name: {{ template "postgres-operator.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ template "postgres-operator.name" . }} helm.sh/chart: {{ template "postgres-operator.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} configuration: -{{ toYaml .Values.configGeneral | indent 2 }} +{{ tpl (toYaml .Values.configGeneral) . | indent 2 }} users: -{{ toYaml .Values.configUsers | indent 4 }} +{{ tpl (toYaml .Values.configUsers) . | indent 4 }} major_version_upgrade: -{{ toYaml .Values.configMajorVersionUpgrade | indent 4 }} +{{ tpl (toYaml .Values.configMajorVersionUpgrade) . | indent 4 }} kubernetes: - {{- if .Values.podPriorityClassName }} - pod_priority_class_name: {{ .Values.podPriorityClassName }} + {{- if .Values.podPriorityClassName.name }} + pod_priority_class_name: {{ .Values.podPriorityClassName.name }} {{- end }} pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} oauth_token_secret_name: {{ template "postgres-operator.fullname" . }} -{{ toYaml .Values.configKubernetes | indent 4 }} +{{ tpl (toYaml .Values.configKubernetes) . | indent 4 }} postgres_pod_resources: -{{ toYaml .Values.configPostgresPodResources | indent 4 }} +{{ tpl (toYaml .Values.configPostgresPodResources) . | indent 4 }} timeouts: -{{ toYaml .Values.configTimeouts | indent 4 }} +{{ tpl (toYaml .Values.configTimeouts) . | indent 4 }} load_balancer: -{{ toYaml .Values.configLoadBalancer | indent 4 }} +{{ tpl (toYaml .Values.configLoadBalancer) . | indent 4 }} aws_or_gcp: -{{ toYaml .Values.configAwsOrGcp | indent 4 }} +{{ tpl (toYaml .Values.configAwsOrGcp) . | indent 4 }} logical_backup: -{{ toYaml .Values.configLogicalBackup | indent 4 }} +{{ tpl (toYaml .Values.configLogicalBackup) . | indent 4 }} debug: -{{ toYaml .Values.configDebug | indent 4 }} +{{ tpl (toYaml .Values.configDebug) . | indent 4 }} teams_api: -{{ toYaml .Values.configTeamsApi | indent 4 }} +{{ tpl (toYaml .Values.configTeamsApi) . | indent 4 }} logging_rest_api: -{{ toYaml .Values.configLoggingRestApi | indent 4 }} +{{ tpl (toYaml .Values.configLoggingRestApi) . | indent 4 }} connection_pooler: -{{ toYaml .Values.configConnectionPooler | indent 4 }} +{{ tpl (toYaml .Values.configConnectionPooler) . | indent 4 }} + patroni: +{{ tpl (toYaml .Values.configPatroni) . | indent 4 }} {{- end }} diff --git a/charts/postgres-operator/templates/postgres-pod-priority-class.yaml b/charts/postgres-operator/templates/postgres-pod-priority-class.yaml index 7ee0f2e55..de78b501c 100644 --- a/charts/postgres-operator/templates/postgres-pod-priority-class.yaml +++ b/charts/postgres-operator/templates/postgres-pod-priority-class.yaml @@ -1,4 +1,4 @@ -{{- if .Values.podPriorityClassName }} +{{- if .Values.podPriorityClassName.create }} apiVersion: scheduling.k8s.io/v1 description: 'Use only for databases controlled by Postgres operator' kind: PriorityClass @@ -8,8 +8,9 @@ metadata: helm.sh/chart: {{ template "postgres-operator.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} - name: {{ .Values.podPriorityClassName }} + name: {{ include "postgres-pod.priorityClassName" . }} + namespace: {{ .Release.Namespace }} preemptionPolicy: PreemptLowerPriority globalDefault: false -value: 1000000 +value: {{ .Values.podPriorityClassName.priority }} {{- end }} diff --git a/charts/postgres-operator/templates/service.yaml b/charts/postgres-operator/templates/service.yaml index 38ea9a062..c1b52744c 100644 --- a/charts/postgres-operator/templates/service.yaml +++ b/charts/postgres-operator/templates/service.yaml @@ -7,6 +7,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} name: {{ template "postgres-operator.fullname" . }} + namespace: {{ .Release.Namespace }} spec: type: ClusterIP ports: diff --git a/charts/postgres-operator/templates/serviceaccount.yaml b/charts/postgres-operator/templates/serviceaccount.yaml index e04e8ad62..4f42559c9 100644 --- a/charts/postgres-operator/templates/serviceaccount.yaml +++ b/charts/postgres-operator/templates/serviceaccount.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "postgres-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ template "postgres-operator.name" . }} helm.sh/chart: {{ template "postgres-operator.chart" . }} diff --git a/charts/postgres-operator/templates/user-facing-clusterroles.yaml b/charts/postgres-operator/templates/user-facing-clusterroles.yaml new file mode 100644 index 000000000..d7db347b2 --- /dev/null +++ b/charts/postgres-operator/templates/user-facing-clusterroles.yaml @@ -0,0 +1,71 @@ +{{ if .Values.rbac.createAggregateClusterRoles }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "postgres-operator.fullname" . }}:users:admin +rules: +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + - postgresqls/status + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "postgres-operator.fullname" . }}:users:edit +rules: +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + verbs: + - create + - update + - patch + - delete + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "postgres-operator.fullname" . }}:users:view +rules: +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + - postgresqls/status + verbs: + - get + - list + - watch +{{ end }} diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index d9c2d0e92..bb2831dd3 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,14 +1,14 @@ image: - registry: registry.opensource.zalan.do - repository: acid/postgres-operator - tag: v1.6.3 + registry: ghcr.io + repository: zalando/postgres-operator + tag: v1.15.1 pullPolicy: "IfNotPresent" - # Optionally specify an array of imagePullSecrets. - # Secrets must be manually created in the namespace. - # ref: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod - # imagePullSecrets: - # - name: myRegistryKeySecretName +# Optionally specify an array of imagePullSecrets. +# Secrets must be manually created in the namespace. +# ref: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod +# imagePullSecrets: +# - name: myRegistryKeySecretName podAnnotations: {} podLabels: {} @@ -18,24 +18,44 @@ configTarget: "OperatorConfigurationCRD" # JSON logging format enableJsonLogging: false +# Command-line options for the operator +extraArgs: [] + # general configuration parameters configGeneral: - # choose if deployment creates/updates CRDs with OpenAPIV3Validation - enable_crd_validation: true + # the deployment should create/update the CRDs + enable_crd_registration: true + # specify categories under which crds should be listed + crd_categories: + - "all" # update only the statefulsets without immediately doing the rolling update enable_lazy_spilo_upgrade: false + # toogle to use maintenance windows feature + enable_maintenance_windows: true # set the PGVERSION env var instead of providing the version via postgresql.bin_dir in SPILO_CONFIGURATION enable_pgversion_env_var: true # start any new database pod without limitations on shm memory enable_shm_volume: true - # enables backwards compatible path between Spilo 12 and Spilo 13 images - enable_spilo_wal_path_compat: false + # operator will sync only clusters where name starts with teamId prefix + enable_team_id_clustername_prefix: false # etcd connection string for Patroni. Empty uses K8s-native DCS. etcd_host: "" + # Spilo docker image + docker_image: ghcr.io/zalando/spilo-18:4.1-p1 + + # key name for annotation to ignore globally configured instance limits + # ignore_instance_limits_annotation_key: "" + + # key name for annotation to ignore globally configured resources thresholds + # ignore_resources_limits_annotation_key: "" + # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: false - # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-13:2.0-p7 + + # maintenance windows applied to all Postgres clusters unless overridden in the manifest + # maintenance_windows: + # - "Sun:01:00-06:00" + # min number of instances in Postgres cluster. -1 = no limit min_instances: -1 # max number of instances in Postgres cluster. -1 = no limit @@ -56,6 +76,16 @@ configGeneral: # parameters describing Postgres users configUsers: + # roles to be granted to database owners + # additional_owner_roles: + # - cron_admin + + # enable password rotation for app users that are not database owners + enable_password_rotation: false + # rotation interval for updating credentials in K8s secrets of app users + password_rotation_interval: 90 + # retention interval to keep rotation users + password_rotation_user_retention: 180 # postgres username used for replication between instances replication_username: standby # postgres superuser name to be created by initdb @@ -63,11 +93,15 @@ configUsers: configMajorVersionUpgrade: # "off": no upgrade, "manual": manifest triggers action, "full": minimal version violation triggers too - major_version_upgrade_mode: "off" + major_version_upgrade_mode: "manual" + # upgrades will only be carried out for clusters of listed teams when mode is "off" + # major_version_upgrade_team_allow_list: + # - acid + # minimal Postgres major version that will not automatically be upgraded - minimal_major_version: "9.5" + minimal_major_version: "14" # target Postgres major version when upgrading clusters automatically - target_major_version: "13" + target_major_version: "18" configKubernetes: # list of additional capabilities for postgres container @@ -97,14 +131,33 @@ configKubernetes: # - deployment-time # - downscaler/* + # allow user secrets in other namespaces than the Postgres cluster + enable_cross_namespace_secret: false + # use finalizers to ensure all managed resources are deleted prior to the postgresql CR + # this avoids stale resources in case the operator misses a delete event or is not running + # during deletion + enable_finalizers: false # enables initContainers to run actions before Spilo is started enable_init_containers: true + # toggles if child resources should have an owner reference to the postgresql CR + enable_owner_references: false + # toggles if operator should delete PVCs on cluster deletion + enable_persistent_volume_claim_deletion: true # toggles pod anti affinity on the Postgres pods enable_pod_antiaffinity: false # toggles PDB to set to MinAvailabe 0 or 1 enable_pod_disruption_budget: true + # toogles readiness probe for database pods + enable_readiness_probe: false + # toggles if operator should delete secrets on cluster deletion + enable_secrets_deletion: true # enables sidecar containers to run alongside Spilo in the same pod enable_sidecars: true + + # annotations to be ignored when comparing statefulsets, services etc. + # ignored_annotations: + # - k8s.v1.cni.cncf.io/network-status + # namespaced name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles @@ -124,11 +177,22 @@ configKubernetes: # node_readiness_label: # status: ready + # defines how nodeAffinity from manifest should be merged with node_readiness_label + # node_readiness_label_merge: "OR" + # namespaced name of the secret containing the OAuth2 token to pass to the teams API # oauth_token_secret_name: postgresql-operator - # defines the template for PDB (Pod Disruption Budget) names + # toggle if `spilo-role=master` selector should be added to the PDB (Pod Disruption Budget) + pdb_master_label_selector: true + # defines the template for PDB names pdb_name_format: "postgres-{cluster}-pdb" + # specify the PVC retention policy when scaling down and/or deleting + persistent_volume_claim_retention_policy: + when_deleted: "retain" + when_scaled: "retain" + # switches pod anti affinity type to `preferredDuringSchedulingIgnoredDuringExecution` + pod_antiaffinity_preferred_during_scheduling: false # override topology key for pod anti affinity pod_antiaffinity_topology_key: "kubernetes.io/hostname" # namespaced name of the ConfigMap with environment variables to populate on every pod @@ -148,11 +212,17 @@ configKubernetes: # Postgres pods are terminated forcefully after this timeout pod_terminate_grace_period: 5m - # template for database user secrets generated by the operator + # template for database user secrets generated by the operator, + # here username contains the namespace in the format namespace.username + # if the user is in different namespace than cluster and cross namespace secrets + # are enabled via `enable_cross_namespace_secret` flag in the configuration. secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + # sharing unix socket of PostgreSQL (`pg_socket`) with the sidecars + share_pgsocket_with_sidecars: false # set user and group for the spilo container (required to run Spilo as non-root process) # spilo_runasuser: 101 # spilo_runasgroup: 103 + # group ID with write-access to volumes (required to run Spilo as non-root process) # spilo_fsgroup: 103 @@ -161,8 +231,27 @@ configKubernetes: # whether the Spilo container should run with additional permissions other than parent. # required by cron which needs setuid spilo_allow_privilege_escalation: true - # storage resize strategy, available options are: ebs, pvc, off + + # liveness probe for the spilo pod + # liveness_probe: + # httpGet: + # scheme: HTTP + # path: /liveness + # port: 8008 + # initialDelaySeconds: 10 + # periodSeconds: 10 + # timeoutSeconds: 5 + # successThreshold: 1 + # failureThreshold: 3 + + # storage resize strategy, available options are: ebs, pvc, off or mixed storage_resize_mode: pvc + # pod toleration assigned to instances of every Postgres cluster + # toleration: + # key: db-only + # operator: Exists + # effect: NoSchedule + # operator watches for postgres objects in the given namespace watched_namespace: "*" # listen to all namespaces @@ -176,6 +265,12 @@ configPostgresPodResources: default_memory_limit: 500Mi # memory request value for the postgres containers default_memory_request: 100Mi + # optional upper boundary for CPU request + # max_cpu_request: "1" + + # optional upper boundary for memory request + # max_memory_request: 4Gi + # hard CPU minimum required to properly run a Postgres cluster min_cpu_limit: 250m # hard memory minimum required to properly run a Postgres cluster @@ -183,6 +278,10 @@ configPostgresPodResources: # timeouts related to some operator actions configTimeouts: + # interval between consecutive attempts of operator calling the Patroni API + patroni_api_check_interval: 1s + # timeout when waiting for successful response from Patroni API + patroni_api_check_timeout: 5s # timeout when waiting for the Postgres pods to be deleted pod_deletion_wait_timeout: 10m # timeout when waiting for pod role and cluster labels @@ -207,14 +306,22 @@ configLoadBalancer: # toggles service type load balancer pointing to the master pod of the cluster enable_master_load_balancer: false + # toggles service type load balancer pointing to the master pooler pod of the cluster + enable_master_pooler_load_balancer: false # toggles service type load balancer pointing to the replica pod of the cluster enable_replica_load_balancer: false + # toggles service type load balancer pointing to the replica pooler pod of the cluster + enable_replica_pooler_load_balancer: false # define external traffic policy for the load balancer external_traffic_policy: "Cluster" # defines the DNS name string template for the master load balancer cluster - master_dns_name_format: "{cluster}.{team}.{hostedzone}" + master_dns_name_format: "{cluster}.{namespace}.{hostedzone}" + # deprecated DNS template for master load balancer using team name + master_legacy_dns_name_format: "{cluster}.{team}.{hostedzone}" # defines the DNS name string template for the replica load balancer cluster - replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}" + replica_dns_name_format: "{cluster}-repl.{namespace}.{hostedzone}" + # deprecated DNS template for replica load balancer using team name + replica_legacy_dns_name_format: "{cluster}-repl.{team}.{hostedzone}" # options to aid debugging of the operator itself configDebug: @@ -240,7 +347,7 @@ configAwsOrGcp: # Path to mount the above Secret in the filesystem of the container(s) # additional_secret_mount_path: "/some/dir" - # AWS region used to store ESB volumes + # AWS region used to store EBS volumes aws_region: eu-central-1 # enable automatic migration on AWS from gp2 to gp3 volumes @@ -263,21 +370,37 @@ configAwsOrGcp: # GCS bucket to use for shipping WAL segments with WAL-E # wal_gs_bucket: "" + # Azure Storage Account to use for shipping WAL segments with WAL-G + # wal_az_storage_account: "" + # configure K8s cron job managed by the operator configLogicalBackup: + # Azure Storage Account specs to store backup results + # logical_backup_azure_storage_account_name: "" + # logical_backup_azure_storage_container: "" + # logical_backup_azure_storage_account_key: "" + + # resources for logical backup pod, if empty configPostgresPodResources will be used + # logical_backup_cpu_limit: "" + # logical_backup_cpu_request: "" + # logical_backup_memory_limit: "" + # logical_backup_memory_request: "" + # image for pods of the logical backup job (example runs pg_dumpall) - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.6.3" + logical_backup_docker_image: "ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1" # path of google cloud service account json file # logical_backup_google_application_credentials: "" # prefix for the backup job name logical_backup_job_prefix: "logical-backup-" - # storage provider - either "s3" or "gcs" + # storage provider - either "s3", "gcs" or "az" logical_backup_provider: "s3" # S3 Access Key ID logical_backup_s3_access_key_id: "" # S3 bucket to store backup results logical_backup_s3_bucket: "my-bucket-url" + # S3 bucket prefix to use + logical_backup_s3_bucket_prefix: "spilo" # S3 region of bucket logical_backup_s3_region: "" # S3 endpoint url when not using AWS @@ -286,8 +409,18 @@ configLogicalBackup: logical_backup_s3_secret_access_key: "" # S3 server side encryption logical_backup_s3_sse: "AES256" + # S3 retention time for stored backups for example "2 week" or "7 days" + logical_backup_s3_retention_time: "" # backup schedule in the cron format logical_backup_schedule: "30 00 * * *" + # secret to be used as reference for env variables in cronjob + logical_backup_cronjob_environment_secret: "" + # number of successful backup jobs to keep in cronjob history + logical_backup_successful_jobs_history_limit: 3 + # number of failed backup jobs to keep in cronjob history + logical_backup_failed_jobs_history_limit: 3 + # TTL in seconds after which finished backup jobs are automatically deleted + logical_backup_ttl_seconds_after_finished: 86400 # automate creation of human users with teams API service configTeamsApi: @@ -314,6 +447,7 @@ configTeamsApi: # List of roles that cannot be overwritten by an application, team or infrastructure role protected_role_names: - admin + - cron_admin # Suffix to add if members are removed from TeamsAPI or PostgresTeam CRD role_deletion_suffix: "_deleted" # role name to grant to team members created from the Teams API @@ -331,7 +465,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-16" + connection_pooler_image: "ghcr.io/zalando/postgres-operator/pgbouncer:latest" # max db connections the pooler should hold connection_pooler_max_db_connections: 60 # default pooling mode @@ -344,14 +478,18 @@ configConnectionPooler: connection_pooler_default_cpu_limit: "1" connection_pooler_default_memory_limit: 100Mi +configPatroni: + # enable Patroni DCS failsafe_mode feature + enable_patroni_failsafe_mode: false + +# Zalando's internal CDC stream feature +enableStreams: false + rbac: # Specifies whether RBAC resources should be created create: true - -crd: - # Specifies whether custom resource definitions should be created - # When using helm3, this is ignored; instead use "--skip-crds" to skip. - create: true + # Specifies whether ClusterRoles that are aggregated into the K8s default roles should be created. (https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings) + createAggregateClusterRoles: false serviceAccount: # Specifies whether a ServiceAccount should be created @@ -369,7 +507,14 @@ podServiceAccount: priorityClassName: "" # priority class for database pods -podPriorityClassName: "" +podPriorityClassName: + # If create is false with no name set, no podPriorityClassName is specified. + # Hence, the pod priorityClass is the one with globalDefault set. + # If there is no PriorityClass with globalDefault set, the priority of Pods with no priorityClassName is zero. + create: true + # If not set a name is generated using the fullname template and "-pod" suffix + name: "" + priority: 1000000 resources: limits: @@ -385,6 +530,29 @@ securityContext: readOnlyRootFilesystem: true allowPrivilegeEscalation: false +# Allow to setup operator Deployment readiness probe +readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 10 + +# configure extra environment variables +# Extra environment variables are writen in kubernetes format and added "as is" to the pod's env variables +# https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables +extraEnvs: + [] + # Exemple of settings maximum amount of memory / cpu that can be used by go process (to match resources.limits) + # - name: MY_VAR + # value: my-value + # - name: GOMAXPROCS + # valueFrom: + # resourceFieldRef: + # resource: limits.cpu + # - name: GOMEMLIMIT + # valueFrom: + # resourceFieldRef: + # resource: limits.memory + # Affinity for pod assignment # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} diff --git a/cmd/main.go b/cmd/main.go index 376df0bad..adbf0cce5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,13 +2,14 @@ package main import ( "flag" - log "github.com/sirupsen/logrus" "os" "os/signal" "sync" "syscall" "time" + log "github.com/sirupsen/logrus" + "github.com/zalando/postgres-operator/pkg/controller" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/k8sutil" @@ -34,6 +35,8 @@ func init() { flag.BoolVar(&outOfCluster, "outofcluster", false, "Whether the operator runs in- our outside of the Kubernetes cluster.") flag.BoolVar(&config.NoDatabaseAccess, "nodatabaseaccess", false, "Disable all access to the database from the operator side.") flag.BoolVar(&config.NoTeamsAPI, "noteamsapi", false, "Disable all access to the teams API") + flag.IntVar(&config.KubeQPS, "kubeqps", 10, "Kubernetes api requests per second.") + flag.IntVar(&config.KubeBurst, "kubeburst", 20, "Kubernetes api requests burst limit.") flag.Parse() config.EnableJsonLogging = os.Getenv("ENABLE_JSON_LOGGING") == "true" @@ -82,6 +85,9 @@ func main() { log.Fatalf("couldn't get REST config: %v", err) } + config.RestConfig.QPS = float32(config.KubeQPS) + config.RestConfig.Burst = config.KubeBurst + c := controller.NewController(&config, "") c.Run(stop, wg) diff --git a/delivery.yaml b/delivery.yaml index d0a689b8b..d81f4ee5e 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -1,94 +1,136 @@ version: "2017-09-20" +allow_concurrent_steps: true + +build_env: &BUILD_ENV + PYTHON_BASE_IMAGE: container-registry.zalando.net/library/python-3.11-slim + ALPINE_BASE_IMAGE: container-registry.zalando.net/library/alpine-3 + UBUNTU_BASE_IMAGE: container-registry.zalando.net/library/ubuntu-22.04 + MULTI_ARCH_REGISTRY: container-registry-test.zalando.net/acid + pipeline: - - id: build-postgres-operator - type: script - vm: large - cache: - paths: - - /go/pkg/mod - commands: - - desc: 'Update' - cmd: | - apt-get update - - desc: 'Install required build software' - cmd: | - apt-get install -y make git apt-transport-https ca-certificates curl build-essential python3 python3-pip - - desc: 'Install go' - cmd: | - cd /tmp - wget -q https://storage.googleapis.com/golang/go1.15.6.linux-amd64.tar.gz -O go.tar.gz - tar -xf go.tar.gz - mv go /usr/local - ln -s /usr/local/go/bin/go /usr/bin/go - go version - - desc: 'Build docker image' - cmd: | - export PATH=$PATH:$HOME/go/bin - IS_PR_BUILD=${CDP_PULL_REQUEST_NUMBER+"true"} - if [[ ${CDP_TARGET_BRANCH} == "master" && ${IS_PR_BUILD} != "true" ]] - then - IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator - else - IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator-test - fi - export IMAGE - make deps mocks docker - - desc: 'Run unit tests' - cmd: | - export PATH=$PATH:$HOME/go/bin - go test ./... - - desc: 'Run e2e tests' - cmd: | - make e2e - - desc: 'Push docker image' - cmd: | - export PATH=$PATH:$HOME/go/bin - IS_PR_BUILD=${CDP_PULL_REQUEST_NUMBER+"true"} - if [[ ${CDP_TARGET_BRANCH} == "master" && ${IS_PR_BUILD} != "true" ]] - then - IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator - else - IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator-test - fi - export IMAGE - make push + - id: build-postgres-operator + env: + <<: *BUILD_ENV + type: script + vm_config: + type: linux + size: large + image: cdp-runtime/go + cache: + paths: + - /go/pkg/mod # pkg cache for Go modules + - ~/.cache/go-build # Go build cache + commands: + - desc: Run unit tests + cmd: | + make mocks test + if ! git diff --quiet; then + echo "Build resulted in files being changed, likely they were not checked in" + exit 1 + fi + - desc: Build Docker image + cmd: | + if [ -z ${CDP_SOURCE_BRANCH} ]; then + IMAGE=${MULTI_ARCH_REGISTRY}/postgres-operator + else + IMAGE=${MULTI_ARCH_REGISTRY}/postgres-operator-test + fi - - id: build-operator-ui - type: script + docker buildx create --config /etc/cdp-buildkitd.toml --driver-opt network=host --bootstrap --use + docker buildx build --platform "linux/amd64,linux/arm64" \ + --build-arg BASE_IMAGE="${ALPINE_BASE_IMAGE}" \ + -t "${IMAGE}:${CDP_BUILD_VERSION}" \ + -f docker/Dockerfile \ + --push . - commands: - - desc: 'Prepare environment' - cmd: | - apt-get update - apt-get install -y build-essential + - id: build-pooler + env: + <<: *BUILD_ENV + type: script + vm_config: + type: linux - - desc: 'Compile JavaScript app' - cmd: | - cd ui - make appjs + commands: + - desc: Build image + cmd: | + cd pooler + if [ -z ${CDP_SOURCE_BRANCH} ]; then + IMAGE=${MULTI_ARCH_REGISTRY}/pgbouncer + else + IMAGE=${MULTI_ARCH_REGISTRY}/pgbouncer-test + fi - - desc: 'Build and push Docker image' - cmd: | - cd ui - IS_PR_BUILD=${CDP_PULL_REQUEST_NUMBER+"true"} - if [[ ${CDP_TARGET_BRANCH} == "master" && ${IS_PR_BUILD} != "true" ]] - then - IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator-ui - else - IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator-ui-test - fi - export IMAGE - make docker - make push + docker buildx create --config /etc/cdp-buildkitd.toml --driver-opt network=host --bootstrap --use + docker buildx build --platform "linux/amd64,linux/arm64" \ + --build-arg BASE_IMAGE="${ALPINE_BASE_IMAGE}" \ + -t "${IMAGE}:${CDP_BUILD_VERSION}" \ + --push . - - id: build-logical-backup - type: script + if [ -z ${CDP_SOURCE_BRANCH} ]; then + cdp-promote-image ${IMAGE}:${CDP_BUILD_VERSION} + fi - commands: - - desc: Build image - cmd: | - cd docker/logical-backup - export TAG=$(git describe --tags --always --dirty) - IMAGE="registry-write.opensource.zalan.do/acid/logical-backup" - docker build --rm -t "$IMAGE:$TAG$CDP_TAG" . - docker push "$IMAGE:$TAG$CDP_TAG" + - id: build-operator-ui + env: + <<: *BUILD_ENV + type: script + vm_config: + type: linux + + commands: + - desc: 'Prepare environment' + cmd: | + apt-get update + apt-get install -y build-essential + + - desc: 'Compile JavaScript app' + cmd: | + cd ui + make appjs + + - desc: 'Build and push Docker image' + cmd: | + cd ui + if [ -z ${CDP_SOURCE_BRANCH} ]; then + IMAGE=${MULTI_ARCH_REGISTRY}/postgres-operator-ui + else + IMAGE=${MULTI_ARCH_REGISTRY}/postgres-operator-ui-test + fi + + make appjs + docker buildx create --config /etc/cdp-buildkitd.toml --driver-opt network=host --bootstrap --use + docker buildx build --platform linux/amd64,linux/arm64 \ + --build-arg BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ + -t ${IMAGE}:${CDP_BUILD_VERSION} \ + --push . + + if [ -z ${CDP_SOURCE_BRANCH} ]; then + cdp-promote-image ${IMAGE}:${CDP_BUILD_VERSION} + fi + + - id: build-logical-backup + env: + <<: *BUILD_ENV + type: script + vm_config: + type: linux + + commands: + - desc: Build image + cmd: | + cd logical-backup + if [ -z ${CDP_SOURCE_BRANCH} ]; then + IMAGE=${MULTI_ARCH_REGISTRY}/logical-backup + else + IMAGE=${MULTI_ARCH_REGISTRY}/logical-backup-test + fi + + docker buildx create --config /etc/cdp-buildkitd.toml --driver-opt network=host --bootstrap --use + docker buildx build --platform linux/amd64,linux/arm64 \ + --build-arg BASE_IMAGE="${UBUNTU_BASE_IMAGE}" \ + -t ${IMAGE}:${CDP_BUILD_VERSION} \ + --push . + + if [ -z ${CDP_SOURCE_BRANCH} ]; then + cdp-promote-image ${IMAGE}:${CDP_BUILD_VERSION} + fi diff --git a/docker/DebugDockerfile b/docker/DebugDockerfile index e8f51badd..367735d08 100644 --- a/docker/DebugDockerfile +++ b/docker/DebugDockerfile @@ -1,18 +1,14 @@ -FROM registry.opensource.zalan.do/library/alpine-3.12:latest +FROM golang:1.26-alpine LABEL maintainer="Team ACID @ Zalando " # We need root certificates to deal with teams api over https -RUN apk --no-cache add ca-certificates go git musl-dev +RUN apk -U add --no-cache ca-certificates delve COPY build/* / RUN addgroup -g 1000 pgo RUN adduser -D -u 1000 -G pgo -g 'Postgres Operator' pgo -RUN go get github.com/derekparker/delve/cmd/dlv -RUN cp /root/go/bin/dlv /dlv -RUN chown -R pgo:pgo /dlv - USER pgo:pgo RUN ls -l / diff --git a/docker/Dockerfile b/docker/Dockerfile index c1b87caf7..4ba9b7630 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,11 +1,27 @@ -FROM registry.opensource.zalan.do/library/alpine-3.12:latest +ARG BASE_IMAGE=alpine:latest + +FROM --platform=$BUILDPLATFORM golang:1.26-alpine AS builder +ARG VERSION=latest +ARG TARGETOS +ARG TARGETARCH + +COPY . /go/src/github.com/zalando/postgres-operator +WORKDIR /go/src/github.com/zalando/postgres-operator + +RUN go clean -cache -modcache +RUN go mod vendor \ + && CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o build/postgres-operator -v -ldflags "-X=main.version=${VERSION}" cmd/main.go + +ARG BASE_IMAGE +FROM --platform=$TARGETPLATFORM ${BASE_IMAGE} LABEL maintainer="Team ACID @ Zalando " +LABEL org.opencontainers.image.source="https://github.com/zalando/postgres-operator" # We need root certificates to deal with teams api over https -RUN apk --no-cache add curl -RUN apk --no-cache add ca-certificates +RUN apk -U upgrade --no-cache \ + && apk add --no-cache curl ca-certificates -COPY build/* / +COPY --from=builder /go/src/github.com/zalando/postgres-operator/build/* / RUN addgroup -g 1000 pgo RUN adduser -D -u 1000 -G pgo -g 'Postgres Operator' pgo diff --git a/docker/build_operator.sh b/docker/build_operator.sh new file mode 100644 index 000000000..0b1bf4d82 --- /dev/null +++ b/docker/build_operator.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive + +arch=$(dpkg --print-architecture) + +set -ex + +# Install dependencies + +apt-get update +apt-get install -y wget + +( + cd /tmp + wget -q "https://storage.googleapis.com/golang/go1.26.4.linux-${arch}.tar.gz" -O go.tar.gz + tar -xf go.tar.gz + mv go /usr/local + ln -s /usr/local/go/bin/go /usr/bin/go + go version +) + +# Build + +export PATH="$PATH:$HOME/go/bin" +export GOPATH="$HOME/go" +mkdir -p build + +go clean -cache -modcache +go mod vendor +CGO_ENABLED=0 go build -o build/postgres-operator -v -ldflags "$OPERATOR_LDFLAGS" cmd/main.go diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh deleted file mode 100755 index 50f7e6e4c..000000000 --- a/docker/logical-backup/dump.sh +++ /dev/null @@ -1,117 +0,0 @@ -#! /usr/bin/env bash - -# enable unofficial bash strict mode -set -o errexit -set -o nounset -set -o pipefail -IFS=$'\n\t' - -ALL_DB_SIZE_QUERY="select sum(pg_database_size(datname)::numeric) from pg_database;" -PG_BIN=$PG_DIR/$PG_VERSION/bin -DUMP_SIZE_COEFF=5 -ERRORCOUNT=0 - -TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) -K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1 -CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - -function estimate_size { - "$PG_BIN"/psql -tqAc "${ALL_DB_SIZE_QUERY}" -} - -function dump { - # settings are taken from the environment - "$PG_BIN"/pg_dumpall -} - -function compress { - pigz -} - -function aws_upload { - declare -r EXPECTED_SIZE="$1" - - # mimic bucket setup from Spilo - # to keep logical backups at the same path as WAL - # NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator - PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz - - args=() - - [[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE") - [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") - [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION") - [[ ! -z "$LOGICAL_BACKUP_S3_SSE" ]] && args+=("--sse=$LOGICAL_BACKUP_S3_SSE") - - aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" -} - -function gcs_upload { - PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz - - gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP" -} - -function upload { - case $LOGICAL_BACKUP_PROVIDER in - "gcs") - gcs_upload - ;; - *) - aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) - ;; - esac -} - -function get_pods { - declare -r SELECTOR="$1" - - curl "${K8S_API_URL}/namespaces/${POD_NAMESPACE}/pods?$SELECTOR" \ - --cacert $CERT \ - -H "Authorization: Bearer ${TOKEN}" | jq .items[].status.podIP -r -} - -function get_current_pod { - curl "${K8S_API_URL}/namespaces/${POD_NAMESPACE}/pods?fieldSelector=metadata.name%3D${HOSTNAME}" \ - --cacert $CERT \ - -H "Authorization: Bearer ${TOKEN}" -} - -declare -a search_strategy=( - list_all_replica_pods_current_node - list_all_replica_pods_any_node - get_master_pod -) - -function list_all_replica_pods_current_node { - get_pods "labelSelector=${CLUSTER_NAME_LABEL}%3D${SCOPE},spilo-role%3Dreplica&fieldSelector=spec.nodeName%3D${CURRENT_NODENAME}" | head -n 1 -} - -function list_all_replica_pods_any_node { - get_pods "labelSelector=${CLUSTER_NAME_LABEL}%3D${SCOPE},spilo-role%3Dreplica" | head -n 1 -} - -function get_master_pod { - get_pods "labelSelector=${CLUSTER_NAME_LABEL}%3D${SCOPE},spilo-role%3Dmaster" | head -n 1 -} - -CURRENT_NODENAME=$(get_current_pod | jq .items[].spec.nodeName --raw-output) -export CURRENT_NODENAME - -for search in "${search_strategy[@]}"; do - - PGHOST=$(eval "$search") - export PGHOST - - if [ -n "$PGHOST" ]; then - break - fi - -done - -set -x -dump | compress | upload -[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 )) -set +x - -exit $ERRORCOUNT diff --git a/docs/administrator.md b/docs/administrator.md index db3bae6b9..ec0bc28dc 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -3,6 +3,40 @@ Learn how to configure and manage the Postgres Operator in your Kubernetes (K8s) environment. +## CRD registration and validation + +On startup, the operator will try to register the necessary +[CustomResourceDefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions) +`Postgresql` and `OperatorConfiguration`. The latter will only get created if +the `POSTGRES_OPERATOR_CONFIGURATION_OBJECT` [environment variable](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L36) +is set in the deployment yaml and is not empty. If the CRDs already exists they +will only be patched. If you do not wish the operator to create or update the +CRDs set `enable_crd_registration` config option to `false`. + +CRDs are defined with a `openAPIV3Schema` structural schema against which new +manifests of [`postgresql`](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql.crd.yaml) or [`OperatorConfiguration`](https://github.com/zalando/postgres-operator/blob/master/manifests/operatorconfiguration.crd.yaml) +resources will be validated. On creation you can bypass the validation with +`kubectl create --validate=false`. + +By default, the operator will register the CRDs in the `all` category so +that resources are listed on `kubectl get all` commands. The `crd_categories` +config option allows for customization of categories. + +## Upgrading the operator + +The Postgres Operator is upgraded by changing the docker image within the +deployment. Before doing so, it is recommended to check the release notes +for new configuration options or changed behavior you might want to reflect +in the ConfigMap or config CRD. E.g. a new feature might get introduced which +is enabled or disabled by default and you want to change it to the opposite +with the corresponding flag option. + +When using helm, be aware that installing the new chart will not update the +`Postgresql` and `OperatorConfiguration` CRD. Make sure to update them before +with the provided manifests in the `crds` folder. Otherwise, you might face +errors about new Postgres manifest or configuration options being unknown +to the CRD schema validation. + ## Minor and major version upgrade Minor version upgrades for PostgreSQL are handled via updating the Spilo Docker @@ -29,14 +63,20 @@ the `PGVERSION` environment variable is set for the database pods. Since `v1.6.0` the related option `enable_pgversion_env_var` is enabled by default. In-place major version upgrades can be configured to be executed by the -operator with the `major_version_upgrade_mode` option. By default it is set -to `off` which means the cluster version will not change when increased in -the manifest. Still, a rolling update would be triggered updating the -`PGVERSION` variable. But Spilo's [`configure_spilo`](https://github.com/zalando/spilo/blob/master/postgres-appliance/scripts/configure_spilo.py) -script will notice the version mismatch and start the old version again. +operator with the `major_version_upgrade_mode` option. By default, it is +enabled (mode: `manual`). In any case, altering the version in the manifest +will update the desired `PGVERSION`. If `maintenanceWindows` are configured, +major-version-related pod rotation is deferred until the next maintenance +window. Without maintenance windows, the operator will trigger a rolling +update of pods to apply the new `PGVERSION`. +Spilo's [`configure_spilo`](https://github.com/zalando/spilo/blob/master/postgres-appliance/scripts/configure_spilo.py) +script will notice the version mismatch but start the current version again. -In this scenario the major version could then be run by a user from within the -master pod. Exec into the container and run: +Next, the operator would call an updage script inside Spilo. When automatic +upgrades are disabled (mode: `off`) the upgrade could still be run by a user +from within the primary pod. This gives you full control about the point in +time when the upgrade can be started (check also maintenance windows below). +Exec into the container and run: ```bash python3 /scripts/inplace_upgrade.py N ``` @@ -45,32 +85,33 @@ The upgrade is usually fast, well under one minute for most DBs. Note, that changes become irrevertible once `pg_upgrade` is called. To understand the upgrade procedure, refer to the [corresponding PR in Spilo](https://github.com/zalando/spilo/pull/488). -When `major_version_upgrade_mode` is set to `manual` the operator will run -the upgrade script for you after the manifest is updated and pods are rotated. +When `major_version_upgrade_mode` is set to `full` the operator will compare +the version in the manifest with the configured `minimal_major_version`. If it +is lower the operator would start an automatic upgrade as described above. The +configured `major_target_version` will be used as the new version. This option +can be useful if you have to get rid of outdated major versions in your fleet. +Please note, that the operator does not patch the version in the manifest. +Thus, the `full` mode can create drift between desired and actual state. -## CRD Validation +### Upgrade during maintenance windows -[CustomResourceDefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions) -will be registered with schema validation by default when the operator is -deployed. The `OperatorConfiguration` CRD will only get created if the -`POSTGRES_OPERATOR_CONFIGURATION_OBJECT` [environment variable](../manifests/postgres-operator.yaml#L36) -in the deployment yaml is set and not empty. +When `maintenanceWindows` are defined in the Postgres manifest or in the global +config the operator will trigger major-version-related pod rotation and the +major version upgrade only during these periods. Make sure they are at least +twice as long as your configured `resync_period` to guarantee that operator +actions can be triggered. -When submitting manifests of [`postgresql`](../manifests/postgresql.crd.yaml) or -[`OperatorConfiguration`](../manifests/operatorconfiguration.crd.yaml) custom -resources with kubectl, validation can be bypassed with `--validate=false`. The -operator can also be configured to not register CRDs with validation on `ADD` or -`UPDATE` events. Running instances are not affected when enabling the validation -afterwards unless the manifests is not changed then. Note, that the provided CRD -manifests contain the validation for users to understand what schema is -enforced. +### Upgrade annotations -Once the validation is enabled it can only be disabled manually by editing or -patching the CRD manifest: +When an upgrade is executed, the operator sets an annotation in the PostgreSQL +resource, either `last-major-upgrade-success` if the upgrade succeeds, or +`last-major-upgrade-failure` if it fails. The value of the annotation is a +timestamp indicating when the upgrade occurred. -```bash -kubectl patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}' -``` +If a PostgreSQL resource contains a failure annotation, the operator will not +attempt to retry the upgrade during a sync event. To remove the failure +annotation, you can revert the PostgreSQL version back to the current version. +This action will trigger the removal of the failure annotation. ## Non-default cluster domain @@ -94,7 +135,7 @@ kubectl config set-context $(kubectl config current-context) --namespace=test All subsequent `kubectl` commands will work with the `test` namespace. The operator will run in this namespace and look up needed resources - such as its ConfigMap - there. Please note that the namespace for service accounts and -cluster role bindings in [operator RBAC rules](../manifests/operator-service-account-rbac.yaml) +cluster role bindings in [operator RBAC rules](https://github.com/zalando/postgres-operator/blob/master/manifests/operator-service-account-rbac.yaml) needs to be adjusted to the non-default value. ### Specify the namespace to watch @@ -105,9 +146,9 @@ clusters in the namespace such as "increase the number of Postgres replicas to By default, the operator watches the namespace it is deployed to. You can change this by setting the `WATCHED_NAMESPACE` var in the `env` section of the -[operator deployment](../manifests/postgres-operator.yaml) manifest or by +[operator deployment](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml) manifest or by altering the `watched_namespace` field in the operator -[configuration](../manifests/postgresql-operator-default-configuration.yaml#L49). +[configuration](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql-operator-default-configuration.yaml#L49). In the case both are set, the env var takes the precedence. To make the operator listen to all namespaces, explicitly set the field/env var to "`*`". @@ -128,7 +169,7 @@ But, it is also possible to define ownership between operator instances and Postgres clusters running all in the same namespace or K8s cluster without interfering. -First, define the [`CONTROLLER_ID`](../../manifests/postgres-operator.yaml#L38) +First, define the [`CONTROLLER_ID`](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L38) environment variable in the operator deployment manifest. Then specify the ID in every Postgres cluster manifest you want this operator to watch using the `"acid.zalan.do/controller"` annotation: @@ -157,16 +198,28 @@ from numerous escape characters in the latter log entry, view it in CLI with `PodTemplate` used by the operator is yet to be updated with the default values used internally in K8s. -The operator also support lazy updates of the Spilo image. That means the pod -template of a PG cluster's stateful set is updated immediately with the new -image, but no rolling update follows. This feature saves you a switchover - and -hence downtime - when you know pods are re-started later anyway, for instance -due to the node rotation. To force a rolling update, disable this mode by -setting the `enable_lazy_spilo_upgrade` to `false` in the operator configuration -and restart the operator pod. With the standard eager rolling updates the -operator checks during Sync all pods run images specified in their respective -statefulsets. The operator triggers a rolling upgrade for PG clusters that -violate this condition. +The StatefulSet is replaced if the following properties change: + +- annotations +- volumeClaimTemplates +- template volumes + +The StatefulSet is replaced and a rolling updates is triggered if the following +properties differ between the old and new state: + +- container name, ports, image, resources, env, envFrom, securityContext and volumeMounts +- template labels, annotations, service account, securityContext, affinity, priority class and termination grace period + +Note that, changes in `SPILO_CONFIGURATION` env variable under `bootstrap.dcs` +path are ignored for the diff. They will be applied through Patroni's rest api +interface, following a restart of all instances. + +The operator also support lazy updates of the Spilo image. In this case the +StatefulSet is only updated, but no rolling update follows. This feature saves +you a switchover - and hence downtime - when you know pods are re-started later +anyway, for instance due to the node rotation. To force a rolling update, +disable this mode by setting the `enable_lazy_spilo_upgrade` to `false` in the +operator configuration and restart the operator pod. ## Delete protection via annotations @@ -203,9 +256,9 @@ configuration: Now, every cluster manifest must contain the configured annotation keys to trigger the delete process when running `kubectl delete pg`. Note, that the -`Postgresql` resource would still get deleted as K8s' API server does not -block it. Only the operator logs will tell, that the delete criteria wasn't -met. +`Postgresql` resource would still get deleted because the operator does not +instruct K8s' API server to block it. Only the operator logs will tell, that +the delete criteria was not met. **cluster manifest** @@ -223,15 +276,68 @@ spec: In case, the resource has been deleted accidentally or the annotations were simply forgotten, it's safe to recreate the cluster with `kubectl create`. -Existing Postgres cluster are not replaced by the operator. But, as the -original cluster still exists the status will show `CreateFailed` at first. -On the next sync event it should change to `Running`. However, as it is in -fact a new resource for K8s, the UID will differ which can trigger a rolling -update of the pods because the UID is used as part of backup path to S3. +Existing Postgres cluster are not replaced by the operator. But, when the +original cluster still exists the status will be `CreateFailed` at first. On +the next sync event it should change to `Running`. However, because it is in +fact a new resource for K8s, the UID and therefore, the backup path to S3, +will differ and trigger a rolling update of the pods. + +## Owner References and Finalizers + +The Postgres Operator can set [owner references](https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/) to most of a cluster's child resources to improve +monitoring with GitOps tools and enable cascading deletes. There are two +exceptions: + +* Persistent Volume Claims, because they are handled by the [PV Reclaim Policy]https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/ of the Stateful Set +* Cross-namespace secrets, because owner references are not allowed across namespaces by design + +The operator would clean these resources up with its regular delete loop +unless they got synced correctly. If for some reason the initial cluster sync +fails, e.g. after a cluster creation or operator restart, a deletion of the +cluster manifest might leave orphaned resources behind which the user has to +clean up manually. + +Another option is to enable finalizers which first ensures the deletion of all +child resources before the cluster manifest gets removed. There is a trade-off +though: The deletion is only performed after the next two operator SYNC cycles +with the first one setting a `deletionTimestamp` and the latter reacting to it. +The final removal of the custom resource will add a DELETE event to the worker +queue but the child resources are already gone at this point. If you do not +desire this behavior consider enabling owner references instead. + +**postgres-operator ConfigMap** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-operator +data: + enable_finalizers: "false" + enable_owner_references: "true" +``` + +**OperatorConfiguration** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + kubernetes: + enable_finalizers: false + enable_owner_references: true +``` + +:warning: Please note, both options are disabled by default. When enabling owner +references the operator cannot block cascading deletes, even when the [delete protection annotations](administrator.md#delete-protection-via-annotations) +are in place. You would need an K8s admission controller that blocks the actual +`kubectl delete` API call e.g. based on existing annotations. ## Role-based access control for the operator -The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml) +The manifest [`operator-service-account-rbac.yaml`](https://github.com/zalando/postgres-operator/blob/master/manifests/operator-service-account-rbac.yaml) defines the service account, cluster roles and bindings needed for the operator to function under access control restrictions. The file also includes a cluster role `postgres-pod` with privileges for Patroni to watch and manage pods and @@ -266,6 +372,103 @@ kubectl create -f manifests/user-facing-clusterroles.yaml It creates zalando-postgres-operator:user:view, :edit and :admin clusterroles that are aggregated into the K8s [default roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings). +For Helm deployments setting `rbac.createAggregateClusterRoles: true` adds these clusterroles to the deployment. + +## Password rotation in K8s secrets + +The operator regularly updates credentials in the K8s secrets if the +`enable_password_rotation` option is set to `true` in the configuration. +It happens only for `LOGIN` roles with an associated secret (manifest roles, +default users from `preparedDatabases`). Furthermore, there are the following +exceptions: + +1. Infrastructure role secrets since rotation should happen by the infrastructure. +2. Team API roles that connect via OAuth2 and JWT token (no secrets to these roles anyway). +3. Database owners since ownership on database objects can not be inherited. +4. System users such as `postgres`, `standby` and `pooler` user. + +The interval of days can be set with `password_rotation_interval` (default +`90` = 90 days, minimum 1). On each rotation the user name and password values +are replaced in the K8s secret. They belong to a newly created user named after +the original role plus rotation date in YYMMDD format. All privileges are +inherited meaning that migration scripts should still grant and revoke rights +against the original role. The timestamp of the next rotation (in RFC 3339 +format, UTC timezone) is written to the secret as well. Note, if the rotation +interval is decreased it is reflected in the secrets only if the next rotation +date is more days away than the new length of the interval. + +Pods still using the previous secret values which they keep in memory continue +to connect to the database since the password of the corresponding user is not +replaced. However, a retention policy can be configured for users created by +the password rotation feature with `password_rotation_user_retention`. The +operator will ensure that this period is at least twice as long as the +configured rotation interval, hence the default of `180` = 180 days. When +the creation date of a rotated user is older than the retention period it +might not get removed immediately. Only on the next user rotation it is checked +if users can get removed. Therefore, you might want to configure the retention +to be a multiple of the rotation interval. + +### Password rotation for single users + +From the configuration, password rotation is enabled for all secrets with the +mentioned exceptions. If you wish to first test rotation for a single user (or +just have it enabled only for a few secrets) you can specify it in the cluster +manifest. The rotation and retention intervals can only be configured globally. + +``` +spec: + usersWithSecretRotation: + - foo_user + - bar_reader_user +``` + +### Password replacement without extra users + +For some use cases where the secret is only used rarely - think of a `flyway` +user running a migration script on pod start - we do not need to create extra +database users but can replace only the password in the K8s secret. This type +of rotation cannot be configured globally but specified in the cluster +manifest: + +``` +spec: + usersWithInPlaceSecretRotation: + - flyway + - bar_owner_user +``` + +This would be the recommended option to enable rotation in secrets of database +owners, but only if they are not used as application users for regular read +and write operations. + +### Ignore rotation for certain users + +If you wish to globally enable password rotation but need certain users to +opt out from it there are two ways. First, you can remove the user from the +manifest's `users` section. The corresponding secret to this user will no +longer be synced by the operator then. + +Secondly, if you want the operator to continue syncing the secret (e.g. to +recreate if it got accidentally removed) but cannot allow it being rotated, +add the user to the following list in your manifest: + +``` +spec: + usersIgnoringSecretRotation: + - bar_user +``` + +### Turning off password rotation + +When password rotation is turned off again the operator will check if the +`username` value in the secret matches the original username and replace it +with the latter. A new password is assigned and the `nextRotation` field is +cleared. A final lookup for child (rotation) users to be removed is done but +they will only be dropped if the retention policy allows for it. This is to +avoid sudden connection issues in pods which still use credentials of these +users in memory. You have to remove these child users manually or re-enable +password rotation with smaller interval so they get cleaned up. + ## Use taints and tolerations for dedicated PostgreSQL nodes To ensure Postgres pods are running on nodes without any other application pods, @@ -312,6 +515,81 @@ master pods from being evicted by the K8s runtime. To prevent eviction completely, specify the toleration by leaving out the `tolerationSeconds` value (similar to how Kubernetes' own DaemonSets are configured) +## Node readiness labels + +The operator can watch on certain node labels to detect e.g. the start of a +Kubernetes cluster upgrade procedure and move master pods off the nodes to be +decommissioned. Key-value pairs for these node readiness labels can be +specified in the configuration (option name is in singular form): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-operator +data: + node_readiness_label: "status1:ready,status2:ready" +``` + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-configuration +configuration: + kubernetes: + node_readiness_label: + status1: ready + status2: ready +``` + +The operator will create a `nodeAffinity` on the pods. This makes the +`node_readiness_label` option the global configuration for defining node +affinities for all Postgres clusters. You can have both, cluster-specific and +global affinity, defined and they will get merged on the pods. If +`node_readiness_label_merge` is configured to `"AND"` the node readiness +affinity will end up under the same `matchExpressions` section(s) from the +manifest affinity. + +```yaml + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: environment + operator: In + values: + - pci + - key: status1 + operator: In + values: + - ready + - key: status2 + ... +``` + +If `node_readiness_label_merge` is set to `"OR"` (default) the readiness label +affinity will be appended with its own expressions block: + +```yaml + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: environment + ... + - matchExpressions: + - key: storage + ... + - matchExpressions: + - key: status1 + ... + - key: status2 + ... +``` + ## Enable pod anti affinity To ensure Postgres pods are running on different topologies, you can use @@ -341,26 +619,41 @@ configuration: enable_pod_antiaffinity: true ``` +By default the type of pod anti affinity is `requiredDuringSchedulingIgnoredDuringExecution`, +you can switch to `preferredDuringSchedulingIgnoredDuringExecution` by setting `pod_antiaffinity_preferred_during_scheduling: true`. + By default the topology key for the pod anti affinity is set to `kubernetes.io/hostname`, you can set another topology key e.g. `failure-domain.beta.kubernetes.io/zone`. See [built-in node labels](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#interlude-built-in-node-labels) for available topology keys. -## Pod Disruption Budget +## Pod Disruption Budgets -By default the operator uses a PodDisruptionBudget (PDB) to protect the cluster -from voluntarily disruptions and hence unwanted DB downtime. The `MinAvailable` -parameter of the PDB is set to `1` which prevents killing masters in single-node -clusters and/or the last remaining running instance in a multi-node cluster. +By default the operator creates two PodDisruptionBudgets (PDB) to protect the cluster +from voluntarily disruptions and hence unwanted DB downtime: so-called primary PDB and +and PDB for critical operations. + +### Primary PDB +The `MinAvailable` parameter of this PDB is set to `1` and, if `pdb_master_label_selector` +is enabled, label selector includes `spilo-role=master` condition, which prevents killing +masters in single-node clusters and/or the last remaining running instance in a multi-node +cluster. + +## PDB for critical operations +The `MinAvailable` parameter of this PDB is equal to the `numberOfInstances` set in the +cluster manifest, while label selector includes `critical-operation=true` condition. This +allows to protect all pods of a cluster, given they are labeled accordingly. +For example, Operator labels all Spilo pods with `critical-operation=true` during the major +version upgrade run. You may want to protect cluster pods during other critical operations +by assigning the label to pods yourself or using other means of automation. The PDB is only relaxed in two scenarios: * If a cluster is scaled down to `0` instances (e.g. for draining nodes) * If the PDB is disabled in the configuration (`enable_pod_disruption_budget`) -The PDB is still in place having `MinAvailable` set to `0`. If enabled it will -be automatically set to `1` on scale up. Disabling PDBs helps avoiding blocking -Kubernetes upgrades in managed K8s environments at the cost of prolonged DB -downtime. See PR [#384](https://github.com/zalando/postgres-operator/pull/384) +The PDBs are still in place having `MinAvailable` set to `0`. Disabling PDBs +helps avoiding blocking Kubernetes upgrades in managed K8s environments at the +cost of prolonged DB downtime. See PR [#384](https://github.com/zalando/postgres-operator/pull/384) for the use case. ## Add cluster-specific labels @@ -426,15 +719,39 @@ spec: ## Custom Pod Environment Variables -It is possible to configure a ConfigMap as well as a Secret which are used by -the Postgres pods as an additional provider for environment variables. One use -case is a customized Spilo image configured by extra environment variables. -Another case could be to provide custom cloud provider or backup settings. +The operator will assign a set of environment variables to the database pods +that cannot be overridden to guarantee core functionality. Only variables with +'WAL_' and 'LOG_' prefixes can be customized to allow for backup and log +shipping to be specified differently. There are three ways to specify extra +environment variables (or override existing ones) for database pods: -In general the Operator will give preference to the globally configured -variables, to not have the custom ones interfere with core functionality. -Variables with the 'WAL_' and 'LOG_' prefix can be overwritten though, to -allow backup and log shipping to be specified differently. +* [Via ConfigMap](#via-configmap) +* [Via Secret](#via-secret) +* [Via Postgres Cluster Manifest](#via-postgres-cluster-manifest) + +The first two options must be referenced from the operator configuration +making them global settings for all Postgres cluster the operator watches. +One use case is a customized Spilo image that must be configured by extra +environment variables. Another case could be to provide custom cloud +provider or backup settings. + +The last options allows for specifying environment variables individual to +every cluster via the `env` section in the manifest. For example, if you use +individual backup locations for each of your clusters. Or you want to disable +WAL archiving for a certain cluster by setting `WAL_S3_BUCKET`, `WAL_GS_BUCKET` +or `AZURE_STORAGE_ACCOUNT` to an empty string. + +The operator will give precedence to environment variables in the following +order (e.g. a variable defined in 4. overrides a variable with the same name +in 5.): + +1. Assigned by the operator +2. `env` section in cluster manifest +3. Clone section (with WAL settings from operator config when `s3_wal_path` is empty) +4. Standby section +5. Pod environment secret via operator config +6. Pod environment config map via operator config +7. WAL and logical backup settings from operator config ### Via ConfigMap @@ -531,6 +848,29 @@ data: The key-value pairs of the Secret are all accessible as environment variables to the Postgres StatefulSet/pods. +### Via Postgres Cluster Manifest + +It is possible to define environment variables directly in the Postgres cluster +manifest to configure it individually. The variables must be listed under the +`env` section in the same way you would do for [containers](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/). +Global parameters served from a custom config map or secret will be overridden. + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-test-cluster +spec: + env: + - name: wal_s3_bucket + value: my-custom-bucket + - name: minio_secret_key + valueFrom: + secretKeyRef: + name: my-custom-secret + key: minio_secret_key +``` + ## Limiting the number of min and max instances in clusters As a preventive measure, one can restrict the minimum and the maximum number of @@ -551,17 +891,22 @@ cluster manifest. In the case any of these variables are omitted from the manifest, the operator configuration settings `enable_master_load_balancer` and `enable_replica_load_balancer` apply. Note that the operator settings affect all Postgresql services running in all namespaces watched by the operator. -If load balancing is enabled two default annotations will be applied to its -services: +If load balancing is enabled the following default annotation will be applied to +its services: - `external-dns.alpha.kubernetes.io/hostname` with the value defined by the operator configs `master_dns_name_format` and `replica_dns_name_format`. This value can't be overwritten. If any changing in its value is needed, it MUST be done changing the DNS format operator config parameters; and -- `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` with - a default value of "3600". This value can be overwritten with the operator - config parameter `custom_service_annotations` or the cluster parameter - `serviceAnnotations`. + +There are multiple options to specify service annotations that will be merged +with each other and override in the following order (where latter take +precedence): + +1. Default annotations if LoadBalancer is enabled +2. Globally configured `custom_service_annotations` +3. `serviceAnnotations` specified in the cluster manifest +4. `masterServiceAnnotations` and `replicaServiceAnnotations` specified in the cluster manifest To limit the range of IP addresses that can reach a load balancer, specify the desired ranges in the `allowedSourceRanges` field (applies to both master and @@ -575,6 +920,49 @@ lead to K8s removing this field from the manifest due to its Then the resultant manifest will not contain the necessary change, and the operator will respectively do nothing with the existing source ranges. +Load balancer services can also be enabled for the [connection pooler](user.md#connection-pooler) +pods with manifest flags `enableMasterPoolerLoadBalancer` and/or +`enableReplicaPoolerLoadBalancer` or in the operator configuration with +`enable_master_pooler_load_balancer` and/or `enable_replica_pooler_load_balancer`. +For the `external-dns.alpha.kubernetes.io/hostname` annotation the `-pooler` +suffix will be appended to the cluster name used in the template which is +defined in `master|replica_dns_name_format`. + +## Node Ports + +Alternatively to Load Balancers Node Ports can be used. Kubernetes services with type +`NodePort` redirect traffic from a specified port on your kubernetes nodes to your service. +To expose your services to an external network with NodePorts you can set `enableMasterNodePort` and/or `enableReplicaNodePort` to `true` +in your cluster manifest. In the case any of these variables are omitted from the manifest, the operator configuration settings `enable_master_node_port` and `enable_replica_node_port` apply. +Note that the operator settings affect all Postgresql services running in all namespaces watched +by the operator. + +**Enabling a NodePort configuration will override the corresponding LoadBalancer configuration.** + +There are multiple options to specify service annotations that will be merged +with each other and override in the following order (where latter take +precedence): + +1. Globally configured `custom_service_annotations` +2. `serviceAnnotations` specified in the cluster manifest +3. `masterServiceAnnotations` and `replicaServiceAnnotations` specified in the cluster manifest + +Load-Balancer specific annotations are not applied. + +Node port services can also be configured for the [connection pooler](user.md#connection-pooler) pods +with the manifest flags `enableMasterPoolerNodePort` and/or `enableReplicaPoolerNodePort` or in the operator configuration with `enable_master_pooler_node_port` +and/or `enable_replica_pooler_node_port`. + +To configure which ports Kubernetes should use for your NodePort service you can configure ports in your cluster manifest +for each type: + +- masterNodePort +- masterPoolerNodePort +- replicaNodePort +- replicaPoolerNodePort + +When not defined or set to 0 kubernetes will choose a port for you from [your kubernetes cluster's configured range](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport). + ## Running periodic 'autorepair' scans of K8s objects The Postgres Operator periodically scans all K8s objects belonging to each @@ -663,6 +1051,12 @@ if it ends up in your specified WAL backup path: envdir "/run/etc/wal-e.d/env" /scripts/postgres_backup.sh "/home/postgres/pgdata/pgroot/data" ``` +You can also check if Spilo is able to find any backups: + +```bash +envdir "/run/etc/wal-e.d/env" wal-g backup-list +``` + Depending on the cloud storage provider different [environment variables](https://github.com/zalando/spilo/blob/master/ENVIRONMENT.rst) have to be set for Spilo. Not all of them are generated automatically by the operator by changing its configuration. In this case you have to use an @@ -730,14 +1124,99 @@ WALE_S3_ENDPOINT='https+path://s3.eu-central-1.amazonaws.com:443' WALE_S3_PREFIX=$WAL_S3_BUCKET/spilo/{WAL_BUCKET_SCOPE_PREFIX}{SCOPE}{WAL_BUCKET_SCOPE_SUFFIX}/wal/{PGVERSION} ``` -If the prefix is not specified Spilo will generate it from `WAL_S3_BUCKET`. -When the `AWS_REGION` is set `AWS_ENDPOINT` and `WALE_S3_ENDPOINT` are +The operator sets the prefix to an empty string so that spilo will generate it +from the configured `WAL_S3_BUCKET`. + +:warning: When you overwrite the configuration by defining `WAL_S3_BUCKET` in +the [pod_environment_configmap](#custom-pod-environment-variables) you have +to set `WAL_BUCKET_SCOPE_PREFIX = ""`, too. Otherwise Spilo will not find +the physical backups on restore (next chapter). + +When the `AWS_REGION` is set, `AWS_ENDPOINT` and `WALE_S3_ENDPOINT` are generated automatically. `WALG_S3_PREFIX` is identical to `WALE_S3_PREFIX`. `SCOPE` is the Postgres cluster name. +:warning: If both `AWS_REGION` and `AWS_ENDPOINT` or `WALE_S3_ENDPOINT` are +defined backups with WAL-E will fail. You can fix it by switching to WAL-G +with `USE_WALG_BACKUP: "true"`. + ### Google Cloud Platform setup -To configure the operator on GCP these prerequisites that are needed: +When using GCP, there are two authentication methods to allow the postgres +cluster to access buckets to write WAL-E logs: Workload Identity (recommended) +or using a GCP Service Account Key (legacy). + +#### Workload Identity setup + +To configure the operator on GCP using Workload Identity these prerequisites are +needed. + +* [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) enabled on the GKE cluster where the operator will be deployed +* A GCP service account with the proper IAM setup to access the GCS bucket for the WAL-E logs +* An IAM policy granting the Kubernetes service account the + `roles/iam.workloadIdentityUser` role on the GCP service account, e.g.: +```bash +gcloud iam service-accounts add-iam-policy-binding @.iam.gserviceaccount.com \ + --role roles/iam.workloadIdentityUser \ + --member "serviceAccount:PROJECT_ID.svc.id.goog[/postgres-pod-custom]" +``` + +The configuration parameters that we will be using are: + +* `wal_gs_bucket` + +1. Create a custom Kubernetes service account to be used by Patroni running on +the postgres cluster pods, this service account should include an annotation +with the email address of the Google IAM service account used to communicate +with the GCS bucket, e.g. + +```yml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: postgres-pod-custom + namespace: + annotations: + iam.gke.io/gcp-service-account: @.iam.gserviceaccount.com +``` + +2. Specify the new custom service account in your [operator parameters](./reference/operator_parameters.md) + +If using manual deployment or kustomize, this is done by setting +`pod_service_account_name` in your configuration file specified in the +[postgres-operator deployment](../manifests/postgres-operator.yaml#L37) + +If deploying the operator [using Helm](./quickstart.md#helm-chart), this can +be specified in the chart's values file, e.g.: + +```yml +... +podServiceAccount: + name: postgres-pod-custom +``` + +3. Setup your operator configuration values. Ensure that the operator's configuration +is set up like the following: +```yml +... +aws_or_gcp: + # additional_secret_mount: "" + # additional_secret_mount_path: "" + # aws_region: eu-central-1 + # kube_iam_role: "" + # log_s3_bucket: "" + # wal_s3_bucket: "" + wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs + # gcp_credentials: "" +... +``` + +Continue to shared steps below. + +#### GCP Service Account Key setup + +To configure the operator on GCP using a GCP service account key these +prerequisites are needed. * A service account with the proper IAM setup to access the GCS bucket for the WAL-E logs * The credentials file for the service account. @@ -770,7 +1249,7 @@ is set up like the following: ```yml ... aws_or_gcp: - additional_secret_mount: "pgsql-wale-creds" + additional_secret_mount: "psql-wale-creds" additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file # aws_region: eu-central-1 # kube_iam_role: "" @@ -781,7 +1260,10 @@ aws_or_gcp: ... ``` -3. Setup pod environment configmap that instructs the operator to use WAL-G, +Once you have set up authentication using one of the two methods above, continue +with the remaining shared steps: + +1. Setup pod environment configmap that instructs the operator to use WAL-G, instead of WAL-E, for backup and restore. ```yml apiVersion: v1 @@ -796,7 +1278,7 @@ data: CLONE_USE_WALG_RESTORE: "true" ``` -4. Then provide this configmap in postgres-operator settings: +2. Then provide this configmap in postgres-operator settings: ```yml ... # namespaced name of the ConfigMap with environment variables to populate on every pod @@ -804,15 +1286,110 @@ pod_environment_configmap: "postgres-operator-system/pod-env-overrides" ... ``` +### Azure setup + +To configure the operator on Azure these prerequisites are needed: + +* A storage account in the same region as the Kubernetes cluster. + +The configuration parameters that we will be using are: + +* `pod_environment_secret` +* `wal_az_storage_account` + +1. Generate the K8s secret resource that will contain your storage account's +access key. You will need a copy of this secret in every namespace you want to +create postgresql clusters. + +The latest version of WAL-G (v1.0) supports the use of a SASS token, but you'll +have to make due with using the primary or secondary access token until the +version of WAL-G is updated in the postgres-operator. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: psql-backup-creds + namespace: default +type: Opaque +stringData: + AZURE_STORAGE_ACCESS_KEY: +``` + +2. Setup pod environment configmap that instructs the operator to use WAL-G, +instead of WAL-E, for backup and restore. +```yml +apiVersion: v1 +kind: ConfigMap +metadata: + name: pod-env-overrides + namespace: postgres-operator-system +data: + # Any env variable used by spilo can be added + USE_WALG_BACKUP: "true" + USE_WALG_RESTORE: "true" + CLONE_USE_WALG_RESTORE: "true" + WALG_AZ_PREFIX: "azure://container-name/$(SCOPE)/$(PGVERSION)" # Enables Azure Backups (SCOPE = Cluster name) (PGVERSION = Postgres version) +``` + +3. Setup your operator configuration values. With the `psql-backup-creds` +and `pod-env-overrides` resources applied to your cluster, ensure that the operator's configuration +is set up like the following: +```yml +... +kubernetes: + pod_environment_secret: "psql-backup-creds" + pod_environment_configmap: "postgres-operator-system/pod-env-overrides" +aws_or_gcp: + wal_az_storage_account: "postgresbackupsbucket28302F2" # name of storage account to save the WAL-G logs +... +``` + ### Restoring physical backups If cluster members have to be (re)initialized restoring physical backups happens automatically either from the backup location or by running -[pg_basebackup](https://www.postgresql.org/docs/13/app-pgbasebackup.html) +[pg_basebackup](https://www.postgresql.org/docs/18/app-pgbasebackup.html) on one of the other running instances (preferably replicas if they do not lag behind). You can test restoring backups by [cloning](user.md#how-to-clone-an-existing-postgresql-cluster) clusters. +If you need to provide a [custom clone environment](#custom-pod-environment-variables) +copy existing variables about your setup (backup location, prefix, access +keys etc.) and prepend the `CLONE_` prefix to get them copied to the correct +directory within Spilo. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-pod-config +data: + AWS_REGION: "eu-west-1" + AWS_ACCESS_KEY_ID: "****" + AWS_SECRET_ACCESS_KEY: "****" + ... + CLONE_AWS_REGION: "eu-west-1" + CLONE_AWS_ACCESS_KEY_ID: "****" + CLONE_AWS_SECRET_ACCESS_KEY: "****" + ... +``` + +### Standby clusters + +The setup for [standby clusters](user.md#setting-up-a-standby-cluster) is +similar to cloning when they stream changes from a WAL archive (S3 or GCS). +If you are using [additional environment variables](#custom-pod-environment-variables) +to access your backup location you have to copy those variables and prepend +the `STANDBY_` prefix for Spilo to find the backups and WAL files to stream. + +Standby clusters can also stream from a remote primary cluster. +You have to specify the host address. Port is optional and defaults to 5432. +You can combine `standby_host` with either `s3_wal_path` or `gs_wal_path` +for additional redundancy. Note that `s3_wal_path` and `gs_wal_path` are +mutually exclusive. At least one of `s3_wal_path`, `gs_wal_path`, or +`standby_host` must be specified under the `standby` top-level key. + ## Logical backups The operator can manage K8s cron jobs to run logical backups (SQL dumps) of @@ -840,7 +1417,7 @@ but only snapshots of your data. In its current state, see logical backups as a way to quickly create SQL dumps that you can easily restore in an empty test cluster. -2. The [example image](../docker/logical-backup/Dockerfile) implements the backup +2. The [example image](https://github.com/zalando/postgres-operator/blob/master/logical-backup/Dockerfile) implements the backup via `pg_dumpall` and upload of compressed and encrypted results to an S3 bucket. `pg_dumpall` requires a `superuser` access to a DB and runs on the replica when possible. @@ -859,7 +1436,11 @@ of the backup cron job. 6. For that feature to work, your RBAC policy must enable operations on the `cronjobs` resource from the `batch` API group for the operator service account. -See [example RBAC](../manifests/operator-service-account-rbac.yaml) +See [example RBAC](https://github.com/zalando/postgres-operator/blob/master/manifests/operator-service-account-rbac.yaml) + +7. Resources of the pod template in the cron job can be configured. When left +empty [default values of spilo pods](reference/operator_parameters.md#kubernetes-resource-requests) +will be used. ## Sidecars for Postgres clusters @@ -878,6 +1459,10 @@ configuration: volumeMounts: - mountPath: /custom-pgdata-mountpoint name: pgdata + env: + - name: "ENV_VAR_NAME" + value: "any-k8s-env-things" + command: ['sh', '-c', 'echo "logging" > /opt/logs.txt'] - ... ``` @@ -913,11 +1498,13 @@ default. Alternatively, a list can also be passed when starting the Python application with the `--cluster` option. The Operator API endpoint can be configured via the `OPERATOR_API_URL` -environment variables in the [deployment manifest](../ui/manifests/deployment.yaml#L40). -You can also expose the operator API through a [service](../manifests/api-service.yaml). +environment variables in the [deployment manifest](https://github.com/zalando/postgres-operator/blob/master/ui/manifests/deployment.yaml#L40). +You can also expose the operator API through a [service](https://github.com/zalando/postgres-operator/blob/master/manifests/api-service.yaml). Some displayed options can be disabled from UI using simple flags under the `OPERATOR_UI_CONFIG` field in the deployment. +The viewing and creation of clusters within the UI is limited to the namespace specified by the `TARGET_NAMESPACE` option. To allow the creation and viewing of clusters in all namespaces, set `TARGET_NAMESPACE` to `*`. + ### Deploy the UI on K8s Now, apply all manifests from the `ui/manifests` folder to deploy the Postgres @@ -950,7 +1537,7 @@ make docker # build in image in minikube docker env eval $(minikube docker-env) -docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.6.3 . +docker build -t ghcr.io/zalando/postgres-operator-ui:v1.15.1 . # apply UI manifests next to a running Postgres Operator kubectl apply -f manifests/ diff --git a/docs/developer.md b/docs/developer.md index a9b3b143e..141ee63de 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -7,7 +7,7 @@ features and tests. Postgres Operator is written in Go. Use the [installation instructions](https://golang.org/doc/install#install) if you don't have Go on your system. You won't be able to compile the operator -with Go older than 1.15. We recommend installing [the latest one](https://golang.org/dl/). +with Go older than 1.17. We recommend installing [the latest one](https://golang.org/dl/). Go projects expect their source code and all the dependencies to be located under the [GOPATH](https://github.com/golang/go/wiki/GOPATH). Normally, one @@ -16,7 +16,7 @@ under the ~/go/src sub directories. Given the schema above, the Postgres Operator source code located at `github.com/zalando/postgres-operator` should be put at --`~/go/src/github.com/zalando/postgres-operator`. +`~/go/src/github.com/zalando/postgres-operator`. ```bash export GOPATH=~/go @@ -33,17 +33,14 @@ by setting the `GO111MODULE` environment variable to `on`. The make targets do this for you, so simply run ```bash -make deps +make ``` -This would take a while to complete. You have to redo `make deps` every time -your dependencies list changes, i.e. after adding a new library dependency. - Build the operator with the `make docker` command. You may define the TAG variable to assign an explicit tag to your Docker image and the IMAGE to set the image name. By default, the tag is computed with `git describe --tags --always --dirty` and the image is -`registry.opensource.zalan.do/acid/postgres-operator` +`ghcr.io/zalando/postgres-operator` ```bash export TAG=$(git describe --tags --always --dirty) @@ -72,22 +69,32 @@ make docker # kind make docker -kind load docker-image --name +kind load docker-image ghcr.io/zalando/postgres-operator:${TAG} --name ``` -Then create a new Postgres Operator deployment. You can reuse the provided -manifest but replace the version and tag. Don't forget to also apply +Then create a new Postgres Operator deployment. + +### Deploying manually with manifests and kubectl + +You can reuse the provided manifest but replace the version and tag. Don't forget to also apply configuration and RBAC manifests first, e.g.: ```bash kubectl create -f manifests/configmap.yaml kubectl create -f manifests/operator-service-account-rbac.yaml -sed -e "s/\(image\:.*\:\).*$/\1$TAG/" manifests/postgres-operator.yaml | kubectl create -f - +sed -e "s/\(image\:.*\:\).*$/\1$TAG/" -e "s/\(imagePullPolicy\:\).*$/\1 Never/" manifests/postgres-operator.yaml | kubectl create -f - # check if the operator is coming up kubectl get pod -l name=postgres-operator ``` +### Deploying with Helm chart + +Yoy can reuse the provided Helm chart to deploy local operator build with the following command: +```bash +helm install postgres-operator ./charts/postgres-operator --namespace zalando-operator --set image.tag=${TAG} --set image.pullPolicy=Never +``` + ## Code generation The operator employs K8s-provided code generation to obtain deep copy methods @@ -95,6 +102,7 @@ and K8s-like APIs for its custom resource definitions, namely the Postgres CRD and the operator CRD. The usage of the code generation follows conventions from the K8s community. Relevant scripts live in the `hack` directory: + * `update-codegen.sh` triggers code generation for the APIs defined in `pkg/apis/acid.zalan.do/`, * `verify-codegen.sh` checks if the generated code is up-to-date (to be used within CI). @@ -102,6 +110,7 @@ The `/pkg/generated/` contains the resultant code. To make these scripts work, you may need to `export GOPATH=$(go env GOPATH)` References for code generation are: + * [Relevant pull request](https://github.com/zalando/postgres-operator/pull/369) See comments there for minor issues that can sometimes broke the generation process. * [Code generator source code](https://github.com/kubernetes/code-generator) @@ -208,10 +217,16 @@ dlv connect 127.0.0.1:DLV_PORT ## Unit tests +Prerequisites: + +```bash +make mocks +``` + To run all unit tests, you can simply do: ```bash -go test ./pkg/... +make test ``` In case if you need to debug your unit test, it's possible to use delve: @@ -257,17 +272,17 @@ Examples for fake K8s objects can be found in: The operator provides reference end-to-end (e2e) tests to ensure various infrastructure parts work smoothly together. The test code is available at `e2e/tests`. -The special `registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner` image is used to run the tests. The container mounts the local `e2e/tests` directory at runtime, so whatever you modify in your local copy of the tests will be executed by a test runner. By maintaining a separate test runner image we avoid the need to re-build the e2e test image on every build. +The special `ghcr.io/zalando/postgres-operator-e2e-tests-runner` image is used to run the tests. The container mounts the local `e2e/tests` directory at runtime, so whatever you modify in your local copy of the tests will be executed by a test runner. By maintaining a separate test runner image we avoid the need to re-build the e2e test image on every build. -Each e2e execution tests a Postgres Operator image built from the current git branch. The test -runner creates a new local K8s cluster using [kind](https://kind.sigs.k8s.io/), +Each e2e execution tests a Postgres Operator image built from the current git branch. +The test runner creates a new local K8s cluster using [kind](https://kind.sigs.k8s.io/), utilizes provided manifest examples, and runs e2e tests contained in the `tests` folder. The K8s API client in the container connects to the `kind` cluster via the standard Docker `bridge` network. The kind cluster is deleted if tests finish successfully or on each new run in case it still exists. End-to-end tests are executed automatically during builds (for more details, -see the [README](../e2e/README.md) in the `e2e` folder): +see the [README](https://github.com/zalando/postgres-operator/blob/master/e2e/README.md) in the `e2e` folder): ```bash make e2e @@ -291,40 +306,44 @@ parameters (with exceptions for certain Patroni/Postgres options) and variables if you feel a per-cluster configuration is necessary. Note: If one option is defined in the operator configuration and in the cluster -[manifest](../manifests/complete-postgres-manifest.yaml), the latter takes +[manifest](https://github.com/zalando/postgres-operator/blob/master/manifests/complete-postgres-manifest.yaml), the latter takes precedence. ### Go code Update the following Go files that obtain the configuration parameter from the manifest files: -* [operator_configuration_type.go](../pkg/apis/acid.zalan.do/v1/operator_configuration_type.go) -* [operator_config.go](../pkg/controller/operator_config.go) -* [config.go](../pkg/util/config/config.go) -Postgres manifest parameters are defined in the [api package](../pkg/apis/acid.zalan.do/v1/postgresql_type.go). -The operator behavior has to be implemented at least in [k8sres.go](../pkg/cluster/k8sres.go). -Validation of CRD parameters is controlled in [crds.go](../pkg/apis/acid.zalan.do/v1/crds.go). +* [operator_configuration_type.go](https://github.com/zalando/postgres-operator/blob/master/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go) +* [operator_config.go](https://github.com/zalando/postgres-operator/blob/master/pkg/controller/operator_config.go) +* [config.go](https://github.com/zalando/postgres-operator/blob/master/pkg/util/config/config.go) + +Postgres manifest parameters are defined in the [api package](https://github.com/zalando/postgres-operator/blob/master/pkg/apis/acid.zalan.do/v1/postgresql_type.go). +The operator behavior has to be implemented at least in [k8sres.go](https://github.com/zalando/postgres-operator/blob/master/pkg/cluster/k8sres.go). +Validation of CRD parameters is controlled in [crds.go](https://github.com/zalando/postgres-operator/blob/master/pkg/apis/acid.zalan.do/v1/crds.go). Please, reflect your changes in tests, for example in: -* [config_test.go](../pkg/util/config/config_test.go) -* [k8sres_test.go](../pkg/cluster/k8sres_test.go) -* [util_test.go](../pkg/apis/acid.zalan.do/v1/util_test.go) + +* [config_test.go](https://github.com/zalando/postgres-operator/blob/master/pkg/util/config/config_test.go) +* [k8sres_test.go](https://github.com/zalando/postgres-operator/blob/master/pkg/cluster/k8sres_test.go) +* [util_test.go](https://github.com/zalando/postgres-operator/blob/master/pkg/apis/acid.zalan.do/v1/util_test.go) ### Updating manifest files For the CRD-based configuration, please update the following files: -* the default [OperatorConfiguration](../manifests/postgresql-operator-default-configuration.yaml) -* the CRD's [validation](../manifests/operatorconfiguration.crd.yaml) -* the CRD's validation in the [Helm chart](../charts/postgres-operator/crds/operatorconfigurations.yaml) -Add new options also to the Helm chart's [values file](../charts/postgres-operator/values.yaml) file. +* the default [OperatorConfiguration](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql-operator-default-configuration.yaml) +* the CRD's [validation](https://github.com/zalando/postgres-operator/blob/master/manifests/operatorconfiguration.crd.yaml) +* the CRD's validation in the [Helm chart](https://github.com/zalando/postgres-operator/blob/master/charts/postgres-operator/crds/operatorconfigurations.yaml) + +Add new options also to the Helm chart's [values file](https://github.com/zalando/postgres-operator/blob/master/charts/postgres-operator/values.yaml) file. It follows the OperatorConfiguration CRD layout. Nested values will be flattened for the ConfigMap. -Last but no least, update the [ConfigMap](../manifests/configmap.yaml) manifest example as well. +Last but no least, update the [ConfigMap](https://github.com/zalando/postgres-operator/blob/master/manifests/configmap.yaml) manifest example as well. ### Updating documentation Finally, add a section for each new configuration option and/or cluster manifest parameter in the reference documents: + * [config reference](reference/operator_parameters.md) * [manifest reference](reference/cluster_manifest.md) diff --git a/docs/diagrams/neutral_operator_dark.png b/docs/diagrams/neutral_operator_dark.png new file mode 100644 index 000000000..f5eb3cb83 Binary files /dev/null and b/docs/diagrams/neutral_operator_dark.png differ diff --git a/docs/diagrams/neutral_operator_light.png b/docs/diagrams/neutral_operator_light.png new file mode 100644 index 000000000..229874faf Binary files /dev/null and b/docs/diagrams/neutral_operator_light.png differ diff --git a/docs/index.md b/docs/index.md index d0b4e4940..1aeac0ccb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,10 +8,10 @@ manages PostgreSQL clusters on Kubernetes (K8s): user submits a new manifest, the operator fetches that manifest and spawns a new Postgres cluster along with all necessary entities such as K8s StatefulSets and Postgres roles. See this - [Postgres cluster manifest](../manifests/complete-postgres-manifest.yaml) + [Postgres cluster manifest](https://github.com/zalando/postgres-operator/blob/master/manifests/complete-postgres-manifest.yaml) for settings that a manifest may contain. -2. The operator also watches updates to [its own configuration](../manifests/configmap.yaml) +2. The operator also watches updates to [its own configuration](https://github.com/zalando/postgres-operator/blob/master/manifests/configmap.yaml) and alters running Postgres clusters if necessary. For instance, if the Docker image in a pod is changed, the operator carries out the rolling update, which means it re-spawns pods of each managed StatefulSet one-by-one @@ -71,6 +71,8 @@ Please, report any issues discovered to https://github.com/zalando/postgres-oper ## Talks +- "Watching after your PostGIS herd" talk by Felix Kunde, FOSS4G 2021: [video](https://www.youtube.com/watch?v=T96FvjSv98A) | [slides](https://docs.google.com/presentation/d/1IICz2RsjNAcosKVGFna7io-65T2zBbGcBHFFtca24cc/edit?usp=sharing) + - "PostgreSQL on K8S at Zalando: Two years in production" talk by Alexander Kukushkin, FOSSDEM 2020: [video](https://fosdem.org/2020/schedule/event/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/) | [slides](https://fosdem.org/2020/schedule/event/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/attachments/slides/3883/export/events/attachments/postgresql_postgresql_on_k8s_at_zalando_two_years_in_production/slides/3883/PostgreSQL_on_K8s_at_Zalando_Two_years_in_production.pdf) - "Postgres as a Service at Zalando" talk by Jan Mußler, DevOpsDays Poznań 2019: [video](https://www.youtube.com/watch?v=FiWS5m72XI8) @@ -87,10 +89,20 @@ Please, report any issues discovered to https://github.com/zalando/postgres-oper ## Posts +- Series of blog posts on how to use the Zalando Operator, configure backups and use etcd as DCS by [thedatabaseme](https://thedatabaseme.de/tag/zalando-operator/), Mar. 2022-23. + +- "Zalando Postgres Operator in Production: the way of Helm" by Zangir Kapishov on [medium](https://medium.com/@zkapishov/zalando-postgres-operator-in-production-the-way-of-helm-ccfd639ccb2d), Jan. 2023. + +- "Chaos testing of a Postgres cluster managed by the Zalando Postgres Operator" by Nikolay Sivko on [coroot](https://coroot.com/blog/chaos-testing-zalando-postgres-operator), Aug. 2022. + +- "Getting started with the Zalando Operator for PostgreSQL" by Daniel Westermann on [dbi services blog](https://www.dbi-services.com/blog/getting-started-with-the-zalando-operator-for-postgresql/), Mar. 2021. + +- "Our experience with Postgres Operator for Kubernetes by Zalando" by Nikolay Bogdanov on [Palark blog](https://blog.palark.com/our-experience-with-postgres-operator-for-kubernetes-by-zalando/), Feb. 2021. + - "How to set up continuous backups and monitoring" by Pål Kristensen on [GitHub](https://github.com/zalando/postgres-operator/issues/858#issuecomment-608136253), Mar. 2020. - "Postgres on Kubernetes with the Zalando operator" by Vito Botta on [has_many :code](https://vitobotta.com/2020/02/05/postgres-kubernetes-zalando-operator/), Feb. 2020. -- "Running PostgreSQL in Google Kubernetes Engine" by Kenneth Rørvik on [Repill Linpro](https://www.redpill-linpro.com/techblog/2019/09/28/postgres-in-kubernetes.html), Sep. 2019. +- "Running PostgreSQL in Google Kubernetes Engine" by Kenneth Rørvik on [Repill Linpro blog](https://www.redpill-linpro.com/techblog/2019/09/28/postgres-in-kubernetes.html), Sep. 2019. - "Zalando Postgres Operator: One Year Later" by Sergey Dudoladov on [Open Source Zalando](https://opensource.zalando.com/blog/2018/11/postgres-operator/), Nov. 2018 diff --git a/docs/operator-ui.md b/docs/operator-ui.md index 7912e8264..69da2d231 100644 --- a/docs/operator-ui.md +++ b/docs/operator-ui.md @@ -33,8 +33,8 @@ status page. Usually, the startup should only take up to 1 minute. If you feel the process got stuck click on the "Logs" button to inspect the operator logs. If the logs look fine, but the UI seems to got stuck, check if you are have configured the -same [cluster name label](../ui/manifests/deployment.yaml#L45) like for the -[operator](../manifests/configmap.yaml#L13). +same [cluster name label](https://github.com/zalando/postgres-operator/blob/master/ui/manifests/deployment.yaml#L45) like for the +[operator](https://github.com/zalando/postgres-operator/blob/master/manifests/configmap.yaml#L13). From the "Status" field in the top menu you can also retrieve the logs and queue of each worker the operator is using. The number of concurrent workers can be diff --git a/docs/quickstart.md b/docs/quickstart.md index a90c90f42..fa16d1813 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -10,17 +10,17 @@ hence set it up first. For local tests we recommend to use one of the following solutions: * [minikube](https://github.com/kubernetes/minikube/releases), which creates a - single-node K8s cluster inside a VM (requires KVM or VirtualBox), + K8s cluster inside a container or VM (requires Docker, KVM, Hyper-V, HyperKit, VirtualBox, or similar), * [kind](https://kind.sigs.k8s.io/) and [k3d](https://k3d.io), which allows creating multi-nodes K8s clusters running on Docker (requires Docker) -To interact with the K8s infrastructure install it's CLI runtime [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl). +To interact with the K8s infrastructure install its CLI runtime [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-binary-via-curl). This quickstart assumes that you have started minikube or created a local kind cluster. Note that you can also use built-in K8s support in the Docker Desktop for Mac to follow the steps of this tutorial. You would have to replace `minikube start` and `minikube delete` with your launch actions for the Docker -built-in K8s support. +Desktop built-in K8s support. ## Configuration Options @@ -37,7 +37,7 @@ The Postgres Operator can be deployed in the following ways: * Kustomization * Helm chart -### Manual deployment setup +### Manual deployment setup on Kubernetes The Postgres Operator can be installed simply by applying yaml manifests. Note, we provide the `/manifests` directory as an example only; you should consider @@ -56,7 +56,7 @@ kubectl create -f manifests/api-service.yaml # operator API to be used by UI ``` There is a [Kustomization](https://github.com/kubernetes-sigs/kustomize) -manifest that [combines the mentioned resources](../manifests/kustomization.yaml) +manifest that [combines the mentioned resources](https://github.com/zalando/postgres-operator/blob/master/manifests/kustomization.yaml) (except for the CRD) - it can be used with kubectl 1.14 or newer as easy as: ```bash @@ -64,26 +64,45 @@ kubectl apply -k github.com/zalando/postgres-operator/manifests ``` For convenience, we have automated starting the operator with minikube using the -`run_operator_locally` script. It applies the [`acid-minimal-cluster`](../manifests/minimal-postgres-manifest.yaml). +`run_operator_locally` script. It applies the [`acid-minimal-cluster`](https://github.com/zalando/postgres-operator/blob/master/manifests/minimal-postgres-manifest.yaml). manifest. ```bash ./run_operator_locally.sh ``` -### Helm chart +### Manual deployment setup on OpenShift -Alternatively, the operator can be installed by using the provided [Helm](https://helm.sh/) -chart which saves you the manual steps. Clone this repo and change directory to -the repo root. With Helm v3 installed you should be able to run: +To install the Postgres Operator in OpenShift you have to change the config +parameter `kubernetes_use_configmaps` to `"true"`. Otherwise, the operator +and Patroni will store leader and config keys in `Endpoints` that are not +supported in OpenShift. This requires also a slightly different set of rules +for the `postgres-operator` and `postgres-pod` cluster roles. ```bash -helm install postgres-operator ./charts/postgres-operator +oc create -f manifests/operator-service-account-rbac-openshift.yaml ``` -The chart works with both Helm 2 and Helm 3. The `crd-install` hook from v2 will -be skipped with warning when using v3. Documentation for installing applications -with Helm 2 can be found in the [v2 docs](https://v2.helm.sh/docs/). +### Helm chart + +Alternatively, the operator can be installed by using the provided +[Helm](https://helm.sh/) chart which saves you the manual steps. The charts +for both the Postgres Operator and its UI are hosted via the `gh-pages` branch. +They only work only with Helm v3. Helm v2 support was dropped with v1.8.0. + +```bash +# add repo for postgres-operator +helm repo add postgres-operator-charts https://opensource.zalando.com/postgres-operator/charts/postgres-operator + +# install the postgres-operator +helm install postgres-operator postgres-operator-charts/postgres-operator + +# add repo for postgres-operator-ui +helm repo add postgres-operator-ui-charts https://opensource.zalando.com/postgres-operator/charts/postgres-operator-ui + +# install the postgres-operator-ui +helm install postgres-operator-ui postgres-operator-ui-charts/postgres-operator-ui +``` ## Check if Postgres Operator is running @@ -112,8 +131,8 @@ In the following paragraphs we describe how to access and manage PostgreSQL clusters from the command line with kubectl. But it can also be done from the browser-based [Postgres Operator UI](operator-ui.md). Before deploying the UI make sure the operator is running and its REST API is reachable through a -[K8s service](../manifests/api-service.yaml). The URL to this API must be -configured in the [deployment manifest](../ui/manifests/deployment.yaml#L43) +[K8s service](https://github.com/zalando/postgres-operator/blob/master/manifests/api-service.yaml). The URL to this API must be +configured in the [deployment manifest](https://github.com/zalando/postgres-operator/blob/master/ui/manifests/deployment.yaml#L43) of the UI. To deploy the UI simply apply all its manifests files or use the UI helm chart: @@ -197,7 +216,7 @@ Non-encrypted connections are rejected by default, so set the SSL mode to require: ```bash -export PGPASSWORD=$(kubectl get secret postgres.acid-minimal-cluster.credentials -o 'jsonpath={.data.password}' | base64 -d) +export PGPASSWORD=$(kubectl get secret postgres.acid-minimal-cluster.credentials.postgresql.acid.zalan.do -o 'jsonpath={.data.password}' | base64 -d) export PGSSLMODE=require psql -U postgres ``` @@ -211,7 +230,7 @@ kubectl delete postgresql acid-minimal-cluster ``` This should remove the associated StatefulSet, database Pods, Services and -Endpoints. The PersistentVolumes are released and the PodDisruptionBudget is +Endpoints. The PersistentVolumes are released and the PodDisruptionBudgets are deleted. Secrets however are not deleted and backups will remain in place. When deleting a cluster while it is still starting up or got stuck during that diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 1b2d71a66..0717e411f 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -4,9 +4,9 @@ Individual Postgres clusters are described by the Kubernetes *cluster manifest* that has the structure defined by the `postgresql` CRD (custom resource definition). The following section describes the structure of the manifest and the purpose of individual keys. You can take a look at the examples of the -[minimal](../../manifests/minimal-postgres-manifest.yaml) +[minimal](https://github.com/zalando/postgres-operator/blob/master/manifests/minimal-postgres-manifest.yaml) and the -[complete](../../manifests/complete-postgres-manifest.yaml) +[complete](https://github.com/zalando/postgres-operator/blob/master/manifests/complete-postgres-manifest.yaml) cluster manifests. When Kubernetes resources, such as memory, CPU or volumes, are configured, @@ -53,8 +53,7 @@ Those parameters are grouped under the `metadata` top-level key. These parameters are grouped directly under the `spec` key in the manifest. * **teamId** - name of the team the cluster belongs to. Changing it after the cluster - creation is not supported. Required field. + name of the team the cluster belongs to. Required field. * **numberOfInstances** total number of instances for a given cluster. The operator parameters @@ -86,30 +85,125 @@ These parameters are grouped directly under the `spec` key in the manifest. requires a custom Spilo image. Note the FSGroup of a Pod cannot be changed without recreating a new Pod. Optional. +* **livenessProbe** + Allows for adding a liveness probe to the Spilo container to detect if it's + running properly. + * **enableMasterLoadBalancer** boolean flag to override the operator defaults (set by the `enable_master_load_balancer` parameter) to define whether to enable the load balancer pointing to the Postgres primary. Optional. +* **enableMasterPoolerLoadBalancer** + boolean flag to override the operator defaults (set by the + `enable_master_pooler_load_balancer` parameter) to define whether to enable + the load balancer for master pooler pods pointing to the Postgres primary. + Optional. + * **enableReplicaLoadBalancer** boolean flag to override the operator defaults (set by the `enable_replica_load_balancer` parameter) to define whether to enable the load balancer pointing to the Postgres standby instances. Optional. +* **enableReplicaPoolerLoadBalancer** + boolean flag to override the operator defaults (set by the + `enable_replica_pooler_load_balancer` parameter) to define whether to enable + the load balancer for replica pooler pods pointing to the Postgres standby + instances. Optional. + * **allowedSourceRanges** when one or more load balancers are enabled for the cluster, this parameter - defines the comma-separated range of IP networks (in CIDR-notation). The - corresponding load balancer is accessible only to the networks defined by - this parameter. Optional, when empty the load balancer service becomes - inaccessible from outside of the Kubernetes cluster. + defines the comma-separated range of IP networks (in CIDR-notation). Both + IPv4 (e.g. `192.168.1.0/24`) and IPv6 (e.g. `fd01::/48`) CIDR ranges are + supported. The corresponding load balancer is accessible only to the networks + defined by this parameter. Optional, when empty the load balancer service + becomes inaccessible from outside of the Kubernetes cluster. + +* **enableMasterNodePort** + boolean flag to override the operator defaults (set by the + `enable_master_node_port` parameter) to define whether to enable the node + port pointing to the Postgres primary. Optional. Overrides `enableMasterLoadBalancer`. + +* **enableMasterPoolerNodePort** + boolean flag to override the operator defaults (set by the + `enable_master_pooler_node_port` parameter) to define whether to enable + the node port for master pooler pods pointing to the Postgres primary. + Optional. Overrides `enableMasterPoolerLoadBalancer`. + +* **enableReplicaNodePort** + boolean flag to override the operator defaults (set by the + `enable_replica_node_port` parameter) to define whether to enable the node + port pointing to the Postgres standby instances. Optional. Overrides `enableReplicaLoadBalancer`. + +* **enableReplicaPoolerNodePort** + boolean flag to override the operator defaults (set by the + `enable_replica_pooler_node_port` parameter) to define whether to enable + the node port for replica pooler pods pointing to the Postgres standby + instances. Optional. Overrides `enableReplicaPoolerLoadBalancer`. + +* **masterNodePort** + integer flag to specify a port number for the node port to the Postgres primary. + Only used when `enableMasterNodePort` or `enable_master_node_port` are enabled. + Optional. Kubernetes will provide a port number for you if not specified. + +* **masterPoolerNodePort** + integer flag to specify a port number for the node port for the master pooler pods pointing to the Postgres primary. + Only used when `enableMasterPoolerNodePort` or `enable_master_pooler_node_port` are enabled. + Optional. Kubernetes will provide a port number for you if not specified. + +* **replicaNodePort** + integer flag to specify a port number for the node port pointing to the Postgres standby instances. + Only used when `enableReplicaNodePort` or `enable_replica_node_port` are enabled. + Optional. Kubernetes will provide a port number for you if not specified. + +* **replicaPoolerNodePort** + integer flag to specify a port number for the node port for the replica pooler pods pointing to the Postgres standby instances + Only used when `enableReplicaPoolerNodePort` or `enable_replica_pooler_node_port` are enabled. + Optional. Kubernetes will provide a port number for you if not specified. + +* **maintenanceWindows** + a list which defines specific time frames when certain maintenance operations + such as automatic major upgrades or master pod migration are allowed to happen. + Accepted formats are "01:00-06:00" for daily maintenance windows or + "Sat:00:00-04:00" for specific days, with all times in UTC. Note, when the + global config option `enable_maintenance_windows` is false, the specified + windows will be ignored. * **users** a map of usernames to user flags for the users that should be created in the cluster by the operator. User flags are a list, allowed elements are `SUPERUSER`, `REPLICATION`, `INHERIT`, `LOGIN`, `NOLOGIN`, `CREATEROLE`, - `CREATEDB`, `BYPASSURL`. A login user is created by default unless NOLOGIN is + `CREATEDB`, `BYPASSRLS`. A login user is created by default unless NOLOGIN is specified, in which case the operator creates a role. One can specify empty - flags by providing a JSON empty array '*[]*'. Optional. + flags by providing a JSON empty array '*[]*'. If the config option + `enable_cross_namespace_secret` is enabled you can specify the namespace in + the user name in the form `{namespace}.{username}` and the operator will + create the K8s secret in that namespace. The part after the first `.` is + considered to be the user name. Optional. + +* **usersWithSecretRotation** + list of users to enable credential rotation in K8s secrets. The rotation + interval can only be configured globally. On each rotation a new user will + be added in the database replacing the `username` value in the secret of + the listed user. Although, rotation users inherit all rights from the + original role, keep in mind that ownership is not transferred. See more + details in the [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#password-rotation-in-k8s-secrets). + +* **usersWithInPlaceSecretRotation** + list of users to enable in-place password rotation in K8s secrets. The + rotation interval can only be configured globally. On each rotation the + password value will be replaced in the secrets which the operator reflects + in the database, too. List only users here that rarely connect to the + database, like a flyway user running a migration on Pod start. See more + details in the [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#password-replacement-without-extra-users). + +* **usersIgnoringSecretRotation** + if you have secret rotation enabled globally you can define a list of + of users that should opt out from it, for example if you store credentials + outside of K8s, too, and corresponding deployments cannot dynamically + reference secrets. Note, you can also opt out from the rotation by removing + users from the manifest's `users` section. The operator will not drop them + from the database. Optional. * **databases** a map of database names to database owners for the databases that should be @@ -142,6 +236,22 @@ These parameters are grouped directly under the `spec` key in the manifest. [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#load-balancers-and-allowed-ip-ranges) for more information regarding default values and overwrite rules. +* **masterServiceAnnotations** + A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) + to the master service created for the database cluster. Check the + [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#load-balancers-and-allowed-ip-ranges) + for more information regarding default values and overwrite rules. + This field overrides `serviceAnnotations` with the same key for the master + service if not empty. + +* **replicaServiceAnnotations** + A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) + to the replica service created for the database cluster. Check the + [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#load-balancers-and-allowed-ip-ranges) + for more information regarding default values and overwrite rules. + This field overrides `serviceAnnotations` with the same key for the replica + service if not empty. + * **enableShmVolume** Start a database pod without limitations on shm memory. By default Docker limit `/dev/shm` to `64M` (see e.g. the [docker @@ -168,10 +278,17 @@ These parameters are grouped directly under the `spec` key in the manifest. Determines if the logical backup of this cluster should be taken and uploaded to S3. Default: false. Optional. +* **logicalBackupRetention** + You can set a retention time for the logical backup cron job to remove old backup + files after a new backup has been uploaded. Example values are "3 days", "2 weeks", or + "1 month". It takes precedence over the global `logical_backup_s3_retention_time` + configuration. Currently only supported for AWS. Optional. + * **logicalBackupSchedule** Schedule for the logical backup K8s cron job. Please take [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) - into account. Optional. Default is: "30 00 \* \* \*" + into account. It takes precedence over the global `logical_backup_schedule` + configuration. Optional. * **additionalVolumes** List of additional volumes to mount in each container of the statefulset pod. @@ -179,12 +296,42 @@ These parameters are grouped directly under the `spec` key in the manifest. [kubernetes volumeSource](https://godoc.org/k8s.io/api/core/v1#VolumeSource). It allows you to mount existing PersistentVolumeClaims, ConfigMaps and Secrets inside the StatefulSet. Also an `emptyDir` volume can be shared between initContainer and statefulSet. - Additionaly, you can provide a `SubPath` for volume mount (a file in a configMap source volume, for example). + Additionally, you can provide a `SubPath` for volume mount (a file in a configMap source volume, for example). + Set `isSubPathExpr` to true if you want to include [API environment variables](https://kubernetes.io/docs/concepts/storage/volumes/#using-subpath-expanded-environment). You can also specify in which container the additional Volumes will be mounted with the `targetContainers` array option. If `targetContainers` is empty, additional volumes will be mounted only in the `postgres` container. If you set the `all` special item, it will be mounted in all containers (postgres + sidecars). Else you can set the list of target containers in which the additional volumes will be mounted (eg : postgres, telegraf) +## Prepared Databases + +The operator can create databases with default owner, reader and writer roles +without the need to specify them under `users` or `databases` sections. Those +parameters are grouped under the `preparedDatabases` top-level key. For more +information, see [user docs](../user.md#prepared-databases-with-roles-and-default-privileges). + +* **defaultUsers** + The operator will always create default `NOLOGIN` roles for defined prepared + databases, but if `defaultUsers` is set to `true` three additional `LOGIN` + roles with `_user` suffix will get created. Default is `false`. + +* **extensions** + map of extensions with target database schema that the operator will install + in the database. Optional. + +* **schemas** + map of schemas that the operator will create. Optional - if no schema is + listed, the operator will create a schema called `data`. Under each schema + key, it can be defined if `defaultRoles` (NOLOGIN) and `defaultUsers` (LOGIN) + roles shall be created that have schema-exclusive privileges. + By default, `defaultRoles` is `true` and `defaultUsers` is false. + +* **secretNamespace** + for each default LOGIN role the operator will create a secret. You can + specify the namespace in which these secrets will get created, if + `enable_cross_namespace_secret` is set to `true` in the config. Otherwise, + the cluster namespace is used. + ## Postgres parameters Those parameters are grouped under the `postgresql` top-level key, which is @@ -208,7 +355,7 @@ documentation](https://patroni.readthedocs.io/en/latest/SETTINGS.html) for the explanation of `ttl` and `loop_wait` parameters. * **initdb** - a map of key-value pairs describing initdb parameters. For `data-checksum`, + a map of key-value pairs describing initdb parameters. For `data-checksums`, `debug`, `no-locale`, `noclean`, `nosync` and `sync-only` parameters use `true` as the value if you want to set them. Changes to this option do not affect the already initialized clusters. Optional. @@ -254,11 +401,22 @@ explanation of `ttl` and `loop_wait` parameters. * **synchronous_mode_strict** Patroni `synchronous_mode_strict` parameter value. Can be used in addition to `synchronous_mode`. The default is set to `false`. Optional. +* **synchronous_node_count** + Patroni `synchronous_node_count` parameter value. Note, this option is only available for Spilo images with Patroni 2.0+. The default is set to `1`. Optional. + +* **failsafe_mode** + Patroni `failsafe_mode` parameter value. If enabled, Patroni will cope + with DCS outages by avoiding leader demotion. See the Patroni documentation + [here](https://patroni.readthedocs.io/en/master/dcs_failsafe_mode.html) for more details. + This feature is included since Patroni 3.0.0. Hence, check the container + image in use if this feature is included in the used Patroni version. The + default is set to `false`. Optional. + ## Postgres container resources Those parameters define [CPU and memory requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the Postgres container. They are grouped under the `resources` top-level -key with subgroups `requests` and `limits`. +key with subgroups `requests` and `limits`. ### Requests @@ -266,11 +424,19 @@ CPU and memory requests for the Postgres container. * **cpu** CPU requests for the Postgres container. Optional, overrides the - `default_cpu_requests` operator configuration parameter. Optional. + `default_cpu_requests` operator configuration parameter. * **memory** memory requests for the Postgres container. Optional, overrides the - `default_memory_request` operator configuration parameter. Optional. + `default_memory_request` operator configuration parameter. + +* **hugepages-2Mi** + hugepages-2Mi requests for the sidecar container. + Optional, defaults to not set. + +* **hugepages-1Gi** + 1Gi hugepages requests for the sidecar container. + Optional, defaults to not set. ### Limits @@ -278,11 +444,19 @@ CPU and memory limits for the Postgres container. * **cpu** CPU limits for the Postgres container. Optional, overrides the - `default_cpu_limits` operator configuration parameter. Optional. + `default_cpu_limits` operator configuration parameter. * **memory** memory limits for the Postgres container. Optional, overrides the - `default_memory_limits` operator configuration parameter. Optional. + `default_memory_limits` operator configuration parameter. + +* **hugepages-2Mi** + hugepages-2Mi requests for the sidecar container. + Optional, defaults to not set. + +* **hugepages-1Gi** + 1Gi hugepages requests for the sidecar container. + Optional, defaults to not set. ## Parameters defining how to clone the cluster from another one @@ -331,12 +505,31 @@ under the `clone` top-level key and do not affect the already running cluster. ## Standby cluster On startup, an existing `standby` top-level key creates a standby Postgres -cluster streaming from a remote location. So far only streaming from a S3 WAL -archive is supported. +cluster streaming from a remote location - either from a S3 or GCS WAL +archive, a remote primary, or a combination of both. At least one of +`s3_wal_path`, `gs_wal_path`, or `standby_host` must be specified. +Note that `s3_wal_path` and `gs_wal_path` are mutually exclusive. * **s3_wal_path** the url to S3 bucket containing the WAL archive of the remote primary. - Required when the `standby` section is present. + Can be combined with `standby_host` for additional redundancy. + +* **gs_wal_path** + the url to GS bucket containing the WAL archive of the remote primary. + Can be combined with `standby_host` for additional redundancy. + +* **standby_host** + hostname or IP address of the primary to stream from. + Can be specified alone or combined with either `s3_wal_path` or `gs_wal_path`. + +* **standby_port** + TCP port on which the primary is listening for connections. Patroni will + use `"5432"` if not set. + +* **standby_primary_slot_name** + name of the replication slot to use on the primary server when streaming + from a remote primary. See the Patroni documentation + [here](https://patroni.readthedocs.io/en/latest/standby_cluster.html) for more details. Optional. ## Volume properties @@ -356,6 +549,9 @@ properties of the persistent storage that stores Postgres data. * **subPath** Subpath to use when mounting volume into Spilo container. Optional. +* **isSubPathExpr** + Set it to true if the specified subPath is an expression. Optional. + * **iops** When running the operator on AWS the latest generation of EBS volumes (`gp3`) allows for configuring the number of IOPS. Maximum is 16000. Optional. @@ -364,6 +560,11 @@ properties of the persistent storage that stores Postgres data. When running the operator on AWS the latest generation of EBS volumes (`gp3`) allows for configuring the throughput in MB/s. Maximum is 1000. Optional. +* **selector** + A label query over PVs to consider for binding. See the [Kubernetes + documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) + for details on using `matchLabels` and `matchExpressions`. Optional + ## Sidecar definitions Those parameters are defined under the `sidecars` key. They consist of a list @@ -398,6 +599,14 @@ CPU and memory requests for the sidecar container. memory requests for the sidecar container. Optional, overrides the `default_memory_request` operator configuration parameter. Optional. +* **hugepages-2Mi** + hugepages-2Mi requests for the sidecar container. + Optional, defaults to not set. + +* **hugepages-1Gi** + 1Gi hugepages requests for the sidecar container. + Optional, defaults to not set. + ### Limits CPU and memory limits for the sidecar container. @@ -410,6 +619,14 @@ CPU and memory limits for the sidecar container. memory limits for the sidecar container. Optional, overrides the `default_memory_limits` operator configuration parameter. Optional. +* **hugepages-2Mi** + hugepages-2Mi requests for the sidecar container. + Optional, defaults to not set. + +* **hugepages-1Gi** + 1Gi hugepages requests for the sidecar container. + Optional, defaults to not set. + ## Connection pooler Parameters are grouped under the `connectionPooler` top-level key and specify @@ -444,7 +661,9 @@ for both master and replica pooler services (if `enableReplicaConnectionPooler` ## Custom TLS certificates -Those parameters are grouped under the `tls` top-level key. +Those parameters are grouped under the `tls` top-level key. Note, you have to +define `spiloFSGroup` in the Postgres cluster manifest or `spilo_fsgroup` in +the global configuration before adding the `tls` section'. * **secretName** By setting the `secretName` value, the cluster will switch to load the given @@ -473,3 +692,70 @@ Those parameters are grouped under the `tls` top-level key. relative to the "/tls/", which is mount path of the tls secret. If `caSecretName` is defined, the ca.crt path is relative to "/tlsca/", otherwise to the same "/tls/". + +## Change data capture streams + +This sections enables change data capture (CDC) streams via Postgres' +[logical decoding](https://www.postgresql.org/docs/18/logicaldecoding.html) +feature and `pgoutput` plugin. While the Postgres operator takes responsibility +for providing the setup to publish change events, it relies on external tools +to consume them. At Zalando, we are using a workflow based on +[Debezium Connector](https://debezium.io/documentation/reference/stable/connectors/postgresql.html) +which can feed streams into Zalando’s distributed event broker [Nakadi](https://nakadi.io/) +among others. + +The Postgres Operator creates custom resources for Zalando's internal CDC +operator which will be used to set up the consumer part. Each stream object +can have the following properties: + +* **applicationId** + The application name to which the database and CDC belongs to. For each + set of streams with a distinct `applicationId` a separate stream resource as + well as a separate logical replication slot will be created. This means there + can be different streams in the same database and streams with the same + `applicationId` are bundled in one stream resource. The stream resource will + be called like the Postgres cluster plus "-" suffix. Required. + +* **database** + Name of the database from where events will be published via Postgres' + logical decoding feature. The operator will take care of updating the + database configuration (setting `wal_level: logical`, creating logical + replication slots, using output plugin `pgoutput` and creating a dedicated + replication user). Required. + +* **tables** + Defines a map of table names and their properties (`eventType`, `idColumn` + and `payloadColumn`). Required. + The CDC operator is following the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/). + The application is responsible for putting events into a (JSON/B or VARCHAR) + payload column of the outbox table in the structure of the specified target + event type. The operator will create a [PUBLICATION](https://www.postgresql.org/docs/18/logical-replication-publication.html) + in Postgres for all tables specified for one `database` and `applicationId`. + The CDC operator will consume from it shortly after transactions are + committed to the outbox table. The `idColumn` will be used in telemetry for + the CDC operator. The names for `idColumn` and `payloadColumn` can be + configured. Defaults are `id` and `payload`. The target `eventType` has to + be defined. One can also specify a `recoveryEventType` that will be used + for a dead letter queue. By enabling `ignoreRecovery`, you can choose to + ignore failing events. + +* **filter** + Streamed events can be filtered by a jsonpath expression for each table. + Optional. + +* **enableRecovery** + Flag to enable a dead letter queue recovery for all streams tables. + Alternatively, recovery can also be enable for single outbox tables by only + specifying a `recoveryEventType` and no `enableRecovery` flag. When set to + false or missing, events will be retried until consuming succeeded. You can + use a `filter` expression to get rid of poison pills. Optional. + +* **batchSize** + Defines the size of batches in which events are consumed. Optional. + Defaults to 1. + +* **cpu** + CPU requests to be set as an annotation on the stream resource. Optional. + +* **memory** + memory requests to be set as an annotation on the stream resource. Optional. diff --git a/docs/reference/command_line_and_environment.md b/docs/reference/command_line_and_environment.md index 35f47cabf..c8aab90b5 100644 --- a/docs/reference/command_line_and_environment.md +++ b/docs/reference/command_line_and_environment.md @@ -23,6 +23,12 @@ The following command-line options are supported for the operator: off can can be overridden by the aforementioned operator configuration option. +* **-kubeqps** + set the maximum number of Kubernetes API requests per second. Default is 10. + +* **-kubeburst** + set the burst limit for Kubernetes API requests, allowing temporary spikes beyond the configured QPS. Default is 20. + In addition to that, standard [glog flags](https://godoc.org/github.com/golang/glog) are also supported. For instance, one may want to add `-alsologtostderr` and `-v=8` to debug the diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 395007c91..13d29c950 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -3,32 +3,46 @@ There are two mutually-exclusive methods to set the Postgres Operator configuration. -* ConfigMaps-based, the legacy one. The configuration is supplied in a - key-value configmap, defined by the `CONFIG_MAP_NAME` environment variable. - Non-scalar values, i.e. lists or maps, are encoded in the value strings using - the comma-based syntax for lists and coma-separated `key:value` syntax for - maps. String values containing ':' should be enclosed in quotes. The - configuration is flat, parameter group names below are not reflected in the - configuration structure. There is an - [example](../../manifests/configmap.yaml) +* ConfigMaps-based, the legacy one +* CRD-based configuration -* CRD-based configuration. The configuration is stored in a custom YAML - manifest. The manifest is an instance of the custom resource definition (CRD) - called `OperatorConfiguration`. The operator registers this CRD during the - start and uses it for configuration if the [operator deployment manifest](../../manifests/postgres-operator.yaml#L36) - sets the `POSTGRES_OPERATOR_CONFIGURATION_OBJECT` env variable to a non-empty - value. The variable should point to the `postgresql-operator-configuration` - object in the operator's namespace. +Variable names are underscore-separated words. - The CRD-based configuration is a regular YAML document; non-scalar keys are - simply represented in the usual YAML way. There are no default values built-in - in the operator, each parameter that is not supplied in the configuration - receives an empty value. In order to create your own configuration just copy - the [default one](../../manifests/postgresql-operator-default-configuration.yaml) - and change it. +### ConfigMaps-based +The configuration is supplied in a +key-value configmap, defined by the `CONFIG_MAP_NAME` environment variable. +Non-scalar values, i.e. lists or maps, are encoded in the value strings using +the comma-based syntax for lists and coma-separated `key:value` syntax for +maps. String values containing ':' should be enclosed in quotes. The +configuration is flat, parameter group names below are not reflected in the +configuration structure. There is an +[example](https://github.com/zalando/postgres-operator/blob/master/manifests/configmap.yaml) - To test the CRD-based configuration locally, use the following - ```bash +For the configmap configuration, the [default parameter values](https://github.com/zalando/postgres-operator/blob/master/pkg/util/config/config.go#L14) +mentioned here are likely to be overwritten in your local operator installation +via your local version of the operator configmap. In the case you use the +operator CRD, all the CRD defaults are provided in the +[operator's default configuration manifest](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql-operator-default-configuration.yaml) + +### CRD-based configuration +The configuration is stored in a custom YAML +manifest. The manifest is an instance of the custom resource definition (CRD) +called `OperatorConfiguration`. The operator registers this CRD during the +start and uses it for configuration if the [operator deployment manifest](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L36) +sets the `POSTGRES_OPERATOR_CONFIGURATION_OBJECT` env variable to a non-empty +value. The variable should point to the `postgresql-operator-configuration` +object in the operator's namespace. + +The CRD-based configuration is a regular YAML document; non-scalar keys are +simply represented in the usual YAML way. There are no default values built-in +in the operator, each parameter that is not supplied in the configuration +receives an empty value. In order to create your own configuration just copy +the [default one](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql-operator-default-configuration.yaml) +and change it. + +To test the CRD-based configuration locally, use the following + +```bash kubectl create -f manifests/operatorconfiguration.crd.yaml # registers the CRD kubectl create -f manifests/postgresql-operator-default-configuration.yaml @@ -36,7 +50,7 @@ configuration. kubectl create -f manifests/postgres-operator.yaml # set the env var as mentioned above kubectl get operatorconfigurations postgresql-operator-default-configuration -o yaml - ``` +``` The CRD-based configuration is more powerful than the one based on ConfigMaps and should be used unless there is a compatibility requirement to use an already @@ -57,24 +71,17 @@ parameters, those parameters have no effect and are replaced by the `CRD_READY_WAIT_INTERVAL` and `CRD_READY_WAIT_TIMEOUT` environment variables. They will be deprecated and removed in the future. -For the configmap configuration, the [default parameter values](../../pkg/util/config/config.go#L14) -mentioned here are likely to be overwritten in your local operator installation -via your local version of the operator configmap. In the case you use the -operator CRD, all the CRD defaults are provided in the -[operator's default configuration manifest](../../manifests/postgresql-operator-default-configuration.yaml) - -Variable names are underscore-separated words. - - ## General Those are top-level keys, containing both leaf keys and groups. -* **enable_crd_validation** - toggles if the operator will create or update CRDs with - [OpenAPI v3 schema validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation) +* **enable_crd_registration** + Instruct the operator to create/update the CRDs. If disabled the operator will rely on the CRDs being managed separately. The default is `true`. +* **crd_categories** + The operator will register CRDs in the `all` category by default so that they will be returned by a `kubectl get all` call. You are free to change categories or leave them empty. + * **enable_lazy_spilo_upgrade** Instruct operator to update only the statefulsets with new images (Spilo and InitContainers) without immediately doing the rolling update. The assumption is pods will be re-started later with new images, for example due to the node rotation. The default is `false`. @@ -82,8 +89,10 @@ Those are top-level keys, containing both leaf keys and groups. * **enable_pgversion_env_var** With newer versions of Spilo, it is preferable to use `PGVERSION` pod environment variable instead of the setting `postgresql.bin_dir` in the `SPILO_CONFIGURATION` env variable. When this option is true, the operator sets `PGVERSION` and omits `postgresql.bin_dir` from `SPILO_CONFIGURATION`. When false, the `postgresql.bin_dir` is set. This setting takes precedence over `PGVERSION`; see PR 222 in Spilo. The default is `true`. -* **enable_spilo_wal_path_compat** - enables backwards compatible path between Spilo 12 and Spilo 13 images. The default is `false`. +* **enable_team_id_clustername_prefix** + To lower the risk of name clashes between clusters of different teams you + can turn on this flag and the operator will sync only clusters where the + name starts with the `teamId` (from `spec`) plus `-`. Default is `false`. * **etcd_host** Etcd connection string for Patroni defined as `host:port`. Not required when @@ -93,8 +102,13 @@ Those are top-level keys, containing both leaf keys and groups. * **kubernetes_use_configmaps** Select if setup uses endpoints (default), or configmaps to manage leader when DCS is kubernetes (not etcd or similar). In OpenShift it is not possible to - use endpoints option, and configmaps is required. By default, - `kubernetes_use_configmaps: false`, meaning endpoints will be used. + use endpoints option, and configmaps is required. Starting with K8s 1.33, + endpoints are marked as deprecated. It's recommended to switch to config maps + instead. But, to do so make sure you scale the Postgres cluster down to just + one primary pod (e.g. using `max_instances` option). Otherwise, you risk + running into a split-brain scenario. + By default, `kubernetes_use_configmaps: false`, meaning endpoints will be used. + Starting from v1.16.0 the default will be changed to `true`. * **docker_image** Spilo Docker image for Postgres instances. For production, don't rely on the @@ -140,6 +154,31 @@ Those are top-level keys, containing both leaf keys and groups. When `-1` is specified for `min_instances`, no limits are applied. The default is `-1`. +* **ignore_instance_limits_annotation_key** + for some clusters it might be required to scale beyond the limits that can be + configured with `min_instances` and `max_instances` options. You can define + an annotation key that can be used as a toggle in cluster manifests to ignore + globally configured instance limits. The value must be `"true"` to be + effective. The default is empty which means the feature is disabled. + +* **ignore_resources_limits_annotation_key** + for some clusters it might be required to request resources beyond the globally + configured thresholds for maximum requests and minimum limits. You can define + an annotation key that can be used as a toggle in cluster manifests to ignore + the thresholds. The value must be `"true"` to be effective. The default is empty + which means the feature is disabled. + +* **enable_maintenance_windows** + toggle for using the maintenance windows feature. Default is `"true"`. + +* **maintenance_windows** + a list which defines specific time frames when certain maintenance + operations such as automatic major upgrades or master pod migration are + allowed to happen for all database clusters. Accepted formats are + "01:00-06:00" for daily maintenance windows or "Sat:00:00-04:00" for + specific days, with all times in UTC. Locally defined maintenance + windows take precedence over globally configured ones. + * **resync_period** period between consecutive sync requests. The default is `30m`. @@ -148,13 +187,14 @@ Those are top-level keys, containing both leaf keys and groups. * **set_memory_request_to_limit** Set `memory_request` to `memory_limit` for all Postgres clusters (the default - value is also increased). This prevents certain cases of memory overcommitment - at the cost of overprovisioning memory and potential scheduling problems for - containers with high memory limits due to the lack of memory on Kubernetes - cluster nodes. This affects all containers created by the operator (Postgres, - Scalyr sidecar, and other sidecars except **sidecars** defined in the operator - configuration); to set resources for the operator's own container, change the - [operator deployment manually](../../manifests/postgres-operator.yaml#L20). + value is also increased but configured `max_memory_request` can not be + bypassed). This prevents certain cases of memory overcommitment at the cost + of overprovisioning memory and potential scheduling problems for containers + with high memory limits due to the lack of memory on Kubernetes cluster + nodes. This affects all containers created by the operator (Postgres, + connection pooler, logical backup, scalyr sidecar, and other sidecars except + **sidecars** defined in the operator configuration); to set resources for the + operator's own container, change the [operator deployment manually](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L20). The default is `false`. ## Postgres users @@ -170,28 +210,68 @@ under the `users` key. Postgres username used for replication between instances. The default is `standby`. +* **additional_owner_roles** + Specifies database roles that will be granted to all database owners. Owners + can then use `SET ROLE` to obtain privileges of these roles to e.g. create + or update functionality from extensions as part of a migration script. One + such role can be `cron_admin` which is provided by the Spilo docker image to + set up cron jobs inside the `postgres` database. In general, roles listed + here should be preconfigured in the docker image and already exist in the + database cluster on startup. Otherwise, syncing roles will return an error + on each cluster sync process. Alternatively, you have to create the role and + do the GRANT manually. Note, the operator will not allow additional owner + roles to be members of database owners because it should be vice versa. If + the operator cannot set up the correct membership it tries to revoke all + additional owner roles from database owners. Default is `empty`. + +* **enable_password_rotation** + For all `LOGIN` roles that are not database owners the operator can rotate + credentials in the corresponding K8s secrets by replacing the username and + password. This means, new users will be added on each rotation inheriting + all privileges from the original roles. The rotation date (in YYMMDD format) + is appended to the names of the new user. The timestamp of the next rotation + is written to the secret. The default is `false`. + +* **password_rotation_interval** + If password rotation is enabled (either from config or cluster manifest) the + interval can be configured with this parameter. The measure is in days which + means daily rotation (`1`) is the most frequent interval possible. + Default is `90`. + +* **password_rotation_user_retention** + To avoid an ever growing amount of new users due to password rotation the + operator will remove the created users again after a certain amount of days + has passed. The number can be configured with this parameter. However, the + operator will check that the retention policy is at least twice as long as + the rotation interval and update to this minimum in case it is not. + Default is `180`. + ## Major version upgrades -Parameters configuring automatic major version upgrades. In a +Parameters configuring automatic major version upgrades. In a CRD-configuration, they are grouped under the `major_version_upgrade` key. * **major_version_upgrade_mode** - Postgres Operator supports [in-place major version upgrade](../administrator.md#in-place-major-version-upgrade) + Postgres Operator supports [in-place major version upgrade](../administrator.md#in-place-major-version-upgrade) with three different modes: `"off"` = no upgrade by the operator, `"manual"` = manifest triggers action, `"full"` = manifest and minimal version violation trigger upgrade. Note, that with all three modes increasing the version in the manifest will - trigger a rolling update of the pods. The default is `"off"`. + trigger a rolling update of the pods. The default is `"manual"`. + +* **major_version_upgrade_team_allow_list** + Upgrades will only be carried out for clusters of listed teams when mode is + set to "off". The default is empty. * **minimal_major_version** The minimal Postgres major version that will not automatically be upgraded - when `major_version_upgrade_mode` is set to `"full"`. The default is `"9.5"`. + when `major_version_upgrade_mode` is set to `"full"`. The default is `"14"`. * **target_major_version** The target Postgres major version when upgrading clusters automatically which violate the configured allowed `minimal_major_version` when - `major_version_upgrade_mode` is set to `"full"`. The default is `"13"`. + `major_version_upgrade_mode` is set to `"full"`. The default is `"18"`. ## Kubernetes resources @@ -199,6 +279,31 @@ Parameters to configure cluster-related Kubernetes objects created by the operator, as well as some timeouts associated with them. In a CRD-based configuration they are grouped under the `kubernetes` key. +* **enable_finalizers** + By default, a deletion of the Postgresql resource will trigger an event + that leads to a cleanup of all child resources. However, if the database + cluster is in a broken state (e.g. failed initialization) and the operator + cannot fully sync it, there can be leftovers. By enabling finalizers the + operator will ensure all managed resources are deleted prior to the + Postgresql resource. See also [admin docs](../administrator.md#owner-references-and-finalizers) + for more information The default is `false`. + +* **enable_owner_references** + The operator can set owner references on its child resources (except PVCs, + Patroni config service/endpoint, cross-namespace secrets) to improve cluster + monitoring and enable cascading deletion. The default is `false`. Warning, + enabling this option disables configured delete protection checks (see below). + +* **delete_annotation_date_key** + key name for annotation that compares manifest value with current date in the + YYYY-MM-DD format. Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`. + The default is empty which also disables this delete protection check. + +* **delete_annotation_name_key** + key name for annotation that compares manifest value with Postgres cluster name. + Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`. The default is + empty which also disables this delete protection check. + * **pod_service_account_name** service account used by Patroni running on individual Pods to communicate with the operator. Required even if native Kubernetes support in Patroni is @@ -217,33 +322,33 @@ configuration they are grouped under the `kubernetes` key. sufficient for the pods to start and for Patroni to access K8s endpoints; service account on its own lacks any such rights starting with K8s v1.8. If not explicitly defined by the user, a simple definition that binds the - account to the 'postgres-pod' [cluster role](../../manifests/operator-service-account-rbac.yaml#L198) + account to the 'postgres-pod' [cluster role](https://github.com/zalando/postgres-operator/blob/master/manifests/operator-service-account-rbac.yaml#L198) will be used. The default is empty. * **pod_terminate_grace_period** - Postgres pods are [terminated forcefully](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) + Postgres pods are [terminated forcefully](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination) after this timeout. The default is `5m`. +* **liveness_probe** + Allows for adding a liveness probe to the Spilo container to detect if it's + running properly. Cannot be configured via ConfigMap. Default is empty. + * **custom_pod_annotations** This key/value map provides a list of annotations that get attached to each pod of a database created by the operator. If the annotation key is also provided by the database definition, the database definition value is used. -* **delete_annotation_date_key** - key name for annotation that compares manifest value with current date in the - YYYY-MM-DD format. Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`. - The default is empty which also disables this delete protection check. - -* **delete_annotation_name_key** - key name for annotation that compares manifest value with Postgres cluster name. - Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`. The default is - empty which also disables this delete protection check. - * **downscaler_annotations** An array of annotations that should be passed from Postgres CRD on to the statefulset and, if exists, to the connection pooler deployment as well. Regular expressions like `downscaler/*` etc. are also accepted. Can be used - with [kube-downscaler](https://github.com/hjacobs/kube-downscaler). + with [kube-downscaler](https://codeberg.org/hjacobs/kube-downscaler). + +* **ignored_annotations** + Some K8s tools inject and update annotations out of the Postgres Operator + control. This can cause rolling updates on each cluster sync cycle. With + this option you can specify an array of annotation keys that should be + ignored when comparing K8s resources on sync. The default is empty. * **watched_namespace** The operator watches for Postgres objects in the given namespace. If not @@ -252,11 +357,40 @@ configuration they are grouped under the `kubernetes` key. pod namespace). * **pdb_name_format** - defines the template for PDB (Pod Disruption Budget) names created by the + defines the template for primary PDB (Pod Disruption Budget) name created by the operator. The default is `postgres-{cluster}-pdb`, where `{cluster}` is replaced by the cluster name. Only the `{cluster}` placeholders is allowed in the template. +* **pdb_master_label_selector** + By default the primary PDB will match the master role hence preventing nodes to be + drained if the node_readiness_label is not used. If this option if set to + `false` the `spilo-role=master` selector will not be added to the PDB. + +* **persistent_volume_claim_retention_policy** + The operator tries to protect volumes as much as possible. If somebody + accidentally deletes the statefulset or scales in the `numberOfInstances` the + Persistent Volume Claims and thus Persistent Volumes will be retained. + However, this can have some consequences when you scale out again at a much + later point, for example after the cluster's Postgres major version has been + upgraded, because the old volume runs the old Postgres version with stale data. + Even if the version has not changed the replication lag could be massive. In + this case a reinitialization of the re-added member would make sense. You can + also modify the [retention policy of PVCs](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention) in the operator configuration. + The behavior can be changed for two scenarios: `when_deleted` - default is + `"retain"` - or `when_scaled` - default is also `"retain"`. The other possible + option is `delete`. + +* **enable_secrets_deletion** + By default, the operator deletes secrets when removing the Postgres cluster + manifest. To keep secrets, set this option to `false`. The default is `true`. + +* **enable_persistent_volume_claim_deletion** + By default, the operator deletes persistent volume claims when removing the + Postgres cluster manifest, no matter if `persistent_volume_claim_retention_policy` + on the statefulset is set to `retain`. To keep PVCs set this option to `false`. + The default is `true`. + * **enable_pod_disruption_budget** PDB is enabled by default to protect the cluster from voluntarily disruptions and hence unwanted DB downtime. However, on some cloud providers it could be @@ -264,6 +398,11 @@ configuration they are grouped under the `kubernetes` key. [admin docs](../administrator.md#pod-disruption-budget) for more information. Default is true. +* **enable_cross_namespace_secret** + To allow secrets in a different namespace other than the Postgres cluster + namespace. Once enabled, specify the namespace in the user name under the + `users` section in the form `{namespace}.{username}`. The default is `false`. + * **enable_init_containers** global option to allow for creating init containers in the cluster manifest to run actions before Spilo is started. Default is true. @@ -273,13 +412,21 @@ configuration they are grouped under the `kubernetes` key. to run alongside Spilo on the same pod. Globally defined sidecars are always enabled. Default is true. +* **share_pgsocket_with_sidecars** + global option to create an emptyDir volume named `postgresql-run`. This is + mounted by all containers at `/var/run/postgresql` sharing the unix socket of + PostgreSQL (`pg_socket`) with the sidecars this way. + Default is `false`. + * **secret_name_template** a template for the name of the database user secrets generated by the - operator. `{username}` is replaced with name of the secret, `{cluster}` with - the name of the cluster, `{tprkind}` with the kind of CRD (formerly known as - TPR) and `{tprgroup}` with the group of the CRD. No other placeholders are - allowed. The default is - `{username}.{cluster}.credentials.{tprkind}.{tprgroup}`. + operator. `{namespace}` is replaced with name of the namespace if + `enable_cross_namespace_secret` is set, otherwise the + secret is in cluster's namespace. `{username}` is replaced with name of the + secret, `{cluster}` with the name of the cluster, `{tprkind}` with the kind + of CRD (formerly known as TPR) and `{tprgroup}` with the group of the CRD. + No other placeholders are allowed. The default is + `{namespace}.{username}.{cluster}.credentials.{tprkind}.{tprgroup}`. * **cluster_domain** defines the default DNS domain for the kubernetes cluster the operator is @@ -329,11 +476,16 @@ configuration they are grouped under the `kubernetes` key. * **node_readiness_label** a set of labels that a running and active node should possess to be - considered `ready`. The operator uses values of those labels to detect the - start of the Kubernetes cluster upgrade procedure and move master pods off - the nodes to be decommissioned. When the set is not empty, the operator also - assigns the `Affinity` clause to the Postgres pods to be scheduled only on - `ready` nodes. The default is empty. + considered `ready`. When the set is not empty, the operator assigns the + `nodeAffinity` clause to the Postgres pods to be scheduled only on `ready` + nodes. The default is empty. + +* **node_readiness_label_merge** + If a `nodeAffinity` is also specified in the postgres cluster manifest + it will get merged with the `node_readiness_label` affinity on the pods. + The merge strategy can be configured - it can either be "AND" or "OR". + See [user docs](../user.md#use-taints-tolerations-and-node-affinity-for-dedicated-postgresql-nodes) + for more details. Default is "OR". * **toleration** a dictionary that should contain `key`, `operator`, `value` and @@ -343,10 +495,16 @@ configuration they are grouped under the `kubernetes` key. * **pod_environment_configmap** namespaced name of the ConfigMap with environment variables to populate on - every pod. Right now this ConfigMap is searched in the namespace of the - Postgres cluster. All variables from that ConfigMap are injected to the pod's - environment, on conflicts they are overridden by the environment variables - generated by the operator. The default is empty. + every pod. All variables from that ConfigMap are injected to the pod's + environment if they not if conflict with the environment variables generated + by the operator. The WAL location (bucket path) can be overridden, though. + The default is empty. + +* **pod_environment_secret** + similar to pod_environment_configmap but referencing a secret with custom + environment variables. Because the secret is not allowed to exist in a + different namespace than a Postgres cluster you can only use it in a single + namespace. The default is empty. * **pod_priority_class_name** a name of the [priority class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) @@ -401,18 +559,32 @@ configuration they are grouped under the `kubernetes` key. override [topology key](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#built-in-node-labels) for pod anti affinity. The default is `kubernetes.io/hostname`. +* **pod_antiaffinity_preferred_during_scheduling** + when scaling the number of pods beyond the available number of topology + keys the anti affinity has to be configured to preferred during scheduling. + The default is `false` which means the pod anti affinity will use + `requiredDuringSchedulingIgnoredDuringExecution`. + * **pod_management_policy** specify the [pod management policy](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies) of stateful sets of PG clusters. The default is `ordered_ready`, the second possible value is `parallel`. +* **enable_readiness_probe** + the operator can set a readiness probe on the statefulset for the database + pods with `InitialDelaySeconds: 6`, `PeriodSeconds: 10`, `TimeoutSeconds: 5`, + `SuccessThreshold: 1` and `FailureThreshold: 3`. When enabling readiness + probes it is recommended to switch the `pod_management_policy` to `parallel` + to avoid unnecessary waiting times in case of multiple instances failing. + The default is `false`. + * **storage_resize_mode** defines how operator handles the difference between the requested volume size and the actual size. Available options are: - 1. `ebs` : operator resizes EBS volumes directly and executes `resizefs` within a pod - 2. `pvc` : operator only changes PVC definition - 3. `off` : disables resize of the volumes. - 4. `mixed` :operator uses AWS API to adjust size, throughput, and IOPS, and calls pvc change for file system resize + 1. `ebs` : operator resizes EBS volumes directly and executes `resizefs` within a pod + 2. `pvc` : operator only changes PVC definition + 3. `off` : disables resize of the volumes. + 4. `mixed` : operator uses AWS API to adjust size, throughput, and IOPS, and calls pvc change for file system resize Default is "pvc". ## Kubernetes resource requests @@ -423,27 +595,46 @@ CRD-based configuration. * **default_cpu_request** CPU request value for the Postgres containers, unless overridden by - cluster-specific settings. The default is `100m`. + cluster-specific settings. Empty string or `0` disables the default. * **default_memory_request** memory request value for the Postgres containers, unless overridden by - cluster-specific settings. The default is `100Mi`. + cluster-specific settings. Empty string or `0` disables the default. * **default_cpu_limit** CPU limits for the Postgres containers, unless overridden by cluster-specific - settings. The default is `1`. + settings. Empty string or `0` disables the default. * **default_memory_limit** memory limits for the Postgres containers, unless overridden by cluster-specific - settings. The default is `500Mi`. + settings. Empty string or `0` disables the default. + +* **max_cpu_request** + optional upper boundary for CPU request + +* **max_memory_request** + optional upper boundary for memory request * **min_cpu_limit** hard CPU minimum what we consider to be required to properly run Postgres - clusters with Patroni on Kubernetes. The default is `250m`. + clusters with Patroni on Kubernetes. * **min_memory_limit** hard memory minimum what we consider to be required to properly run Postgres - clusters with Patroni on Kubernetes. The default is `250Mi`. + clusters with Patroni on Kubernetes. + +## Patroni options + +Parameters configuring Patroni. In the CRD-based configuration they are grouped +under the `patroni` key. + +* **enable_patroni_failsafe_mode** + If enabled, Patroni copes with DCS outages by avoiding leader demotion. + See the Patroni documentation [here](https://patroni.readthedocs.io/en/master/dcs_failsafe_mode.html) for more details. + This feature is included since Patroni 3.0.0. Hence, check the container image + in use if this feature is included in the used Patroni version. It can also be + enabled cluster-wise with the `failsafe_mode` flag under the `patroni` section + in the manifest. The default for the global config option is set to `false`. ## Operator timeouts @@ -453,6 +644,13 @@ configuration `resource_check_interval` and `resource_check_timeout` have no effect, and the parameters are grouped under the `timeouts` key in the CRD-based configuration. +* **PatroniAPICheckInterval** + the interval between consecutive attempts waiting for the return of + Patroni Api. The default is `1s`. + +* **PatroniAPICheckTimeout** + the timeout for a response from Patroni Api. The default is `5s`. + * **resource_check_interval** interval to wait between consecutive attempts to check for the presence of some Kubernetes resource (i.e. `StatefulSet` or `PodDisruptionBudget`). The @@ -501,27 +699,62 @@ In the CRD-based configuration they are grouped under the `load_balancer` key. toggles service type load balancer pointing to the master pod of the cluster. Can be overridden by individual cluster settings. The default is `true`. -* **enable_replica_load_balancer** - toggles service type load balancer pointing to the replica pod of the - cluster. Can be overridden by individual cluster settings. The default is +* **enable_master_pooler_load_balancer** + toggles service type load balancer pointing to the master pooler pod of the + cluster. Can be overridden by individual cluster settings. The default is `false`. -* **external_traffic_policy** defines external traffic policy for load +* **enable_replica_load_balancer** + toggles service type load balancer pointing to the replica pod(s) of the + cluster. Can be overridden by individual cluster settings. The default is + `false`. + +* **enable_replica_pooler_load_balancer** + toggles service type load balancer pointing to the replica pooler pod(s) of + the cluster. Can be overridden by individual cluster settings. The default + is `false`. + +* **external_traffic_policy** + defines external traffic policy for load balancers. Allowed values are `Cluster` (default) and `Local`. -* **master_dns_name_format** defines the DNS name string template for the - master load balancer cluster. The default is - `{cluster}.{team}.{hostedzone}`, where `{cluster}` is replaced by the cluster - name, `{team}` is replaced with the team name and `{hostedzone}` is replaced - with the hosted zone (the value of the `db_hosted_zone` parameter). No other - placeholders are allowed. +* **master_dns_name_format** + defines the DNS name string template for the master load balancer cluster. + The default is `{cluster}.{namespace}.{hostedzone}`, where `{cluster}` is + replaced by the cluster name, `{namespace}` is replaced with the namespace + and `{hostedzone}` is replaced with the hosted zone (the value of the + `db_hosted_zone` parameter). The `{team}` placeholder can still be used, + although it is not recommended because the team of a cluster can change. + If the cluster name starts with the `teamId` it will also be part of the + DNS, aynway. No other placeholders are allowed! -* **replica_dns_name_format** defines the DNS name string template for the - replica load balancer cluster. The default is - `{cluster}-repl.{team}.{hostedzone}`, where `{cluster}` is replaced by the - cluster name, `{team}` is replaced with the team name and `{hostedzone}` is - replaced with the hosted zone (the value of the `db_hosted_zone` parameter). - No other placeholders are allowed. +* **master_legacy_dns_name_format** + *deprecated* default master DNS template `{cluster}.{team}.{hostedzone}` as + of pre `v1.9.0`. If cluster name starts with `teamId` then a second DNS + entry will be created using the template defined here to provide backwards + compatibility. The `teamId` prefix will be extracted from the clustername + because it follows later in the DNS string. When using a customized + `master_dns_name_format` make sure to define the legacy DNS format when + switching to v1.9.0. + +* **replica_dns_name_format** + defines the DNS name string template for the replica load balancer cluster. + The default is `{cluster}-repl.{namespace}.{hostedzone}`, where `{cluster}` + is replaced by the cluster name, `{namespace}` is replaced with the + namespace and `{hostedzone}` is replaced with the hosted zone (the value of + the `db_hosted_zone` parameter). The `{team}` placeholder can still be used, + although it is not recommended because the team of a cluster can change. + If the cluster name starts with the `teamId` it will also be part of the + DNS, aynway. No other placeholders are allowed! + +* **replica_legacy_dns_name_format** + *deprecated* default master DNS template `{cluster}-repl.{team}.{hostedzone}` + as of pre `v1.9.0`. If cluster name starts with `teamId` then a second DNS + entry will be created using the template defined here to provide backwards + compatibility. The `teamId` prefix will be extracted from the clustername + because it follows later in the DNS string. When using a customized + `master_dns_name_format` make sure to define the legacy DNS format when + switching to v1.9.0. ## AWS or GCP interaction @@ -550,6 +783,12 @@ yet officially supported. [service accounts](https://cloud.google.com/kubernetes-engine/docs/tutorials/authenticating-to-cloud-platform). The default is empty +* **wal_az_storage_account** + Azure Storage Account to use for shipping WAL segments with WAL-G. The + storage account must exist and be accessible by Postgres pods. Note, only the + name of the storage account is required. + The default is empty. + * **log_s3_bucket** S3 bucket to use for shipping Postgres daily logs. Works only with S3 on AWS. The bucket has to be present and accessible by Postgres pods. The default is @@ -562,7 +801,10 @@ yet officially supported. empty. * **aws_region** - AWS region used to store EBS volumes. The default is `eu-central-1`. + AWS region used to store EBS volumes. The default is `eu-central-1`. Note, + this option is not meant for specifying the AWS region for backups and + restore, since it can be separate from the EBS region. You have to define + AWS_REGION as a [custom environment variable](../administrator.md#custom-pod-environment-variables). * **additional_secret_mount** Additional Secret (aws or gcp credentials) to mount in the pod. @@ -588,12 +830,19 @@ These parameters configure a K8s cron job managed by the operator to produce Postgres logical backups. In the CRD-based configuration those parameters are grouped under the `logical_backup` key. +* **logical_backup_cpu_limit** + **logical_backup_cpu_request** + **logical_backup_memory_limit** + **logical_backup_memory_request** + Resource configuration for pod template in logical backup cron job. If empty + default values from `postgres_pod_resources` will be used. + * **logical_backup_docker_image** - An image for pods of the logical backup job. The [example image](../../docker/logical-backup/Dockerfile) + An image for pods of the logical backup job. The [example image](https://github.com/zalando/postgres-operator/blob/master/logical-backup/Dockerfile) runs `pg_dumpall` on a replica if possible and uploads compressed results to - an S3 bucket under the key `/spilo/pg_cluster_name/cluster_k8s_uuid/logical_backups`. + an S3 bucket under the key `////logical_backups`. The default image is the same image built with the Zalando-internal CI - pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup:v1.6.3" + pipeline. Default: "ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1" * **logical_backup_google_application_credentials** Specifies the path of the google cloud service account json file. Default is empty. @@ -602,8 +851,17 @@ grouped under the `logical_backup` key. The prefix to be prepended to the name of a k8s CronJob running the backups. Beware the prefix counts towards the name length restrictions imposed by k8s. Empty string is a legitimate value. Operator does not do the actual renaming: It simply creates the job with the new prefix. You will have to delete the old cron job manually. Default: "logical-backup-". * **logical_backup_provider** - Specifies the storage provider to which the backup should be uploaded (`s3` or `gcs`). - Default: "s3" + Specifies the storage provider to which the backup should be uploaded + (`s3`, `gcs` or `az`). Default: "s3" + +* **logical_backup_azure_storage_account_name** + Storage account name used to upload logical backups to when using Azure. Default: "" + +* **logical_backup_azure_storage_container** + Storage container used to upload logical backups to when using Azure. Default: "" + +* **logical_backup_azure_storage_account_key** + Storage account key used to authenticate with Azure when uploading logical backups. Default: "" * **logical_backup_s3_access_key_id** When set, value will be in AWS_ACCESS_KEY_ID env variable. The Default is empty. @@ -612,6 +870,9 @@ grouped under the `logical_backup` key. S3 bucket to store backup results. The bucket has to be present and accessible by Postgres pods. Default: empty. +* **logical_backup_s3_bucket_prefix** + S3 bucket prefix to use in configured bucket. Default: "spilo" + * **logical_backup_s3_endpoint** When using non-AWS S3 storage, endpoint can be set as a ENV variable. The default is empty. @@ -625,11 +886,41 @@ grouped under the `logical_backup` key. Specify server side encryption that S3 storage is using. If empty string is specified, no argument will be passed to `aws s3` command. Default: "AES256". +* **logical_backup_s3_retention_time** + Specify a retention time for logical backups stored in S3. Backups older than the specified retention + time will be deleted after a new backup was uploaded. If empty, all backups will be kept. Example values are + "3 days", "2 weeks", or "1 month". The default is empty. + * **logical_backup_schedule** Backup schedule in the cron format. Please take the [reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) into account. Default: "30 00 \* \* \*" +* **logical_backup_cronjob_environment_secret** + Reference to a Kubernetes secret, which keys will be added as environment variables to the cronjob. Default: "" + +* **logical_backup_successful_jobs_history_limit** + number of successful backup jobs to keep in cronjob history. The default is `3`. + +* **logical_backup_failed_jobs_history_limit** + number of failed backup jobs to keep in cronjob history. The default is `3`. + +* **logical_backup_ttl_seconds_after_finished** + TTL in seconds after which finished backup jobs are automatically deleted. The default is `86400`. + +The following environment variables can be passed to the logical backup +cronjob via `logical_backup_cronjob_environment_secret` to control +connectivity checks before the backup starts: + +* **LOGICAL_BACKUP_CONNECT_RETRIES** + Number of times to retry connecting to the target PostgreSQL pod before + giving up. This is useful when NetworkPolicy enforcement introduces a + short delay before a newly-created pod's IP is allowed through ingress + rules on the destination node. Default: "10" + +* **LOGICAL_BACKUP_CONNECT_RETRY_DELAY** + Delay in seconds between connectivity retries. Default: "2" + ## Debugging the operator Options to aid debugging of the operator itself. Grouped under the `debug` key. @@ -697,7 +988,7 @@ key. * **protected_role_names** List of roles that cannot be overwritten by an application, team or - infrastructure role. The default is `admin`. + infrastructure role. The default list is `admin` and `cron_admin`. * **postgres_superuser_teams** List of teams which members need the superuser role in each PG database @@ -792,7 +1083,7 @@ operator being able to provide some reasonable defaults. * **connection_pooler_image** Docker image to use for connection pooler deployment. - Default: "registry.opensource.zalan.do/acid/pgbouncer" + Default: "ghcr.io/zalando/postgres-operator/pgbouncer:latest" * **connection_pooler_max_db_connections** How many connections the pooler can max hold. This value is divided among the @@ -806,5 +1097,4 @@ operator being able to provide some reasonable defaults. **connection_pooler_default_memory_reques** **connection_pooler_default_cpu_limit** **connection_pooler_default_memory_limit** - Default resource configuration for connection pooler deployment. The internal - default for memory request and limit is `100Mi`, for CPU it is `500m` and `1`. + Default resource configuration for connection pooler deployment. diff --git a/docs/user.md b/docs/user.md index 8e406ec00..293ddf8c2 100644 --- a/docs/user.md +++ b/docs/user.md @@ -5,7 +5,7 @@ Learn how to work with the Postgres Operator in a Kubernetes (K8s) environment. ## Create a manifest for a new PostgreSQL cluster Make sure you have [set up](quickstart.md) the operator. Then you can create a -new Postgres cluster by applying manifest like this [minimal example](../manifests/minimal-postgres-manifest.yaml): +new Postgres cluster by applying manifest like this [minimal example](https://github.com/zalando/postgres-operator/blob/master/manifests/minimal-postgres-manifest.yaml): ```yaml apiVersion: "acid.zalan.do/v1" @@ -30,7 +30,7 @@ spec: databases: foo: zalando postgresql: - version: "13" + version: "18" ``` Once you cloned the Postgres Operator [repository](https://github.com/zalando/postgres-operator) @@ -45,11 +45,12 @@ Make sure, the `spec` section of the manifest contains at least a `teamId`, the The minimum volume size to run the `postgresql` resource on Elastic Block Storage (EBS) is `1Gi`. -Note, that the name of the cluster must start with the `teamId` and `-`. At -Zalando we use team IDs (nicknames) to lower the chance of duplicate cluster -names and colliding entities. The team ID would also be used to query an API to -get all members of a team and create [database roles](#teams-api-roles) for -them. Besides, the maximum cluster name length is 53 characters. +Note, that when `enable_team_id_clustername_prefix` is set to `true` the name +of the cluster must start with the `teamId` and `-`. At Zalando we use team IDs +(nicknames) to lower chances of duplicate cluster names and colliding entities. +The team ID would also be used to query an API to get all members of a team +and create [database roles](#teams-api-roles) for them. Besides, the maximum +cluster name length is 53 characters. ## Watch pods being created @@ -83,16 +84,33 @@ kubectl port-forward $PGMASTER 6432:5432 -n default ``` Open another CLI and connect to the database using e.g. the psql client. -When connecting with the `postgres` user read its password from the K8s secret -which was generated when creating the `acid-minimal-cluster`. As non-encrypted -connections are rejected by default set the SSL mode to `require`: +When connecting with a manifest role like `foo_user` user, read its password +from the K8s secret which was generated when creating `acid-minimal-cluster`. +As non-encrypted connections are rejected by default set SSL mode to `require`: ```bash -export PGPASSWORD=$(kubectl get secret postgres.acid-minimal-cluster.credentials -o 'jsonpath={.data.password}' | base64 -d) +export PGPASSWORD=$(kubectl get secret postgres.acid-minimal-cluster.credentials.postgresql.acid.zalan.do -o 'jsonpath={.data.password}' | base64 -d) export PGSSLMODE=require psql -U postgres -h localhost -p 6432 ``` +## Password encryption + +Passwords are encrypted using the `scram-sha-256` hashing method by default. Other methods can be configured by changing the `password_encryption` parameter in the cluster manifest: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-minimal-cluster +spec: + [...] + postgresql: + version: "18" + parameters: + password_encryption: scram-sha-256 +``` + ## Defining database roles in the operator Postgres Operator allows defining roles to be created in the resulting database @@ -113,7 +131,7 @@ chapter. ### Manifest roles Manifest roles are defined directly in the cluster manifest. See -[minimal postgres manifest](../manifests/minimal-postgres-manifest.yaml) +[minimal postgres manifest](https://github.com/zalando/postgres-operator/blob/master/manifests/minimal-postgres-manifest.yaml) for an example of `zalando` role, defined with `superuser` and `createdb` flags. Manifest roles are defined as a dictionary, with a role name as a key and a @@ -131,7 +149,7 @@ specified explicitly. The operator automatically generates a password for each manifest role and places it in the secret named -`{username}.{team}-{clustername}.credentials.postgresql.acid.zalan.do` in the +`{username}.{clustername}.credentials.postgresql.acid.zalan.do` in the same namespace as the cluster. This way, the application running in the K8s cluster and connecting to Postgres can obtain the password right from the secret, without ever sharing it outside of the cluster. @@ -139,6 +157,30 @@ secret, without ever sharing it outside of the cluster. At the moment it is not possible to define membership of the manifest role in other roles. +To define the secrets for the users in a different namespace than that of the +cluster, one can set `enable_cross_namespace_secret` and declare the namespace +for the secrets in the manifest in the following manner (note, that it has to +be reflected in the `database` section, too), + +```yaml +spec: + users: + # users with secret in different namespace + appspace.db_user: + - createdb + databases: + # namespace notation is part of user name + app_db: appspace.db_user +``` + +Here, anything before the first dot is considered the namespace and the text after +the first dot is the username. Also, the postgres roles of these usernames would +be in the form of `namespace.username`. + +For such usernames, the secret is created in the given namespace and its name is +of the following form, +`{namespace}.{username}.{clustername}.credentials.postgresql.acid.zalan.do` + ### Infrastructure roles An infrastructure role is a role that should be present on every PostgreSQL @@ -178,7 +220,7 @@ the user name, password etc. The secret itself is referenced by the above list them separately. ```yaml -apiVersion: v1 +apiVersion: "acid.zalan.do/v1" kind: OperatorConfiguration metadata: name: postgresql-operator-configuration @@ -198,7 +240,7 @@ Note, only the CRD-based configuration allows for referencing multiple secrets. As of now, the ConfigMap is restricted to either one or the existing template option with `infrastructure_roles_secret_name`. Please, refer to the example manifests to understand how `infrastructure_roles_secrets` has to be configured -for the [configmap](../manifests/configmap.yaml) or [CRD configuration](../manifests/postgresql-operator-default-configuration.yaml). +for the [configmap](https://github.com/zalando/postgres-operator/blob/master/manifests/configmap.yaml) or [CRD configuration](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql-operator-default-configuration.yaml). If both `infrastructure_roles_secret_name` and `infrastructure_roles_secrets` are defined the operator will create roles for both of them. So make sure, @@ -243,8 +285,8 @@ Since an infrastructure role is created uniformly on all clusters managed by the operator, it makes no sense to define it without the password. Such definitions will be ignored with a prior warning. -See [infrastructure roles secret](../manifests/infrastructure-roles.yaml) -and [infrastructure roles configmap](../manifests/infrastructure-roles-configmap.yaml) +See [infrastructure roles secret](https://github.com/zalando/postgres-operator/blob/master/manifests/infrastructure-roles.yaml) +and [infrastructure roles configmap](https://github.com/zalando/postgres-operator/blob/master/manifests/infrastructure-roles-configmap.yaml) for the examples. ### Teams API roles @@ -260,7 +302,7 @@ returns usernames. A minimal Teams API should work like this: /.../ -> ["name","anothername"] ``` -A ["fake" Teams API](../manifests/fake-teams-api.yaml) deployment is provided +A ["fake" Teams API](https://github.com/zalando/postgres-operator/blob/master/manifests/fake-teams-api.yaml) deployment is provided in the manifests folder to set up a basic API around whatever services is used for user management. The Teams API's URL is set in the operator's [configuration](reference/operator_parameters.md#automatic-creation-of-human-users-in-the-database) @@ -275,12 +317,12 @@ Postgres clusters are associated with one team by providing the `teamID` in the manifest. Additional superuser teams can be configured as mentioned in the previous paragraph. However, this is a global setting. To assign additional teams, superuser teams and single users to clusters of a given -team, use the [PostgresTeam CRD](../manifests/postgresteam.crd.yaml). +team, use the [PostgresTeam CRD](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresteam.crd.yaml). Note, by default the `PostgresTeam` support is disabled in the configuration. Switch `enable_postgres_team_crd` flag to `true` and the operator will start to watch for this CRD. Make sure, the cluster role is up to date and contains a -section for [PostgresTeam](../manifests/operator-service-account-rbac.yaml#L30). +section for [PostgresTeam](https://github.com/zalando/postgres-operator/blob/master/manifests/operator-service-account-rbac.yaml#L30). #### Additional teams @@ -330,7 +372,7 @@ spec: This creates roles for members of the `c-team` team not only in all clusters owned by `a-team`, but as well in cluster owned by `b-team`, as `a-team` is -an `additionalTeam` to `b-team` +an `additionalTeam` to `b-team` Not, you can also define `additionalSuperuserTeams` in the `PostgresTeam` manifest. By default, this option is disabled and must be configured with @@ -472,7 +514,7 @@ Postgres Operator will create the following NOLOGIN roles: The `_owner` role is the database owner and should be used when creating new database objects. All members of the `admin` role, e.g. teams API roles, can -become the owner with the `SET ROLE` command. [Default privileges](https://www.postgresql.org/docs/12/sql-alterdefaultprivileges.html) +become the owner with the `SET ROLE` command. [Default privileges](https://www.postgresql.org/docs/18/sql-alterdefaultprivileges.html) are configured for the owner role so that the `_reader` role automatically gets read-access (SELECT) to new tables and sequences and the `_writer` receives write-access (INSERT, UPDATE, DELETE on tables, @@ -501,9 +543,10 @@ Then, the schemas are owned by the database owner, too. The roles described in the previous paragraph can be granted to LOGIN roles from the `users` section in the manifest. Optionally, the Postgres Operator can also -create default LOGIN roles for the database an each schema individually. These +create default LOGIN roles for the database and each schema individually. These roles will get the `_user` suffix and they inherit all rights from their NOLOGIN -counterparts. +counterparts. Therefore, you cannot have `defaultRoles` set to `false` and enable +`defaultUsers` at the same time. | Role name | Member of | Admin | | ------------------- | -------------- | ------------- | @@ -526,9 +569,29 @@ spec: defaultUsers: true ``` +Default access privileges are also defined for LOGIN roles on database and +schema creation. This means they are currently not set when `defaultUsers` +(or `defaultRoles` for schemas) are enabled at a later point in time. + +For all LOGIN roles the operator will create K8s secrets in the namespace +specified in `secretNamespace`, if `enable_cross_namespace_secret` is set to +`true` in the config. Otherwise, they are created in the same namespace like +the Postgres cluster. Unlike roles specified with `namespace.username` under +`users`, the namespace will not be part of the role name here. Keep in mind +that the underscores in a role name are replaced with dashes in the K8s +secret name. + +```yaml +spec: + preparedDatabases: + foo: + defaultUsers: true + secretNamespace: appspace +``` + ### Schema `search_path` for default roles -The schema [`search_path`](https://www.postgresql.org/docs/13/ddl-schemas.html#DDL-SCHEMAS-PATH) +The schema [`search_path`](https://www.postgresql.org/docs/18/ddl-schemas.html#DDL-SCHEMAS-PATH) for each role will include the role name and the schemas, this role should have access to. So `foo_bar_writer` does not have to schema-qualify tables from schemas `foo_bar_writer, bar`, while `foo_writer` can look up `foo_writer` and @@ -560,10 +623,9 @@ spec: ``` Some extensions require SUPERUSER rights on creation unless they are not -whitelisted by the [pgextwlist](https://github.com/dimitri/pgextwlist) -extension, that is shipped with the Spilo image. To see which extensions are -on the list check the `extwlist.extension` parameter in the postgresql.conf -file. +allowed by the [pgextwlist](https://github.com/dimitri/pgextwlist) extension, +that is shipped with the Spilo image. To see which extensions are on the list +check the `extwlist.extension` parameter in the postgresql.conf file. ```bash SHOW extwlist.extensions; @@ -624,12 +686,38 @@ The minimum limits to properly run the `postgresql` resource are configured to manifest the operator will raise the limits to the configured minimum values. If no resources are defined in the manifest they will be obtained from the configured [default requests](reference/operator_parameters.md#kubernetes-resource-requests). +If neither defaults nor minimum limits are configured the operator will not +specify any resources and it's up to K8s (or your own) admission hooks to +handle it. -## Use taints, tolerations and node affinity for dedicated PostgreSQL nodes +### HugePages support + +The operator supports [HugePages](https://www.postgresql.org/docs/18/kernel-resources.html#LINUX-HUGEPAGES). +To enable HugePages, set the matching resource requests and/or limits in the manifest: + +```yaml +spec: + resources: + requests: + hugepages-2Mi: 250Mi + hugepages-1Gi: 1Gi + limits: + hugepages-2Mi: 500Mi + hugepages-1Gi: 2Gi +``` + +There are no minimums or maximums and the default is 0 for both HugePage sizes, +but Kubernetes will not spin up the pod if the requested HugePages cannot be allocated. +For more information on HugePages in Kubernetes, see also +[https://kubernetes.io/docs/tasks/manage-hugepages/scheduling-hugepages/](https://kubernetes.io/docs/tasks/manage-hugepages/scheduling-hugepages/) + +## Use taints, tolerations, node affinity and topology spread constraint for dedicated PostgreSQL nodes To ensure Postgres pods are running on nodes without any other application pods, you can use [taints and tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) -and configure the required toleration in the manifest. +and configure the required toleration in the manifest. Tolerations can also be +defined in the [operator config](administrator.md#use-taints-and-tolerations-for-dedicated-postgresql-nodes) +to apply for all Postgres clusters. ```yaml spec: @@ -661,10 +749,30 @@ spec: - pci ``` +If you need to define a `nodeAffinity` for all your Postgres clusters use the +`node_readiness_label` [configuration](administrator.md#node-readiness-labels). + +If you need PostgreSQL Pods to run on separate nodes, you can use the +[topologySpreadConstraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/) to control how they are distributed across your cluster. +This ensures they are spread among failure domains such as +regions, zones, nodes, or other user-defined topology domains. + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-minimal-cluster +spec: + topologySpreadConstraints: + - maxskew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule +``` + ## In-place major version upgrade -Starting with Spilo 13, operator supports in-place major version upgrade to a -higher major version (e.g. from PG 10 to PG 12). To trigger the upgrade, +Starting with Spilo 14, operator supports in-place major version upgrade to a +higher major version (e.g. from PG 14 to PG 16). To trigger the upgrade, simply increase the version in the manifest. It is your responsibility to test your applications against the new version before the upgrade; downgrading is not supported. The easiest way to do so is to try the upgrade on the cloned @@ -690,28 +798,41 @@ source cluster. If you create it in the same Kubernetes environment, use a different name. ```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-minimal-cluster-clone spec: clone: uid: "efd12e58-5786-11e8-b5a7-06148230260c" - cluster: "acid-batman" - timestamp: "2017-12-19T12:40:33+01:00" + cluster: "acid-minimal-cluster" + timestamp: "2025-12-19T12:40:33+01:00" ``` Here `cluster` is a name of a source cluster that is going to be cloned. A new cluster will be cloned from S3, using the latest backup before the `timestamp`. -Note, that a time zone is required for `timestamp` in the format of +00:00 which -is UTC. The `uid` field is also mandatory. The operator will use it to find a -correct key inside an S3 bucket. You can find this field in the metadata of the -source cluster: +Note, a time zone is required for `timestamp` in the format of `+00:00` (UTC). + +The operator will try to find the WAL location based on the configured +`wal_[s3|gs]_bucket` or `wal_az_storage_account` and the specified `uid`. +You can find the UID of the source cluster in its metadata: ```yaml apiVersion: acid.zalan.do/v1 kind: postgresql metadata: - name: acid-test-cluster + name: acid-minimal-cluster uid: efd12e58-5786-11e8-b5a7-06148230260c ``` +If your source cluster uses a WAL location different from the global +configuration you can specify the full path under `s3_wal_path`. For +[Google Cloud Platform](administrator.md#google-cloud-platform-setup) +or [Azure](administrator.md#azure-setup) +it can only be set globally with [custom Pod environment variables](administrator.md#custom-pod-environment-variables) +or locally in the Postgres manifest's [`env`](administrator.md#via-postgres-cluster-manifest) section. + + For non AWS S3 following settings can be set to support cloning from other S3 implementations: @@ -719,8 +840,9 @@ implementations: spec: clone: uid: "efd12e58-5786-11e8-b5a7-06148230260c" - cluster: "acid-batman" - timestamp: "2017-12-19T12:40:33+01:00" + cluster: "acid-minimal-cluster" + timestamp: "2025-12-19T12:40:33+01:00" + s3_wal_path: "s3://custom/path/to/bucket" s3_endpoint: https://s3.acme.org s3_access_key_id: 0123456789abcdef0123456789abcdef s3_secret_access_key: 0123456789abcdef0123456789abcdef @@ -730,7 +852,7 @@ spec: ### Clone directly Another way to get a fresh copy of your source DB cluster is via -[pg_basebackup](https://www.postgresql.org/docs/13/app-pgbasebackup.html). To +[pg_basebackup](https://www.postgresql.org/docs/18/app-pgbasebackup.html). To use this feature simply leave out the timestamp field from the clone section. The operator will connect to the service of the source cluster by name. If the cluster is called test, then the connection string will look like host=test @@ -740,89 +862,152 @@ namespace. ```yaml spec: clone: - cluster: "acid-batman" + cluster: "acid-minimal-cluster" ``` Be aware that on a busy source database this can result in an elevated load! +## Restore in place + +There is also a possibility to restore a database without cloning it. The +advantage to this is that there is no need to change anything on the +application side. However, as it involves deleting the database first, this +process is of course riskier than cloning (which involves adjusting the +connection parameters of the app). + +First, make sure there is no writing activity on your DB, and save the UID. +Then delete the `postgresql` K8S resource: + +```bash +zkubectl delete postgresql acid-test-restore +``` + +Then deploy a new manifest with the same name, referring to itself +(both name and UID) in the `clone` section: + +```yaml +metadata: + name: acid-minimal-cluster + # [...] +spec: + # [...] + clone: + cluster: "acid-minimal-cluster" # the same as metadata.name above! + uid: "" + timestamp: "2022-04-01T10:11:12.000+00:00" +``` + +This will create a new database cluster with the same name but different UID, +whereas the database will be in the state it was at the specified time. + +:warning: The backups and WAL files for the original DB are retained under the +original UID, making it possible retry restoring. However, it is probably +better to create a temporary clone for experimenting or finding out to which +point you should restore. + ## Setting up a standby cluster Standby cluster is a [Patroni feature](https://github.com/zalando/patroni/blob/master/docs/replica_bootstrap.rst#standby-cluster) -that first clones a database, and keeps replicating changes afterwards. As the -replication is happening by the means of archived WAL files (stored on S3 or -the equivalent of other cloud providers), the standby cluster can exist in a -different location than its source database. Unlike cloning, the PostgreSQL -version between source and target cluster has to be the same. +that first clones a database, and keeps replicating changes afterwards. It can +exist in a different location than its source database, but unlike cloning, +the PostgreSQL version between source and target cluster has to be the same. To start a cluster as standby, add the following `standby` section in the YAML -file and specify the S3 bucket path. An empty path will result in an error and -no statefulset will be created. +file. You can stream changes from archived WAL files (AWS S3 or Google Cloud +Storage), from a remote primary, or combine a remote primary with a WAL archive. +At least one of `s3_wal_path`, `gs_wal_path`, or `standby_host` must be specified. +Note that `s3_wal_path` and `gs_wal_path` are mutually exclusive. ```yaml spec: standby: - s3_wal_path: "s3 bucket path to the master" + s3_wal_path: "s3:///spilo///wal/" ``` -At the moment, the operator only allows to stream from the WAL archive of the -master. Thus, it is recommended to deploy standby clusters with only [one pod](../manifests/standby-manifest.yaml#L10). -You can raise the instance count when detaching. Note, that the same pod role -labels like for normal clusters are used: The standby leader is labeled as -`master`. +For GCS, you have to define STANDBY_GOOGLE_APPLICATION_CREDENTIALS as a +[custom pod environment variable](administrator.md#custom-pod-environment-variables). +It is not set from the config to allow for overriding. + +```yaml +spec: + standby: + gs_wal_path: "gs:///spilo///wal/" +``` + +For a remote primary you specify the host address and optionally the port. +If you leave out the port Patroni will use `"5432"`. + +```yaml +spec: + standby: + standby_host: "acid-minimal-cluster.default" + standby_port: "5433" +``` + +You can also combine a remote primary with a WAL archive for additional redundancy: + +```yaml +spec: + standby: + standby_host: "acid-minimal-cluster.default" + standby_port: "5433" + s3_wal_path: "s3:///spilo///wal/" +``` + +Note, that the pods and services use the same role labels like for normal clusters: +The standby leader is labeled as `master`. When using the `standby_host` option +you have to copy the credentials from the source cluster's secrets to successfully +bootstrap a standby cluster (see next chapter). ### Providing credentials of source cluster A standby cluster is replicating the data (including users and passwords) from the source database and is read-only. The system and application users (like standby, postgres etc.) all have a password that does not match the credentials -stored in secrets which are created by the operator. One solution is to create -secrets beforehand and paste in the credentials of the source cluster. +stored in secrets which are created by the operator. You have two options: + +a. Create secrets manually beforehand and paste the credentials of the source + cluster +b. Let the operator create the secrets when it bootstraps the standby cluster. + Patch the secrets with the credentials of the source cluster. Replace the + spilo pods. + Otherwise, you will see errors in the Postgres logs saying users cannot log in and the operator logs will complain about not being able to sync resources. +If you stream changes from a remote primary you have to align the secrets or +the standby cluster will not start up. -When you only run a standby leader, you can safely ignore this, as it will be -sorted out once the cluster is detached from the source. It is also harmless if -you don’t plan it. But, when you created a standby replica, too, fix the -credentials right away. WAL files will pile up on the standby leader if no -connection can be established between standby replica(s). You can also edit the -secrets after their creation. Find them by: - -```bash -kubectl get secrets --all-namespaces | grep -``` +If you stream changes from WAL files and you only run a standby leader, you +can safely ignore the secret mismatch, as it will be sorted out once the +cluster is detached from the source. It is also harmless if you do not plan it. +But, when you create a standby replica, too, fix the credentials right away. +WAL files will pile up on the standby leader if no connection can be +established between standby replica(s). ### Promote the standby One big advantage of standby clusters is that they can be promoted to a proper database cluster. This means it will stop replicating changes from the source, and start accept writes itself. This mechanism makes it possible to move -databases from one place to another with minimal downtime. Currently, the -operator does not support promoting a standby cluster. It has to be done -manually using `patronictl edit-config` inside the postgres container of the -standby leader pod. Remove the following lines from the YAML structure and the -leader promotion happens immediately. Before doing so, make sure that the -standby is not behind the source database. +databases from one place to another with minimal downtime. -```yaml -standby_cluster: - create_replica_methods: - - bootstrap_standby_with_wale - - basebackup_fast_xlog - restore_command: envdir "/home/postgres/etc/wal-e.d/env-standby" /scripts/restore_command.sh - "%f" "%p" -``` +Before promoting a standby cluster, make sure that the standby is not behind +the source database. You should ideally stop writes to your source cluster and +then create a dummy database object that you check for being replicated in the +target to verify all data has been copied. -Finally, remove the `standby` section from the postgres cluster manifest. +To promote, remove the `standby` section from the postgres cluster manifest. +A rolling update will be triggered removing the `STANDBY_*` environment +variables from the pods, followed by a Patroni config update that promotes the +cluster. -### Turn a normal cluster into a standby +### Adding standby section after promotion -There is no way to transform a non-standby cluster to a standby cluster through -the operator. Adding the `standby` section to the manifest of a running -Postgres cluster will have no effect. But, as explained in the previous -paragraph it can be done manually through `patronictl edit-config`. This time, -by adding the `standby_cluster` section to the Patroni configuration. However, -the transformed standby cluster will not be doing any streaming. It will be in -standby mode and allow read-only transactions only. +Turning a running cluster into a standby is not easily possible and should be +avoided. The best way is to remove the cluster and resubmit the manifest +after a short wait of a few minutes. Adding the `standby` section would turn +the database cluster in read-only mode on next operator SYNC cycle but it +does not sync automatically with the source cluster again. ## Sidecar Support @@ -845,6 +1030,7 @@ spec: env: - name: "ENV_VAR_NAME" value: "any-k8s-env-things" + command: ['sh', '-c', 'echo "logging" > /opt/logs.txt'] ``` In addition to any environment variables you specify, the following environment @@ -864,6 +1050,42 @@ option must be set to `true`. If you want to add a sidecar to every cluster managed by the operator, you can specify it in the [operator configuration](administrator.md#sidecars-for-postgres-clusters) instead. +### Accessing the PostgreSQL socket from sidecars + +If enabled by the `share_pgsocket_with_sidecars` option in the operator +configuration the PostgreSQL socket is placed in a volume of type `emptyDir` +named `postgresql-run`. To allow access to the socket from any sidecar +container simply add a VolumeMount to this volume to your sidecar spec. + +```yaml + - name: "container-name" + image: "company/image:tag" + volumeMounts: + - mountPath: /var/run + name: postgresql-run +``` + +If you do not want to globally enable this feature and only use it for single +Postgres clusters, specify an `EmptyDir` volume under `additionalVolumes` in +the manifest: + +```yaml +spec: + additionalVolumes: + - name: postgresql-run + mountPath: /var/run/postgresql + targetContainers: + - all + volumeSource: + emptyDir: {} + sidecars: + - name: "container-name" + image: "company/image:tag" + volumeMounts: + - mountPath: /var/run + name: postgresql-run +``` + ## InitContainers Support Each cluster can specify arbitrary init containers to run. These containers can @@ -888,9 +1110,9 @@ specified but globally disabled in the configuration. The ## Increase volume size -Postgres operator supports statefulset volume resize if you're using the -operator on top of AWS. For that you need to change the size field of the -volume description in the cluster manifest and apply the change: +Postgres operator supports statefulset volume resize without doing a rolling +update. For that you need to change the size field of the volume description +in the cluster manifest and apply the change: ```yaml spec: @@ -899,22 +1121,29 @@ spec: ``` The operator compares the new value of the size field with the previous one and -acts on differences. +acts on differences. The `storage_resize_mode` can be configured. By default, +the operator will adjust the PVCs and leave it to K8s and the infrastructure to +apply the change. -You can only enlarge the volume with the process described above, shrinking is -not supported and will emit a warning. After this update all the new volumes in -the statefulset are allocated according to the new size. To enlarge persistent -volumes attached to the running pods, the operator performs the following -actions: +When using AWS with gp3 volumes you should set the mode to `mixed` because it +will also adjust the IOPS and throughput that can be defined in the manifest. +Check the [AWS docs](https://aws.amazon.com/ebs/general-purpose/) to learn +about default and maximum values. Keep in mind that AWS rate-limits updating +volume specs to no more than once every 6 hours. -* call AWS API to change the volume size +```yaml +spec: + volume: + size: 5Gi # new volume size + iops: 4000 + throughput: 500 +``` -* connect to pod using `kubectl exec` and resize filesystem with `resize2fs`. - -Fist step has a limitation, AWS rate-limits this operation to no more than once -every 6 hours. Note, that if the statefulset is scaled down before resizing the -new size is only applied to the volumes attached to the running pods. The -size of volumes that correspond to the previously running pods is not changed. +The operator can only enlarge volumes. Shrinking is not supported and will emit +a warning. However, it can be done manually after updating the manifest. You +have to delete the PVC, which will hang until you also delete the corresponding +pod. Proceed with the next pod when the cluster is healthy again and replicas +are streaming. ## Logical backups @@ -1012,14 +1241,19 @@ don't know the value, use `103` which is the GID from the default Spilo image OpenShift allocates the users and groups dynamically (based on scc), and their range is different in every namespace. Due to this dynamic behaviour, it's not trivial to know at deploy time the uid/gid of the user in the cluster. -Therefore, instead of using a global `spilo_fsgroup` setting, use the -`spiloFSGroup` field per Postgres cluster. +Therefore, instead of using a global `spilo_fsgroup` setting in operator +configuration or use the `spiloFSGroup` field per Postgres cluster manifest. + +For testing purposes, you can generate a self-signed certificate with openssl: +```sh +openssl req -x509 -nodes -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=acid.zalan.do" +``` Upload the cert as a kubernetes secret: ```sh kubectl create secret tls pg-tls \ - --key pg-tls.key \ - --cert pg-tls.crt + --key tls.key \ + --cert tls.crt ``` When doing client auth, CA can come optionally from the same secret: @@ -1046,8 +1280,7 @@ spec: Optionally, the CA can be provided by a different secret: ```sh -kubectl create secret generic pg-tls-ca \ - --from-file=ca.crt=ca.crt +kubectl create secret generic pg-tls-ca --from-file=ca.crt=ca.crt ``` Then configure the postgres resource with the TLS secret: @@ -1070,3 +1303,16 @@ Alternatively, it is also possible to use Certificate rotation is handled in the Spilo image which checks every 5 minutes if the certificates have changed and reloads postgres accordingly. + +### TLS certificates for connection pooler + +By default, the pgBouncer image generates its own TLS certificate like Spilo. +When the `tls` section is specified in the manifest it will be used for the +connection pooler pod(s) as well. The security context options are hard coded +to `runAsUser: 100` and `runAsGroup: 101`. The `fsGroup` will be the same +like for Spilo. + +As of now, the operator does not sync the pooler deployment automatically +which means that changes in the pod template are not caught. You need to +toggle `enableConnectionPooler` to set environment variables, volumes, secret +mounts and securityContext required for TLS support in the pooler pod. diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 3eb8c9d70..1f4083116 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,28 +1,20 @@ # An image to run e2e tests. # The image does not include the tests; all necessary files are bind-mounted when a container starts. -FROM ubuntu:20.04 +FROM python:3.11-slim LABEL maintainer="Team ACID @ Zalando " ENV TERM xterm-256color -COPY requirements.txt ./ -COPY scm-source.json ./ - -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ - python3 \ - python3-setuptools \ - python3-pip \ - curl \ - vim \ - && pip3 install --no-cache-dir -r requirements.txt \ - && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl \ +RUN apt-get -qq -y update \ + # https://www.psycopg.org/docs/install.html#psycopg-vs-psycopg-binary + && apt-get -qq -y install --no-install-recommends curl vim python3-dev \ + && curl -LO https://dl.k8s.io/release/v1.36.1/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl \ - && apt-get clean \ + && apt-get -qq -y clean \ && rm -rf /var/lib/apt/lists/* -# working line -# python3 -m unittest discover -v --failfast -k test_e2e.EndToEndTestCase.test_lazy_spilo_upgrade --start-directory tests -ENTRYPOINT ["python3", "-m", "unittest"] -CMD ["discover","-v","--failfast","--start-directory","/tests"] \ No newline at end of file +COPY requirements.txt ./ +RUN pip install -r ./requirements.txt + +CMD ["python", "-m", "unittest", "discover", "-v", "--failfast", "--start-directory", "/tests"] \ No newline at end of file diff --git a/e2e/Makefile b/e2e/Makefile index b904ad763..09ac74986 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -11,7 +11,7 @@ endif LOCAL_BUILD_FLAGS ?= $(BUILD_FLAGS) LDFLAGS ?= -X=main.version=$(VERSION) -IMAGE ?= registry.opensource.zalan.do/acid/$(BINARY) +IMAGE ?= ghcr.io/zalando/$(BINARY) VERSION ?= $(shell git describe --tags --always --dirty) TAG ?= $(VERSION) GITHEAD = $(shell git rev-parse --short HEAD) @@ -29,25 +29,24 @@ default: tools clean: rm -rf manifests + rm -rf tls copy: clean mkdir manifests - cp ../manifests -r . + cp -r ../manifests . + mkdir tls -docker: scm-source.json +docker: docker build -t "$(IMAGE):$(TAG)" . -scm-source.json: ../.git - echo '{\n "url": "git:$(GITURL)",\n "revision": "$(GITHEAD)",\n "author": "$(USER)",\n "status": "$(GITSTATUS)"\n}' > scm-source.json - push: docker docker push "$(IMAGE):$(TAG)" tools: # install pinned version of 'kind' - # go get must run outside of a dir with a (module-based) Go project ! - # otherwise go get updates project's dependencies and/or behaves differently - cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.9.0 + # go install must run outside of a dir with a (module-based) Go project ! + # otherwise go install updates project's dependencies and/or behaves differently + cd "/tmp" && go install sigs.k8s.io/kind@v0.27.0 e2etest: tools copy clean ./run.sh main diff --git a/e2e/exec_into_env.sh b/e2e/exec_into_env.sh index ef12ba18a..4d017bc35 100755 --- a/e2e/exec_into_env.sh +++ b/e2e/exec_into_env.sh @@ -2,8 +2,9 @@ export cluster_name="postgres-operator-e2e-tests" export kubeconfig_path="/tmp/kind-config-${cluster_name}" -export operator_image="registry.opensource.zalan.do/acid/postgres-operator:latest" -export e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3" +export operator_image="ghcr.io/zalando/postgres-operator:latest" +export pooler_image="ghcr.io/zalando/postgres-operator/pgbouncer:latest" +export e2e_test_runner_image="ghcr.io/zalando/postgres-operator-e2e-tests-runner:latest" docker run -it --entrypoint /bin/bash --network=host -e "TERM=xterm-256color" \ --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \ @@ -11,4 +12,5 @@ docker run -it --entrypoint /bin/bash --network=host -e "TERM=xterm-256color" \ --mount type=bind,source="$(readlink -f tests)",target=/tests \ --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \ --mount type=bind,source="$(readlink -f scripts)",target=/scripts \ - -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" + -e OPERATOR_IMAGE="${operator_image}" -e POOLER_IMAGE="${pooler_image}" \ + "${e2e_test_runner_image}" diff --git a/e2e/kind-cluster-postgres-operator-e2e-tests.yaml b/e2e/kind-cluster-postgres-operator-e2e-tests.yaml index 752e993cd..da633db82 100644 --- a/e2e/kind-cluster-postgres-operator-e2e-tests.yaml +++ b/e2e/kind-cluster-postgres-operator-e2e-tests.yaml @@ -4,3 +4,5 @@ nodes: - role: control-plane - role: worker - role: worker +featureGates: + StatefulSetAutoDeletePVC: true diff --git a/e2e/requirements.txt b/e2e/requirements.txt index b276d2537..30b656552 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -1,3 +1,3 @@ -kubernetes==11.0.0 -timeout_decorator==0.4.1 -pyyaml==5.4.1 +kubernetes==31.0.0 +timeout_decorator==0.5.0 +pyyaml==6.0.3 diff --git a/e2e/run.sh b/e2e/run.sh index 2d5708778..c5daf81f6 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -7,23 +7,52 @@ set -o pipefail IFS=$'\n\t' readonly cluster_name="postgres-operator-e2e-tests" -readonly kubeconfig_path="/tmp/kind-config-${cluster_name}" -readonly spilo_image="registry.opensource.zalan.do/acid/spilo-13-e2e:0.3" -readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3" +readonly kubeconfig_path="${HOME}/kind-config-${cluster_name}" +readonly spilo_image="ghcr.io/zalando/spilo-18:4.1-p1" +readonly e2e_test_runner_image="ghcr.io/zalando/postgres-operator-e2e-tests-runner:latest" export GOPATH=${GOPATH-~/go} export PATH=${GOPATH}/bin:$PATH +# detect system architecture for pulling the correct Spilo image +case "$(uname -m)" in + x86_64) readonly PLATFORM="linux/amd64" ;; + aarch64|arm64) readonly PLATFORM="linux/arm64" ;; + *) echo "Unsupported architecture: $(uname -m)"; exit 1 ;; +esac + echo "Clustername: ${cluster_name}" echo "Kubeconfig path: ${kubeconfig_path}" function pull_images(){ operator_tag=$(git describe --tags --always --dirty) - if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator:${operator_tag}) ]] - then - docker pull registry.opensource.zalan.do/acid/postgres-operator:latest - fi - operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) + components=("postgres-operator" "pooler") + image_urls=("ghcr.io/zalando/postgres-operator:${operator_tag}" "ghcr.io/zalando/postgres-operator/pgbouncer:${operator_tag}") + + for i in "${!components[@]}"; do + component="${components[$i]}" + image="${image_urls[$i]}" + + if [[ -z $(docker images -q "$image") ]]; then + echo "Pulling $component image: $image" + if ! docker pull "$image"; then + echo "Failed to pull $component image: $image" + exit 1 + fi + else + echo "$component image already exists: $image" + fi + + # Set variables for later use + if [[ "$component" == "postgres-operator" ]]; then + operator_image="$image" + elif [[ "$component" == "pooler" ]]; then + pooler_image="$image" + fi + done + + echo "Using operator image: $operator_image" + echo "Using pooler image: $pooler_image" } function start_kind(){ @@ -36,14 +65,17 @@ function start_kind(){ export KUBECONFIG="${kubeconfig_path}" kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml - docker pull "${spilo_image}" + + echo "Pulling Spilo image for platform ${PLATFORM}" + docker pull --platform ${PLATFORM} "${spilo_image}" kind load docker-image "${spilo_image}" --name ${cluster_name} } -function load_operator_image() { - echo "Loading operator image" +function load_operator_images() { + echo "Loading operator images" export KUBECONFIG="${kubeconfig_path}" kind load docker-image "${operator_image}" --name ${cluster_name} + kind load docker-image "${pooler_image}" --name ${cluster_name} } function set_kind_api_server_ip(){ @@ -52,7 +84,11 @@ function set_kind_api_server_ip(){ # but update the IP address of the API server to the one from the Docker 'bridge' network readonly local kind_api_server_port=6443 # well-known in the 'kind' codebase readonly local kind_api_server=$(docker inspect --format "{{ .NetworkSettings.Networks.kind.IPAddress }}:${kind_api_server_port}" "${cluster_name}"-control-plane) - sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" "${kubeconfig_path}" + sed "s/server.*$/server: https:\/\/$kind_api_server/g" "${kubeconfig_path}" > "${kubeconfig_path}".tmp && mv "${kubeconfig_path}".tmp "${kubeconfig_path}" +} + +function generate_certificate(){ + openssl req -x509 -nodes -newkey rsa:2048 -keyout tls/tls.key -out tls/tls.crt -subj "/CN=acid.zalan.do" } function run_tests(){ @@ -62,10 +98,12 @@ function run_tests(){ docker run --rm --network=host -e "TERM=xterm-256color" \ --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \ --mount type=bind,source="$(readlink -f manifests)",target=/manifests \ + --mount type=bind,source="$(readlink -f tls)",target=/tls \ --mount type=bind,source="$(readlink -f tests)",target=/tests \ --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \ --mount type=bind,source="$(readlink -f scripts)",target=/scripts \ - -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" ${E2E_TEST_CASE-} $@ + -e OPERATOR_IMAGE="${operator_image}" -e POOLER_IMAGE="${pooler_image}" \ + "${e2e_test_runner_image}" ${E2E_TEST_CASE-} $@ } function cleanup(){ @@ -80,8 +118,9 @@ function main(){ [[ -z ${NOCLEANUP-} ]] && trap "cleanup" QUIT TERM EXIT pull_images [[ ! -f ${kubeconfig_path} ]] && start_kind - load_operator_image + load_operator_images set_kind_api_server_ip + generate_certificate shift run_tests $@ diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py index 85bcb6245..0ef3d6315 100644 --- a/e2e/tests/k8s_api.py +++ b/e2e/tests/k8s_api.py @@ -20,12 +20,13 @@ class K8sApi: self.config = config.load_kube_config() self.k8s_client = client.ApiClient() + self.rbac_api = client.RbacAuthorizationV1Api() self.core_v1 = client.CoreV1Api() self.apps_v1 = client.AppsV1Api() - self.batch_v1_beta1 = client.BatchV1beta1Api() + self.batch_v1 = client.BatchV1Api() self.custom_objects_api = client.CustomObjectsApi() - self.policy_v1_beta1 = client.PolicyV1beta1Api() + self.policy_v1 = client.PolicyV1Api() self.storage_v1_api = client.StorageV1Api() @@ -53,7 +54,7 @@ class K8s: return master_pod_node, replica_pod_nodes - def get_cluster_nodes(self, cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'): + def get_cluster_nodes(self, cluster_labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): m = [] r = [] podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=cluster_labels) @@ -156,6 +157,30 @@ class K8s: while not get_services(): time.sleep(self.RETRY_TIMEOUT_SEC) + def count_pods_with_volume_mount(self, mount_name, labels, namespace='default'): + pod_count = 0 + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items + for pod in pods: + for mount in pod.spec.containers[0].volume_mounts: + if mount.name == mount_name: + pod_count += 1 + + return pod_count + + def count_pods_with_env_variable(self, env_variable_key, labels, namespace='default'): + pod_count = 0 + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items + for pod in pods: + for env in pod.spec.containers[0].env: + if env.name == env_variable_key: + pod_count += 1 + + return pod_count + + def count_pods_with_rolling_update_flag(self, labels, namespace='default'): + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items + return len(list(filter(lambda x: "zalando-postgres-operator-rolling-update-required" in x.metadata.annotations, pods))) + def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) @@ -175,9 +200,12 @@ class K8s: return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items) def count_pdbs_with_label(self, labels, namespace='default'): - return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( + return len(self.api.policy_v1.list_namespaced_pod_disruption_budget( namespace, label_selector=labels).items) + def count_pvcs_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_persistent_volume_claim(namespace, label_selector=labels).items) + def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items return len(list(filter(lambda x: x.status.phase == 'Running', pods))) @@ -189,16 +217,30 @@ class K8s: def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): pod_phase = 'Failing over' new_pod_node = '' - + pods_with_update_flag = self.count_pods_with_rolling_update_flag(labels, namespace) while (pod_phase != 'Running') or (new_pod_node not in failover_targets): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items if pods: new_pod_node = pods[0].spec.node_name pod_phase = pods[0].status.phase time.sleep(self.RETRY_TIMEOUT_SEC) + + while pods_with_update_flag != 0: + pods_with_update_flag = self.count_pods_with_rolling_update_flag(labels, namespace) + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_namespace_creation(self, namespace='default'): + ns_found = False + while ns_found != True: + ns = self.api.core_v1.list_namespace().items + for n in ns: + if n.metadata.name == namespace: + ns_found = True + break + time.sleep(self.RETRY_TIMEOUT_SEC) def get_logical_backup_job(self, namespace='default'): - return self.api.batch_v1_beta1.list_namespaced_cron_job(namespace, label_selector="application=spilo") + return self.api.batch_v1.list_namespaced_cron_job(namespace, label_selector="application=spilo-logical-backup") def wait_for_logical_backup_job(self, expected_num_of_jobs): while (len(self.get_logical_backup_job().items) != expected_num_of_jobs): @@ -222,6 +264,18 @@ class K8s: def patch_pod(self, data, pod_name, namespace="default"): self.api.core_v1.patch_namespaced_pod(pod_name, namespace, data) + def create_tls_secret_with_kubectl(self, secret_name): + return subprocess.run( + ["kubectl", "create", "secret", "tls", secret_name, "--key=tls/tls.key", "--cert=tls/tls.crt"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + def create_tls_ca_secret_with_kubectl(self, secret_name): + return subprocess.run( + ["kubectl", "create", "secret", "generic", secret_name, "--from-file=ca.crt=tls/ca.crt"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + def create_with_kubectl(self, path): return subprocess.run( ["kubectl", "apply", "-f", path], @@ -233,6 +287,13 @@ class K8s: stdout=subprocess.PIPE, stderr=subprocess.PIPE) + def patroni_rest(self, pod, path): + r = self.exec_with_kubectl(pod, "curl localhost:8008/" + path) + if not r.returncode == 0 or not r.stdout.decode()[0:1] == "{": + return None + + return json.loads(r.stdout.decode()) + def get_patroni_state(self, pod): r = self.exec_with_kubectl(pod, "patronictl list -f json") if not r.returncode == 0 or not r.stdout.decode()[0:1] == "[": @@ -253,7 +314,7 @@ class K8s: def get_patroni_running_members(self, pod="acid-minimal-cluster-0"): result = self.get_patroni_state(pod) - return list(filter(lambda x: "State" in x and x["State"] == "running", result)) + return list(filter(lambda x: "State" in x and x["State"] in ["running", "streaming"], result)) def get_deployment_replica_count(self, name="acid-minimal-cluster-pooler", namespace="default"): try: @@ -295,6 +356,15 @@ class K8s: def get_cluster_replica_pod(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): return self.get_cluster_pod('replica', labels, namespace) + def get_secret(self, username, clustername='acid-minimal-cluster', namespace='default'): + secret = self.api.core_v1.read_namespaced_secret( + "{}.{}.credentials.postgresql.acid.zalan.do".format(username.replace("_","-"), clustername), namespace) + secret.metadata.resource_version = None + secret.metadata.uid = None + return secret + + def create_secret(self, secret, namespace='default'): + return self.api.core_v1.create_namespaced_secret(namespace, secret) class K8sBase: ''' @@ -413,6 +483,10 @@ class K8sBase: while not get_services(): time.sleep(self.RETRY_TIMEOUT_SEC) + def count_pods_with_rolling_update_flag(self, labels, namespace='default'): + pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items + return len(list(filter(lambda x: "zalando-postgres-operator-rolling-update-required" in x.metadata.annotations, pods))) + def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) @@ -432,9 +506,12 @@ class K8sBase: return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items) def count_pdbs_with_label(self, labels, namespace='default'): - return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( + return len(self.api.policy_v1.list_namespaced_pod_disruption_budget( namespace, label_selector=labels).items) + def count_pvcs_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_persistent_volume_claim(namespace, label_selector=labels).items) + def count_running_pods(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items return len(list(filter(lambda x: x.status.phase == 'Running', pods))) @@ -446,7 +523,7 @@ class K8sBase: def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): pod_phase = 'Failing over' new_pod_node = '' - + pods_with_update_flag = self.count_pods_with_rolling_update_flag(labels, namespace) while (pod_phase != 'Running') or (new_pod_node not in failover_targets): pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items if pods: @@ -454,8 +531,12 @@ class K8sBase: pod_phase = pods[0].status.phase time.sleep(self.RETRY_TIMEOUT_SEC) + while pods_with_update_flag != 0: + pods_with_update_flag = self.count_pods_with_rolling_update_flag(labels, namespace) + time.sleep(self.RETRY_TIMEOUT_SEC) + def get_logical_backup_job(self, namespace='default'): - return self.api.batch_v1_beta1.list_namespaced_cron_job(namespace, label_selector="application=spilo") + return self.api.batch_v1.list_namespaced_cron_job(namespace, label_selector="application=spilo") def wait_for_logical_backup_job(self, expected_num_of_jobs): while (len(self.get_logical_backup_job().items) != expected_num_of_jobs): @@ -486,6 +567,13 @@ class K8sBase: stdout=subprocess.PIPE, stderr=subprocess.PIPE) + def patroni_rest(self, pod, path): + r = self.exec_with_kubectl(pod, "curl localhost:8008/" + path) + if not r.returncode == 0 or not r.stdout.decode()[0:1] == "{": + return None + + return json.loads(r.stdout.decode()) + def get_patroni_state(self, pod): r = self.exec_with_kubectl(pod, "patronictl list -f json") if not r.returncode == 0 or not r.stdout.decode()[0:1] == "[": @@ -494,7 +582,7 @@ class K8sBase: def get_patroni_running_members(self, pod): result = self.get_patroni_state(pod) - return list(filter(lambda x: x["State"] == "running", result)) + return list(filter(lambda x: x["State"] in ["running", "streaming"], result)) def get_statefulset_image(self, label_selector="application=spilo,cluster-name=acid-minimal-cluster", namespace='default'): ssets = self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=label_selector, limit=1) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 2b9e3ad28..7ce80a6c9 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -4,16 +4,17 @@ import time import timeout_decorator import os import yaml +import base64 -from datetime import datetime +from datetime import datetime, date, timedelta from kubernetes import client from tests.k8s_api import K8s from kubernetes.client.rest import ApiException -SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-13-e2e:0.3" -SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-13-e2e:0.4" - +SPILO_CURRENT = "ghcr.io/zalando/spilo-e2e:dev-18.3" +SPILO_LAZY = "ghcr.io/zalando/spilo-e2e:dev-18.4" +SPILO_FULL_IMAGE = "ghcr.io/zalando/spilo-18:4.1-p1" def to_selector(labels): return ",".join(["=".join(lbl) for lbl in labels.items()]) @@ -85,6 +86,7 @@ class EndToEndTestCase(unittest.TestCase): # set a single K8s wrapper for all tests k8s = cls.k8s = K8s() + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' # remove existing local storage class and create hostpath class try: @@ -93,7 +95,7 @@ class EndToEndTestCase(unittest.TestCase): print("Failed to delete the 'standard' storage class: {0}".format(e)) # operator deploys pod service account there on start up - # needed for test_multi_namespace_support() + # needed for test_multi_namespace_support and test_owner_references cls.test_namespace = "test" try: v1_namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=cls.test_namespace)) @@ -113,6 +115,8 @@ class EndToEndTestCase(unittest.TestCase): configmap = yaml.safe_load(f) configmap["data"]["workers"] = "1" configmap["data"]["docker_image"] = SPILO_CURRENT + configmap["data"]["major_version_upgrade_mode"] = "full" + configmap["data"]["connection_pooler_image"] = os.environ['POOLER_IMAGE'] with open("manifests/configmap.yaml", 'w') as f: yaml.dump(configmap, f, Dumper=yaml.Dumper) @@ -127,7 +131,8 @@ class EndToEndTestCase(unittest.TestCase): "infrastructure-roles.yaml", "infrastructure-roles-new.yaml", "custom-team-membership.yaml", - "e2e-storage-class.yaml"]: + "e2e-storage-class.yaml", + "fes.crd.yaml"]: result = k8s.create_with_kubectl("manifests/" + filename) print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) @@ -147,20 +152,64 @@ class EndToEndTestCase(unittest.TestCase): 'default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish + # load minimal Postgres manifest and wait for cluster to be up and running result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") print('stdout: {}, stderr: {}'.format(result.stdout, result.stderr)) try: - k8s.wait_for_pod_start('spilo-role=master') - k8s.wait_for_pod_start('spilo-role=replica') + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_additional_owner_roles(self): + ''' + Test granting additional roles to existing database owners + ''' + k8s = self.k8s + + # first test - wait for the operator to get in sync and set everything up + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + leader = k8s.get_cluster_leader_pod() + + # produce wrong membership for cron_admin + grant_dbowner = """ + GRANT bar_owner TO cron_admin; + """ + self.query_database(leader.metadata.name, "postgres", grant_dbowner) + + # enable PostgresTeam CRD and lower resync + owner_roles = { + "data": { + "additional_owner_roles": "cron_admin", + }, + } + k8s.update_config(owner_roles) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + + owner_query = """ + SELECT a2.rolname + FROM pg_catalog.pg_authid a + JOIN pg_catalog.pg_auth_members am + ON a.oid = am.member + AND a.rolname IN ('zalando', 'bar_owner', 'bar_data_owner') + JOIN pg_catalog.pg_authid a2 + ON a2.oid = am.roleid + WHERE a2.rolname = 'cron_admin'; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", owner_query)), 3, + "Not all additional users found in database", 10, 5) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_additional_pod_capabilities(self): ''' Extend postgres container capabilities ''' + k8s = self.k8s cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' capabilities = ["SYS_NICE","CHOWN"] patch_capabilities = { @@ -170,18 +219,17 @@ class EndToEndTestCase(unittest.TestCase): } # get node and replica (expected target of new master) - _, replica_nodes = self.k8s.get_pg_nodes(cluster_label) + _, replica_nodes = k8s.get_pg_nodes(cluster_label) try: - self.k8s.update_config(patch_capabilities) - self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") + k8s.update_config(patch_capabilities) - # changed security context of postrges container should trigger a rolling update - self.k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) - self.k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + # changed security context of postgres container should trigger a rolling update + k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - self.eventuallyEqual(lambda: self.k8s.count_pods_with_container_capabilities(capabilities, cluster_label), + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_pods_with_container_capabilities(capabilities, cluster_label), 2, "Container capabilities not updated") except timeout_decorator.TimeoutError: @@ -193,20 +241,23 @@ class EndToEndTestCase(unittest.TestCase): ''' Test PostgresTeam CRD with extra teams and members ''' + k8s = self.k8s + # enable PostgresTeam CRD and lower resync enable_postgres_team_crd = { "data": { "enable_postgres_team_crd": "true", "enable_team_member_deprecation": "true", "role_deletion_suffix": "_delete_me", - "resync_period": "15s" + "resync_period": "15s", + "repair_period": "15s", }, } - self.k8s.update_config(enable_postgres_team_crd) - self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") + k8s.update_config(enable_postgres_team_crd) - self.k8s.api.custom_objects_api.patch_namespaced_custom_object( + # add team and member to custom-team-membership + # contains already elephant user + k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', 'postgresteams', 'custom-team-membership', { @@ -224,7 +275,7 @@ class EndToEndTestCase(unittest.TestCase): } }) - leader = self.k8s.get_cluster_leader_pod() + leader = k8s.get_cluster_leader_pod() user_query = """ SELECT rolname FROM pg_catalog.pg_roles @@ -234,7 +285,7 @@ class EndToEndTestCase(unittest.TestCase): "Not all additional users found in database", 10, 5) # replace additional member and check if the removed member's role is renamed - self.k8s.api.custom_objects_api.patch_namespaced_custom_object( + k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', 'postgresteams', 'custom-team-membership', { @@ -253,11 +304,18 @@ class EndToEndTestCase(unittest.TestCase): WHERE (rolname = 'tester' AND rolcanlogin) OR (rolname = 'kind_delete_me' AND NOT rolcanlogin); """ - self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, "Database role of replaced member in PostgresTeam not renamed", 10, 5) + # create fake deletion user so operator fails renaming + # but altering role to NOLOGIN will succeed + create_fake_deletion_user = """ + CREATE USER tester_delete_me NOLOGIN; + """ + self.query_database(leader.metadata.name, "postgres", create_fake_deletion_user) + # re-add additional member and check if the role is renamed back - self.k8s.api.custom_objects_api.patch_namespaced_custom_object( + k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', 'postgresteams', 'custom-team-membership', { @@ -273,55 +331,344 @@ class EndToEndTestCase(unittest.TestCase): user_query = """ SELECT rolname FROM pg_catalog.pg_roles - WHERE (rolname = 'kind' AND rolcanlogin) + WHERE rolname = 'kind' AND rolcanlogin; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 1, + "Database role of recreated member in PostgresTeam not renamed back to original name", 10, 5) + + user_query = """ + SELECT rolname + FROM pg_catalog.pg_roles + WHERE rolname IN ('tester','tester_delete_me') AND NOT rolcanlogin; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, + "Database role of replaced member in PostgresTeam not denied from login", 10, 5) + + # re-add other additional member, operator should grant LOGIN back to tester + # but nothing happens to deleted role + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresteams', 'custom-team-membership', + { + 'spec': { + 'additionalMembers': { + 'e2e': [ + 'kind', + 'tester' + ] + }, + } + }) + + user_query = """ + SELECT rolname + FROM pg_catalog.pg_roles + WHERE (rolname IN ('tester', 'kind') + AND rolcanlogin) OR (rolname = 'tester_delete_me' AND NOT rolcanlogin); """ - self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, - "Database role of recreated member in PostgresTeam not renamed back to original name", 10, 5) + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 3, + "Database role of deleted member in PostgresTeam not removed when recreated manually", 10, 5) # revert config change revert_resync = { "data": { - "resync_period": "30m", + "resync_period": "4m", + "repair_period": "1m", }, } - self.k8s.update_config(revert_resync) - self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, + k8s.update_config(revert_resync) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_overwrite_pooler_deployment(self): - self.k8s.create_with_kubectl("manifests/minimal-fake-pooler-deployment.yaml") - self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: self.k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler"), 1, - "Initial broken deployment not rolled out") + def test_config_update(self): + ''' + Change Postgres config under Spec.Postgresql.Parameters and Spec.Patroni + and query Patroni config endpoint to check if manifest changes got applied + via restarting cluster through Patroni's rest api + ''' + k8s = self.k8s + leader = k8s.get_cluster_leader_pod() + replica = k8s.get_cluster_replica_pod() + masterCreationTimestamp = leader.metadata.creation_timestamp + replicaCreationTimestamp = replica.metadata.creation_timestamp + new_max_connections_value = "50" - self.k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': True + # adjust Postgres config + pg_patch_config = { + "spec": { + "postgresql": { + "parameters": { + "max_connections": new_max_connections_value, + "wal_level": "logical" + } + }, + "patroni": { + "slots": { + "first_slot": { + "type": "physical" + } + }, + "ttl": 29, + "loop_wait": 9, + "retry_timeout": 9, + "synchronous_mode": True, + "failsafe_mode": True, + } } - }) + } - self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: self.k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler"), 2, - "Operator did not succeed in overwriting labels") + try: + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config) - self.k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': False + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + def compare_config(): + effective_config = k8s.patroni_rest(leader.metadata.name, "config") + desired_config = pg_patch_config["spec"]["patroni"] + desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"] + effective_parameters = effective_config["postgresql"]["parameters"] + self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"], + "max_connections not updated") + self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added") + self.assertEqual(desired_config["ttl"], effective_config["ttl"], + "ttl not updated") + self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"], + "loop_wait not updated") + self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"], + "retry_timeout not updated") + self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"], + "synchronous_mode not updated") + self.assertEqual(desired_config["failsafe_mode"], effective_config["failsafe_mode"], + "failsafe_mode not updated") + self.assertEqual(desired_config["slots"], effective_config["slots"], + "slots not updated") + return True + + # check if Patroni config has been updated + self.eventuallyTrue(compare_config, "Postgres config not applied") + + # make sure that pods were not recreated + leader = k8s.get_cluster_leader_pod() + replica = k8s.get_cluster_replica_pod() + self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp, + "Master pod creation timestamp is updated") + self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp, + "Master pod creation timestamp is updated") + + # query max_connections setting + setting_query = """ + SELECT setting + FROM pg_settings + WHERE name = 'max_connections'; + """ + self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value, + "New max_connections setting not applied on master", 10, 5) + self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value, + "Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5) + + # the next sync should restart the replica because it has pending_restart flag set + # force next sync by deleting the operator pod + k8s.delete_operator_pod() + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value, + "New max_connections setting not applied on replica", 10, 5) + + # decrease max_connections again + # this time restart will be correct and new value should appear on both instances + lower_max_connections_value = "30" + pg_patch_max_connections = { + "spec": { + "postgresql": { + "parameters": { + "max_connections": lower_max_connections_value + } + } + } } - }) - self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: self.k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), - 0, "Pooler pods not scaled down") + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # check Patroni config again + pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value + self.eventuallyTrue(compare_config, "Postgres config not applied") + + # and query max_connections setting again + self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value, + "Previous max_connections setting not applied on master", 10, 5) + self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value, + "Previous max_connections setting not applied on replica", 10, 5) + + # patch new slot via Patroni REST + patroni_slot = "test_patroni_slot" + patch_slot_command = """curl -s -XPATCH -d '{"slots": {"test_patroni_slot": {"type": "physical"}}}' localhost:8008/config""" + pg_patch_config["spec"]["patroni"]["slots"][patroni_slot] = {"type": "physical"} + + k8s.exec_with_kubectl(leader.metadata.name, patch_slot_command) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyTrue(compare_config, "Postgres config not applied") + + # test adding new slots + pg_add_new_slots_patch = { + "spec": { + "patroni": { + "slots": { + "test_slot": { + "type": "logical", + "database": "foo", + "plugin": "pgoutput" + }, + "test_slot_2": { + "type": "physical" + } + } + } + } + } + + for slot_name, slot_details in pg_add_new_slots_patch["spec"]["patroni"]["slots"].items(): + pg_patch_config["spec"]["patroni"]["slots"][slot_name] = slot_details + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_add_new_slots_patch) + + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyTrue(compare_config, "Postgres config not applied") + + # delete test_slot_2 from config and change the database type for test_slot + slot_to_change = "test_slot" + slot_to_remove = "test_slot_2" + pg_delete_slot_patch = { + "spec": { + "patroni": { + "slots": { + "test_slot": { + "type": "logical", + "database": "bar", + "plugin": "pgoutput" + }, + "test_slot_2": None + } + } + } + } + + pg_patch_config["spec"]["patroni"]["slots"][slot_to_change]["database"] = "bar" + del pg_patch_config["spec"]["patroni"]["slots"][slot_to_remove] + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_delete_slot_patch) + + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyTrue(compare_config, "Postgres config not applied") + + get_slot_query = """ + SELECT %s + FROM pg_replication_slots + WHERE slot_name = '%s'; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", get_slot_query%("slot_name", slot_to_remove))), 0, + "The replication slot cannot be deleted", 10, 5) + + self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", get_slot_query%("database", slot_to_change))[0], "bar", + "The replication slot cannot be updated", 10, 5) + + # make sure slot from Patroni didn't get deleted + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", get_slot_query%("slot_name", patroni_slot))), 1, + "The replication slot from Patroni gets deleted", 10, 5) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + # make sure cluster is in a good state for further tests + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, + "No 2 pods running") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_cross_namespace_secrets(self): + ''' + Test secrets in different namespace + ''' + k8s = self.k8s + + # enable secret creation in separate namespace + patch_cross_namespace_secret = { + "data": { + "enable_cross_namespace_secret": "true" + } + } + k8s.update_config(patch_cross_namespace_secret, + step="cross namespace secrets enabled") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + + # create secret in test namespace + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'users':{ + 'test.db_user': [], + } + } + }) + + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label("cluster-name=acid-minimal-cluster,application=spilo", self.test_namespace), + 1, "Secret not created for user in namespace") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_custom_ssl_certificate(self): + ''' + Test if spilo uses a custom SSL certificate + ''' + + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + tls_secret = "pg-tls" + + # get nodes of master and replica(s) (expected target of new master) + _, replica_nodes = k8s.get_pg_nodes(cluster_label) + self.assertNotEqual(replica_nodes, []) + + try: + # create secret containing ssl certificate + result = self.k8s.create_tls_secret_with_kubectl(tls_secret) + print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) + + # enable load balancer services + pg_patch_tls = { + "spec": { + "spiloFSGroup": 103, + "tls": { + "secretName": tls_secret + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_tls) + + # wait for switched over + k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + self.eventuallyEqual(lambda: k8s.count_pods_with_env_variable("SSL_CERTIFICATE_FILE", cluster_label), 2, "TLS env variable SSL_CERTIFICATE_FILE missing in Spilo pods") + self.eventuallyEqual(lambda: k8s.count_pods_with_env_variable("SSL_PRIVATE_KEY_FILE", cluster_label), 2, "TLS env variable SSL_PRIVATE_KEY_FILE missing in Spilo pods") + self.eventuallyEqual(lambda: k8s.count_pods_with_volume_mount(tls_secret, cluster_label), 2, "TLS volume mount missing in Spilo pods") + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_enable_disable_connection_pooler(self): @@ -332,6 +679,9 @@ class EndToEndTestCase(unittest.TestCase): the end turn connection pooler off to not interfere with other tests. ''' k8s = self.k8s + pooler_label = 'application=db-connection-pooler,cluster-name=acid-minimal-cluster' + master_pooler_label = 'connection-pooler=acid-minimal-cluster-pooler' + replica_pooler_label = master_pooler_label + '-repl' self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -343,20 +693,47 @@ class EndToEndTestCase(unittest.TestCase): 'enableReplicaConnectionPooler': True, } }) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, - "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods( - "connection-pooler=acid-minimal-cluster-pooler"), - 2, "No pooler pods found") - self.eventuallyEqual(lambda: k8s.count_running_pods( - "connection-pooler=acid-minimal-cluster-pooler-repl"), - 2, "No pooler replica pods found") - self.eventuallyEqual(lambda: k8s.count_services_with_label( - 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'), - 2, "No pooler service found") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), - 1, "Pooler secret not created") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, "Deployment replicas is 2 default") + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 2, "No pooler pods found") + self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 2, "No pooler replica pods found") + self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 2, "No pooler service found") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), 3, "Not all pooler secrets found") + + # TLS still enabled so check existing env variables and volume mounts + self.eventuallyEqual(lambda: k8s.count_pods_with_env_variable("CONNECTION_POOLER_CLIENT_TLS_CRT", pooler_label), 4, "TLS env variable CONNECTION_POOLER_CLIENT_TLS_CRT missing in pooler pods") + self.eventuallyEqual(lambda: k8s.count_pods_with_env_variable("CONNECTION_POOLER_CLIENT_TLS_KEY", pooler_label), 4, "TLS env variable CONNECTION_POOLER_CLIENT_TLS_KEY missing in pooler pods") + self.eventuallyEqual(lambda: k8s.count_pods_with_volume_mount("pg-tls", pooler_label), 4, "TLS volume mount missing in pooler pods") + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableMasterPoolerLoadBalancer': True, + 'enableReplicaPoolerLoadBalancer': True, + } + }) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label), + 'LoadBalancer', + "Expected LoadBalancer service type for master pooler pod, found {}") + self.eventuallyEqual(lambda: k8s.get_service_type(replica_pooler_label+","+pooler_label), + 'LoadBalancer', + "Expected LoadBalancer service type for replica pooler pod, found {}") + + master_annotations = { + "external-dns.alpha.kubernetes.io/hostname": "acid-minimal-cluster-pooler.default.db.example.com", + } + self.eventuallyTrue(lambda: k8s.check_service_annotations( + master_pooler_label+","+pooler_label, master_annotations), "Wrong annotations") + + replica_annotations = { + "external-dns.alpha.kubernetes.io/hostname": "acid-minimal-cluster-pooler-repl.default.db.example.com", + } + self.eventuallyTrue(lambda: k8s.check_service_annotations( + replica_pooler_label+","+pooler_label, replica_annotations), "Wrong annotations") # Turn off only master connection pooler k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -369,21 +746,18 @@ class EndToEndTestCase(unittest.TestCase): } }) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler-repl"), 2, "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods( - "connection-pooler=acid-minimal-cluster-pooler"), + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 0, "Master pooler pods not deleted") - self.eventuallyEqual(lambda: k8s.count_running_pods( - "connection-pooler=acid-minimal-cluster-pooler-repl"), + self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 2, "Pooler replica pods not found") - self.eventuallyEqual(lambda: k8s.count_services_with_label( - 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 1, "No pooler service found") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), - 1, "Secret not created") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), + 2, "Not all pooler secrets created") # Turn off only replica connection pooler k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -393,21 +767,25 @@ class EndToEndTestCase(unittest.TestCase): 'spec': { 'enableConnectionPooler': True, 'enableReplicaConnectionPooler': False, + 'enableMasterPoolerLoadBalancer': False, } }) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 2, "Master pooler pods not found") - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler-repl"), + self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 0, "Pooler replica pods not deleted") - self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 1, "No pooler service found") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), - 1, "Secret not created") + self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label), + 'ClusterIP', + "Expected LoadBalancer service type for master, found {}") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), + 2, "Not all pooler secrets created") # scale up connection pooler deployment k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -423,7 +801,7 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 3, "Deployment replicas is scaled to 3") - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 3, "Scale up of pooler pods does not work") # turn it off, keeping config should be overwritten by false @@ -434,15 +812,16 @@ class EndToEndTestCase(unittest.TestCase): 'spec': { 'enableConnectionPooler': False, 'enableReplicaConnectionPooler': False, + 'enableReplicaPoolerLoadBalancer': False, } }) - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 0, "Pooler pods not scaled down") - self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 0, "Pooler service not removed") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=spilo,cluster-name=acid-minimal-cluster'), - 4, "Secrets not deleted") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), + 0, "Not all pooler secrets deleted") # Verify that all the databases have pooler schema installed. # Do this via psql, since otherwise we need to deal with @@ -526,6 +905,54 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_ignored_annotations(self): + ''' + Test if injected annotation does not cause replacement of resources when listed under ignored_annotations + ''' + k8s = self.k8s + + + try: + patch_config_ignored_annotations = { + "data": { + "ignored_annotations": "k8s-status", + } + } + k8s.update_config(patch_config_ignored_annotations) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + sts = k8s.api.apps_v1.read_namespaced_stateful_set('acid-minimal-cluster', 'default') + svc = k8s.api.core_v1.read_namespaced_service('acid-minimal-cluster', 'default') + + annotation_patch = { + "metadata": { + "annotations": { + "k8s-status": "healthy" + }, + } + } + + old_sts_creation_timestamp = sts.metadata.creation_timestamp + k8s.api.apps_v1.patch_namespaced_stateful_set(sts.metadata.name, sts.metadata.namespace, annotation_patch) + old_svc_creation_timestamp = svc.metadata.creation_timestamp + k8s.api.core_v1.patch_namespaced_service(svc.metadata.name, svc.metadata.namespace, annotation_patch) + + k8s.delete_operator_pod() + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + sts = k8s.api.apps_v1.read_namespaced_stateful_set('acid-minimal-cluster', 'default') + new_sts_creation_timestamp = sts.metadata.creation_timestamp + svc = k8s.api.core_v1.read_namespaced_service('acid-minimal-cluster', 'default') + new_svc_creation_timestamp = svc.metadata.creation_timestamp + + self.assertEqual(old_sts_creation_timestamp, new_sts_creation_timestamp, "unexpected replacement of statefulset on sync") + self.assertEqual(old_svc_creation_timestamp, new_svc_creation_timestamp, "unexpected replacement of master service on sync") + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_infrastructure_roles(self): ''' @@ -568,12 +995,16 @@ class EndToEndTestCase(unittest.TestCase): role.pop("Password", None) self.assertDictEqual(role, { "Name": "robot_zmon_acid_monitoring_new", + "Namespace":"", "Flags": None, "MemberOf": ["robot_zmon"], "Parameters": None, "AdminRole": "", "Origin": 2, - "Deleted": False + "IsDbOwner": False, + "Deleted": False, + "Rotated": False, + "Degraded": False, }) return True except: @@ -594,7 +1025,6 @@ class EndToEndTestCase(unittest.TestCase): but lets pods run with the old image until they are recreated for reasons other than operator's activity. That works because the operator configures stateful sets to use "onDelete" pod update policy. - The test covers: 1) enabling lazy upgrade in existing operator deployment 2) forcing the normal rolling upgrade by changing the operator @@ -689,7 +1119,6 @@ class EndToEndTestCase(unittest.TestCase): Ensure we can (a) create the cron job at user request for a specific PG cluster (b) update the cluster-wide image for the logical backup pod (c) delete the job at user request - Limitations: (a) Does not run the actual batch job because there is no S3 mock to upload backups to (b) Assumes 'acid-minimal-cluster' exists as defined in setUp @@ -754,24 +1183,234 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_min_resource_limits(self): + def test_major_version_upgrade(self): + """ + Test major version upgrade: with full upgrade, maintenance window, and annotation + """ + def check_version(): + p = k8s.patroni_rest("acid-upgrade-test-0", "") or {} + version = p.get("server_version", 0) // 10000 + return version + + def get_annotations(): + pg_manifest = k8s.api.custom_objects_api.get_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test") + annotations = pg_manifest["metadata"]["annotations"] + return annotations + + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-upgrade-test' + + with open("manifests/minimal-postgres-lowest-version-manifest.yaml", 'r+') as f: + upgrade_manifest = yaml.safe_load(f) + upgrade_manifest["spec"]["dockerImage"] = SPILO_FULL_IMAGE + + with open("manifests/minimal-postgres-lowest-version-manifest.yaml", 'w') as f: + yaml.dump(upgrade_manifest, f, Dumper=yaml.Dumper) + + k8s.create_with_kubectl("manifests/minimal-postgres-lowest-version-manifest.yaml") + self.eventuallyEqual(lambda: k8s.count_running_pods(labels=cluster_label), 2, "No 2 pods running") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(check_version, 14, "Version is not correct") + + master_nodes, _ = k8s.get_cluster_nodes(cluster_labels=cluster_label) + # should upgrade immediately + pg_patch_version_higher_version = { + "spec": { + "postgresql": { + "version": "15" + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_higher_version) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + self.eventuallyEqual(check_version, 15, "Version should be upgraded from 14 to 15") + + # check if annotation for last upgrade's success is set + annotations = get_annotations() + self.assertIsNotNone(annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") + + # should not upgrade because current time is not in maintenanceWindow + current_time = datetime.now() + maintenance_window_future = f"{(current_time+timedelta(minutes=60)).strftime('%H:%M')}-{(current_time+timedelta(minutes=120)).strftime('%H:%M')}" + pg_patch_version_higher_version_outside_mw = { + "spec": { + "postgresql": { + "version": "16" + }, + "maintenanceWindows": [ + maintenance_window_future + ] + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_higher_version_outside_mw) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # no pod replacement outside of the maintenance window + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + self.eventuallyEqual(check_version, 15, "Version should not be upgraded") + + second_annotations = get_annotations() + self.assertIsNone(second_annotations.get("last-major-upgrade-failure"), "Annotation for last upgrade's failure should not be set") + + # change maintenanceWindows to current + maintenance_window_current = f"{(current_time-timedelta(minutes=30)).strftime('%H:%M')}-{(current_time+timedelta(minutes=30)).strftime('%H:%M')}" + pg_patch_version_higher_version_in_mw = { + "spec": { + "postgresql": { + "version": "16" + }, + "maintenanceWindows": [ + maintenance_window_current + ] + } + } + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_higher_version_in_mw) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + self.eventuallyEqual(check_version, 16, "Version should be upgraded from 15 to 16") + + # check if annotation for last upgrade's success is updated after second upgrade + third_annotations = get_annotations() + self.assertIsNotNone(third_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") + self.assertNotEqual(annotations.get("last-major-upgrade-success"), third_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated") + + # test upgrade with failed upgrade annotation + pg_patch_version_highest_version = { + "metadata": { + "annotations": { + "last-major-upgrade-failure": "2024-01-02T15:04:05Z" + }, + }, + "spec": { + "postgresql": { + "version": "18" + }, + }, + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_highest_version) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + self.eventuallyEqual(check_version, 16, "Version should not be upgraded because annotation for last upgrade's failure is set") + + # change the version back to 16 and should remove failure annotation + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_higher_version_in_mw) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + self.eventuallyEqual(check_version, 16, "Version should not be upgraded from 16") + fourth_annotations = get_annotations() + self.assertIsNone(fourth_annotations.get("last-major-upgrade-failure"), "Annotation for last upgrade's failure is not removed") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_persistent_volume_claim_retention_policy(self): ''' - Lower resource limits below configured minimum and let operator fix it + Test the retention policy for persistent volume claim ''' k8s = self.k8s - # self.eventuallyEqual(lambda: k8s.pg_get_status(), "Running", "Cluster not healthy at start") + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - # configure minimum boundaries for CPU and memory limits + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_pvcs_with_label(cluster_label), 2, "PVCs is not equal to number of instance") + + # patch the pvc retention policy to enable delete when scale down + patch_scaled_policy_delete = { + "data": { + "persistent_volume_claim_retention_policy": "when_deleted:retain,when_scaled:delete" + } + } + k8s.update_config(patch_scaled_policy_delete) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + pg_patch_scale_down_instances = { + 'spec': { + 'numberOfInstances': 1 + } + } + # decrease the number of instances + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', pg_patch_scale_down_instances) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},"Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_pvcs_with_label(cluster_label), 1, "PVCs is not deleted when scaled down") + + pg_patch_scale_up_instances = { + 'spec': { + 'numberOfInstances': 2 + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', pg_patch_scale_up_instances) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},"Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_pvcs_with_label(cluster_label), 2, "PVCs is not equal to number of instances") + + # reset retention policy to retain + patch_scaled_policy_retain = { + "data": { + "persistent_volume_claim_retention_policy": "when_deleted:retain,when_scaled:retain" + } + } + k8s.update_config(patch_scaled_policy_retain) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # decrease the number of instances + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', pg_patch_scale_down_instances) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},"Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 1, "Scale down to 1 failed") + self.eventuallyEqual(lambda: k8s.count_pvcs_with_label(cluster_label), 2, "PVCs is deleted when scaled down") + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', pg_patch_scale_up_instances) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"},"Operator does not get in sync") + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + self.eventuallyEqual(lambda: k8s.count_pvcs_with_label(cluster_label), 2, "PVCs is not equal to number of instances") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_resource_generation(self): + ''' + Lower resource limits below configured minimum and let operator fix it. + It will try to raise requests to limits which is capped with max_memory_request. + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # get nodes of master and replica(s) (expected target of new master) + _, replica_nodes = k8s.get_pg_nodes(cluster_label) + self.assertNotEqual(replica_nodes, []) + + # configure maximum memory request and minimum boundaries for CPU and memory limits + maxMemoryRequest = '300Mi' minCPULimit = '503m' minMemoryLimit = '502Mi' - patch_min_resource_limits = { + patch_pod_resources = { "data": { + "max_memory_request": maxMemoryRequest, "min_cpu_limit": minCPULimit, - "min_memory_limit": minMemoryLimit + "min_memory_limit": minMemoryLimit, + "set_memory_request_to_limit": "true" } } - k8s.update_config(patch_min_resource_limits, "Minimum resource test") + k8s.update_config(patch_pod_resources, "Pod resource test") # lower resource limits below minimum pg_patch_resources = { @@ -790,23 +1429,27 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No two pods running after lazy rolling upgrade") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members()), 2, "Postgres status did not enter running") + # wait for switched over + k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - def verify_pod_limits(): + def verify_pod_resources(): pods = k8s.api.core_v1.list_namespaced_pod('default', label_selector="cluster-name=acid-minimal-cluster,application=spilo").items if len(pods) < 2: return False - r = pods[0].spec.containers[0].resources.limits['memory'] == minMemoryLimit + r = pods[0].spec.containers[0].resources.requests['memory'] == maxMemoryRequest + r = r and pods[0].spec.containers[0].resources.limits['memory'] == minMemoryLimit r = r and pods[0].spec.containers[0].resources.limits['cpu'] == minCPULimit + r = r and pods[1].spec.containers[0].resources.requests['memory'] == maxMemoryRequest r = r and pods[1].spec.containers[0].resources.limits['memory'] == minMemoryLimit r = r and pods[1].spec.containers[0].resources.limits['cpu'] == minCPULimit return r - self.eventuallyTrue(verify_pod_limits, "Pod limits where not adjusted") + self.eventuallyTrue(verify_pod_resources, "Pod resources where not adjusted") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_multi_namespace_support(self): @@ -823,18 +1466,454 @@ class EndToEndTestCase(unittest.TestCase): try: k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") k8s.wait_for_pod_start("spilo-role=master", self.test_namespace) + k8s.wait_for_pod_start("spilo-role=replica", self.test_namespace) self.assert_master_is_unique(self.test_namespace, "acid-test-cluster") + # acid-test-cluster will be deleted in test_owner_references test except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise - finally: - # delete the new cluster so that the k8s_api.get_operator_state works correctly in subsequent tests - # ideally we should delete the 'test' namespace here but - # the pods inside the namespace stuck in the Terminating state making the test time out + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + @unittest.skip("Skipping this test until fixed") + def test_node_affinity(self): + ''' + Add label to a node and update postgres cluster spec to deploy only on a node with that label + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # verify we are in good state from potential previous tests + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + + # get nodes of master and replica(s) + master_nodes, replica_nodes = k8s.get_cluster_nodes() + self.assertNotEqual(master_nodes, []) + self.assertNotEqual(replica_nodes, []) + + # label node with environment=postgres + node_label_body = { + "metadata": { + "labels": { + "node-affinity-test": "postgres" + } + } + } + + try: + # patch master node with the label + k8s.api.core_v1.patch_node(master_nodes[0], node_label_body) + + # add node affinity to cluster + patch_node_affinity_config = { + "spec": { + "nodeAffinity" : { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "node-affinity-test", + "operator": "In", + "values": [ + "postgres" + ] + } + ] + } + ] + } + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + group="acid.zalan.do", + version="v1", + namespace="default", + plural="postgresqls", + name="acid-minimal-cluster", + body=patch_node_affinity_config) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + + # node affinity change should cause replica to relocate from replica node to master node due to node affinity requirement + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + # next master will be switched over and pod needs to be replaced as well to finish the rolling update + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + podsList = k8s.api.core_v1.list_namespaced_pod('default', label_selector=cluster_label) + for pod in podsList.items: + if pod.metadata.labels.get('spilo-role') == 'replica': + self.assertEqual(master_nodes[0], pod.spec.node_name, + "Sanity check: expected replica to relocate to master node {}, but found on {}".format(master_nodes[0], pod.spec.node_name)) + + # check that pod has correct node affinity + key = pod.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms[0].match_expressions[0].key + value = pod.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms[0].match_expressions[0].values[0] + self.assertEqual("node-affinity-test", key, + "Sanity check: expect node selector key to be equal to 'node-affinity-test' but got {}".format(key)) + self.assertEqual("postgres", value, + "Sanity check: expect node selector value to be equal to 'postgres' but got {}".format(value)) + + patch_node_remove_affinity_config = { + "spec": { + "nodeAffinity" : None + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + group="acid.zalan.do", + version="v1", + namespace="default", + plural="postgresqls", + name="acid-minimal-cluster", + body=patch_node_remove_affinity_config) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # node affinity change should cause another rolling update and relocation of replica + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + # toggle pod anti affinity to make sure replica and master run on separate nodes + self.assert_distributed_pods(master_nodes) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + @unittest.skip("Skipping this test until fixed") + def test_node_readiness_label(self): + ''' + Remove node readiness label from master node. This must cause a failover. + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + readiness_label = 'lifecycle-status' + readiness_value = 'ready' + + # verify we are in good state from potential previous tests + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + + # get nodes of master and replica(s) (expected target of new master) + master_nodes, replica_nodes = k8s.get_cluster_nodes() + self.assertNotEqual(master_nodes, []) + self.assertNotEqual(replica_nodes, []) + + try: + # add node_readiness_label to potential failover nodes + patch_readiness_label = { + "metadata": { + "labels": { + readiness_label: readiness_value + } + } + } + for replica_node in replica_nodes: + k8s.api.core_v1.patch_node(replica_node, patch_readiness_label) + + # define node_readiness_label in config map which should trigger a rolling update + patch_readiness_label_config = { + "data": { + "node_readiness_label": readiness_label + ':' + readiness_value, + "node_readiness_label_merge": "AND", + } + } + k8s.update_config(patch_readiness_label_config, "setting readiness label") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # first replica will be replaced and get the new affinity + # however, it might not start due to a volume node affinity conflict + # in this case only if the pvc and pod are deleted it can be scheduled + replica = k8s.get_cluster_replica_pod() + if replica.status.phase == 'Pending': + k8s.api.core_v1.delete_namespaced_persistent_volume_claim('pgdata-' + replica.metadata.name, 'default') + k8s.api.core_v1.delete_namespaced_pod(replica.metadata.name, 'default') + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + # next master will be switched over and pod needs to be replaced as well to finish the rolling update + k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + # patch also node where master ran before + k8s.api.core_v1.patch_node(master_nodes[0], patch_readiness_label) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(master_nodes) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_overwrite_pooler_deployment(self): + pooler_name = 'acid-minimal-cluster-pooler' + k8s = self.k8s + k8s.create_with_kubectl("manifests/minimal-fake-pooler-deployment.yaml") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name=pooler_name), 1, + "Initial broken deployment not rolled out") + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': True + } + }) + + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name=pooler_name), 2, + "Operator did not succeed in overwriting labels") + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': False + } + }) + + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler="+pooler_name), + 0, "Pooler pods not scaled down") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_owner_references(self): + ''' + Enable owner references, test if resources get updated and test cascade deletion of test cluster. + ''' + k8s = self.k8s + cluster_name = 'acid-test-cluster' + cluster_label = 'application=spilo,cluster-name={}'.format(cluster_name) + default_test_cluster = 'acid-minimal-cluster' + + try: + # enable owner references in config + enable_owner_refs = { + "data": { + "enable_owner_references": "true" + } + } + k8s.update_config(enable_owner_refs) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + time.sleep(5) # wait for the operator to sync the cluster and update resources + + # check if child resources were updated with owner references + self.assertTrue(self.check_cluster_child_resources_owner_references(cluster_name, self.test_namespace), "Owner references not set on all child resources of {}".format(cluster_name)) + self.assertTrue(self.check_cluster_child_resources_owner_references(default_test_cluster), "Owner references not set on all child resources of {}".format(default_test_cluster)) + + # delete the new cluster to test owner references + # and also to make k8s_api.get_operator_state work better in subsequent tests + # ideally we should delete the 'test' namespace here but the pods + # inside the namespace stuck in the Terminating state making the test time out k8s.api.custom_objects_api.delete_namespaced_custom_object( - "acid.zalan.do", "v1", self.test_namespace, "postgresqls", "acid-test-cluster") - time.sleep(5) + "acid.zalan.do", "v1", self.test_namespace, "postgresqls", cluster_name) + + # child resources with owner references should be deleted via owner references + self.eventuallyEqual(lambda: k8s.count_pods_with_label(cluster_label), 0, "Pods not deleted") + self.eventuallyEqual(lambda: k8s.count_statefulsets_with_label(cluster_label), 0, "Statefulset not deleted") + self.eventuallyEqual(lambda: k8s.count_services_with_label(cluster_label), 0, "Services not deleted") + self.eventuallyEqual(lambda: k8s.count_endpoints_with_label(cluster_label), 0, "Endpoints not deleted") + self.eventuallyEqual(lambda: k8s.count_pdbs_with_label(cluster_label), 0, "Pod disruption budget not deleted") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(cluster_label), 0, "Secrets were not deleted") + + time.sleep(5) # wait for the operator to also delete the PVCs + + # pvcs do not have an owner reference but will deleted by the operator almost immediately + self.eventuallyEqual(lambda: k8s.count_pvcs_with_label(cluster_label), 0, "PVCs not deleted") + + # disable owner references in config + disable_owner_refs = { + "data": { + "enable_owner_references": "false" + } + } + k8s.update_config(disable_owner_refs) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + time.sleep(5) # wait for the operator to remove owner references + + # check if child resources were updated without Postgresql owner references + self.assertTrue(self.check_cluster_child_resources_owner_references(default_test_cluster, "default", True), "Owner references still present on some child resources of {}".format(default_test_cluster)) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_password_rotation(self): + ''' + Test password rotation and removal of users due to retention policy + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + leader = k8s.get_cluster_leader_pod() + today = date.today() + + # remember number of secrets to make sure it stays the same + secret_count = k8s.count_secrets_with_label(cluster_label) + + # enable password rotation for owner of foo database + pg_patch_rotation_single_users = { + "spec": { + "usersIgnoringSecretRotation": [ + "test.db_user" + ], + "usersWithInPlaceSecretRotation": [ + "zalando" + ] + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_rotation_single_users) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # check if next rotation date was set in secret + zalando_secret = k8s.get_secret("zalando") + next_rotation_timestamp = datetime.strptime(str(base64.b64decode(zalando_secret.data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ") + today90days = today+timedelta(days=90) + self.assertEqual(today90days, next_rotation_timestamp.date(), + "Unexpected rotation date in secret of zalando user: expected {}, got {}".format(today90days, next_rotation_timestamp.date())) + + # create fake rotation users that should be removed by operator + # but have one that would still fit into the retention period + create_fake_rotation_user = """ + CREATE USER foo_user201031 IN ROLE foo_user; + CREATE USER foo_user211031 IN ROLE foo_user; + CREATE USER foo_user"""+(today-timedelta(days=40)).strftime("%y%m%d")+""" IN ROLE foo_user; + """ + self.query_database(leader.metadata.name, "postgres", create_fake_rotation_user) + + # patch foo_user secret with outdated rotation date + fake_rotation_date = today.isoformat() + 'T00:00:00Z' + fake_rotation_date_encoded = base64.b64encode(fake_rotation_date.encode('utf-8')) + secret_fake_rotation = { + "data": { + "nextRotation": str(fake_rotation_date_encoded, 'utf-8'), + }, + } + k8s.api.core_v1.patch_namespaced_secret( + name="foo-user.acid-minimal-cluster.credentials.postgresql.acid.zalan.do", + namespace="default", + body=secret_fake_rotation) + + # update rolconfig for foo_user that will be copied for new rotation user + alter_foo_user_search_path = """ + ALTER ROLE foo_user SET search_path TO data; + """ + self.query_database(leader.metadata.name, "postgres", alter_foo_user_search_path) + + # enable password rotation for all other users (foo_user) + # this will force a sync of secrets for further assertions + enable_password_rotation = { + "data": { + "enable_password_rotation": "true", + "inherited_annotations": "environment", + "password_rotation_interval": "30", + "password_rotation_user_retention": "30", # should be set to 60 + }, + } + k8s.update_config(enable_password_rotation) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + + # check if next rotation date and username have been replaced + foo_user_secret = k8s.get_secret("foo_user") + secret_username = str(base64.b64decode(foo_user_secret.data["username"]), 'utf-8') + next_rotation_timestamp = datetime.strptime(str(base64.b64decode(foo_user_secret.data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ") + rotation_user = "foo_user"+today.strftime("%y%m%d") + today30days = today+timedelta(days=30) + + self.assertEqual(rotation_user, secret_username, + "Unexpected username in secret of foo_user: expected {}, got {}".format(rotation_user, secret_username)) + self.assertEqual(today30days, next_rotation_timestamp.date(), + "Unexpected rotation date in secret of foo_user: expected {}, got {}".format(today30days, next_rotation_timestamp.date())) + + # check if oldest fake rotation users were deleted + # there should only be foo_user, foo_user+today and foo_user+today-40days + user_query = """ + SELECT rolname + FROM pg_catalog.pg_roles + WHERE rolname LIKE 'foo_user%'; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 3, + "Found incorrect number of rotation users", 10, 5) + + # check if rolconfig was passed from foo_user to foo_user+today + # and that no foo_user has been deprecated (can still login) + user_query = """ + SELECT rolname + FROM pg_catalog.pg_roles + WHERE rolname LIKE 'foo_user%' + AND rolconfig = ARRAY['search_path=data']::text[] + AND rolcanlogin; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, + "Rolconfig not applied to new rotation user", 10, 5) + + # test that rotation_user can connect to the database + self.eventuallyEqual(lambda: len(self.query_database_with_user(leader.metadata.name, "postgres", "SELECT 1", "foo_user")), 1, + "Could not connect to the database with rotation user {}".format(rotation_user), 10, 5) + + # add annotation which triggers syncSecrets call + pg_annotation_patch = { + "metadata": { + "annotations": { + "environment": "test", + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_annotation_patch) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + time.sleep(10) + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(cluster_label), secret_count, "Unexpected number of secrets") + + # check if rotation has been ignored for user from test_cross_namespace_secrets test + db_user_secret = k8s.get_secret(username="test.db_user", namespace="test") + secret_username = str(base64.b64decode(db_user_secret.data["username"]), 'utf-8') + self.assertEqual("test.db_user", secret_username, + "Unexpected username in secret of test.db_user: expected {}, got {}".format("test.db_user", secret_username)) + + # check if annotation for secret has been updated + self.assertTrue("environment" in db_user_secret.metadata.annotations, "Added annotation was not propagated to secret") + + # disable password rotation for all other users (foo_user) + # and pick smaller intervals to see if the third fake rotation user is dropped + enable_password_rotation = { + "data": { + "enable_password_rotation": "false", + "password_rotation_interval": "15", + "password_rotation_user_retention": "30", # 2 * rotation interval + }, + } + k8s.update_config(enable_password_rotation) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + + # check if username in foo_user secret is reset + foo_user_secret = k8s.get_secret("foo_user") + secret_username = str(base64.b64decode(foo_user_secret.data["username"]), 'utf-8') + next_rotation_timestamp = str(base64.b64decode(foo_user_secret.data["nextRotation"]), 'utf-8') + self.assertEqual("foo_user", secret_username, + "Unexpected username in secret of foo_user: expected {}, got {}".format("foo_user", secret_username)) + self.assertEqual('', next_rotation_timestamp, + "Unexpected rotation date in secret of foo_user: expected empty string, got {}".format(next_rotation_timestamp)) + + # check roles again, there should only be foo_user and foo_user+today + user_query = """ + SELECT rolname + FROM pg_catalog.pg_roles + WHERE rolname LIKE 'foo_user%'; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, + "Found incorrect number of rotation users after disabling password rotation", 10, 5) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_rolling_update_flag(self): @@ -875,6 +1954,7 @@ class EndToEndTestCase(unittest.TestCase): # operator should now recreate the master pod and do a switchover before k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) # check if the former replica is now the new master leader = k8s.get_cluster_leader_pod() @@ -885,12 +1965,12 @@ class EndToEndTestCase(unittest.TestCase): replica = k8s.get_cluster_replica_pod() self.assertTrue(replica.metadata.creation_timestamp > old_creation_timestamp, "Old master pod was not recreated") - except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + @unittest.skip("Skipping this test until fixed") def test_rolling_update_label_timeout(self): ''' Simulate case when replica does not receive label in time and rolling update does not finish @@ -920,7 +2000,8 @@ class EndToEndTestCase(unittest.TestCase): patch_resync_config = { "data": { "pod_label_wait_timeout": "2s", - "resync_period": "20s", + "resync_period": "30s", + "repair_period": "30s", } } @@ -944,6 +2025,7 @@ class EndToEndTestCase(unittest.TestCase): # wait for next sync, replica should be running normally by now and be ready for switchover k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) # check if the former replica is now the new master leader = k8s.get_cluster_leader_pod() @@ -953,14 +2035,15 @@ class EndToEndTestCase(unittest.TestCase): k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) # status should again be "SyncFailed" but turn into "Running" on the next sync - time.sleep(10) + time.sleep(30) self.eventuallyEqual(lambda: k8s.pg_get_status(), "Running", "Expected running cluster after two syncs") # revert config changes patch_resync_config = { "data": { "pod_label_wait_timeout": "10m", - "resync_period": "30m", + "resync_period": "4m", + "repair_period": "2m", } } k8s.update_config(patch_resync_config, "revert resync interval and pod_label_wait_timeout") @@ -970,54 +2053,6 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_zz_node_readiness_label(self): - ''' - Remove node readiness label from master node. This must cause a failover. - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - readiness_label = 'lifecycle-status' - readiness_value = 'ready' - - try: - # get nodes of master and replica(s) (expected target of new master) - current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - num_replicas = len(current_replica_nodes) - failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # add node_readiness_label to potential failover nodes - patch_readiness_label = { - "metadata": { - "labels": { - readiness_label: readiness_value - } - } - } - self.assertTrue(len(failover_targets) > 0, "No failover targets available") - for failover_target in failover_targets: - k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) - - # define node_readiness_label in config map which should trigger a failover of the master - patch_readiness_label_config = { - "data": { - "node_readiness_label": readiness_label + ':' + readiness_value, - } - } - k8s.update_config(patch_readiness_label_config, "setting readiness label") - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) - - # patch also node where master ran before - k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) - - # toggle pod anti affinity to move replica away from master node - self.eventuallyTrue(lambda: self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label), "Pods are redistributed") - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): ''' @@ -1086,7 +2121,7 @@ class EndToEndTestCase(unittest.TestCase): patch_sset_propagate_annotations = { "data": { "downscaler_annotations": "deployment-time,downscaler/*", - "inherited_annotations": "owned-by", + "inherited_annotations": "environment,owned-by", } } k8s.update_config(patch_sset_propagate_annotations) @@ -1112,35 +2147,194 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - @unittest.skip("Skipping this test until fixed") - def test_zaa_test_major_version_upgrade(self): + def test_standby_cluster(self): + ''' + Create standby cluster streaming from remote primary + ''' k8s = self.k8s - result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest-12.yaml") - self.eventuallyEqual(lambda: k8s.count_running_pods(labels="application=spilo,cluster-name=acid-upgrade-test"), 2, "No 2 pods running") - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + standby_cluster_name = 'acid-standby-cluster' + cluster_name_label = 'cluster-name' + cluster_label = 'application=spilo,{}={}'.format(cluster_name_label, standby_cluster_name) + superuser_name = 'postgres' + replication_user = 'standby' + secret_suffix = 'credentials.postgresql.acid.zalan.do' - pg_patch_version = { - "spec": { - "postgres": { - "version": "13" - } - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version) + # copy secrets from remote cluster before operator creates them when bootstrapping the standby cluster + postgres_secret = k8s.get_secret(superuser_name) + postgres_secret.metadata.name = '{}.{}.{}'.format(superuser_name, standby_cluster_name, secret_suffix) + postgres_secret.metadata.labels[cluster_name_label] = standby_cluster_name + k8s.create_secret(postgres_secret) + standby_secret = k8s.get_secret(replication_user) + standby_secret.metadata.name = '{}.{}.{}'.format(replication_user, standby_cluster_name, secret_suffix) + standby_secret.metadata.labels[cluster_name_label] = standby_cluster_name + k8s.create_secret(standby_secret) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + try: + k8s.create_with_kubectl("manifests/standby-manifest.yaml") + k8s.wait_for_pod_start("spilo-role=master," + cluster_label) - def check_version_13(): - p = k8s.get_patroni_state("acid-upgrade-test-0") - version = p["server_version"][0:2] - return version - - self.evantuallyEqual(check_version_13, "13", "Version was not upgrade to 13") + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + finally: + # delete the standby cluster so that the k8s_api.get_operator_state works correctly in subsequent tests + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-standby-cluster") + time.sleep(5) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - @unittest.skip("Skipping this test until fixed") - def test_zzz_taint_based_eviction(self): + def test_stream_resources(self): + ''' + Create and delete fabric event streaming resources. + ''' + k8s = self.k8s + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + leader = k8s.get_cluster_leader_pod() + + # patch ClusterRole with CRUD privileges on FES resources + cluster_role = k8s.api.rbac_api.read_cluster_role("postgres-operator") + fes_cluster_role_rule = client.V1PolicyRule( + api_groups=["zalando.org"], + resources=["fabriceventstreams"], + verbs=["create", "delete", "deletecollection", "get", "list", "patch", "update", "watch"] + ) + cluster_role.rules.append(fes_cluster_role_rule) + + try: + k8s.api.rbac_api.patch_cluster_role("postgres-operator", cluster_role) + + # create a table in one of the database of acid-minimal-cluster + create_stream_table = """ + CREATE TABLE test_table (id int, payload jsonb); + """ + self.query_database(leader.metadata.name, "foo", create_stream_table) + + # update the manifest with the streams section + patch_streaming_config = { + "spec": { + "patroni": { + "slots": { + "manual_slot": { + "type": "physical" + } + } + }, + "streams": [ + { + "applicationId": "test-app", + "batchSize": 100, + "cpu": "100m", + "memory": "200Mi", + "database": "foo", + "enableRecovery": True, + "tables": { + "test_table": { + "eventType": "test-event", + "idColumn": "id", + "payloadColumn": "payload", + "recoveryEventType": "test-event-dlq" + } + } + }, + { + "applicationId": "test-app2", + "batchSize": 100, + "database": "foo", + "enableRecovery": True, + "tables": { + "test_non_exist_table": { + "eventType": "test-event", + "idColumn": "id", + "payloadColumn": "payload", + "ignoreRecovery": True + } + } + } + ] + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', patch_streaming_config) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # check if publication, slot, and fes resource are created + get_publication_query = """ + SELECT * FROM pg_publication WHERE pubname = 'fes_foo_test_app'; + """ + get_slot_query = """ + SELECT * FROM pg_replication_slots WHERE slot_name = 'fes_foo_test_app'; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "foo", get_publication_query)), 1, + "Publication is not created", 10, 5) + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "foo", get_slot_query)), 1, + "Replication slot is not created", 10, 5) + self.eventuallyEqual(lambda: len(k8s.api.custom_objects_api.list_namespaced_custom_object( + "zalando.org", "v1", "default", "fabriceventstreams", label_selector="cluster-name=acid-minimal-cluster")["items"]), 1, + "Could not find Fabric Event Stream resource", 10, 5) + + # check if the non-existing table in the stream section does not create a publication and slot + get_publication_query_not_exist_table = """ + SELECT * FROM pg_publication WHERE pubname = 'fes_foo_test_app2'; + """ + get_slot_query_not_exist_table = """ + SELECT * FROM pg_replication_slots WHERE slot_name = 'fes_foo_test_app2'; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "foo", get_publication_query_not_exist_table)), 0, + "Publication is created for non-existing tables", 10, 5) + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "foo", get_slot_query_not_exist_table)), 0, + "Replication slot is created for non-existing tables", 10, 5) + + # grant create and ownership of test_table to foo_user, reset search path to default + grant_permission_foo_user = """ + GRANT CREATE ON DATABASE foo TO foo_user; + ALTER TABLE test_table OWNER TO foo_user; + ALTER ROLE foo_user RESET search_path; + """ + self.query_database(leader.metadata.name, "foo", grant_permission_foo_user) + # non-postgres user creates a publication + create_nonstream_publication = """ + CREATE PUBLICATION mypublication FOR TABLE test_table; + """ + self.query_database_with_user(leader.metadata.name, "foo", create_nonstream_publication, "foo_user") + + # remove the streams section from the manifest + patch_streaming_config_removal = { + "spec": { + "streams": [] + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', 'postgresqls', 'acid-minimal-cluster', patch_streaming_config_removal) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # check if publication, slot, and fes resource are removed + self.eventuallyEqual(lambda: len(k8s.api.custom_objects_api.list_namespaced_custom_object( + "zalando.org", "v1", "default", "fabriceventstreams", label_selector="cluster-name=acid-minimal-cluster")["items"]), 0, + 'Could not delete Fabric Event Stream resource', 10, 5) + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "foo", get_publication_query)), 0, + "Publication is not deleted", 10, 5) + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "foo", get_slot_query)), 0, + "Replication slot is not deleted", 10, 5) + + # check the manual_slot and mypublication should not get deleted + get_manual_slot_query = """ + SELECT * FROM pg_replication_slots WHERE slot_name = 'manual_slot'; + """ + get_nonstream_publication_query = """ + SELECT * FROM pg_publication WHERE pubname = 'mypublication'; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", get_manual_slot_query)), 1, + "Slot defined in patroni config is deleted", 10, 5) + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "foo", get_nonstream_publication_query)), 1, + "Publication defined not in stream section is deleted", 10, 5) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_taint_based_eviction(self): ''' Add taint "postgres=:NoExecute" to node with master. This must cause a failover. ''' @@ -1153,7 +2347,6 @@ class EndToEndTestCase(unittest.TestCase): # get nodes of master and replica(s) (expected target of new master) master_nodes, replica_nodes = k8s.get_cluster_nodes() - self.assertNotEqual(master_nodes, []) self.assertNotEqual(replica_nodes, []) @@ -1168,10 +2361,7 @@ class EndToEndTestCase(unittest.TestCase): ] } } - k8s.api.core_v1.patch_node(master_nodes[0], body) - self.eventuallyTrue(lambda: k8s.get_cluster_nodes()[0], replica_nodes) - self.assertNotEqual(lambda: k8s.get_cluster_nodes()[0], master_nodes) # add toleration to pods patch_toleration_config = { @@ -1180,126 +2370,107 @@ class EndToEndTestCase(unittest.TestCase): } } - k8s.update_config(patch_toleration_config, step="allow tainted nodes") - - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") - - # toggle pod anti affinity to move replica away from master node - nm, new_replica_nodes = k8s.get_cluster_nodes() - new_master_node = nm[0] - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_node_affinity(self): - ''' - Add label to a node and update postgres cluster spec to deploy only on a node with that label - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # verify we are in good state from potential previous tests - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") - - # get nodes of master and replica(s) - master_node, replica_nodes = k8s.get_pg_nodes(cluster_label) - - self.assertNotEqual(master_node, []) - self.assertNotEqual(replica_nodes, []) - - # label node with environment=postgres - node_label_body = { - "metadata": { - "labels": { - "node-affinity-test": "postgres" - } - } - } - try: - # patch current master node with the label - print('patching master node: {}'.format(master_node)) - k8s.api.core_v1.patch_node(master_node, node_label_body) - - # add node affinity to cluster - patch_node_affinity_config = { - "spec": { - "nodeAffinity" : { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "node-affinity-test", - "operator": "In", - "values": [ - "postgres" - ] - } - ] - } - ] - } - } - } - } - - k8s.api.custom_objects_api.patch_namespaced_custom_object( - group="acid.zalan.do", - version="v1", - namespace="default", - plural="postgresqls", - name="acid-minimal-cluster", - body=patch_node_affinity_config) - self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - # node affinity change should cause replica to relocate from replica node to master node due to node affinity requirement - k8s.wait_for_pod_failover(master_node, 'spilo-role=replica,' + cluster_label) - k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - - podsList = k8s.api.core_v1.list_namespaced_pod('default', label_selector=cluster_label) - for pod in podsList.items: - if pod.metadata.labels.get('spilo-role') == 'replica': - self.assertEqual(master_node, pod.spec.node_name, - "Sanity check: expected replica to relocate to master node {}, but found on {}".format(master_node, pod.spec.node_name)) - - # check that pod has correct node affinity - key = pod.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms[0].match_expressions[0].key - value = pod.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms[0].match_expressions[0].values[0] - self.assertEqual("node-affinity-test", key, - "Sanity check: expect node selector key to be equal to 'node-affinity-test' but got {}".format(key)) - self.assertEqual("postgres", value, - "Sanity check: expect node selector value to be equal to 'postgres' but got {}".format(value)) - - patch_node_remove_affinity_config = { - "spec": { - "nodeAffinity" : None - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - group="acid.zalan.do", - version="v1", - namespace="default", - plural="postgresqls", - name="acid-minimal-cluster", - body=patch_node_remove_affinity_config) - self.eventuallyEqual(lambda: self.k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + k8s.update_config(patch_toleration_config, step="allow tainted nodes") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") - # remove node affinity to move replica away from master node - nm, new_replica_nodes = k8s.get_cluster_nodes() - new_master_node = nm[0] - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(master_nodes) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_zzzz_cluster_deletion(self): + def test_topology_spread_constraints(self): + ''' + Enable topologySpreadConstraints for pods + ''' + k8s = self.k8s + cluster_labels = "application=spilo,cluster-name=acid-minimal-cluster" + + # Verify we are in good state from potential previous tests + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + + # patch the pvc retention policy to enable delete when scale down + patch_scaled_policy_delete = { + "data": { + "persistent_volume_claim_retention_policy": "when_deleted:retain,when_scaled:delete" + } + } + k8s.update_config(patch_scaled_policy_delete) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + master_nodes, replica_nodes = k8s.get_cluster_nodes() + self.assertNotEqual(master_nodes, []) + self.assertNotEqual(replica_nodes, []) + + # Patch label to nodes for topologySpreadConstraints + patch_node_label = { + "metadata": { + "labels": { + "topology.kubernetes.io/zone": "zalando" + } + } + } + k8s.api.core_v1.patch_node(master_nodes[0], patch_node_label) + k8s.api.core_v1.patch_node(replica_nodes[0], patch_node_label) + + # Patch topologySpreadConstraint and scale-out postgresql pods to postgresqls manifest. + patch_topologySpreadConstraint_config = { + "spec": { + "numberOfInstances": 6, + "topologySpreadConstraint": [ + { + "maxskew": 1, + "topologyKey": "topology.kubernetes.io/zone", + "whenUnsatisfiable": "DoNotSchedule" + } + ] + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", + "postgresqls", "acid-minimal-cluster", + patch_topologySpreadConstraint_config) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_pods_with_label(cluster_labels), 6, "Postgresql StatefulSet are scale to 6") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 6, "All pods are running") + + worker_node_1 = 0 + worker_node_2 = 0 + pods = k8s.api.core_v1.list_namespaced_pod('default', label_selector=cluster_labels) + for pod in pods.items: + if pod.spec.node_name == 'postgres-operator-e2e-tests-worker': + worker_node_1 += 1 + elif pod.spec.node_name == 'postgres-operator-e2e-tests-worker2': + worker_node_2 += 1 + + self.assertEqual(worker_node_1, worker_node_2) + self.assertEqual(worker_node_1, 3) + self.assertEqual(worker_node_2, 3) + + # Reset configurations + patch_topologySpreadConstraint_config = { + "spec": { + "numberOfInstances": 2, + "topologySpreadConstraint": [] + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", + "postgresqls", "acid-minimal-cluster", + patch_topologySpreadConstraint_config) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_pods_with_label(cluster_labels), 2, "Postgresql StatefulSet are scale to 2") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "All pods are running") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_zz_cluster_deletion(self): ''' Test deletion with configured protection ''' @@ -1310,7 +2481,9 @@ class EndToEndTestCase(unittest.TestCase): patch_delete_annotations = { "data": { "delete_annotation_date_key": "delete-date", - "delete_annotation_name_key": "delete-clustername" + "delete_annotation_name_key": "delete-clustername", + "enable_secrets_deletion": "false", + "enable_persistent_volume_claim_deletion": "false" } } k8s.update_config(patch_delete_annotations) @@ -1361,6 +2534,8 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: len(k8s.api.custom_objects_api.list_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", label_selector="cluster-name=acid-minimal-cluster")["items"]), 0, "Manifest not deleted") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + # check if everything has been deleted self.eventuallyEqual(lambda: k8s.count_pods_with_label(cluster_label), 0, "Pods not deleted") self.eventuallyEqual(lambda: k8s.count_services_with_label(cluster_label), 0, "Service not deleted") @@ -1368,7 +2543,8 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: k8s.count_statefulsets_with_label(cluster_label), 0, "Statefulset not deleted") self.eventuallyEqual(lambda: k8s.count_deployments_with_label(cluster_label), 0, "Deployments not deleted") self.eventuallyEqual(lambda: k8s.count_pdbs_with_label(cluster_label), 0, "Pod disruption budget not deleted") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label(cluster_label), 0, "Secrets not deleted") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(cluster_label), 8, "Secrets were deleted although disabled in config") + self.eventuallyEqual(lambda: k8s.count_pvcs_with_label(cluster_label), 2, "PVCs were deleted although disabled in config") except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) @@ -1383,39 +2559,6 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_delete_annotations) - def get_failover_targets(self, master_node, replica_nodes): - ''' - If all pods live on the same node, failover will happen to other worker(s) - ''' - k8s = self.k8s - k8s_master_exclusion = 'kubernetes.io/hostname!=postgres-operator-e2e-tests-control-plane' - - failover_targets = [x for x in replica_nodes if x != master_node] - if len(failover_targets) == 0: - nodes = k8s.api.core_v1.list_node(label_selector=k8s_master_exclusion) - for n in nodes.items: - if n.metadata.name != master_node: - failover_targets.append(n.metadata.name) - - return failover_targets - - def assert_failover(self, current_master_node, num_replicas, failover_targets, cluster_label): - ''' - Check if master is failing over. The replica should move first to be the switchover target - ''' - k8s = self.k8s - k8s.wait_for_pod_failover(failover_targets, 'spilo-role=master,' + cluster_label) - k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - - new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label) - self.assertNotEqual(current_master_node, new_master_node, - "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) - self.assertEqual(num_replicas, len(new_replica_nodes), - "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) - self.assert_master_is_unique() - - return new_master_node, new_replica_nodes - def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"): ''' Check that there is a single pod in the k8s cluster with the label "spilo-role=master" @@ -1427,13 +2570,23 @@ class EndToEndTestCase(unittest.TestCase): num_of_master_pods = k8s.count_pods_with_label(labels, namespace) self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) - def assert_distributed_pods(self, master_node, replica_nodes, cluster_label): + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def assert_distributed_pods(self, target_nodes, cluster_labels='cluster-name=acid-minimal-cluster'): ''' Other tests can lead to the situation that master and replica are on the same node. Toggle pod anti affinty to distribute pods accross nodes (replica in particular). ''' k8s = self.k8s - failover_targets = self.get_failover_targets(master_node, replica_nodes) + cluster_labels = 'application=spilo,cluster-name=acid-minimal-cluster' + + # get nodes of master and replica(s) + master_nodes, replica_nodes = k8s.get_cluster_nodes() + self.assertNotEqual(master_nodes, []) + self.assertNotEqual(replica_nodes, []) + + # if nodes are different we can quit here + if master_nodes[0] not in replica_nodes: + return True # enable pod anti affintiy in config map which should trigger movement of replica patch_enable_antiaffinity = { @@ -1441,20 +2594,82 @@ class EndToEndTestCase(unittest.TestCase): "enable_pod_antiaffinity": "true" } } - k8s.update_config(patch_enable_antiaffinity, "enable antiaffinity") - self.assert_failover(master_node, len(replica_nodes), failover_targets, cluster_label) - # now disable pod anti affintiy again which will cause yet another failover - patch_disable_antiaffinity = { - "data": { - "enable_pod_antiaffinity": "false" + try: + k8s.update_config(patch_enable_antiaffinity, "enable antiaffinity") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_labels) + k8s.wait_for_running_pods(cluster_labels, 2) + + # now disable pod anti affintiy again which will cause yet another failover + patch_disable_antiaffinity = { + "data": { + "enable_pod_antiaffinity": "false" + } } - } - k8s.update_config(patch_disable_antiaffinity, "disable antiaffinity") - k8s.wait_for_pod_start('spilo-role=master') - k8s.wait_for_pod_start('spilo-role=replica') + k8s.update_config(patch_disable_antiaffinity, "disable antiaffinity") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_labels) + k8s.wait_for_running_pods(cluster_labels, 2) + + master_nodes, replica_nodes = k8s.get_cluster_nodes() + self.assertNotEqual(master_nodes, []) + self.assertNotEqual(replica_nodes, []) + + # if nodes are different we can quit here + for target_node in target_nodes: + if (target_node not in master_nodes or target_node not in replica_nodes) and master_nodes[0] in replica_nodes: + print('Pods run on the same node') + return False + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + return True + def check_cluster_child_resources_owner_references(self, cluster_name, cluster_namespace='default', inverse=False): + k8s = self.k8s + + # check if child resources were updated with owner references + sset = k8s.api.apps_v1.read_namespaced_stateful_set(cluster_name, cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(sset.metadata.owner_references, inverse), "statefulset owner reference check failed") + + svc = k8s.api.core_v1.read_namespaced_service(cluster_name, cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(svc.metadata.owner_references, inverse), "primary service owner reference check failed") + replica_svc = k8s.api.core_v1.read_namespaced_service(cluster_name + "-repl", cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(replica_svc.metadata.owner_references, inverse), "replica service owner reference check failed") + config_svc = k8s.api.core_v1.read_namespaced_service(cluster_name + "-config", cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(config_svc.metadata.owner_references, inverse), "config service owner reference check failed") + + ep = k8s.api.core_v1.read_namespaced_endpoints(cluster_name, cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(ep.metadata.owner_references, inverse), "primary endpoint owner reference check failed") + replica_ep = k8s.api.core_v1.read_namespaced_endpoints(cluster_name + "-repl", cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(replica_ep.metadata.owner_references, inverse), "replica endpoint owner reference check failed") + config_ep = k8s.api.core_v1.read_namespaced_endpoints(cluster_name + "-config", cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(config_ep.metadata.owner_references, inverse), "config endpoint owner reference check failed") + + pdb = k8s.api.policy_v1.read_namespaced_pod_disruption_budget("postgres-{}-pdb".format(cluster_name), cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(pdb.metadata.owner_references, inverse), "primary pod disruption budget owner reference check failed") + + pdb = k8s.api.policy_v1.read_namespaced_pod_disruption_budget("postgres-{}-critical-op-pdb".format(cluster_name), cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(pdb.metadata.owner_references, inverse), "pod disruption budget for critical operations owner reference check failed") + + pg_secret = k8s.api.core_v1.read_namespaced_secret("postgres.{}.credentials.postgresql.acid.zalan.do".format(cluster_name), cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(pg_secret.metadata.owner_references, inverse), "postgres secret owner reference check failed") + standby_secret = k8s.api.core_v1.read_namespaced_secret("standby.{}.credentials.postgresql.acid.zalan.do".format(cluster_name), cluster_namespace) + self.assertTrue(self.has_postgresql_owner_reference(standby_secret.metadata.owner_references, inverse), "standby secret owner reference check failed") + + return True + + def has_postgresql_owner_reference(self, owner_references, inverse): + if inverse: + return owner_references is None or owner_references[0].kind != 'postgresql' + + return owner_references is not None and owner_references[0].kind == 'postgresql' and owner_references[0].controller + def list_databases(self, pod_name): ''' Get list of databases we might want to iterate over @@ -1468,12 +2683,8 @@ class EndToEndTestCase(unittest.TestCase): try: q = exec_query.format(db_list_query, "postgres") q = "su postgres -c \"{}\"".format(q) - print('Get databases: {}'.format(q)) result = k8s.exec_with_kubectl(pod_name, q) db_list = clean_list(result.stdout.split(b'\n')) - print('db_list: {}, stdout: {}, stderr {}'.format( - db_list, result.stdout, result.stderr - )) except Exception as ex: print('Could not get databases: {}'.format(ex)) print('Stdout: {}'.format(result.stdout)) @@ -1497,12 +2708,31 @@ class EndToEndTestCase(unittest.TestCase): try: q = exec_query.format(query, db_name) q = "su postgres -c \"{}\"".format(q) - print('Send query: {}'.format(q)) result = k8s.exec_with_kubectl(pod_name, q) result_set = clean_list(result.stdout.split(b'\n')) - print('result: {}, stdout: {}, stderr {}'.format( - result_set, result.stdout, result.stderr - )) + except Exception as ex: + print('Error on query execution: {}'.format(ex)) + print('Stdout: {}'.format(result.stdout)) + print('Stderr: {}'.format(result.stderr)) + + return result_set + + def query_database_with_user(self, pod_name, db_name, query, user_name): + ''' + Query database and return result as a list + ''' + k8s = self.k8s + result_set = [] + exec_query = r"PGPASSWORD={} psql -h localhost -U {} -tAq -c \"{}\" -d {}" + + try: + user_secret = k8s.get_secret(user_name) + secret_user = str(base64.b64decode(user_secret.data["username"]), 'utf-8') + secret_pw = str(base64.b64decode(user_secret.data["password"]), 'utf-8') + q = exec_query.format(secret_pw, secret_user, query, db_name) + q = "su postgres -c \"{}\"".format(q) + result = k8s.exec_with_kubectl(pod_name, q) + result_set = clean_list(result.stdout.split(b'\n')) except Exception as ex: print('Error on query execution: {}'.format(ex)) print('Stdout: {}'.format(result.stdout)) diff --git a/go.mod b/go.mod index 1759d9573..37c2d2b65 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,101 @@ module github.com/zalando/postgres-operator -go 1.15 +go 1.26.4 require ( - github.com/aws/aws-sdk-go v1.36.29 - github.com/golang/mock v1.4.4 - github.com/lib/pq v1.9.0 + github.com/Masterminds/semver/v3 v3.5.0 + github.com/aws/aws-sdk-go-v2 v1.42.0 + github.com/aws/aws-sdk-go-v2/config v1.32.24 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.305.3 + github.com/golang/mock v1.6.0 + github.com/lib/pq v1.12.3 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d + github.com/pkg/errors v0.9.1 github.com/r3labs/diff v1.1.0 - github.com/sirupsen/logrus v1.7.0 - github.com/stretchr/testify v1.6.1 - golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c - golang.org/x/mod v0.4.0 // indirect - gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.20.6 - k8s.io/apiextensions-apiserver v0.20.6 - k8s.io/apimachinery v0.20.6 - k8s.io/client-go v0.20.6 - k8s.io/code-generator v0.20.6 + github.com/sirupsen/logrus v1.9.4 + github.com/stretchr/testify v1.11.1 + golang.org/x/crypto v0.51.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.36.1 + k8s.io/apiextensions-apiserver v0.36.1 + k8s.io/apimachinery v0.36.1 + k8s.io/client-go v0.36.1 + sigs.k8s.io/yaml v1.6.0 +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.1.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 // indirect + github.com/aws/smithy-go v1.27.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gobuffalo/flect v1.0.3 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/moby/spdystream v0.5.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.14.0 // indirect + golang.org/x/tools v0.45.0 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/code-generator v0.36.1 // indirect + k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/streaming v0.36.1 // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/controller-tools v0.17.3 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect +) + +tool ( + github.com/golang/mock/mockgen + k8s.io/code-generator + k8s.io/code-generator/cmd/client-gen + k8s.io/code-generator/cmd/deepcopy-gen + k8s.io/code-generator/cmd/informer-gen + k8s.io/code-generator/cmd/lister-gen + sigs.k8s.io/controller-tools/cmd/controller-gen ) diff --git a/go.sum b/go.sum index 81b242b7a..c4927d42e 100644 --- a/go.sum +++ b/go.sum @@ -1,677 +1,243 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.36.29 h1:lM1G3AF1+7vzFm0n7hfH8r2+750BTo+6Lo6FtPB7kzk= -github.com/aws/aws-sdk-go v1.36.29/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/bymhA= +github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= +github.com/aws/aws-sdk-go-v2/config v1.32.24 h1:aEDEj533yGdVvEHfkCY0D/1FbDrjnZr4pIulxRjqpHs= +github.com/aws/aws-sdk-go-v2/config v1.32.24/go.mod h1:yZtrGKJGlqfEW+/m2uTsJK+Jz7xF5R0eZfgcIG9m1ss= +github.com/aws/aws-sdk-go-v2/credentials v1.19.23 h1:Zhu3GOpRCkNjtE/gJpuPDsytSnaCCTQk8neAGsgzG5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.19.23/go.mod h1:VsJF2ropPB37gDr7M2rLSpCE8IQWdpl62uae7qxZmqU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 h1:r6qZHbT+wxgWO/e9vYNUEtg7lv5+UN3pRqKhLXvnArg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29/go.mod h1:QRnaRcTVGKPGRy8w78HMQtKUGRYcnMZAANATkeVA6Mo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 h1:f3vKqSo13fhTYb+JEcXwXefZQE26I1FB5eTSniU67ko= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29/go.mod h1:MzoLFUArKGpGD+ukmPiTPG1X5x4o6M2kq4v2dr1FiEc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 h1:RdwIf/CuUsvJX3RgJagbOyotl/cxoLY4xviKuE7p2GY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29/go.mod h1:71wt8W2EgswdZy9Mf9KNnzxZ3TiZlv4caKghPktDOkA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 h1:VTGy885W5DKBxWRUJbym9hytNaYzsyaPkCHGRRMAOhU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30/go.mod h1:AS0HycUvJRFvTt613AYDOgO2jzw+00cVSMny8XB3yMY= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.305.3 h1:rEay0b3E0qqyY+W0c3ox8wGRsVK+GjxXadj9B9vpg4c= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.305.3/go.mod h1:8mrDF7OtbuL0QpwP4YCvLuoOE4/5lL7D33MXgp069/Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 h1:ZD2+BSw9vFsNlKYIasSNt3uDbjqqXIBcM13UJv/Lx2k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12/go.mod h1:Ms4zlcVBbXbiP7EVLhl+lgjvA/a7YphqQ3Ih3174EmI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 h1:DRebniUGZ2MqiiIVmQJ04vIXr918hubdHMnarSLEWyU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29/go.mod h1:LfRkPCD8YHDM2E5eTkos2UpwYeZnBcVarTa8L59bJHA= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.5 h1:6Xt6Ztjkwdia/7EtEaG7ki/qZUYlCcd7tGUotQed1QE= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.5/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 h1:ey1XLTYXb9PcLt4535632o5kCGXNXEhNb620Dqwuylo= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.3/go.mod h1:Lk7PlmoTYryQmyBG0EXqj5BcUbj3whXdU2s3yGI3EAc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 h1:yLr03zQE/5Eu5l3QU0Si+xMbLMbSDF2YXsigqXngs6g= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6/go.mod h1:Q5N6icH+KJZDLh+ESNwzdv6cZ6vLFF/egy3IOxWhmz4= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 h1:VrIhKRCSK1umelSgB9RghvA9RTUYeQffyAS5ApXehNI= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.3/go.mod h1:r8wkDOuLaaMFqFiYAb8dGY2A3gJCOujMc6CFOVC4Zhc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4= +github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= -github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= +github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d h1:LznySqW8MqVeFh+pW6rOkFdld9QQ7jRydBKKM6jyPVI= github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d/go.mod h1:u3hJ0kqCQu/cPpsu3RbCOPZ0d7V3IjPjv1adNRleM9I= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/r3labs/diff v1.1.0 h1:V53xhrbTHrWFWq3gI4b94AjgEJOerO1+1l0xyHOBi8M= github.com/r3labs/diff v1.1.0/go.mod h1:7WjXasNzi0vJetRcB/RqNl5dlIsmXcTTLmF5IoH6Xig= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c h1:9HhBz5L/UjnK9XLtiZhYAdue5BVKep3PMmS2LuPDt8k= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.6 h1:bgdZrW++LqgrLikWYNruIKAtltXbSCX2l5mJu11hrVE= -k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/apiextensions-apiserver v0.20.6 h1:3ZJiXMif7W/32RnfyHN8HygTAZaN7o+FvHxCyuQ7VOo= -k8s.io/apiextensions-apiserver v0.20.6/go.mod h1:qO8YMqeMmZH+lV21LUNzV41vfpoE9QVAJRA+MNqj0mo= -k8s.io/apimachinery v0.20.6 h1:R5p3SlhaABYShQSO6LpPsYHjV05Q+79eBUR0Ut/f4tk= -k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/client-go v0.20.6 h1:nJZOfolnsVtDtbGJNCxzOtKUAu7zvXjB8+pMo9UNxZo= -k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/code-generator v0.20.6 h1:kp65Y6kF6A4+5PvSNvXWSI5p5vuA9tUxEqEZciPw+7Q= -k8s.io/code-generator v0.20.6/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU= -k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded h1:JApXBKYyB7l9xx+DK7/+mFjC7A9Bt5A93FPvFD0HIFE= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3 h1:4oyYo8NREp49LBBhKxEqCulFjg26rawYKrnCmg+Sr6c= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= +k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= +k8s.io/apiextensions-apiserver v0.36.1 h1:6JfYmPUsuUIHuN+3QxutXYWj492RqF5fBSx67GYK5Ks= +k8s.io/apiextensions-apiserver v0.36.1/go.mod h1:pLzZin90riwisdzKwv/GoTwENooytoIx5zWJb4Hkby8= +k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= +k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= +k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= +k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= +k8s.io/code-generator v0.36.1 h1:5bHQ7NbBcFFLHcoyo/hgU3m2tQV5RLz2nv4QNDlsbXc= +k8s.io/code-generator v0.36.1/go.mod h1:oCv8WmrW2RGdcMyvSk1aYbBfSs51ggtSFQr1YNeuAuo= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= +k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/controller-tools v0.17.3 h1:lwFPLicpBKLgIepah+c8ikRBubFW5kOQyT88r3EwfNw= +sigs.k8s.io/controller-tools v0.17.3/go.mod h1:1ii+oXcYZkxcBXzwv3YZBlzjt1fvkrCGjVF73blosJI= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/adjust_postgresql_crd.sh b/hack/adjust_postgresql_crd.sh new file mode 100755 index 000000000..d06b74a2d --- /dev/null +++ b/hack/adjust_postgresql_crd.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Hack to adjust the generated postgresql CRD YAML file and add missing field +# settings which can not be expressed via kubebuilder markers. +# +# Injections: +# +# * oneOf: for the standby field to enforce validation rules: +# - s3_wal_path and gs_wal_path are mutually exclusive +# - standby_host can be specified alone or with either s3_wal_path OR gs_wal_path +# - at least one of s3_wal_path, gs_wal_path, or standby_host must be set +# * type: string and pattern for the maintenanceWindows items. + +file="${1:-"manifests/postgresql.crd.yaml"}" + +sed -i '/^[[:space:]]*standby:$/{ + # Capture the indentation + s/^\([[:space:]]*\)standby:$/\1standby:\n\1 anyOf:\n\1 - required:\n\1 - s3_wal_path\n\1 - required:\n\1 - gs_wal_path\n\1 - required:\n\1 - standby_host\n\1 not:\n\1 required:\n\1 - s3_wal_path\n\1 - gs_wal_path/ +}' "$file" + +sed -i '/^[[:space:]]*maintenanceWindows:$/{ + # Capture the indentation + s/^\([[:space:]]*\)maintenanceWindows:$/\1maintenanceWindows:\n\1 items:\n\1 pattern: '\''^\\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))-((2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))\\ *$'\''\n\1 type: string/ +}' "$file" diff --git a/hack/tools.go b/hack/tools.go deleted file mode 100644 index 18432c1b1..000000000 --- a/hack/tools.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build tools - -/* -Copyright 2019 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This package imports things required by build scripts, to force `go mod` to see them as dependencies -package tools - -import _ "k8s.io/code-generator" diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 280da9385..fa3efb599 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -1,13 +1,66 @@ #!/usr/bin/env bash +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -o errexit set -o nounset set -o pipefail -SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. -CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)} +SRC="github.com" +GOPKG="$SRC/zalando/postgres-operator" +CUSTOM_RESOURCE_NAME_ZAL="zalando.org" +CUSTOM_RESOURCE_NAME_ACID="acid.zalan.do" +CUSTOM_RESOURCE_VERSION="v1" -bash "${CODEGEN_PKG}/generate-groups.sh" all \ - github.com/zalando/postgres-operator/pkg/generated github.com/zalando/postgres-operator/pkg/apis \ - "acid.zalan.do:v1" \ - --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt +SCRIPT_ROOT="$(dirname "${BASH_SOURCE[0]}")/.." + +OUTPUT_DIR="pkg/generated" +OUTPUT_PKG="${GOPKG}/${OUTPUT_DIR}" +APIS_PKG="${GOPKG}/pkg/apis" +GROUPS_WITH_VERSIONS="${CUSTOM_RESOURCE_NAME_ZAL}:${CUSTOM_RESOURCE_VERSION},${CUSTOM_RESOURCE_NAME_ACID}:${CUSTOM_RESOURCE_VERSION}" + +echo "Generating deepcopy funcs" +go tool deepcopy-gen \ + --output-file zz_generated.deepcopy.go \ + --go-header-file "${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt" \ + "${APIS_PKG}/${CUSTOM_RESOURCE_NAME_ZAL}/${CUSTOM_RESOURCE_VERSION}" \ + "${APIS_PKG}/${CUSTOM_RESOURCE_NAME_ACID}/${CUSTOM_RESOURCE_VERSION}" + +echo "Generating clientset for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset}" +go tool client-gen \ + --clientset-name versioned \ + --input-base "${APIS_PKG}" \ + --input "${CUSTOM_RESOURCE_NAME_ZAL}/${CUSTOM_RESOURCE_VERSION},${CUSTOM_RESOURCE_NAME_ACID}/${CUSTOM_RESOURCE_VERSION}" \ + --output-pkg "${OUTPUT_PKG}/clientset" \ + --go-header-file "${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt" \ + --output-dir "${OUTPUT_DIR}/clientset" + +echo "Generating listers for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/listers" +go tool lister-gen \ + --output-pkg "${OUTPUT_PKG}/listers" \ + --go-header-file "${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt" \ + --output-dir "${OUTPUT_DIR}/listers" \ + "${APIS_PKG}/${CUSTOM_RESOURCE_NAME_ZAL}/${CUSTOM_RESOURCE_VERSION}" \ + "${APIS_PKG}/${CUSTOM_RESOURCE_NAME_ACID}/${CUSTOM_RESOURCE_VERSION}" + +echo "Generating informers for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/informers" +go tool informer-gen \ + --versioned-clientset-package "${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset}/${CLIENTSET_NAME_VERSIONED:-versioned}" \ + --listers-package "${OUTPUT_PKG}/listers" \ + --output-pkg "${OUTPUT_PKG}/informers" \ + --go-header-file "${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt" \ + --output-dir "${OUTPUT_DIR}/informers" \ + "${APIS_PKG}/${CUSTOM_RESOURCE_NAME_ZAL}/${CUSTOM_RESOURCE_VERSION}" \ + "${APIS_PKG}/${CUSTOM_RESOURCE_NAME_ACID}/${CUSTOM_RESOURCE_VERSION}" diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh index 68710015e..451ac76b7 100755 --- a/hack/verify-codegen.sh +++ b/hack/verify-codegen.sh @@ -19,15 +19,14 @@ cleanup mkdir -p "${TMP_DIFFROOT}" cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}" -"${SCRIPT_ROOT}/hack/update-codegen.sh" +"${SCRIPT_ROOT}/hack/update-codegen.sh" "${TMP_DIFFROOT}" echo "diffing ${DIFFROOT} against freshly generated codegen" ret=0 diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$? -cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}" if [[ $ret -eq 0 ]] then echo "${DIFFROOT} up to date." else - echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh" + echo "${DIFFROOT} is out of date. Please run 'make codegen'" exit 1 fi diff --git a/kubectl-pg/README.md b/kubectl-pg/README.md deleted file mode 100644 index 78102cf9c..000000000 --- a/kubectl-pg/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# Kubectl Plugin for Zalando's Postgres Operator - -This plugin is a prototype developed as a part of **Google Summer of Code 2019** under the [Postgres Operator](https://summerofcode.withgoogle.com/archive/2019/organizations/6187982082539520/) organization. - -## Installation of kubectl pg plugin - -This project uses Go Modules for dependency management to build locally. -Install go and enable go modules with ```export GO111MODULE=on```. -From Go >=1.13 Go modules will be enabled by default. - -```bash -# Assumes you have a working KUBECONFIG -$ GO111MODULE="on" -$ GOPATH/src/github.com/zalando/postgres-operator/kubectl-pg go mod vendor -# This generate a vendor directory with all dependencies needed by the plugin. -$ $GOPATH/src/github.com/zalando/postgres-operator/kubectl-pg go install -# This will place the kubectl-pg binary in your $GOPATH/bin -``` - -## Before using the kubectl pg plugin make sure to set KUBECONFIG env variable - -Ideally KUBECONFIG is found in $HOME/.kube/config else specify the KUBECONFIG path here. -```export KUBECONFIG=$HOME/.kube/config``` - -## List all commands available in kubectl pg - -```kubectl pg --help``` (or) ```kubectl pg``` - -## Check if Postgres Operator is installed and running - -```kubectl pg check``` - -## Create a new cluster using manifest file - -```kubectl pg create -f acid-minimal-cluster.yaml``` - -## List postgres resources - -```kubectl pg list``` - -List clusters across namespaces -```kubectl pg list all``` - -## Update existing cluster using manifest file - -```kubectl pg update -f acid-minimal-cluster.yaml``` - -## Delete existing cluster - -Using the manifest file: -```kubectl pg delete -f acid-minimal-cluster.yaml``` - -Or by specifying the cluster name: -```kubectl pg delete acid-minimal-cluster``` - -Use `--namespace` or `-n` flag if your cluster is in a different namespace to where your current context is pointing to: -```kubectl pg delete acid-minimal-cluster -n namespace01``` - -## Adding manifest roles to an existing cluster - -```kubectl pg add-user USER01 -p CREATEDB,LOGIN -c acid-minimal-cluster``` - -Privileges can only be [SUPERUSER, REPLICATION, INHERIT, LOGIN, NOLOGIN, CREATEROLE, CREATEDB, BYPASSURL] -Note: By default, a LOGIN user is created (unless NOLOGIN is specified). - -## Adding databases to an existing cluster - -You have to specify an owner of the new database and this role must already exist in the cluster: -```kubectl pg add-db DB01 -o OWNER01 -c acid-minimal-cluster``` - -## Extend the volume of an existing pg cluster - -```kubectl pg ext-volume 2Gi -c acid-minimal-cluster``` - -## Print the version of Postgres Operator and kubectl pg plugin - -```kubectl pg version``` - -## Connect to the shell of a postgres pod - -Connect to the master pod: -```kubectl pg connect -c CLUSTER -m``` - -Connect to a random replica pod: -```kubectl pg connect -c CLUSTER``` - -Connect to a certain replica pod: -```kubectl pg connect -c CLUSTER -r 0``` - -## Connect to a database via psql - -Adding the `-p` flag allows you to directly connect to a given database with the psql client. -With `-u` you specify the user. If left out the name of the current OS user is taken. -`-d` lets you specify the database. If no database is specified, it will be the same as the user name. - -Connect to `app_db` database on the master with role `app_user`: -```kubectl pg connect -c CLUSTER -m -p -u app_user -d app_db``` - -Connect to the `postgres` database on a random replica with role `postgres`: -```kubectl pg connect -c CLUSTER -p -u postgres``` - -Connect to a certain replica assuming name of OS user, database role and name are all the same: -```kubectl pg connect -c CLUSTER -r 0 -p``` - - -## Access Postgres Operator logs - -```kubectl pg logs -o``` - -## Access Patroni logs of different database pods - -Fetch logs of master: -```kubectl pg logs -c CLUSTER -m``` - -Fetch logs of a random replica pod: -```kubectl pg logs -c CLUSTER``` - -Fetch logs of specified replica -```kubectl pg logs -c CLUSTER -r 2``` - -## Development - -When making changes to the plugin make sure to change the (major/patch) version of plugin in `build.sh` script and run `./build.sh`. - -## Google Summer of Code 2019 - -### GSoC Proposal - -[kubectl pg proposal](https://docs.google.com/document/d/1-WMy9HkfZ1XnnMbzplMe9rCzKrRMGaMz4owLVXXPb7w/edit) - -### Weekly Reports - -https://github.com/VineethReddy02/GSoC-Kubectl-Plugin-for-Postgres-Operator-tracker - -### Final Project Report - -https://gist.github.com/VineethReddy02/159283bd368a710379eaf0f6bd60a40a diff --git a/kubectl-pg/build.sh b/kubectl-pg/build.sh deleted file mode 100755 index a81bf54fc..000000000 --- a/kubectl-pg/build.sh +++ /dev/null @@ -1,4 +0,0 @@ - -VERSION=1.0 -sed -i "s/KubectlPgVersion string = \"[^\"]*\"/KubectlPgVersion string = \"${VERSION}\"/" cmd/version.go -go install \ No newline at end of file diff --git a/kubectl-pg/cmd/addDb.go b/kubectl-pg/cmd/addDb.go deleted file mode 100644 index 1c33579d9..000000000 --- a/kubectl-pg/cmd/addDb.go +++ /dev/null @@ -1,114 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "log" - - "github.com/spf13/cobra" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -// addDbCmd represents the addDb command -var addDbCmd = &cobra.Command{ - Use: "add-db", - Short: "Adds a DB and its owner to a Postgres cluster. The owner role is created if it does not exist", - Long: `Adds a new DB to the Postgres cluster. Owner needs to be specified by the -o flag, cluster with -c flag.`, - Run: func(cmd *cobra.Command, args []string) { - if len(args) > 0 { - dbName := args[0] - dbOwner, _ := cmd.Flags().GetString("owner") - clusterName, _ := cmd.Flags().GetString("cluster") - addDb(dbName, dbOwner, clusterName) - } else { - fmt.Println("database name can't be empty. Use kubectl pg add-db [-h | --help] for more info") - } - - }, - Example: ` -kubectl pg add-db db01 -o owner01 -c cluster01 -`, -} - -// add db and it's owner to the cluster -func addDb(dbName string, dbOwner string, clusterName string) { - config := getConfig() - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - namespace := getCurrentNamespace() - postgresql, err := postgresConfig.Postgresqls(namespace).Get(context.TODO(), clusterName, metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - - var dbOwnerExists bool - dbUsers := postgresql.Spec.Users - for key := range dbUsers { - if key == dbOwner { - dbOwnerExists = true - } - } - var patch []byte - // validating reserved DB names - if dbOwnerExists && dbName != "postgres" && dbName != "template0" && dbName != "template1" { - patch = dbPatch(dbName, dbOwner) - } else if !dbOwnerExists { - log.Fatal("The provided db-owner doesn't exist") - } else { - log.Fatal("The provided db-name is reserved by postgres") - } - - updatedPostgres, err := postgresConfig.Postgresqls(namespace).Patch(context.TODO(), postgresql.Name, types.MergePatchType, patch, metav1.PatchOptions{}) - if err != nil { - log.Fatal(err) - } - - if updatedPostgres.ResourceVersion != postgresql.ResourceVersion { - fmt.Printf("Created new database %s with owner %s in PostgreSQL cluster %s.\n", dbName, dbOwner, updatedPostgres.Name) - } else { - fmt.Printf("postgresql %s is unchanged.\n", updatedPostgres.Name) - } -} - -func dbPatch(dbname string, owner string) []byte { - ins := map[string]map[string]map[string]string{"spec": {"databases": {dbname: owner}}} - patchInstances, err := json.Marshal(ins) - if err != nil { - log.Fatal(err, "unable to parse patch for add-db") - } - return patchInstances -} - -func init() { - addDbCmd.Flags().StringP("owner", "o", "", "provide owner of the database.") - addDbCmd.Flags().StringP("cluster", "c", "", "provide a postgres cluster name.") - rootCmd.AddCommand(addDbCmd) -} diff --git a/kubectl-pg/cmd/addUser.go b/kubectl-pg/cmd/addUser.go deleted file mode 100644 index 288af0836..000000000 --- a/kubectl-pg/cmd/addUser.go +++ /dev/null @@ -1,144 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "log" - "strings" - - "github.com/spf13/cobra" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -var allowedPrivileges = []string{"SUPERUSER", "REPLICATION", "INHERIT", "LOGIN", "NOLOGIN", "CREATEROLE", "CREATEDB", "BYPASSURL"} - -// addUserCmd represents the addUser command -var addUserCmd = &cobra.Command{ - Use: "add-user", - Short: "Adds a user to the postgres cluster with given privileges", - Long: `Adds a user to the postgres cluster. You can add privileges as well with -p flag.`, - Run: func(cmd *cobra.Command, args []string) { - clusterName, _ := cmd.Flags().GetString("cluster") - privileges, _ := cmd.Flags().GetString("privileges") - - if len(args) > 0 { - user := args[0] - var permissions []string - var perms []string - - if privileges != "" { - parsedRoles := strings.Replace(privileges, ",", " ", -1) - parsedRoles = strings.ToUpper(parsedRoles) - permissions = strings.Fields(parsedRoles) - var invalidPerms []string - - for _, userPrivilege := range permissions { - validPerm := false - for _, privilege := range allowedPrivileges { - if privilege == userPrivilege { - perms = append(perms, userPrivilege) - validPerm = true - } - } - if !validPerm { - invalidPerms = append(invalidPerms, userPrivilege) - } - } - - if len(invalidPerms) > 0 { - fmt.Printf("Invalid privileges %s\n", invalidPerms) - return - } - } - addUser(user, clusterName, perms) - } - }, - Example: ` -kubectl pg add-user user01 -p login,createdb -c cluster01 -`, -} - -// add user to the cluster with provided permissions -func addUser(user string, clusterName string, permissions []string) { - config := getConfig() - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - namespace := getCurrentNamespace() - postgresql, err := postgresConfig.Postgresqls(namespace).Get(context.TODO(), clusterName, metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - - setUsers := make(map[string]bool) - for _, k := range permissions { - setUsers[k] = true - } - - if existingRoles, key := postgresql.Spec.Users[user]; key { - for _, k := range existingRoles { - setUsers[k] = true - } - } - - Privileges := []string{} - for keys, values := range setUsers { - if values { - Privileges = append(Privileges, keys) - } - } - - patch := applyUserPatch(user, Privileges) - updatedPostgresql, err := postgresConfig.Postgresqls(namespace).Patch(context.TODO(), postgresql.Name, types.MergePatchType, patch, metav1.PatchOptions{}) - if err != nil { - log.Fatal(err) - } - - if updatedPostgresql.ResourceVersion != postgresql.ResourceVersion { - fmt.Printf("postgresql %s is updated with new user %s and with privileges %s.\n", updatedPostgresql.Name, user, permissions) - } else { - fmt.Printf("postgresql %s is unchanged.\n", updatedPostgresql.Name) - } -} - -func applyUserPatch(user string, value []string) []byte { - ins := map[string]map[string]map[string][]string{"spec": {"users": {user: value}}} - patchInstances, err := json.Marshal(ins) - if err != nil { - log.Fatal(err, "unable to parse number of instances json") - } - return patchInstances -} - -func init() { - addUserCmd.Flags().StringP("cluster", "c", "", "add user to the provided cluster.") - addUserCmd.Flags().StringP("privileges", "p", "", "add privileges separated by commas without spaces") - rootCmd.AddCommand(addUserCmd) -} diff --git a/kubectl-pg/cmd/check.go b/kubectl-pg/cmd/check.go deleted file mode 100644 index 6068c35bb..000000000 --- a/kubectl-pg/cmd/check.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "fmt" - "log" - - "github.com/spf13/cobra" - postgresConstants "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apiextv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// checkCmd represent kubectl pg check. -var checkCmd = &cobra.Command{ - Use: "check", - Short: "Checks the Postgres operator is installed in the k8s cluster", - Long: `Checks that the Postgres CRD is registered in a k8s cluster. -This means that the operator pod was able to start normally.`, - Run: func(cmd *cobra.Command, args []string) { - check() - }, - Example: ` -kubectl pg check -`, -} - -// check validates postgresql CRD registered or not. -func check() *v1.CustomResourceDefinition { - config := getConfig() - apiExtClient, err := apiextv1.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - crdInfo, err := apiExtClient.CustomResourceDefinitions().Get(context.TODO(), postgresConstants.PostgresCRDResouceName, metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - - if crdInfo.Name == postgresConstants.PostgresCRDResouceName { - fmt.Printf("Postgres Operator is installed in the k8s cluster.\n") - } else { - fmt.Printf("Postgres Operator is not installed in the k8s cluster.\n") - } - return crdInfo -} - -func init() { - rootCmd.AddCommand(checkCmd) -} diff --git a/kubectl-pg/cmd/connect.go b/kubectl-pg/cmd/connect.go deleted file mode 100644 index 2c6d87835..000000000 --- a/kubectl-pg/cmd/connect.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "log" - "os" - user "os/user" - - "github.com/spf13/cobra" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/remotecommand" -) - -// connectCmd represents the kubectl pg connect command -var connectCmd = &cobra.Command{ - Use: "connect", - Short: "Connects to the shell prompt, psql prompt of postgres cluster", - Long: `Connects to the shell prompt, psql prompt of postgres cluster and also to specified replica or master.`, - Run: func(cmd *cobra.Command, args []string) { - clusterName, _ := cmd.Flags().GetString("cluster") - master, _ := cmd.Flags().GetBool("master") - replica, _ := cmd.Flags().GetString("replica") - psql, _ := cmd.Flags().GetBool("psql") - userName, _ := cmd.Flags().GetString("user") - dbName, _ := cmd.Flags().GetString("database") - - if psql { - if userName == "" { - userInfo, err := user.Current() - if err != nil { - log.Fatal(err) - } - userName = userInfo.Username - } - } - if dbName == "" { - dbName = userName - } - - connect(clusterName, master, replica, psql, userName, dbName) - }, - Example: ` -#connects to the master of postgres cluster -kubectl pg connect -c cluster -m - -#connects to the random replica of postgres cluster -kubectl pg connect -c cluster - -#connects to the provided replica number of postgres cluster -kubectl pg connect -c cluster -r 2 - -#connects to psql prompt of master for provided postgres cluster with current shell user -kubectl pg connect -c cluster -p -m - -#connects to psql prompt of random replica for provided postgres cluster with provided user and db -kubectl pg connect -c cluster -p -u user01 -d db01 -`, -} - -func connect(clusterName string, master bool, replica string, psql bool, user string, dbName string) { - config := getConfig() - client, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - podName := getPodName(clusterName, master, replica) - var execRequest *rest.Request - - if psql { - execRequest = client.CoreV1().RESTClient().Post().Resource("pods"). - Name(podName). - Namespace(getCurrentNamespace()). - SubResource("exec"). - Param("container", "postgres"). - Param("command", "psql"). - Param("command", dbName). - Param("command", user). - Param("stdin", "true"). - Param("stdout", "true"). - Param("stderr", "true"). - Param("tty", "true") - } else { - execRequest = client.CoreV1().RESTClient().Post().Resource("pods"). - Name(podName). - Namespace(getCurrentNamespace()). - SubResource("exec"). - Param("container", "postgres"). - Param("command", "su"). - Param("command", "postgres"). - Param("stdin", "true"). - Param("stdout", "true"). - Param("stderr", "true"). - Param("tty", "true") - } - - exec, err := remotecommand.NewSPDYExecutor(config, "POST", execRequest.URL()) - if err != nil { - log.Fatal(err) - } - - err = exec.Stream(remotecommand.StreamOptions{ - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - Tty: true, - }) - if err != nil { - log.Fatal(err) - } -} - -func init() { - connectCmd.Flags().StringP("cluster", "c", "", "provide the cluster name.") - connectCmd.Flags().BoolP("master", "m", false, "connect to master.") - connectCmd.Flags().StringP("replica", "r", "", "connect to replica. Specify replica number.") - connectCmd.Flags().BoolP("psql", "p", false, "connect to psql prompt.") - connectCmd.Flags().StringP("user", "u", "", "provide user.") - connectCmd.Flags().StringP("database", "d", "", "provide database name.") - rootCmd.AddCommand(connectCmd) -} diff --git a/kubectl-pg/cmd/create.go b/kubectl-pg/cmd/create.go deleted file mode 100644 index 00ee7ac24..000000000 --- a/kubectl-pg/cmd/create.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "fmt" - "io/ioutil" - "log" - - "github.com/spf13/cobra" - v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// createCmd kubectl pg create. -var createCmd = &cobra.Command{ - Use: "create", - Short: "Creates postgres object using manifest file", - Long: `Creates postgres custom resource objects from a manifest file.`, - Run: func(cmd *cobra.Command, args []string) { - fileName, _ := cmd.Flags().GetString("file") - create(fileName) - }, - Example: ` -kubectl pg create -f cluster-manifest.yaml -`, -} - -// Create postgresql resources. -func create(fileName string) { - config := getConfig() - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - ymlFile, err := ioutil.ReadFile(fileName) - if err != nil { - log.Fatal(err) - } - - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(ymlFile), nil, &v1.Postgresql{}) - if err != nil { - log.Fatal(err) - } - - postgresSql := obj.(*v1.Postgresql) - _, err = postgresConfig.Postgresqls(postgresSql.Namespace).Create(context.TODO(), postgresSql, metav1.CreateOptions{}) - if err != nil { - log.Fatal(err) - } - - fmt.Printf("postgresql %s created.\n", postgresSql.Name) -} - -func init() { - createCmd.Flags().StringP("file", "f", "", "manifest file with the cluster definition.") - rootCmd.AddCommand(createCmd) -} diff --git a/kubectl-pg/cmd/delete.go b/kubectl-pg/cmd/delete.go deleted file mode 100644 index 7737212b9..000000000 --- a/kubectl-pg/cmd/delete.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "fmt" - "io/ioutil" - "log" - - "github.com/spf13/cobra" - v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// deleteCmd represents kubectl pg delete. -var deleteCmd = &cobra.Command{ - Use: "delete", - Short: "Deletes postgresql object by cluster-name/manifest file", - Long: `Deletes the postgres objects identified by a manifest file or cluster-name. -Deleting the manifest is sufficient to delete the cluster.`, - Run: func(cmd *cobra.Command, args []string) { - namespace, _ := cmd.Flags().GetString("namespace") - file, _ := cmd.Flags().GetString("file") - - if file != "" { - deleteByFile(file) - } else if namespace != "" { - if len(args) != 0 { - clusterName := args[0] - deleteByName(clusterName, namespace) - } else { - fmt.Println("cluster name can't be empty") - } - } else { - fmt.Println("use the flag either -n or -f to delete a resource.") - } - }, - Example: ` -#Deleting the postgres cluster using manifest file -kubectl pg delete -f cluster-manifest.yaml - -#Deleting the postgres cluster using cluster name in current namespace. -kubectl pg delete cluster01 - -#Deleting the postgres cluster using cluster name in provided namespace -kubectl pg delete cluster01 -n namespace01 -`, -} - -func deleteByFile(file string) { - config := getConfig() - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - ymlFile, err := ioutil.ReadFile(file) - if err != nil { - log.Fatal(err) - } - - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(ymlFile), nil, &v1.Postgresql{}) - if err != nil { - log.Fatal(err) - } - - postgresSql := obj.(*v1.Postgresql) - _, err = postgresConfig.Postgresqls(postgresSql.Namespace).Get(context.TODO(), postgresSql.Name, metav1.GetOptions{}) - if err != nil { - fmt.Printf("Postgresql %s not found with the provided namespace %s : %s \n", postgresSql.Name, postgresSql.Namespace, err) - return - } - fmt.Printf("Are you sure you want to remove this PostgreSQL cluster? If so, please type (%s/%s) and hit Enter\n", postgresSql.Namespace, postgresSql.Name) - - confirmAction(postgresSql.Name, postgresSql.Namespace) - err = postgresConfig.Postgresqls(postgresSql.Namespace).Delete(context.TODO(), postgresSql.Name, metav1.DeleteOptions{}) - if err != nil { - log.Fatal(err) - } - fmt.Printf("Postgresql %s deleted from %s.\n", postgresSql.Name, postgresSql.Namespace) -} - -func deleteByName(clusterName string, namespace string) { - config := getConfig() - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - _, err = postgresConfig.Postgresqls(namespace).Get(context.TODO(), clusterName, metav1.GetOptions{}) - if err != nil { - fmt.Printf("Postgresql %s not found with the provided namespace %s : %s \n", clusterName, namespace, err) - return - } - fmt.Printf("Are you sure you want to remove this PostgreSQL cluster? If so, please type (%s/%s) and hit Enter\n", namespace, clusterName) - - confirmAction(clusterName, namespace) - err = postgresConfig.Postgresqls(namespace).Delete(context.TODO(), clusterName, metav1.DeleteOptions{}) - if err != nil { - log.Fatal(err) - } - fmt.Printf("Postgresql %s deleted from %s.\n", clusterName, namespace) -} - -func init() { - namespace := getCurrentNamespace() - deleteCmd.Flags().StringP("namespace", "n", namespace, "namespace of the cluster to be deleted.") - deleteCmd.Flags().StringP("file", "f", "", "manifest file with the cluster definition.") - rootCmd.AddCommand(deleteCmd) -} diff --git a/kubectl-pg/cmd/extVolume.go b/kubectl-pg/cmd/extVolume.go deleted file mode 100644 index 02ccc372d..000000000 --- a/kubectl-pg/cmd/extVolume.go +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "log" - "strconv" - - "github.com/spf13/cobra" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -// extVolumeCmd represents the extVolume command -var extVolumeCmd = &cobra.Command{ - Use: "ext-volume", - Short: "Increases the volume size of a given Postgres cluster", - Long: `Extends the volume of the postgres cluster. But volume cannot be shrinked.`, - Run: func(cmd *cobra.Command, args []string) { - clusterName, _ := cmd.Flags().GetString("cluster") - if len(args) > 0 { - volume := args[0] - extVolume(volume, clusterName) - } else { - fmt.Println("please enter the cluster name with -c flag & volume in desired units") - } - }, - Example: ` -#Extending the volume size of provided cluster -kubectl pg ext-volume 2Gi -c cluster01 -`, -} - -// extend volume with provided size & cluster name -func extVolume(increasedVolumeSize string, clusterName string) { - config := getConfig() - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - namespace := getCurrentNamespace() - postgresql, err := postgresConfig.Postgresqls(namespace).Get(context.TODO(), clusterName, metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - - oldSize, err := resource.ParseQuantity(postgresql.Spec.Volume.Size) - if err != nil { - log.Fatal(err) - } - - newSize, err := resource.ParseQuantity(increasedVolumeSize) - if err != nil { - log.Fatal(err) - } - - _, err = strconv.Atoi(newSize.String()) - if err == nil { - fmt.Println("provide the valid volume size with respective units i.e Ki, Mi, Gi") - return - } - - if newSize.Value() > oldSize.Value() { - patchInstances := volumePatch(newSize) - response, err := postgresConfig.Postgresqls(namespace).Patch(context.TODO(), postgresql.Name, types.MergePatchType, patchInstances, metav1.PatchOptions{}) - if err != nil { - log.Fatal(err) - } - if postgresql.ResourceVersion != response.ResourceVersion { - fmt.Printf("%s volume is extended to %s.\n", response.Name, increasedVolumeSize) - } else { - fmt.Printf("%s volume %s is unchanged.\n", response.Name, postgresql.Spec.Volume.Size) - } - } else if newSize.Value() == oldSize.Value() { - fmt.Println("volume already has the desired size.") - } else { - fmt.Printf("volume %s size cannot be shrinked.\n", postgresql.Spec.Volume.Size) - } -} - -func volumePatch(volume resource.Quantity) []byte { - patchData := map[string]map[string]map[string]resource.Quantity{"spec": {"volume": {"size": volume}}} - patch, err := json.Marshal(patchData) - if err != nil { - log.Fatal(err, "unable to parse patch to extend volume") - } - return patch -} - -func init() { - extVolumeCmd.Flags().StringP("cluster", "c", "", "provide cluster name.") - rootCmd.AddCommand(extVolumeCmd) -} diff --git a/kubectl-pg/cmd/list.go b/kubectl-pg/cmd/list.go deleted file mode 100644 index 4fd6de3ba..000000000 --- a/kubectl-pg/cmd/list.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "fmt" - "log" - "strconv" - "time" - - "github.com/spf13/cobra" - v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - TrimCreateTimestamp = 6000000000 -) - -// listCmd represents kubectl pg list. -var listCmd = &cobra.Command{ - Use: "list", - Short: "Lists all the resources of kind postgresql", - Long: `Lists all the info specific to postgresql objects.`, - Run: func(cmd *cobra.Command, args []string) { - allNamespaces, _ := cmd.Flags().GetBool("all-namespaces") - namespace, _ := cmd.Flags().GetString("namespace") - if allNamespaces { - list(allNamespaces, "") - } else { - list(allNamespaces, namespace) - } - - }, - Example: ` -#Lists postgres cluster in current namespace -kubectl pg list - -#Lists postgres clusters in all namespaces -kubectl pg list -A -`, -} - -// list command to list postgres. -func list(allNamespaces bool, namespace string) { - config := getConfig() - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - listPostgres, err := postgresConfig.Postgresqls(namespace).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - log.Fatal(err) - } - - if len(listPostgres.Items) == 0 { - if namespace != "" { - fmt.Printf("No Postgresql clusters found in namespace: %v\n", namespace) - } else { - fmt.Println("No Postgresql clusters found in all namespaces") - } - return - } - - if allNamespaces { - listAll(listPostgres) - } else { - listWithNamespace(listPostgres) - } -} - -func listAll(listPostgres *v1.PostgresqlList) { - template := "%-32s%-16s%-12s%-12s%-12s%-12s%-12s\n" - fmt.Printf(template, "NAME", "STATUS", "INSTANCES", "VERSION", "AGE", "VOLUME", "NAMESPACE") - for _, pgObjs := range listPostgres.Items { - fmt.Printf(template, pgObjs.Name, - pgObjs.Status.PostgresClusterStatus, - strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), - pgObjs.Spec.PostgresqlParam.PgVersion, - time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), - pgObjs.Spec.Size, pgObjs.Namespace) - } -} - -func listWithNamespace(listPostgres *v1.PostgresqlList) { - template := "%-32s%-16s%-12s%-12s%-12s%-12s\n" - fmt.Printf(template, "NAME", "STATUS", "INSTANCES", "VERSION", "AGE", "VOLUME") - for _, pgObjs := range listPostgres.Items { - fmt.Printf(template, pgObjs.Name, - pgObjs.Status.PostgresClusterStatus, - strconv.Itoa(int(pgObjs.Spec.NumberOfInstances)), - pgObjs.Spec.PostgresqlParam.PgVersion, - time.Since(pgObjs.CreationTimestamp.Time).Truncate(TrimCreateTimestamp), - pgObjs.Spec.Size) - } -} - -func init() { - listCmd.Flags().BoolP("all-namespaces", "A", false, "list pg resources across all namespaces.") - listCmd.Flags().StringP("namespace", "n", getCurrentNamespace(), "provide the namespace") - rootCmd.AddCommand(listCmd) -} diff --git a/kubectl-pg/cmd/logs.go b/kubectl-pg/cmd/logs.go deleted file mode 100644 index 21a4fd6ec..000000000 --- a/kubectl-pg/cmd/logs.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "io" - "log" - "os" - - "github.com/spf13/cobra" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -// logsCmd represents the logs command -var logsCmd = &cobra.Command{ - Use: "logs", - Short: "This will fetch the logs of the specified postgres cluster & postgres operator", - Long: `Fetches the logs of the postgres cluster (i.e master( with -m flag) & replica with (-r 1 pod number) and without -m or -r connects to random replica`, - Run: func(cmd *cobra.Command, args []string) { - opLogs, _ := cmd.Flags().GetBool("operator") - clusterName, _ := cmd.Flags().GetString("cluster") - master, _ := cmd.Flags().GetBool("master") - replica, _ := cmd.Flags().GetString("replica") - - if opLogs { - operatorLogs() - } else { - clusterLogs(clusterName, master, replica) - } - }, - Example: ` -#Fetch the logs of the postgres operator -kubectl pg logs -o - -#Fetch the logs of the master for provided cluster -kubectl pg logs -c cluster01 -m - -#Fetch the logs of the random replica for provided cluster -kubectl pg logs -c cluster01 - -#Fetch the logs of the provided replica number of the cluster -kubectl pg logs -c cluster01 -r 3 -`, -} - -func operatorLogs() { - config := getConfig() - client, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - operator := getPostgresOperator(client) - allPods, err := client.CoreV1().Pods(operator.Namespace).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - log.Fatal(err) - } - - var operatorPodName string - for _, pod := range allPods.Items { - for key, value := range pod.Labels { - if (key == "name" && value == OperatorName) || (key == "app.kubernetes.io/name" && value == OperatorName) { - operatorPodName = pod.Name - break - } - } - } - - execRequest := client.CoreV1().RESTClient().Get().Namespace(operator.Namespace). - Name(operatorPodName). - Resource("pods"). - SubResource("log"). - Param("follow", "--follow"). - Param("container", OperatorName) - - readCloser, err := execRequest.Stream(context.TODO()) - if err != nil { - log.Fatal(err) - } - - defer readCloser.Close() - _, err = io.Copy(os.Stdout, readCloser) - if err != nil { - log.Fatal(err) - } -} - -func clusterLogs(clusterName string, master bool, replica string) { - config := getConfig() - client, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - podName := getPodName(clusterName, master, replica) - execRequest := client.CoreV1().RESTClient().Get().Namespace(getCurrentNamespace()). - Name(podName). - Resource("pods"). - SubResource("log"). - Param("follow", "--follow"). - Param("container", "postgres") - - readCloser, err := execRequest.Stream(context.TODO()) - if err != nil { - log.Fatal(err) - } - - defer readCloser.Close() - _, err = io.Copy(os.Stdout, readCloser) - if err != nil { - log.Fatal(err) - } -} - -func init() { - rootCmd.AddCommand(logsCmd) - logsCmd.Flags().BoolP("operator", "o", false, "logs of operator") - logsCmd.Flags().StringP("cluster", "c", "", "logs for the provided cluster") - logsCmd.Flags().BoolP("master", "m", false, "Patroni logs of master") - logsCmd.Flags().StringP("replica", "r", "", "Patroni logs of replica. Specify replica number.") -} diff --git a/kubectl-pg/cmd/scale.go b/kubectl-pg/cmd/scale.go deleted file mode 100644 index 0a7bdc60f..000000000 --- a/kubectl-pg/cmd/scale.go +++ /dev/null @@ -1,194 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "log" - "strconv" - - "github.com/spf13/cobra" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -// scaleCmd represents the scale command -var scaleCmd = &cobra.Command{ - Use: "scale", - Short: "Add/remove pods to a Postgres cluster", - Long: `Scales the postgres objects using cluster-name. -Scaling to 0 leads to down time.`, - Run: func(cmd *cobra.Command, args []string) { - clusterName, err := cmd.Flags().GetString("cluster") - if err != nil { - log.Fatal(err) - } - namespace, err := cmd.Flags().GetString("namespace") - if err != nil { - log.Fatal(err) - } - - if len(args) > 0 { - numberOfInstances, err := strconv.Atoi(args[0]) - if err != nil { - log.Fatal(err) - } - scale(int32(numberOfInstances), clusterName, namespace) - } else { - fmt.Println("Please enter number of instances to scale.") - } - - }, - Example: ` -#Usage -kubectl pg scale [NUMBER-OF-INSTANCES] -c [CLUSTER-NAME] -n [NAMESPACE] - -#Scales the number of instances of the provided cluster -kubectl pg scale 5 -c cluster01 -`, -} - -func scale(numberOfInstances int32, clusterName string, namespace string) { - config := getConfig() - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - postgresql, err := postgresConfig.Postgresqls(namespace).Get(context.TODO(), clusterName, metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - - minInstances, maxInstances := allowedMinMaxInstances(config) - - if minInstances == -1 && maxInstances == -1 { - postgresql.Spec.NumberOfInstances = numberOfInstances - } else if numberOfInstances <= maxInstances && numberOfInstances >= minInstances { - postgresql.Spec.NumberOfInstances = numberOfInstances - } else if minInstances == -1 && numberOfInstances < postgresql.Spec.NumberOfInstances || - maxInstances == -1 && numberOfInstances > postgresql.Spec.NumberOfInstances { - postgresql.Spec.NumberOfInstances = numberOfInstances - } else { - log.Fatalf("cannot scale to the provided instances as they don't adhere to MIN_INSTANCES: %v and MAX_INSTANCES: %v provided in configmap or operatorconfiguration", maxInstances, minInstances) - } - - if numberOfInstances == 0 { - fmt.Printf("Scaling to zero leads to down time. please type %s/%s and hit Enter this serves to confirm the action\n", namespace, clusterName) - confirmAction(clusterName, namespace) - } - - patchInstances := scalePatch(numberOfInstances) - UpdatedPostgres, err := postgresConfig.Postgresqls(namespace).Patch(context.TODO(), postgresql.Name, types.MergePatchType, patchInstances, metav1.PatchOptions{}) - if err != nil { - log.Fatal(err) - } - - if UpdatedPostgres.ResourceVersion != postgresql.ResourceVersion { - fmt.Printf("scaled postgresql %s/%s to %d instances\n", UpdatedPostgres.Namespace, UpdatedPostgres.Name, UpdatedPostgres.Spec.NumberOfInstances) - return - } - fmt.Printf("postgresql %s is unchanged.\n", postgresql.Name) -} - -func scalePatch(value int32) []byte { - instances := map[string]map[string]int32{"spec": {"numberOfInstances": value}} - patchInstances, err := json.Marshal(instances) - if err != nil { - log.Fatal(err, "unable to parse patch for scale") - } - return patchInstances -} - -func allowedMinMaxInstances(config *rest.Config) (int32, int32) { - k8sClient, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - operator := getPostgresOperator(k8sClient) - - operatorContainer := operator.Spec.Template.Spec.Containers - var configMapName, operatorConfigName string - // -1 indicates no limitations for min/max instances - minInstances := -1 - maxInstances := -1 - for _, envData := range operatorContainer[0].Env { - if envData.Name == "CONFIG_MAP_NAME" { - configMapName = envData.Value - } - if envData.Name == "POSTGRES_OPERATOR_CONFIGURATION_OBJECT" { - operatorConfigName = envData.Value - } - } - - if operatorConfigName == "" { - configMap, err := k8sClient.CoreV1().ConfigMaps(operator.Namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - - configMapData := configMap.Data - for key, value := range configMapData { - if key == "min_instances" { - minInstances, err = strconv.Atoi(value) - if err != nil { - log.Fatalf("invalid min instances in configmap %v", err) - } - } - - if key == "max_instances" { - maxInstances, err = strconv.Atoi(value) - if err != nil { - log.Fatalf("invalid max instances in configmap %v", err) - } - } - } - } else if configMapName == "" { - pgClient, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - operatorConfig, err := pgClient.OperatorConfigurations(operator.Namespace).Get(context.TODO(), operatorConfigName, metav1.GetOptions{}) - if err != nil { - log.Fatalf("unable to read operator configuration %v", err) - } - - minInstances = int(operatorConfig.Configuration.MinInstances) - maxInstances = int(operatorConfig.Configuration.MaxInstances) - } - return int32(minInstances), int32(maxInstances) -} - -func init() { - namespace := getCurrentNamespace() - scaleCmd.Flags().StringP("namespace", "n", namespace, "namespace of the cluster to be scaled") - scaleCmd.Flags().StringP("cluster", "c", "", "provide the cluster name.") - rootCmd.AddCommand(scaleCmd) -} diff --git a/kubectl-pg/cmd/update.go b/kubectl-pg/cmd/update.go deleted file mode 100644 index 6a5f4e36d..000000000 --- a/kubectl-pg/cmd/update.go +++ /dev/null @@ -1,96 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "fmt" - "io/ioutil" - "log" - - "github.com/spf13/cobra" - v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// updateCmd represents kubectl pg update -var updateCmd = &cobra.Command{ - Use: "update", - Short: "Updates postgresql object using manifest file", - Long: `Updates the state of cluster using manifest file to reflect the changes on the cluster.`, - Run: func(cmd *cobra.Command, args []string) { - fileName, _ := cmd.Flags().GetString("file") - updatePgResources(fileName) - }, - Example: ` -#usage -kubectl pg update -f [File-NAME] - -#update the postgres cluster with updated manifest file -kubectl pg update -f cluster-manifest.yaml -`, -} - -// Update postgresql resources. -func updatePgResources(fileName string) { - config := getConfig() - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - ymlFile, err := ioutil.ReadFile(fileName) - if err != nil { - log.Fatal(err) - } - - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(ymlFile), nil, &v1.Postgresql{}) - if err != nil { - log.Fatal(err) - } - - newPostgresObj := obj.(*v1.Postgresql) - oldPostgresObj, err := postgresConfig.Postgresqls(newPostgresObj.Namespace).Get(context.TODO(), newPostgresObj.Name, metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - - newPostgresObj.ResourceVersion = oldPostgresObj.ResourceVersion - response, err := postgresConfig.Postgresqls(newPostgresObj.Namespace).Update(context.TODO(), newPostgresObj, metav1.UpdateOptions{}) - if err != nil { - log.Fatal(err) - } - - if newPostgresObj.ResourceVersion != response.ResourceVersion { - fmt.Printf("postgresql %s updated.\n", response.Name) - } else { - fmt.Printf("postgresql %s is unchanged.\n", response.Name) - } -} - -func init() { - updateCmd.Flags().StringP("file", "f", "", "manifest file with the cluster definition.") - rootCmd.AddCommand(updateCmd) -} diff --git a/kubectl-pg/cmd/util.go b/kubectl-pg/cmd/util.go deleted file mode 100644 index fa0eb6d42..000000000 --- a/kubectl-pg/cmd/util.go +++ /dev/null @@ -1,172 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "context" - "flag" - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - - PostgresqlLister "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - v1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" -) - -const ( - OperatorName = "postgres-operator" - DefaultNamespace = "default" -) - -func getConfig() *restclient.Config { - var kubeconfig *string - var config *restclient.Config - envKube := os.Getenv("KUBECONFIG") - if envKube != "" { - kubeconfig = &envKube - } else { - if home := homedir.HomeDir(); home != "" { - kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") - } else { - kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file") - } - } - flag.Parse() - var err error - config, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) - if err != nil { - log.Fatal(err) - } - return config -} - -func getCurrentNamespace() string { - namespace, err := exec.Command("kubectl", "config", "view", "--minify", "--output", "jsonpath={..namespace}").CombinedOutput() - if err != nil { - log.Fatal(err) - } - currentNamespace := string(namespace) - if currentNamespace == "" { - currentNamespace = DefaultNamespace - } - return currentNamespace -} - -func confirmAction(clusterName string, namespace string) { - for { - confirmClusterDetails := "" - _, err := fmt.Scan(&confirmClusterDetails) - if err != nil { - log.Fatalf("couldn't get confirmation from the user %v", err) - } - clusterDetails := strings.Split(confirmClusterDetails, "/") - if clusterDetails[0] != namespace || clusterDetails[1] != clusterName { - fmt.Printf("cluster name or namespace does not match. Please re-enter %s/%s\nHint: Press (ctrl+c) to exit\n", namespace, clusterName) - } else { - return - } - } -} - -func getPodName(clusterName string, master bool, replicaNumber string) string { - config := getConfig() - client, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - postgresConfig, err := PostgresqlLister.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - postgresCluster, err := postgresConfig.Postgresqls(getCurrentNamespace()).Get(context.TODO(), clusterName, metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - - numOfInstances := postgresCluster.Spec.NumberOfInstances - var podName string - var podRole string - replica := clusterName + "-" + replicaNumber - - for ins := 0; ins < int(numOfInstances); ins++ { - pod, err := client.CoreV1().Pods(getCurrentNamespace()).Get(context.TODO(), clusterName+"-"+strconv.Itoa(ins), metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - - podRole = pod.Labels["spilo-role"] - if podRole == "master" && master { - podName = pod.Name - fmt.Printf("connected to %s with pod name as %s\n", podRole, podName) - break - } else if podRole == "replica" && !master && (pod.Name == replica || replicaNumber == "") { - podName = pod.Name - fmt.Printf("connected to %s with pod name as %s\n", podRole, podName) - break - } - } - if podName == "" { - log.Fatal("Provided replica doesn't exist") - } - return podName -} - -func getPostgresOperator(k8sClient *kubernetes.Clientset) *v1.Deployment { - var operator *v1.Deployment - operator, err := k8sClient.AppsV1().Deployments(getCurrentNamespace()).Get(context.TODO(), OperatorName, metav1.GetOptions{}) - if err == nil { - return operator - } - - allDeployments := k8sClient.AppsV1().Deployments("") - listDeployments, err := allDeployments.List(context.TODO(), metav1.ListOptions{}) - if err != nil { - log.Fatal(err) - } - - for _, deployment := range listDeployments.Items { - if deployment.Name == OperatorName { - operator = deployment.DeepCopy() - break - } else { - for key, value := range deployment.Labels { - if key == "app.kubernetes.io/name" && value == OperatorName { - operator = deployment.DeepCopy() - break - } - } - } - } - return operator -} diff --git a/kubectl-pg/cmd/version.go b/kubectl-pg/cmd/version.go deleted file mode 100644 index 91976fbc2..000000000 --- a/kubectl-pg/cmd/version.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright © 2019 Vineeth Pothulapati - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package cmd - -import ( - "fmt" - "github.com/spf13/cobra" - "k8s.io/client-go/kubernetes" - "log" - "strings" -) - -var KubectlPgVersion string = "1.0" - -// versionCmd represents the version command -var versionCmd = &cobra.Command{ - Use: "version", - Short: "version of kubectl-pg & postgres-operator", - Long: `version of kubectl-pg and current running postgres-operator`, - Run: func(cmd *cobra.Command, args []string) { - namespace, err := cmd.Flags().GetString("namespace") - if err != nil { - log.Fatal(err) - } - version(namespace) - }, - Example: ` -#Lists the version of kubectl pg plugin and postgres operator in current namespace -kubectl pg version - -#Lists the version of kubectl pg plugin and postgres operator in provided namespace -kubectl pg version -n namespace01 -`, -} - -func version(namespace string) { - fmt.Printf("kubectl-pg: %s\n", KubectlPgVersion) - - config := getConfig() - client, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatal(err) - } - - operatorDeployment := getPostgresOperator(client) - if operatorDeployment.Name == "" { - log.Fatal("make sure zalando's postgres operator is running") - } - operatorImage := operatorDeployment.Spec.Template.Spec.Containers[0].Image - imageDetails := strings.Split(operatorImage, ":") - imageSplit := len(imageDetails) - imageVersion := imageDetails[imageSplit-1] - fmt.Printf("Postgres-Operator: %s\n", imageVersion) -} - -func init() { - rootCmd.AddCommand(versionCmd) - versionCmd.Flags().StringP("namespace", "n", DefaultNamespace, "provide the namespace.") -} diff --git a/kubectl-pg/go.mod b/kubectl-pg/go.mod deleted file mode 100644 index 6e732936e..000000000 --- a/kubectl-pg/go.mod +++ /dev/null @@ -1,14 +0,0 @@ -module github.com/zalando/postgres-operator/kubectl-pg - -go 1.15 - -require ( - github.com/spf13/cobra v1.1.3 - github.com/spf13/viper v1.7.1 - github.com/zalando/postgres-operator v1.6.0 - k8s.io/api v0.20.2 - k8s.io/apiextensions-apiserver v0.20.2 - k8s.io/apimachinery v0.20.2 - k8s.io/client-go v0.20.2 - k8s.io/klog v1.0.0 // indirect -) diff --git a/kubectl-pg/go.sum b/kubectl-pg/go.sum deleted file mode 100644 index c161c44b8..000000000 --- a/kubectl-pg/go.sum +++ /dev/null @@ -1,768 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.36.3/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.4 h1:ynbQIWjLw7iv6HAFdixb30U7Uvcmx+f4KlLJpmhkTK0= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d h1:LznySqW8MqVeFh+pW6rOkFdld9QQ7jRydBKKM6jyPVI= -github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d/go.mod h1:u3hJ0kqCQu/cPpsu3RbCOPZ0d7V3IjPjv1adNRleM9I= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/r3labs/diff v1.1.0/go.mod h1:7WjXasNzi0vJetRcB/RqNl5dlIsmXcTTLmF5IoH6Xig= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/zalando/postgres-operator v1.6.0 h1:Nypf6vO9T6NA1+PTGun7Oz0dEMCZH6NYp6vtyss/XM0= -github.com/zalando/postgres-operator v1.6.0/go.mod h1:gQC+cdbxdh9YpjDVWLHFgu7/xNVO/GDvTXZaBC1Qv2Q= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c h1:9HhBz5L/UjnK9XLtiZhYAdue5BVKep3PMmS2LuPDt8k= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201207204333-a835c872fcea/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs= -k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk= -k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= -k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= -k8s.io/apiextensions-apiserver v0.19.3/go.mod h1:igVEkrE9TzInc1tYE7qSqxaLg/rEAp6B5+k9Q7+IC8Q= -k8s.io/apiextensions-apiserver v0.20.2 h1:rfrMWQ87lhd8EzQWRnbQ4gXrniL/yTRBgYH1x1+BLlo= -k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= -k8s.io/apimachinery v0.19.3/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= -k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apiserver v0.19.3/go.mod h1:bx6dMm+H6ifgKFpCQT/SAhPwhzoeIMlHIaibomUDec0= -k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= -k8s.io/client-go v0.19.3/go.mod h1:+eEMktZM+MG0KO+PTkci8xnbCZHvj9TqR6Q1XDUIJOM= -k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= -k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= -k8s.io/client-go v1.5.1 h1:XaX/lo2/u3/pmFau8HN+sB5C/b4dc4Dmm2eXjBH4p1E= -k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= -k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= -k8s.io/code-generator v0.19.3/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/code-generator v0.19.4/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/component-base v0.19.3/go.mod h1:WhLWSIefQn8W8jxSLl5WNiR6z8oyMe/8Zywg7alOkRc= -k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/docker/logical-backup/Dockerfile b/logical-backup/Dockerfile similarity index 79% rename from docker/logical-backup/Dockerfile rename to logical-backup/Dockerfile index 62bd5ce8c..804eb65ad 100644 --- a/docker/logical-backup/Dockerfile +++ b/logical-backup/Dockerfile @@ -1,4 +1,5 @@ -FROM registry.opensource.zalan.do/library/ubuntu-18.04:latest +ARG BASE_IMAGE=ubuntu:22.04 +FROM ${BASE_IMAGE} LABEL maintainer="Team ACID @ Zalando " SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -15,6 +16,7 @@ RUN apt-get update \ gnupg \ gcc \ libffi-dev \ + && curl -sL https://aka.ms/InstallAzureCLIDeb | bash \ && pip3 install --upgrade pip \ && pip3 install --no-cache-dir awscli --upgrade \ && pip3 install --no-cache-dir gsutil --upgrade \ @@ -23,12 +25,11 @@ RUN apt-get update \ && curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && apt-get update \ && apt-get install --no-install-recommends -y \ - postgresql-client-13 \ - postgresql-client-12 \ - postgresql-client-11 \ - postgresql-client-10 \ - postgresql-client-9.6 \ - postgresql-client-9.5 \ + postgresql-client-18 \ + postgresql-client-17 \ + postgresql-client-16 \ + postgresql-client-15 \ + postgresql-client-14 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/logical-backup/dump.sh b/logical-backup/dump.sh new file mode 100755 index 000000000..7833de399 --- /dev/null +++ b/logical-backup/dump.sh @@ -0,0 +1,231 @@ +#! /usr/bin/env bash + +# enable unofficial bash strict mode +set -o errexit +set -o nounset +set -o pipefail +IFS=$'\n\t' + +ALL_DB_SIZE_QUERY="select sum(pg_database_size(datname)::numeric) from pg_database;" +PG_BIN=$PG_DIR/$PG_VERSION/bin +DUMP_SIZE_COEFF=5 +ERRORCOUNT=0 + +TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) +KUBERNETES_SERVICE_PORT=${KUBERNETES_SERVICE_PORT:-443} +if [ "$KUBERNETES_SERVICE_HOST" != "${KUBERNETES_SERVICE_HOST#*[0-9].[0-9]}" ]; then + echo "IPv4" + K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1 +elif [ "$KUBERNETES_SERVICE_HOST" != "${KUBERNETES_SERVICE_HOST#*:[0-9a-fA-F]}" ]; then + echo "IPv6" + K8S_API_URL=https://[$KUBERNETES_SERVICE_HOST]:$KUBERNETES_SERVICE_PORT/api/v1 +elif [ -n "$KUBERNETES_SERVICE_HOST" ]; then + echo "Hostname" + K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1 +else + echo "KUBERNETES_SERVICE_HOST was not set" +fi +echo "API Endpoint: ${K8S_API_URL}" +CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + +LOGICAL_BACKUP_PROVIDER=${LOGICAL_BACKUP_PROVIDER:="s3"} +LOGICAL_BACKUP_S3_RETENTION_TIME=${LOGICAL_BACKUP_S3_RETENTION_TIME:=""} + +function estimate_size { + "$PG_BIN"/psql -tqAc "${ALL_DB_SIZE_QUERY}" +} + +function dump { + # settings are taken from the environment + "$PG_BIN"/pg_dumpall +} + +function compress { + pigz +} + +function az_upload { + PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz + + az storage blob upload --file "$1" --account-name "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME" --account-key "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY" -c "$LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER" -n "$PATH_TO_BACKUP" +} + +function aws_delete_objects { + args=( + "--bucket=$LOGICAL_BACKUP_S3_BUCKET" + ) + + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") + [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION") + + aws s3api delete-objects "${args[@]}" --delete Objects=["$(printf {Key=%q}, "$@")"],Quiet=true +} +export -f aws_delete_objects + +function aws_delete_outdated { + if [[ -z "$LOGICAL_BACKUP_S3_RETENTION_TIME" ]] ; then + echo "no retention time configured: skip cleanup of outdated backups" + return 0 + fi + + # define cutoff date for outdated backups (day precision) + cutoff_date=$(date -d "$LOGICAL_BACKUP_S3_RETENTION_TIME ago" +%F) + + # mimic bucket setup from Spilo + prefix=$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/" + + args=( + "--no-paginate" + "--output=text" + "--prefix=$prefix" + "--bucket=$LOGICAL_BACKUP_S3_BUCKET" + ) + + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") + [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION") + + # list objects older than the cutoff date + aws s3api list-objects "${args[@]}" --query="Contents[?LastModified<='$cutoff_date'].[Key]" > /tmp/outdated-backups + + # spare the last backup + sed -i '$d' /tmp/outdated-backups + + count=$(wc -l < /tmp/outdated-backups) + if [[ $count == 0 ]] ; then + echo "no outdated backups to delete" + return 0 + fi + echo "deleting $count outdated backups created before $cutoff_date" + + # deleted outdated files in batches with 100 at a time + tr '\n' '\0' < /tmp/outdated-backups | xargs -0 -P1 -n100 bash -c 'aws_delete_objects "$@"' _ +} + +function aws_upload { + declare -r EXPECTED_SIZE="$1" + + # mimic bucket setup from Spilo + # to keep logical backups at the same path as WAL + # NB: $LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX already contains the leading "/" when set by the Postgres Operator + PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz + + args=() + + [[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE") + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") + [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION") + [[ ! -z "$LOGICAL_BACKUP_S3_SSE" ]] && args+=("--sse=$LOGICAL_BACKUP_S3_SSE") + + aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}" +} + +function gcs_upload { + PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/"$LOGICAL_BACKUP_S3_BUCKET_PREFIX"/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz + + #Set local LOGICAL_GOOGLE_APPLICATION_CREDENTIALS to nothing or + #value of LOGICAL_GOOGLE_APPLICATION_CREDENTIALS env var. Needed + #because `set -o nounset` is globally set + local LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS=${LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS:-} + + GSUTIL_OPTIONS=("-o" "Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS") + + #If GOOGLE_APPLICATION_CREDENTIALS is not set try to get + #creds from metadata + if [[ -z $LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS ]] + then + GSUTIL_OPTIONS[1]="GoogleCompute:service_account=default" + fi + + gsutil ${GSUTIL_OPTIONS[@]} cp - "$PATH_TO_BACKUP" +} + +function upload { + case $LOGICAL_BACKUP_PROVIDER in + "gcs") + gcs_upload + ;; + "s3") + aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) + aws_delete_outdated + ;; + esac +} + +function get_pods { + declare -r SELECTOR="$1" + + curl "${K8S_API_URL}/namespaces/${POD_NAMESPACE}/pods?$SELECTOR" \ + --cacert $CERT \ + -H "Authorization: Bearer ${TOKEN}" | jq .items[].status.podIP -r +} + +function get_current_pod { + curl "${K8S_API_URL}/namespaces/${POD_NAMESPACE}/pods?fieldSelector=metadata.name%3D${HOSTNAME}" \ + --cacert $CERT \ + -H "Authorization: Bearer ${TOKEN}" +} + +declare -a search_strategy=( + list_all_replica_pods_current_node + list_all_replica_pods_any_node + get_master_pod +) + +function list_all_replica_pods_current_node { + get_pods "labelSelector=${CLUSTER_NAME_LABEL}%3D${SCOPE},spilo-role%3Dreplica&fieldSelector=spec.nodeName%3D${CURRENT_NODENAME}" | tee | head -n 1 +} + +function list_all_replica_pods_any_node { + get_pods "labelSelector=${CLUSTER_NAME_LABEL}%3D${SCOPE},spilo-role%3Dreplica" | tee | head -n 1 +} + +function get_master_pod { + get_pods "labelSelector=${CLUSTER_NAME_LABEL}%3D${SCOPE},spilo-role%3Dmaster" | tee | head -n 1 +} + +# Wait for TCP connectivity to the target PostgreSQL pod. +# When NetworkPolicy is enforced via iptables, a newly-created pod's IP may not +# yet be present in the destination node's ingress allow lists, causing +# cross-node connections to be rejected until the next policy sync. +function wait_for_pg { + local retries=${LOGICAL_BACKUP_CONNECT_RETRIES:-10} + local delay=${LOGICAL_BACKUP_CONNECT_RETRY_DELAY:-2} + local i + for (( i=1; i<=retries; i++ )); do + if "$PG_BIN"/pg_isready -h "$PGHOST" -p "${PGPORT:-5432}" -q 2>/dev/null; then + return 0 + fi + echo "waiting for $PGHOST:${PGPORT:-5432} to become reachable (attempt $i/$retries)..." + sleep "$delay" + done + echo "ERROR: $PGHOST:${PGPORT:-5432} not reachable after $((retries * delay))s" + return 1 +} + +CURRENT_NODENAME=$(get_current_pod | jq .items[].spec.nodeName --raw-output) +export CURRENT_NODENAME + +for search in "${search_strategy[@]}"; do + + PGHOST=$(eval "$search") + export PGHOST + + if [ -n "$PGHOST" ]; then + break + fi + +done + +wait_for_pg + +set -x +if [ "$LOGICAL_BACKUP_PROVIDER" == "az" ]; then + dump | compress > /tmp/azure-backup.sql.gz + az_upload /tmp/azure-backup.sql.gz +else + dump | compress | upload + [[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 )) + set +x + + exit $ERRORCOUNT +fi diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 6e2acbdd3..93797e0e1 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -3,23 +3,35 @@ kind: postgresql metadata: name: acid-test-cluster # labels: +# application: test-app # environment: demo # annotations: # "acid.zalan.do/controller": "second-operator" # "delete-date": "2020-08-31" # can only be deleted on that day if "delete-date "key is configured # "delete-clustername": "acid-test-cluster" # can only be deleted when name matches if "delete-clustername" key is configured spec: - dockerImage: registry.opensource.zalan.do/acid/spilo-13:2.0-p7 + dockerImage: ghcr.io/zalando/spilo-18:4.1-p1 teamId: "acid" numberOfInstances: 2 users: # Application/Robot users zalando: - superuser - createdb + foo_user: [] +# flyway: [] +# usersIgnoringSecretRotation: +# - bar_user +# usersWithSecretRotation: +# - foo_user +# usersWithInPlaceSecretRotation: +# - flyway +# - bar_owner_user enableMasterLoadBalancer: false enableReplicaLoadBalancer: false enableConnectionPooler: false # enable/disable connection pooler deployment enableReplicaConnectionPooler: false # set to enable connectionPooler for replica service + enableMasterPoolerLoadBalancer: false + enableReplicaPoolerLoadBalancer: false allowedSourceRanges: # load balancers' source ranges for both master and replica services - 127.0.0.1/32 databases: @@ -36,16 +48,28 @@ spec: defaultRoles: true defaultUsers: false postgresql: - version: "13" + version: "18" parameters: # Expert section shared_buffers: "32MB" max_connections: "10" log_statement: "all" +# env: +# - name: wal_s3_bucket +# value: my-custom-bucket + volume: size: 1Gi # storageClass: my-sc # iops: 1000 # for EBS gp3 # throughput: 250 # in MB/s for EBS gp3 +# selector: +# matchExpressions: +# - { key: flavour, operator: In, values: [ "banana", "chocolate" ] } +# matchLabels: +# environment: dev +# service: postgres +# subPath: $(NODE_NAME)/$(POD_NAME) +# isSubPathExpr: true additionalVolumes: - name: empty mountPath: /opt/empty @@ -61,6 +85,16 @@ spec: # PersistentVolumeClaim: # claimName: pvc-postgresql-data-partitions # readyOnly: false +# - name: data +# mountPath: /home/postgres/pgdata/partitions +# subPath: $(NODE_NAME)/$(POD_NAME) +# isSubPathExpr: true +# targetContainers: +# - postgres +# volumeSource: +# PersistentVolumeClaim: +# claimName: pvc-postgresql-data-partitions +# readyOnly: false # - name: conf # mountPath: /etc/telegraf # subPath: telegraf.conf @@ -87,10 +121,15 @@ spec: requests: cpu: 10m memory: 100Mi +# hugepages-2Mi: 128Mi +# hugepages-1Gi: 1Gi limits: cpu: 500m memory: 500Mi +# hugepages-2Mi: 128Mi +# hugepages-1Gi: 1Gi patroni: + failsafe_mode: false initdb: encoding: "UTF8" locale: "en_US.UTF-8" @@ -106,10 +145,11 @@ spec: # database: foo # plugin: pgoutput ttl: 30 - loop_wait: &loop_wait 10 + loop_wait: 10 retry_timeout: 10 synchronous_mode: false synchronous_mode_strict: false + synchronous_node_count: 1 maximum_lag_on_failover: 33554432 # restore a Postgres DB with point-in-time-recovery @@ -117,12 +157,13 @@ spec: # with an empty/absent timestamp, clone from an existing alive cluster using pg_basebackup # clone: # uid: "efd12e58-5786-11e8-b5a7-06148230260c" -# cluster: "acid-batman" +# cluster: "acid-minimal-cluster" # timestamp: "2017-12-19T12:40:33+01:00" # timezone required (offset relative to UTC, see RFC 3339 section 5.6) # s3_wal_path: "s3://custom/path/to/bucket" # run periodic backups with k8s cron jobs # enableLogicalBackup: true +# logicalBackupRetention: "3 months" # logicalBackupSchedule: "30 00 * * *" # maintenanceWindows: @@ -135,6 +176,7 @@ spec: # mode: "transaction" # schema: "pooler" # user: "pooler" +# maxDBConnections: 60 # resources: # requests: # cpu: 300m @@ -151,7 +193,7 @@ spec: # - name: "telegraf-sidecar" # image: "telegraf:latest" # ports: -# name: metrics +# - name: metrics # containerPort: 8094 # protocol: TCP # resources: @@ -189,3 +231,27 @@ spec: # operator: In # values: # - enabled + +# Add topology spread constraint to distribute PostgreSQL pods across all nodes labeled with "topology.kubernetes.io/zone". +# topologySpreadConstraint: +# - maxSkew: 1 +# topologyKey: topology.kubernetes.io/zone +# whenUnsatisfiable: DoNotSchedule + +# Enables change data capture streams for defined database tables +# streams: +# - applicationId: test-app +# database: foo +# tables: +# data.state_pending_outbox: +# eventType: test-app.status-pending +# data.state_approved_outbox: +# eventType: test-app.status-approved +# data.orders_outbox: +# eventType: test-app.order-completed +# idColumn: o_id +# payloadColumn: o_payload +# # Optional. Filter ignores events before a certain txnId and lsn. Can be used to skip bad events +# filter: +# data.orders_outbox: "[?(@.source.txId > 500 && @.source.lsn > 123456)]" +# batchSize: 1000 diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 7a05135ab..1c663c757 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -3,6 +3,7 @@ kind: ConfigMap metadata: name: postgres-operator data: + # additional_owner_roles: "cron_admin" # additional_pod_capabilities: "SYS_NICE" # additional_secret_mount: "some-secret-name" # additional_secret_mount_path: "/some/dir" @@ -12,84 +13,122 @@ data: cluster_history_entries: "1000" cluster_labels: application:spilo cluster_name_label: cluster-name - # connection_pooler_default_cpu_limit: "1" - # connection_pooler_default_cpu_request: "500m" - # connection_pooler_default_memory_limit: 100Mi - # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-16" - # connection_pooler_max_db_connections: 60 - # connection_pooler_mode: "transaction" - # connection_pooler_number_of_instances: 2 - # connection_pooler_schema: "pooler" - # connection_pooler_user: "pooler" + connection_pooler_default_cpu_limit: "1" + connection_pooler_default_cpu_request: "500m" + connection_pooler_default_memory_limit: 100Mi + connection_pooler_default_memory_request: 100Mi + connection_pooler_image: "ghcr.io/zalando/postgres-operator/pgbouncer:latest" + connection_pooler_max_db_connections: "60" + connection_pooler_mode: "transaction" + connection_pooler_number_of_instances: "2" + connection_pooler_schema: "pooler" + connection_pooler_user: "pooler" + crd_categories: "all" # custom_service_annotations: "keyx:valuez,keya:valuea" # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com debug_logging: "true" - # default_cpu_limit: "1" - # default_cpu_request: 100m - # default_memory_limit: 500Mi - # default_memory_request: 100Mi + default_cpu_limit: "1" + default_cpu_request: 100m + default_memory_limit: 500Mi + default_memory_request: 100Mi # delete_annotation_date_key: delete-date # delete_annotation_name_key: delete-clustername - docker_image: registry.opensource.zalan.do/acid/spilo-13:2.0-p7 + docker_image: ghcr.io/zalando/spilo-18:4.1-p1 # downscaler_annotations: "deployment-time,downscaler/*" - # enable_admin_role_for_users: "true" - # enable_crd_validation: "true" - # enable_database_access: "true" + enable_admin_role_for_users: "true" + enable_crd_registration: "true" + enable_cross_namespace_secret: "false" + enable_finalizers: "false" + enable_database_access: "true" enable_ebs_gp3_migration: "false" - # enable_ebs_gp3_migration_max_size: "1000" - # enable_init_containers: "true" - # enable_lazy_spilo_upgrade: "false" + enable_ebs_gp3_migration_max_size: "1000" + enable_init_containers: "true" + enable_lazy_spilo_upgrade: "false" + enable_maintenance_windows: "true" enable_master_load_balancer: "false" + enable_master_pooler_load_balancer: "false" + enable_password_rotation: "false" + enable_patroni_failsafe_mode: "false" + enable_owner_references: "false" + enable_persistent_volume_claim_deletion: "true" enable_pgversion_env_var: "true" - # enable_pod_antiaffinity: "false" - # enable_pod_disruption_budget: "true" - # enable_postgres_team_crd: "false" - # enable_postgres_team_crd_superusers: "false" + enable_pod_antiaffinity: "false" + enable_pod_disruption_budget: "true" + enable_postgres_team_crd: "false" + enable_postgres_team_crd_superusers: "false" + enable_readiness_probe: "false" enable_replica_load_balancer: "false" - # enable_shm_volume: "true" - # enable_sidecars: "true" + enable_replica_pooler_load_balancer: "false" + enable_secrets_deletion: "true" + enable_shm_volume: "true" + enable_sidecars: "true" enable_spilo_wal_path_compat: "true" + enable_team_id_clustername_prefix: "false" enable_team_member_deprecation: "false" - # enable_team_superuser: "false" + enable_team_superuser: "false" enable_teams_api: "false" - # etcd_host: "" + etcd_host: "" external_traffic_policy: "Cluster" # gcp_credentials: "" - # kubernetes_use_configmaps: "false" + # ignored_annotations: "" # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole" + # ignore_instance_limits_annotation_key: "" + # ignore_resources_limits_annotation_key: "" # inherited_annotations: owned-by # inherited_labels: application,environment # kube_iam_role: "" + kubernetes_use_configmaps: "false" # log_s3_bucket: "" - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.6.3" + # logical_backup_azure_storage_account_name: "" + # logical_backup_azure_storage_container: "" + # logical_backup_azure_storage_account_key: "" + # logical_backup_cpu_limit: "" + # logical_backup_cpu_request: "" + logical_backup_cronjob_environment_secret: "" + logical_backup_docker_image: "ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1" # logical_backup_google_application_credentials: "" logical_backup_job_prefix: "logical-backup-" + # logical_backup_memory_limit: "" + # logical_backup_memory_request: "" logical_backup_provider: "s3" - # logical_backup_s3_access_key_id: "" + logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" - # logical_backup_s3_region: "" - # logical_backup_s3_endpoint: "" - # logical_backup_s3_secret_access_key: "" + logical_backup_s3_bucket_prefix: "spilo" + logical_backup_s3_region: "" + logical_backup_s3_endpoint: "" + logical_backup_s3_secret_access_key: "" logical_backup_s3_sse: "AES256" + logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" + # maintenance_windows: "Sat:22:00-23:59,Sun:00:00-01:00" major_version_upgrade_mode: "manual" - master_dns_name_format: "{cluster}.{team}.{hostedzone}" - # master_pod_move_timeout: 20m - # max_instances: "-1" - # min_instances: "-1" - # min_cpu_limit: 250m - # min_memory_limit: 250Mi - # minimal_major_version: "9.5" - # node_readiness_label: "" - # oauth_token_secret_name: postgresql-operator - # pam_configuration: | - # https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees - # pam_role_name: zalandos + # major_version_upgrade_team_allow_list: "" + master_dns_name_format: "{cluster}.{namespace}.{hostedzone}" + master_legacy_dns_name_format: "{cluster}.{team}.{hostedzone}" + master_pod_move_timeout: 20m + # max_cpu_request: "1" + max_instances: "-1" + # max_memory_request: 4Gi + min_cpu_limit: 250m + min_instances: "-1" + min_memory_limit: 250Mi + minimal_major_version: "14" + # node_readiness_label: "status:ready" + # node_readiness_label_merge: "OR" + oauth_token_secret_name: postgresql-operator + pam_configuration: "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees" + pam_role_name: zalandos + patroni_api_check_interval: "1s" + patroni_api_check_timeout: "5s" + password_rotation_interval: "90" + password_rotation_user_retention: "180" + pdb_master_label_selector: "true" pdb_name_format: "postgres-{cluster}-pdb" - # pod_antiaffinity_topology_key: "kubernetes.io/hostname" + persistent_volume_claim_retention_policy: "when_deleted:retain,when_scaled:retain" + pod_antiaffinity_preferred_during_scheduling: "false" + pod_antiaffinity_topology_key: "kubernetes.io/hostname" pod_deletion_wait_timeout: 10m # pod_environment_configmap: "default/my-custom-config" # pod_environment_secret: "my-custom-secret" @@ -97,25 +136,27 @@ data: pod_management_policy: "ordered_ready" # pod_priority_class_name: "postgres-pod-priority" pod_role_label: spilo-role - # pod_service_account_definition: "" + pod_service_account_definition: "" pod_service_account_name: "postgres-pod" - # pod_service_account_role_binding_definition: "" + pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m - # postgres_superuser_teams: "postgres_superusers" - # protected_role_names: "admin" + postgres_superuser_teams: "postgres_superusers" + protected_role_names: "admin,cron_admin" ready_wait_interval: 3s ready_wait_timeout: 30s repair_period: 5m - replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}" + replica_dns_name_format: "{cluster}-repl.{namespace}.{hostedzone}" + replica_legacy_dns_name_format: "{cluster}-repl.{team}.{hostedzone}" replication_username: standby resource_check_interval: 3s resource_check_timeout: 10m resync_period: 30m ring_log_lines: "100" role_deletion_suffix: "_deleted" - secret_name_template: "{username}.{cluster}.credentials" + secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + share_pgsocket_with_sidecars: "false" # sidecar_docker_images: "" - # set_memory_request_to_limit: "false" + set_memory_request_to_limit: "false" spilo_allow_privilege_escalation: "true" # spilo_runasuser: 101 # spilo_runasgroup: 103 @@ -123,11 +164,12 @@ data: spilo_privileged: "false" storage_resize_mode: "pvc" super_username: postgres - # target_major_version: "13" - # team_admin_role: "admin" - # team_api_role_configuration: "log_statement:all" - # teams_api_url: http://fake-teams-api.default.svc.cluster.local - # toleration: "" + target_major_version: "18" + team_admin_role: "admin" + team_api_role_configuration: "log_statement:all" + teams_api_url: http://fake-teams-api.default.svc.cluster.local + # toleration: "key:db-only,operator:Exists,effect:NoSchedule" + # wal_az_storage_account: "" # wal_gs_bucket: "" # wal_s3_bucket: "" watched_namespace: "*" # listen to all namespaces diff --git a/manifests/e2e-storage-class.yaml b/manifests/e2e-storage-class.yaml index c8d941341..3c70c4020 100644 --- a/manifests/e2e-storage-class.yaml +++ b/manifests/e2e-storage-class.yaml @@ -1,7 +1,6 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - namespace: kube-system name: standard annotations: storageclass.kubernetes.io/is-default-class: "true" diff --git a/manifests/fake-teams-api.yaml b/manifests/fake-teams-api.yaml index 15f7c7576..33f7c6013 100644 --- a/manifests/fake-teams-api.yaml +++ b/manifests/fake-teams-api.yaml @@ -4,6 +4,9 @@ metadata: name: fake-teams-api spec: replicas: 1 + selector: + matchLabels: + name: fake-teams-api template: metadata: labels: @@ -37,8 +40,6 @@ metadata: name: postgresql-operator namespace: default type: Opaque -data: -apiVersion: v1 data: read-only-token-secret: dGVzdHRva2Vu read-only-token-type: QmVhcmVy diff --git a/manifests/fes.crd.yaml b/manifests/fes.crd.yaml new file mode 100644 index 000000000..70a8c9555 --- /dev/null +++ b/manifests/fes.crd.yaml @@ -0,0 +1,23 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: fabriceventstreams.zalando.org +spec: + group: zalando.org + names: + kind: FabricEventStream + listKind: FabricEventStreamList + plural: fabriceventstreams + singular: fabriceventstream + shortNames: + - fes + categories: + - all + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object diff --git a/manifests/infrastructure-roles-new.yaml b/manifests/infrastructure-roles-new.yaml index 64b854c6a..d6b5fa9ad 100644 --- a/manifests/infrastructure-roles-new.yaml +++ b/manifests/infrastructure-roles-new.yaml @@ -8,5 +8,4 @@ data: kind: Secret metadata: name: postgresql-infrastructure-roles-new - namespace: default type: Opaque diff --git a/manifests/infrastructure-roles.yaml b/manifests/infrastructure-roles.yaml index c66d79139..975a02827 100644 --- a/manifests/infrastructure-roles.yaml +++ b/manifests/infrastructure-roles.yaml @@ -21,5 +21,4 @@ data: kind: Secret metadata: name: postgresql-infrastructure-roles - namespace: default type: Opaque diff --git a/manifests/minimal-fake-pooler-deployment.yaml b/manifests/minimal-fake-pooler-deployment.yaml index 037cac0f8..13f2cc68f 100644 --- a/manifests/minimal-fake-pooler-deployment.yaml +++ b/manifests/minimal-fake-pooler-deployment.yaml @@ -23,7 +23,7 @@ spec: serviceAccountName: postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/pgbouncer:master-16 + image: ghcr.io/zalando/postgres-operator/pgbouncer:latest imagePullPolicy: IfNotPresent resources: requests: diff --git a/manifests/minimal-master-replica-svcmonitor.yaml b/manifests/minimal-master-replica-svcmonitor.yaml new file mode 100644 index 000000000..57bbd01d1 --- /dev/null +++ b/manifests/minimal-master-replica-svcmonitor.yaml @@ -0,0 +1,141 @@ +# Here we use https://github.com/prometheus-community/helm-charts/charts/kube-prometheus-stack +# Please keep the ServiceMonitor's label same as the Helm release name of kube-prometheus-stack + +apiVersion: v1 +kind: Namespace +metadata: + name: test-pg +--- +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-minimal-cluster + namespace: test-pg + labels: + app: test-pg +spec: + teamId: "acid" + volume: + size: 1Gi + numberOfInstances: 2 + users: + zalando: # database owner + - superuser + - createdb + foo_user: [] # role for application foo + databases: + foo: zalando # dbname: owner + preparedDatabases: + bar: {} + postgresql: + version: "18" + sidecars: + - name: "exporter" + image: "quay.io/prometheuscommunity/postgres-exporter:v0.15.0" + ports: + - name: exporter + containerPort: 9187 + protocol: TCP + env: + - name: DATA_SOURCE_URI + value: ":5432/?sslmode=disable" + - name: DATA_SOURCE_USER + value: "postgres" + - name: DATA_SOURCE_PASS + valueFrom: + secretKeyRef: + name: postgres.test-pg.credentials.postgresql.acid.zalan.do + key: password + resources: + limits: + cpu: 500m + memory: 256M + requests: + cpu: 100m + memory: 200M +--- +apiVersion: v1 +kind: Service +metadata: + name: acid-minimal-cluster-svc-metrics-master + namespace: test-pg + labels: + app: test-pg + spilo-role: master + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9187" +spec: + type: ClusterIP + ports: + - name: exporter + port: 9187 + targetPort: exporter + selector: + application: spilo + cluster-name: acid-minimal-cluster + spilo-role: master +--- +apiVersion: v1 +kind: Service +metadata: + name: acid-minimal-cluster-svc-metrics-replica + namespace: test-pg + labels: + app: test-pg + spilo-role: replica + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9187" +spec: + type: ClusterIP + ports: + - name: exporter + port: 9187 + targetPort: exporter + selector: + application: spilo + cluster-name: acid-minimal-cluster + spilo-role: replica +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: acid-minimal-cluster-svcm-master + namespace: test-pg + labels: + app: test-pg + spilo-role: master +spec: + endpoints: + - port: exporter + interval: 15s + scrapeTimeout: 10s + namespaceSelector: + matchNames: + - test-pg + selector: + matchLabels: + app: test-pg + spilo-role: master +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: acid-minimal-cluster-svcm-replica + namespace: test-pg + labels: + app: test-pg + spilo-role: replica +spec: + endpoints: + - port: exporter + interval: 15s + scrapeTimeout: 10s + namespaceSelector: + matchNames: + - test-pg + selector: + matchLabels: + app: test-pg + spilo-role: replica diff --git a/manifests/minimal-postgres-manifest-12.yaml b/manifests/minimal-postgres-lowest-version-manifest.yaml similarity index 90% rename from manifests/minimal-postgres-manifest-12.yaml rename to manifests/minimal-postgres-lowest-version-manifest.yaml index 3f89b765d..897925c56 100644 --- a/manifests/minimal-postgres-manifest-12.yaml +++ b/manifests/minimal-postgres-lowest-version-manifest.yaml @@ -2,7 +2,6 @@ apiVersion: "acid.zalan.do/v1" kind: postgresql metadata: name: acid-upgrade-test - namespace: default spec: teamId: "acid" volume: @@ -18,4 +17,4 @@ spec: preparedDatabases: bar: {} postgresql: - version: "12" + version: "14" diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index ff96e392b..086ce553a 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -2,7 +2,6 @@ apiVersion: "acid.zalan.do/v1" kind: postgresql metadata: name: acid-minimal-cluster - namespace: default spec: teamId: "acid" volume: @@ -18,4 +17,4 @@ spec: preparedDatabases: bar: {} postgresql: - version: "13" + version: "18" diff --git a/manifests/operator-service-account-rbac-openshift.yaml b/manifests/operator-service-account-rbac-openshift.yaml new file mode 100644 index 000000000..e716e82b7 --- /dev/null +++ b/manifests/operator-service-account-rbac-openshift.yaml @@ -0,0 +1,285 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: postgres-operator + namespace: default + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: postgres-operator +rules: +# all verbs allowed for custom operator resources +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + - postgresqls/status + - operatorconfigurations + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# operator only reads PostgresTeams +- apiGroups: + - acid.zalan.do + resources: + - postgresteams + verbs: + - get + - list + - watch +# all verbs allowed for event streams (Zalando-internal feature) +# - apiGroups: +# - zalando.org +# resources: +# - fabriceventstreams +# verbs: +# - create +# - delete +# - deletecollection +# - get +# - list +# - patch +# - update +# - watch +# to create or get/update CRDs when starting up +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - patch + - update +# to read configuration and manage ConfigMaps used by Patroni +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# to send events to the CRs +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - update + - watch +# to CRUD secrets for database access +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - patch + - update +# to check nodes for node readiness label +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +# to read or delete existing PVCs. Creation via StatefulSet +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - delete + - get + - list + - patch + - update + # to read existing PVs. Creation should be done via dynamic provisioning +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update # only for resizing AWS volumes +# to watch Spilo pods and do rolling updates. Creation via StatefulSet +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +# to resize the filesystem in Spilo pods when increasing volume size +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +# to CRUD services to point to Postgres cluster instances +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - patch + - update +# to CRUD the StatefulSet which controls the Postgres cluster instances +- apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update +# to CRUD cron jobs for logical backups +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update +# to get namespaces operator resources can run in +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +# to define PDBs. Update happens via delete/create +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - get +# to create ServiceAccounts in each namespace the operator watches +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - create +# to create role bindings to the postgres-pod service account +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - get + - create +# to grant privilege to run privileged pods (not needed by default) +#- apiGroups: +# - extensions +# resources: +# - podsecuritypolicies +# resourceNames: +# - privileged +# verbs: +# - use + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: postgres-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: postgres-operator +subjects: +- kind: ServiceAccount + name: postgres-operator + namespace: default + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: postgres-pod +rules: +# Patroni needs to watch and manage config maps +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# Patroni needs to watch pods +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch +# to let Patroni create a headless service +- apiGroups: + - "" + resources: + - services + verbs: + - create +# to grant privilege to run privileged pods (not needed by default) +#- apiGroups: +# - extensions +# resources: +# - podsecuritypolicies +# resourceNames: +# - privileged +# verbs: +# - use diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index f0307f6a0..2cc1edcd1 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -35,6 +35,20 @@ rules: - get - list - watch +# all verbs allowed for event streams (Zalando-internal feature) +# - apiGroups: +# - zalando.org +# resources: +# - fabriceventstreams +# verbs: +# - create +# - delete +# - deletecollection +# - get +# - list +# - patch +# - update +# - watch # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io @@ -45,13 +59,20 @@ rules: - get - patch - update -# to read configuration from ConfigMaps +# to read configuration from ConfigMaps and help Patroni manage the cluster if endpoints are not used - apiGroups: - "" resources: - configmaps verbs: + - create + - delete + - deletecollection - get + - list + - patch + - update + - watch # to send events to the CRs - apiGroups: - "" @@ -64,7 +85,7 @@ rules: - patch - update - watch -# to manage endpoints which are also used by Patroni +# to manage endpoints which are also used by Patroni (if it is using config maps) - apiGroups: - "" resources: @@ -88,6 +109,7 @@ rules: - delete - get - update + - patch # to check nodes for node readiness label - apiGroups: - "" @@ -159,6 +181,7 @@ rules: - get - list - patch + - update # to CRUD cron jobs for logical backups - apiGroups: - batch @@ -233,7 +256,21 @@ kind: ClusterRole metadata: name: postgres-pod rules: -# Patroni needs to watch and manage endpoints +# Patroni needs to watch and manage config maps (or endpoints) +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# Patroni needs to watch and manage endpoints (or config maps) - apiGroups: - "" resources: diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 806acc8da..5f347f2ac 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -1,564 +1,971 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 name: operatorconfigurations.acid.zalan.do spec: group: acid.zalan.do names: + categories: + - all kind: OperatorConfiguration listKind: OperatorConfigurationList plural: operatorconfigurations - singular: operatorconfiguration shortNames: - opconfig - categories: - - all + singular: operatorconfiguration scope: Namespaced versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - name: Image - type: string - description: Spilo image to be used for Pods + - additionalPrinterColumns: + - description: Spilo image to be used for Pods jsonPath: .configuration.docker_image - - name: Cluster-Label + name: Image type: string - description: Label for K8s resources created by operator + - description: Label for K8s resources created by operator jsonPath: .configuration.kubernetes.cluster_name_label - - name: Service-Account + name: Cluster-Label type: string - description: Name of service account to be used + - description: Name of service account to be used jsonPath: .configuration.kubernetes.pod_service_account_name - - name: Min-Instances - type: integer - description: Minimum number of instances per Postgres cluster + name: Service-Account + type: string + - description: Minimum number of instances per Postgres cluster jsonPath: .configuration.min_instances - - name: Age - type: date + name: Min-Instances + type: integer + - description: Age of the OperatorConfiguration resource jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - configuration + description: OperatorConfiguration defines the specification for the OperatorConfiguration. properties: - kind: - type: string - enum: - - OperatorConfiguration apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string - enum: - - acid.zalan.do/v1 configuration: - type: object + description: OperatorConfigurationData defines the operation config properties: - docker_image: - type: string - default: "registry.opensource.zalan.do/acid/spilo-13:2.0-p7" - enable_crd_validation: - type: boolean - default: true - enable_lazy_spilo_upgrade: - type: boolean - default: false - enable_pgversion_env_var: - type: boolean - default: true - enable_shm_volume: - type: boolean - default: true - enable_spilo_wal_path_compat: - type: boolean - default: false - etcd_host: - type: string - default: "" - kubernetes_use_configmaps: - type: boolean - default: false - max_instances: - type: integer - minimum: -1 # -1 = disabled - default: -1 - min_instances: - type: integer - minimum: -1 # -1 = disabled - default: -1 - resync_period: - type: string - default: "30m" - repair_period: - type: string - default: "5m" - set_memory_request_to_limit: - type: boolean - default: false - sidecar_docker_images: - type: object - additionalProperties: - type: string - sidecars: - type: array - nullable: true - items: - type: object - x-kubernetes-preserve-unknown-fields: true - workers: - type: integer - minimum: 1 - default: 8 - users: - type: object - properties: - replication_username: - type: string - default: standby - super_username: - type: string - default: postgres - major_version_upgrade: - type: object - properties: - major_version_upgrade_mode: - type: string - default: "off" - minimal_major_version: - type: string - default: "9.5" - target_major_version: - type: string - default: "13" - kubernetes: - type: object - properties: - additional_pod_capabilities: - type: array - items: - type: string - cluster_domain: - type: string - default: "cluster.local" - cluster_labels: - type: object - additionalProperties: - type: string - default: - application: spilo - cluster_name_label: - type: string - default: "cluster-name" - custom_pod_annotations: - type: object - additionalProperties: - type: string - delete_annotation_date_key: - type: string - delete_annotation_name_key: - type: string - downscaler_annotations: - type: array - items: - type: string - enable_init_containers: - type: boolean - default: true - enable_pod_antiaffinity: - type: boolean - default: false - enable_pod_disruption_budget: - type: boolean - default: true - enable_sidecars: - type: boolean - default: true - infrastructure_roles_secret_name: - type: string - infrastructure_roles_secrets: - type: array - nullable: true - items: - type: object - required: - - secretname - - userkey - - passwordkey - properties: - secretname: - type: string - userkey: - type: string - passwordkey: - type: string - rolekey: - type: string - defaultuservalue: - type: string - defaultrolevalue: - type: string - details: - type: string - template: - type: boolean - inherited_annotations: - type: array - items: - type: string - inherited_labels: - type: array - items: - type: string - master_pod_move_timeout: - type: string - default: "20m" - node_readiness_label: - type: object - additionalProperties: - type: string - oauth_token_secret_name: - type: string - default: "postgresql-operator" - pdb_name_format: - type: string - default: "postgres-{cluster}-pdb" - pod_antiaffinity_topology_key: - type: string - default: "kubernetes.io/hostname" - pod_environment_configmap: - type: string - pod_environment_secret: - type: string - pod_management_policy: - type: string - enum: - - "ordered_ready" - - "parallel" - default: "ordered_ready" - pod_priority_class_name: - type: string - pod_role_label: - type: string - default: "spilo-role" - pod_service_account_definition: - type: string - default: "" - pod_service_account_name: - type: string - default: "postgres-pod" - pod_service_account_role_binding_definition: - type: string - default: "" - pod_terminate_grace_period: - type: string - default: "5m" - secret_name_template: - type: string - default: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" - spilo_allow_privilege_escalation: - type: boolean - default: true - spilo_runasuser: - type: integer - spilo_runasgroup: - type: integer - spilo_fsgroup: - type: integer - spilo_privileged: - type: boolean - default: false - storage_resize_mode: - type: string - enum: - - "ebs" - - "pvc" - - "off" - default: "pvc" - toleration: - type: object - additionalProperties: - type: string - watched_namespace: - type: string - postgres_pod_resources: - type: object - properties: - default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "1" - default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "100m" - default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "500Mi" - default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "100Mi" - min_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "250m" - min_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "250Mi" - timeouts: - type: object - properties: - pod_label_wait_timeout: - type: string - default: "10m" - pod_deletion_wait_timeout: - type: string - default: "10m" - ready_wait_interval: - type: string - default: "4s" - ready_wait_timeout: - type: string - default: "30s" - resource_check_interval: - type: string - default: "3s" - resource_check_timeout: - type: string - default: "10m" - load_balancer: - type: object - properties: - custom_service_annotations: - type: object - additionalProperties: - type: string - db_hosted_zone: - type: string - default: "db.example.com" - enable_master_load_balancer: - type: boolean - default: true - enable_replica_load_balancer: - type: boolean - default: false - external_traffic_policy: - type: string - enum: - - "Cluster" - - "Local" - default: "Cluster" - master_dns_name_format: - type: string - default: "{cluster}.{team}.{hostedzone}" - replica_dns_name_format: - type: string - default: "{cluster}-repl.{team}.{hostedzone}" aws_or_gcp: - type: object + description: AWSGCPConfiguration defines the configuration for AWS properties: additional_secret_mount: type: string additional_secret_mount_path: type: string - default: "/meta/credentials" aws_region: + default: eu-central-1 type: string - default: "eu-central-1" enable_ebs_gp3_migration: type: boolean - default: false enable_ebs_gp3_migration_max_size: + format: int64 type: integer - default: 1000 gcp_credentials: type: string kube_iam_role: type: string log_s3_bucket: type: string + wal_az_storage_account: + type: string wal_gs_bucket: type: string wal_s3_bucket: type: string - logical_backup: type: object + connection_pooler: + description: ConnectionPoolerConfiguration defines default configuration + for connection pooler properties: - logical_backup_docker_image: + connection_pooler_default_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ type: string - default: "registry.opensource.zalan.do/acid/logical-backup:v1.6.3" + connection_pooler_default_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + connection_pooler_default_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + connection_pooler_default_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + connection_pooler_image: + default: ghcr.io/zalando/postgres-operator/pgbouncer:latest + type: string + connection_pooler_max_db_connections: + format: int32 + type: integer + connection_pooler_mode: + default: transaction + enum: + - session + - transaction + type: string + connection_pooler_number_of_instances: + default: 2 + format: int32 + minimum: 1 + type: integer + connection_pooler_schema: + default: pooler + type: string + connection_pooler_user: + default: pooler + type: string + type: object + crd_categories: + items: + type: string + type: array + debug: + description: OperatorDebugConfiguration defines options for the debug + mode + properties: + debug_logging: + default: true + type: boolean + enable_database_access: + default: true + type: boolean + type: object + docker_image: + default: ghcr.io/zalando/spilo-18:4.1-p1 + type: string + enable_crd_registration: + default: true + type: boolean + enable_lazy_spilo_upgrade: + type: boolean + enable_maintenance_windows: + default: true + type: boolean + enable_pgversion_env_var: + default: true + type: boolean + enable_shm_volume: + default: true + type: boolean + enable_spilo_wal_path_compat: + type: boolean + enable_team_id_clustername_prefix: + type: boolean + etcd_host: + default: "" + type: string + ignore_instance_limits_annotation_key: + type: string + ignore_resources_limits_annotation_key: + type: string + kubernetes: + description: KubernetesMetaConfiguration defines k8s conf required + for all Postgres clusters and the operator itself + properties: + additional_pod_capabilities: + items: + type: string + type: array + cluster_domain: + default: cluster.local + type: string + cluster_labels: + additionalProperties: + type: string + default: + application: spilo + type: object + cluster_name_label: + default: cluster-name + type: string + custom_pod_annotations: + additionalProperties: + type: string + type: object + delete_annotation_date_key: + type: string + delete_annotation_name_key: + type: string + downscaler_annotations: + items: + type: string + type: array + enable_cross_namespace_secret: + type: boolean + enable_finalizers: + type: boolean + enable_init_containers: + default: true + type: boolean + enable_owner_references: + type: boolean + enable_persistent_volume_claim_deletion: + default: true + type: boolean + enable_pod_antiaffinity: + type: boolean + enable_pod_disruption_budget: + default: true + type: boolean + enable_readiness_probe: + type: boolean + enable_secrets_deletion: + default: true + type: boolean + enable_sidecars: + default: true + type: boolean + ignored_annotations: + items: + type: string + type: array + infrastructure_roles_secret_name: + description: |- + NamespacedName comprises a resource name, with a mandatory namespace, + rendered as "/". Being a type captures intent and + helps make sure that UIDs, namespaced names and non-namespaced names + do not get conflated in code. For most use cases, namespace and name + will already have been format validated at the API entry point, so we + don't do that here. Where that's not the case (e.g. in testing), + consider using NamespacedNameOrDie() in testing.go in this package. + + from: https://github.com/kubernetes/apimachinery/blob/master/pkg/types/namespacedname.go + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + infrastructure_roles_secrets: + description: namespaced name of the secret containing infrastructure + roles names and passwords + items: + properties: + defaultrolevalue: + type: string + defaultuservalue: + type: string + details: + description: This field point out the detailed yaml definition + of the role, if exists + type: string + passwordkey: + type: string + rolekey: + type: string + secretname: + description: |- + Name of a secret which describes the role, and optionally name of a + configmap with an extra information + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + template: + type: boolean + userkey: + type: string + type: object + type: array + inherited_annotations: + items: + type: string + type: array + inherited_labels: + items: + type: string + type: array + liveness_probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + master_pod_move_timeout: + default: 20m + description: timeout for successful migration of master pods from + unschedulable node + format: int64 + type: integer + node_readiness_label: + additionalProperties: + type: string + type: object + node_readiness_label_merge: + enum: + - AND + - OR + type: string + oauth_token_secret_name: + default: postgres-operator + description: namespaced name of the secret containing the OAuth2 + token to pass to the teams API + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + pdb_master_label_selector: + default: true + type: boolean + pdb_name_format: + default: postgres-{cluster}-pdb + description: defines the template for PDB names + type: string + persistent_volume_claim_retention_policy: + additionalProperties: + type: string + type: object + pod_antiaffinity_preferred_during_scheduling: + type: boolean + pod_antiaffinity_topology_key: + default: kubernetes.io/hostname + type: string + pod_environment_configmap: + description: namespaced name of the ConfigMap with environment + variables to populate on every pod + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + pod_environment_secret: + type: string + pod_management_policy: + default: ordered_ready + enum: + - ordered_ready + - parallel + type: string + pod_priority_class_name: + type: string + pod_role_label: + default: spilo-role + type: string + pod_service_account_definition: + type: string + pod_service_account_name: + default: postgres-pod + type: string + pod_service_account_role_binding_definition: + type: string + pod_terminate_grace_period: + default: 5m + description: Postgres pods are terminated forcefully after this + timeout + format: int64 + type: integer + secret_name_template: + default: '{username}.{cluster}.credentials.{tprkind}.{tprgroup}' + description: |- + template for database user secrets generated by the operator, + here username contains the namespace in the format namespace.username + if the user is in different namespace than cluster and cross namespace secrets + are enabled via `enable_cross_namespace_secret` flag in the configuration. + type: string + share_pgsocket_with_sidecars: + type: boolean + spilo_allow_privilege_escalation: + default: true + type: boolean + spilo_fsgroup: + format: int64 + type: integer + spilo_privileged: + type: boolean + spilo_runasgroup: + format: int64 + type: integer + spilo_runasuser: + format: int64 + type: integer + storage_resize_mode: + default: pvc + enum: + - ebs + - mixed + - pvc + - "off" + type: string + toleration: + additionalProperties: + type: string + type: object + watched_namespace: + type: string + type: object + kubernetes_use_configmaps: + default: true + type: boolean + load_balancer: + description: LoadBalancerConfiguration defines the LB configuration + properties: + custom_service_annotations: + additionalProperties: + type: string + type: object + db_hosted_zone: + type: string + enable_master_load_balancer: + type: boolean + enable_master_node_port: + type: boolean + enable_master_pooler_load_balancer: + type: boolean + enable_master_pooler_node_port: + type: boolean + enable_replica_load_balancer: + type: boolean + enable_replica_node_port: + type: boolean + enable_replica_pooler_load_balancer: + type: boolean + enable_replica_pooler_node_port: + type: boolean + external_traffic_policy: + default: Cluster + enum: + - Cluster + - Local + type: string + master_dns_name_format: + default: '{cluster}.{namespace}.{hostedzone}' + description: defines the DNS name string template for the master + load balancer cluster + type: string + master_legacy_dns_name_format: + default: '{cluster}.{team}.{hostedzone}' + description: deprecated DNS template for master load balancer + using team name + type: string + replica_dns_name_format: + default: '{cluster}-repl.{namespace}.{hostedzone}' + description: defines the DNS name string template for the replica + load balancer cluster + type: string + replica_legacy_dns_name_format: + default: '{cluster}-repl.{team}.{hostedzone}' + description: deprecated DNS template for replica load balancer + using team name + type: string + type: object + logging_rest_api: + description: LoggingRESTAPIConfiguration defines Logging API conf + properties: + api_port: + default: 8080 + type: integer + cluster_history_entries: + default: 1000 + type: integer + ring_log_lines: + default: 100 + type: integer + type: object + logical_backup: + description: OperatorLogicalBackupConfiguration defines configuration + for logical backup + properties: + logical_backup_azure_storage_account_key: + type: string + logical_backup_azure_storage_account_name: + type: string + logical_backup_azure_storage_container: + type: string + logical_backup_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + logical_backup_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + logical_backup_cronjob_environment_secret: + type: string + logical_backup_docker_image: + default: ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1 + type: string + logical_backup_failed_jobs_history_limit: + default: 3 + format: int32 + minimum: 0 + type: integer logical_backup_google_application_credentials: type: string logical_backup_job_prefix: + default: logical-backup- + type: string + logical_backup_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + logical_backup_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - default: "logical-backup-" logical_backup_provider: + default: s3 + enum: + - az + - gcs + - s3 type: string - default: "s3" logical_backup_s3_access_key_id: type: string logical_backup_s3_bucket: type: string + logical_backup_s3_bucket_prefix: + type: string logical_backup_s3_endpoint: type: string logical_backup_s3_region: type: string + logical_backup_s3_retention_time: + type: string logical_backup_s3_secret_access_key: type: string logical_backup_s3_sse: type: string logical_backup_schedule: + default: 30 00 * * * + pattern: ^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$ type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - default: "30 00 * * *" - debug: + logical_backup_successful_jobs_history_limit: + default: 3 + format: int32 + minimum: 0 + type: integer + logical_backup_ttl_seconds_after_finished: + default: 86400 + format: int32 + minimum: 0 + type: integer type: object + maintenance_windows: + type: array + major_version_upgrade: + description: MajorVersionUpgradeConfiguration defines how to execute + major version upgrades of Postgres. properties: - debug_logging: - type: boolean - default: true - enable_database_access: - type: boolean - default: true - teams_api: - type: object - properties: - enable_admin_role_for_users: - type: boolean - default: true - enable_postgres_team_crd: - type: boolean - default: true - enable_postgres_team_crd_superusers: - type: boolean - default: false - enable_team_member_deprecation: - type: boolean - default: false - enable_team_superuser: - type: boolean - default: false - enable_teams_api: - type: boolean - default: true - pam_configuration: + major_version_upgrade_mode: + default: manual + enum: + - "off" + - manual + - full type: string - default: "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees" - pam_role_name: - type: string - default: "zalandos" - postgres_superuser_teams: - type: array + major_version_upgrade_team_allow_list: items: type: string - protected_role_names: type: array - items: - type: string - default: - - admin - role_deletion_suffix: + minimal_major_version: + default: "14" type: string - default: "_deleted" - team_admin_role: + target_major_version: + default: "18" type: string - default: "admin" - team_api_role_configuration: - type: object - additionalProperties: - type: string - default: - log_statement: all - teams_api_url: - type: string - default: "https://teams.example.com/api/" - logging_rest_api: type: object + max_instances: + default: -1 + description: -1 = disabled + format: int32 + minimum: -1 + type: integer + min_instances: + default: -1 + description: -1 = disabled + format: int32 + minimum: -1 + type: integer + patroni: + description: PatroniConfiguration defines configuration for Patroni properties: - api_port: - type: integer - default: 8080 - cluster_history_entries: - type: integer - default: 1000 - ring_log_lines: - type: integer - default: 100 - scalyr: # deprecated + enable_patroni_failsafe_mode: + type: boolean type: object + postgres_pod_resources: + description: PostgresPodResourcesDefaults defines the spec of default + resources + properties: + default_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + default_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + default_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + default_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + max_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + max_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + min_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + min_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + repair_period: + default: 5m + description: period between consecutive repair requests + format: int64 + type: integer + resync_period: + default: 30m + description: period between consecutive sync requests + format: int64 + type: integer + scalyr: + description: ScalyrConfiguration defines the configuration for ScalyrAPI properties: scalyr_api_key: type: string scalyr_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' default: "1" - scalyr_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + scalyr_cpu_request: + default: 100m + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "100m" scalyr_image: type: string scalyr_memory_limit: + default: 500Mi + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "500Mi" scalyr_memory_request: + default: 50Mi + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "50Mi" scalyr_server_url: + default: https://upload.eu.scalyr.com type: string - default: "https://upload.eu.scalyr.com" - connection_pooler: type: object + set_memory_request_to_limit: + type: boolean + sidecar_docker_images: + additionalProperties: + type: string + type: object + sidecars: + type: object + x-kubernetes-preserve-unknown-fields: true + teams_api: + description: TeamsAPIConfiguration defines the configuration of TeamsAPI properties: - connection_pooler_schema: + enable_admin_role_for_users: + default: true + type: boolean + enable_postgres_team_crd: + default: true + type: boolean + enable_postgres_team_crd_superusers: + type: boolean + enable_team_member_deprecation: + type: boolean + enable_team_superuser: + type: boolean + enable_teams_api: + type: boolean + pam_configuration: + default: https://info.example.com/oauth2/tokeninfo?access_token= + uid realm=/employees type: string - default: "pooler" - connection_pooler_user: + pam_role_name: + default: zalandos type: string - default: "pooler" - connection_pooler_image: + postgres_superuser_teams: + items: + type: string + type: array + protected_role_names: + default: '["admin", "cron_admin"]' + items: + type: string + type: array + role_deletion_suffix: + default: _deleted type: string - default: "registry.opensource.zalan.do/acid/pgbouncer:master-16" - connection_pooler_max_db_connections: + team_admin_role: + default: admin + type: string + team_api_role_configuration: + additionalProperties: + type: string + default: + log_statement: all + type: object + teams_api_url: + default: https://teams.example.com/api/ + type: string + type: object + timeouts: + description: OperatorTimeouts defines the timeout of ResourceCheck, + PodWait, ReadyWait + properties: + patroni_api_check_interval: + default: 1s + description: interval between consecutive attempts of operator + calling the Patroni API + format: int64 type: integer - default: 60 - connection_pooler_mode: - type: string - enum: - - "session" - - "transaction" - default: "transaction" - connection_pooler_number_of_instances: + patroni_api_check_timeout: + default: 5s + description: timeout when waiting for successful response from + Patroni API + format: int64 type: integer - minimum: 1 - default: 2 - connection_pooler_default_cpu_limit: + pod_deletion_wait_timeout: + default: 10m + description: timeout when waiting for the Postgres pods to be + deleted + format: int64 + type: integer + pod_label_wait_timeout: + default: 10m + description: timeout when waiting for pod role and cluster labels + format: int64 + type: integer + ready_wait_interval: + default: 4s + description: interval between consecutive attempts waiting for + postgresql CRD to be created + format: int64 + type: integer + ready_wait_timeout: + default: 30s + description: timeout for the complete postgres CRD creation + format: int64 + type: integer + resource_check_interval: + default: 3s + description: interval to wait between consecutive attempts to + check for some K8s resources + format: int64 + type: integer + resource_check_timeout: + default: 10m + description: timeout when waiting for the presence of a certain + K8s resource + format: int64 + type: integer + type: object + users: + description: PostgresUsersConfiguration defines the system users of + Postgres. + properties: + additional_owner_roles: + items: + type: string + type: array + enable_password_rotation: + type: boolean + password_rotation_interval: + default: 90 + format: int32 + type: integer + password_rotation_user_retention: + default: 120 + format: int32 + type: integer + replication_username: + default: standby type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "1" - connection_pooler_default_cpu_request: + super_username: + default: postgres type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "500m" - connection_pooler_default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "100Mi" - connection_pooler_default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "100Mi" - status: + type: object + workers: + default: 8 + format: int32 + minimum: 1 + type: integer type: object - additionalProperties: - type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + required: + - configuration + - metadata + type: object + served: true + storage: true + subresources: + status: {} diff --git a/manifests/platform-credentials.yaml b/manifests/platform-credentials.yaml index 0a320b838..44ecf7f24 100644 --- a/manifests/platform-credentials.yaml +++ b/manifests/platform-credentials.yaml @@ -2,7 +2,6 @@ apiVersion: "zalando.org/v1" kind: PlatformCredentialsSet metadata: name: postgresql-operator - namespace: acid spec: application: postgresql-operator tokens: diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index 3dbe3acb9..253649078 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -19,7 +19,7 @@ spec: serviceAccountName: postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/postgres-operator:v1.6.3 + image: ghcr.io/zalando/postgres-operator:v1.15.1 imagePullPolicy: IfNotPresent resources: requests: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index bd6f321dd..88af48b66 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -3,14 +3,23 @@ kind: OperatorConfiguration metadata: name: postgresql-operator-default-configuration configuration: - docker_image: registry.opensource.zalan.do/acid/spilo-13:2.0-p7 - # enable_crd_validation: true + docker_image: ghcr.io/zalando/spilo-18:4.1-p1 + # enable_crd_registration: true + # crd_categories: + # - all # enable_lazy_spilo_upgrade: false + enable_maintenance_windows: true enable_pgversion_env_var: true # enable_shm_volume: true enable_spilo_wal_path_compat: false + enable_team_id_clustername_prefix: false etcd_host: "" + # ignore_instance_limits_annotation_key: "" + # ignore_resources_limits_annotation_key: "" # kubernetes_use_configmaps: false + # maintenance_windows: + # - "Sat:22:00-23:59" + # - "Sun:00:00-01:00" max_instances: -1 min_instances: -1 resync_period: 30m @@ -24,12 +33,19 @@ configuration: # protocol: TCP workers: 8 users: + # additional_owner_roles: + # - cron_admin + enable_password_rotation: false + password_rotation_interval: 90 + password_rotation_user_retention: 180 replication_username: standby super_username: postgres major_version_upgrade: - major_version_upgrade_mode: "off" - minimal_major_version: "9.5" - target_major_version: "13" + major_version_upgrade_mode: "manual" + # major_version_upgrade_team_allow_list: + # - acid + minimal_major_version: "14" + target_major_version: "18" kubernetes: # additional_pod_capabilities: # - "SYS_NICE" @@ -45,10 +61,18 @@ configuration: # downscaler_annotations: # - deployment-time # - downscaler/* + # enable_cross_namespace_secret: "false" + enable_finalizers: false enable_init_containers: true + enable_owner_references: false + enable_persistent_volume_claim_deletion: true enable_pod_antiaffinity: false enable_pod_disruption_budget: true + enable_readiness_probe: false + enable_secrets_deletion: true enable_sidecars: true + # ignored_annotations: + # - k8s.v1.cni.cncf.io/network-status # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" # infrastructure_roles_secrets: # - secretname: "monitoring-roles" @@ -63,11 +87,27 @@ configuration: # inherited_labels: # - application # - environment + # liveness_probe: + # httpGet: + # scheme: HTTP + # path: /liveness + # port: 8008 + # initialDelaySeconds: 10 + # periodSeconds: 10 + # timeoutSeconds: 5 + # successThreshold: 1 + # failureThreshold: 3 master_pod_move_timeout: 20m # node_readiness_label: # status: ready + # node_readiness_label_merge: "OR" oauth_token_secret_name: postgresql-operator + pdb_master_label_selector: true pdb_name_format: "postgres-{cluster}-pdb" + persistent_volume_claim_retention_policy: + when_deleted: "retain" + when_scaled: "retain" + pod_antiaffinity_preferred_during_scheduling: false pod_antiaffinity_topology_key: "kubernetes.io/hostname" # pod_environment_configmap: "default/my-custom-config" # pod_environment_secret: "my-custom-secret" @@ -79,22 +119,30 @@ configuration: # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + share_pgsocket_with_sidecars: false spilo_allow_privilege_escalation: true # spilo_runasuser: 101 # spilo_runasgroup: 103 # spilo_fsgroup: 103 spilo_privileged: false storage_resize_mode: pvc - # toleration: {} + # toleration: + # key: db-only + # operator: Exists + # effect: NoSchedule # watched_namespace: "" postgres_pod_resources: default_cpu_limit: "1" default_cpu_request: 100m default_memory_limit: 500Mi default_memory_request: 100Mi + # max_cpu_request: "1" + # max_memory_request: 4Gi # min_cpu_limit: 250m # min_memory_limit: 250Mi timeouts: + patroni_api_check_interval: 1s + patroni_api_check_timeout: 5s pod_label_wait_timeout: 10m pod_deletion_wait_timeout: 10m ready_wait_interval: 4s @@ -107,10 +155,14 @@ configuration: # keyy: valuey # db_hosted_zone: "" enable_master_load_balancer: false + enable_master_pooler_load_balancer: false enable_replica_load_balancer: false + enable_replica_pooler_load_balancer: false external_traffic_policy: "Cluster" - master_dns_name_format: "{cluster}.{team}.{hostedzone}" - replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}" + master_dns_name_format: "{cluster}.{namespace}.{hostedzone}" + # master_legacy_dns_name_format: "{cluster}.{team}.{hostedzone}" + replica_dns_name_format: "{cluster}-repl.{namespace}.{hostedzone}" + # replica_dns_old_name_format: "{cluster}-repl.{team}.{hostedzone}" aws_or_gcp: # additional_secret_mount: "some-secret-name" # additional_secret_mount_path: "/some/dir" @@ -120,20 +172,31 @@ configuration: # gcp_credentials: "" # kube_iam_role: "" # log_s3_bucket: "" + # wal_az_storage_account: "" # wal_gs_bucket: "" # wal_s3_bucket: "" logical_backup: - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.6.3" + # logical_backup_azure_storage_account_name: "" + # logical_backup_azure_storage_container: "" + # logical_backup_azure_storage_account_key: "" + # logical_backup_cpu_limit: "" + # logical_backup_cpu_request: "" + # logical_backup_memory_limit: "" + # logical_backup_memory_request: "" + logical_backup_docker_image: "ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1" # logical_backup_google_application_credentials: "" logical_backup_job_prefix: "logical-backup-" logical_backup_provider: "s3" # logical_backup_s3_access_key_id: "" logical_backup_s3_bucket: "my-bucket-url" + # logical_backup_s3_bucket_prefix: "spilo" # logical_backup_s3_endpoint: "" # logical_backup_s3_region: "" # logical_backup_s3_secret_access_key: "" logical_backup_s3_sse: "AES256" + # logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" + # logical_backup_cronjob_environment_secret: "" debug: debug_logging: true enable_database_access: true @@ -150,6 +213,7 @@ configuration: # - postgres_superusers protected_role_names: - admin + - cron_admin role_deletion_suffix: "_deleted" team_admin_role: admin team_api_role_configuration: @@ -164,9 +228,11 @@ configuration: connection_pooler_default_cpu_request: "500m" connection_pooler_default_memory_limit: 100Mi connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-16" + connection_pooler_image: "ghcr.io/zalando/postgres-operator/pgbouncer:latest" # connection_pooler_max_db_connections: 60 connection_pooler_mode: "transaction" connection_pooler_number_of_instances: 2 # connection_pooler_schema: "pooler" # connection_pooler_user: "pooler" + patroni: + enable_patroni_failsafe_mode: false diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 30b41d392..9b855eb68 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -1,524 +1,4697 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 name: postgresqls.acid.zalan.do spec: group: acid.zalan.do names: + categories: + - all kind: postgresql listKind: postgresqlList plural: postgresqls - singular: postgresql shortNames: - pg - categories: - - all + singular: postgresql scope: Namespaced versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - name: Team - type: string - description: Team responsible for Postgres cluster + - additionalPrinterColumns: + - description: Team responsible for Postgres cluster jsonPath: .spec.teamId - - name: Version + name: Team type: string - description: PostgreSQL version + - description: PostgreSQL version jsonPath: .spec.postgresql.version - - name: Pods - type: integer - description: Number of Pods per Postgres cluster + name: Version + type: string + - description: Number of Pods per Postgres cluster jsonPath: .spec.numberOfInstances - - name: Volume - type: string - description: Size of the bound volume + name: Pods + type: integer + - description: Size of the bound volume jsonPath: .spec.volume.size - - name: CPU-Request + name: Volume type: string - description: Requested CPU for Postgres containers + - description: Requested CPU for Postgres containers jsonPath: .spec.resources.requests.cpu - - name: Memory-Request + name: CPU-Request type: string - description: Requested memory for Postgres containers + - description: Requested memory for Postgres containers jsonPath: .spec.resources.requests.memory - - name: Age - type: date - jsonPath: .metadata.creationTimestamp - - name: Status + name: Memory-Request type: string - description: Current sync status of postgresql resource + - description: Age of the PostgreSQL cluster + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Current sync status of postgresql resource jsonPath: .status.PostgresClusterStatus + name: Status + type: string + name: v1 schema: openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - spec + description: Postgresql defines PostgreSQL Custom Resource Definition Object. properties: - kind: - type: string - enum: - - postgresql apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string - enum: - - acid.zalan.do/v1 - spec: + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: type: object - required: - - numberOfInstances - - teamId - - postgresql - - volume + spec: + description: PostgresSpec defines the specification for the PostgreSQL + TPR. properties: additionalVolumes: - type: array items: - type: object - required: - - name - - mountPath - - volumeSource + description: AdditionalVolume specs additional optional volumes + for statefulset properties: - name: - type: string + isSubPathExpr: + type: boolean mountPath: type: string + name: + type: string + subPath: + type: string targetContainers: - type: array - nullable: true items: type: string + nullable: true + type: array volumeSource: type: object x-kubernetes-preserve-unknown-fields: true - subPath: - type: string - allowedSourceRanges: + required: + - mountPath + - name + - volumeSource + type: object type: array - nullable: true + allowedSourceRanges: + description: load balancers' source ranges are the same for master + and replica services items: + pattern: ^((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:))\/(12[0-8]|1[01][0-9]|[1-9]?[0-9]))$ type: string - pattern: '^(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])$' + nullable: true + type: array clone: - type: object - required: - - cluster + description: CloneDescription describes which cluster the new should + clone and up to which point in time properties: cluster: type: string - s3_endpoint: - type: string s3_access_key_id: type: string - s3_secret_access_key: + s3_endpoint: type: string s3_force_path_style: type: boolean + s3_secret_access_key: + type: string s3_wal_path: type: string timestamp: + description: |- + The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + Example: 1996-12-19T16:39:57-08:00 + Note: this field requires a timezone + pattern: ^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$ type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' - # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC - # Example: 1996-12-19T16:39:57-08:00 - # Note: this field requires a timezone uid: format: uuid type: string - connectionPooler: + required: + - cluster type: object + connectionPooler: + description: |- + ConnectionPooler Options for connection pooler + + pgbouncer-large (with higher resources) or odyssey-small (with smaller + resources) + Type string `json:"type,omitempty"` + + makes sense to expose. E.g. pool size (min/max boundaries), max client + connections etc. properties: dockerImage: type: string maxDBConnections: + format: int32 type: integer mode: - type: string enum: - - "session" - - "transaction" + - session + - transaction + type: string numberOfInstances: + format: int32 + minimum: 1 type: integer - minimum: 2 resources: - type: object - required: - - requests - - limits + description: Resources describes requests and limits for the cluster + resouces. properties: limits: - type: object - required: - - cpu - - memory + description: ResourceDescription describes CPU and memory + resources defined for a cluster. properties: cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + type: object requests: - type: object - required: - - cpu - - memory + description: ResourceDescription describes CPU and memory + resources defined for a cluster. properties: cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + type: object + type: object schema: type: string user: type: string - databases: type: object + databases: additionalProperties: type: string - # Note: usernames specified here as database owners must be declared in the users key of the spec key. + description: |- + Note: usernames specified here as database owners must be declared + in the users key of the spec key. + type: object dockerImage: type: string enableConnectionPooler: type: boolean - enableReplicaConnectionPooler: - type: boolean enableLogicalBackup: type: boolean enableMasterLoadBalancer: + description: |- + vars that enable load balancers are pointers because it is important to know if any of them is omitted from the Postgres manifest + in that case the var evaluates to nil and the value is taken from the operator config + type: boolean + enableMasterNodePort: + description: |- + vars to enable and configure nodeport services + set ports to 0 or nil to let kubernetes decide which port to use + overrides loadbalancer configuration + type: boolean + enableMasterPoolerLoadBalancer: + type: boolean + enableMasterPoolerNodePort: + type: boolean + enableReplicaConnectionPooler: type: boolean enableReplicaLoadBalancer: type: boolean + enableReplicaNodePort: + type: boolean + enableReplicaPoolerLoadBalancer: + type: boolean + enableReplicaPoolerNodePort: + type: boolean enableShmVolume: type: boolean - init_containers: # deprecated - type: array - nullable: true + env: items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name type: object - x-kubernetes-preserve-unknown-fields: true + type: array + init_containers: + description: deprecated + items: + description: A single application container that you want to run + within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set + of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a + single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource resize + policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array initContainers: - type: array - nullable: true items: + description: A single application container that you want to run + within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set + of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a + single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource resize + policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name type: object - x-kubernetes-preserve-unknown-fields: true - logicalBackupSchedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - maintenanceWindows: type: array - items: - type: string - pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' - numberOfInstances: - type: integer - minimum: 0 - patroni: - type: object + livenessProbe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. properties: - initdb: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes a custom header to be + used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + logicalBackupRetention: + type: string + logicalBackupSchedule: + pattern: ^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$ + type: string + maintenanceWindows: + items: + pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' + type: string + type: array + masterNodePort: + format: int32 + type: integer + masterPoolerNodePort: + format: int32 + type: integer + masterServiceAnnotations: + additionalProperties: + type: string + description: MasterServiceAnnotations takes precedence over ServiceAnnotations + for master role if not empty + type: object + nodeAffinity: + description: Node affinity is a group of node affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + numberOfInstances: + format: int32 + minimum: 0 + type: integer + patroni: + description: Patroni contains Patroni-specific configuration + properties: + failsafe_mode: + type: boolean + initdb: additionalProperties: type: string + type: object loop_wait: + format: int32 type: integer maximum_lag_on_failover: + format: int64 type: integer pg_hba: - type: array items: type: string + type: array retry_timeout: + format: int32 type: integer slots: - type: object additionalProperties: - type: object additionalProperties: type: string + type: object + type: object synchronous_mode: type: boolean synchronous_mode_strict: type: boolean - ttl: + synchronous_node_count: + format: int32 + type: integer + ttl: + format: int32 type: integer - podAnnotations: type: object + pod_priority_class_name: + description: deprecated + type: string + podAnnotations: additionalProperties: type: string - pod_priority_class_name: # deprecated - type: string + type: object podPriorityClassName: type: string postgresql: - type: object - required: - - version + description: PostgresqlParam describes PostgreSQL version and pairs + of configuration parameter name - values. properties: - version: - type: string - enum: - - "9.3" - - "9.4" - - "9.5" - - "9.6" - - "10" - - "11" - - "12" - - "13" parameters: - type: object additionalProperties: type: string - preparedDatabases: + type: object + version: + enum: + - "14" + - "15" + - "16" + - "17" + - "18" + type: string + required: + - version type: object + preparedDatabases: additionalProperties: - type: object + description: PreparedDatabase describes elements to be bootstrapped properties: defaultUsers: type: boolean extensions: - type: object additionalProperties: type: string - schemas: type: object + schemas: additionalProperties: - type: object + description: PreparedSchema describes elements to be bootstrapped + per schema properties: - defaultUsers: - type: boolean defaultRoles: type: boolean - replicaLoadBalancer: # deprecated - type: boolean - resources: + defaultUsers: + type: boolean + type: object + type: object + secretNamespace: + type: string + type: object type: object - required: - - requests - - limits + replicaLoadBalancer: + description: deprecated + type: boolean + replicaNodePort: + format: int32 + type: integer + replicaPoolerNodePort: + format: int32 + type: integer + replicaServiceAnnotations: + additionalProperties: + type: string + description: ReplicaServiceAnnotations takes precedence over ServiceAnnotations + for replica role if not empty + type: object + resources: + description: Resources describes requests and limits for the cluster + resouces. properties: limits: - type: object - required: - - cpu - - memory + description: ResourceDescription describes CPU and memory resources + defined for a cluster. properties: cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - # Decimal natural followed by m, or decimal natural followed by - # dot followed by up to three decimal digits. - # - # This is because the Kubernetes CPU resource has millis as the - # maximum precision. The actual values are checked in code - # because the regular expression would be huge and horrible and - # not very helpful in validation error messages; this one checks - # only the format of the given number. - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - # Note: the value specified here must not be zero or be lower - # than the corresponding request. memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - # You can express memory as a plain integer or as a fixed-point - # integer using one of these suffixes: E, P, T, G, M, k. You can - # also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki - # - # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero or be higher - # than the corresponding limit. + type: object requests: - type: object - required: - - cpu - - memory + description: ResourceDescription describes CPU and memory resources + defined for a cluster. properties: cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + type: object + type: object schedulerName: type: string serviceAnnotations: - type: object additionalProperties: type: string + type: object sidecars: - type: array - nullable: true items: + description: Sidecar defines a container to be run in the same pod + as the Postgres container. + properties: + command: + items: + type: string + type: array + env: + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + type: string + name: + type: string + ports: + items: + description: ContainerPort represents a network port in a + single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + resources: + description: Resources describes requests and limits for the + cluster resouces. + properties: + limits: + description: ResourceDescription describes CPU and memory + resources defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + requests: + description: ResourceDescription describes CPU and memory + resources defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + type: object type: object - x-kubernetes-preserve-unknown-fields: true - spiloRunAsUser: + type: array + spiloFSGroup: + format: int64 type: integer spiloRunAsGroup: + format: int64 type: integer - spiloFSGroup: + spiloRunAsUser: + format: int64 type: integer standby: - type: object - required: + anyOf: + - required: - s3_wal_path + - required: + - gs_wal_path + - required: + - standby_host + not: + required: + - s3_wal_path + - gs_wal_path + description: |- + StandbyDescription contains remote primary config and/or s3/gs wal path. + standby_host can be specified alone or together with either s3_wal_path OR gs_wal_path (mutually exclusive). + At least one field must be specified. s3_wal_path and gs_wal_path are mutually exclusive. properties: + gs_wal_path: + type: string s3_wal_path: type: string + standby_host: + type: string + standby_port: + type: string + standby_primary_slot_name: + type: string + type: object + streams: + items: + description: Stream defines properties for creating FabricEventStream + resources + properties: + applicationId: + type: string + batchSize: + format: int32 + type: integer + cpu: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + database: + type: string + enableRecovery: + type: boolean + filter: + additionalProperties: + type: string + type: object + memory: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + tables: + additionalProperties: + description: StreamTable defines properties of outbox tables + for FabricEventStreams + properties: + eventType: + type: string + idColumn: + type: string + ignoreRecovery: + type: boolean + payloadColumn: + type: string + recoveryEventType: + type: string + required: + - eventType + type: object + type: object + required: + - applicationId + - database + - tables + type: object + type: array teamId: type: string tls: - type: object - required: - - secretName + description: TLSDescription specs TLS properties properties: - secretName: + caFile: + type: string + caSecretName: type: string certificateFile: type: string privateKeyFile: type: string - caFile: + secretName: type: string - caSecretName: - type: string - nodeAffinity: + required: + - secretName type: object - properties: - preferredDuringSchedulingIgnoredDuringExecution: - type: array - items: - type: object - required: - - weight - - preference - properties: - preference: - type: object - properties: - matchExpressions: - type: array - items: - type: object - required: - - key - - operator - properties: - key: - type: string - operator: - type: string - values: - type: array - items: - type: string - matchFields: - type: array - items: - type: object - required: - - key - - operator - properties: - key: - type: string - operator: - type: string - values: - type: array - items: - type: string - weight: - format: int32 - type: integer - requiredDuringSchedulingIgnoredDuringExecution: - type: object - required: - - nodeSelectorTerms - properties: - nodeSelectorTerms: - type: array - items: - type: object - properties: - matchExpressions: - type: array - items: - type: object - required: - - key - - operator - properties: - key: - type: string - operator: - type: string - values: - type: array - items: - type: string - matchFields: - type: array - items: - type: object - required: - - key - - operator - properties: - key: - type: string - operator: - type: string - values: - type: array - items: - type: string tolerations: - type: array items: - type: object - required: - - key - - operator - - effect + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string - enum: - - Equal - - Exists - value: - type: string - effect: - type: string - enum: - - NoExecute - - NoSchedule - - PreferNoSchedule tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 type: integer - useLoadBalancer: # deprecated + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + useLoadBalancer: + description: |- + deprecated load balancer settings maintained for backward compatibility + see "Load balancers" operator docs type: boolean users: - type: object additionalProperties: - type: array - nullable: true - description: "Role flags specified here must not contradict each other" + description: UserFlags defines flags (such as superuser, nologin) + that could be assigned to individual users items: - type: string enum: - bypassrls - BYPASSRLS @@ -548,24 +4721,116 @@ spec: - SUPERUSER - nosuperuser - NOSUPERUSER - volume: + type: string + type: array type: object - required: - - size + usersIgnoringSecretRotation: + items: + type: string + nullable: true + type: array + usersWithInPlaceSecretRotation: + items: + type: string + nullable: true + type: array + usersWithSecretRotation: + items: + type: string + nullable: true + type: array + volume: + description: Volume describes a single volume in the manifest. properties: iops: + format: int64 type: integer + isSubPathExpr: + type: boolean + selector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic size: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - # Note: the value specified here must not be zero. storageClass: type: string subPath: type: string throughput: + format: int64 type: integer - status: + type: + type: string + required: + - size + type: object + required: + - numberOfInstances + - postgresql + - teamId + - volume type: object - additionalProperties: - type: string + status: + description: PostgresStatus contains status of the PostgreSQL cluster + (running, creation failed etc.) + properties: + PostgresClusterStatus: + type: string + required: + - PostgresClusterStatus + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/manifests/postgresteam.crd.yaml b/manifests/postgresteam.crd.yaml index 2588e53b1..2245c6253 100644 --- a/manifests/postgresteam.crd.yaml +++ b/manifests/postgresteam.crd.yaml @@ -1,68 +1,82 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 name: postgresteams.acid.zalan.do spec: group: acid.zalan.do names: + categories: + - all kind: PostgresTeam listKind: PostgresTeamList plural: postgresteams - singular: postgresteam shortNames: - pgteam - categories: - - all + singular: postgresteam scope: Namespaced versions: - name: v1 + schema: + openAPIV3Schema: + description: PostgresTeam defines Custom Resource Definition Object for team + management. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: PostgresTeamSpec defines the specification for the PostgresTeam + TPR. + properties: + additionalMembers: + additionalProperties: + description: List of users who will also be added to the Postgres + cluster. + items: + type: string + type: array + description: Map for teamId and associated additional users + type: object + additionalSuperuserTeams: + additionalProperties: + description: List of teams to become Postgres superusers + items: + type: string + type: array + description: Map for teamId and associated additional superuser teams + type: object + additionalTeams: + additionalProperties: + description: List of teams whose members will also be added to the + Postgres cluster. + items: + type: string + type: array + description: Map for teamId and associated additional teams + type: object + type: object + required: + - metadata + - spec + type: object served: true storage: true subresources: status: {} - schema: - openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - spec - properties: - kind: - type: string - enum: - - PostgresTeam - apiVersion: - type: string - enum: - - acid.zalan.do/v1 - spec: - type: object - properties: - additionalSuperuserTeams: - type: object - description: "Map for teamId and associated additional superuser teams" - additionalProperties: - type: array - nullable: true - description: "List of teams to become Postgres superusers" - items: - type: string - additionalTeams: - type: object - description: "Map for teamId and associated additional teams" - additionalProperties: - type: array - nullable: true - description: "List of teams whose members will also be added to the Postgres cluster" - items: - type: string - additionalMembers: - type: object - description: "Map for teamId and associated additional users" - additionalProperties: - type: array - nullable: true - description: "List of users who will also be added to the Postgres cluster" - items: - type: string diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml index 593f409ec..de4f9edaf 100644 --- a/manifests/standby-manifest.yaml +++ b/manifests/standby-manifest.yaml @@ -2,14 +2,17 @@ apiVersion: "acid.zalan.do/v1" kind: postgresql metadata: name: acid-standby-cluster - namespace: default spec: teamId: "acid" volume: size: 1Gi numberOfInstances: 1 postgresql: - version: "13" -# Make this a standby cluster and provide the s3 bucket path of source cluster for continuous streaming. + version: "18" + # Make this a standby cluster. You can specify s3_wal_path or gs_wal_path for WAL archive, + # standby_host for remote primary streaming, or combine standby_host with either WAL path. + # Note: s3_wal_path and gs_wal_path are mutually exclusive. standby: - s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/" + # s3_wal_path: "s3://mybucket/spilo/acid-minimal-cluster/abcd1234-2a4b-4b2a-8c9c-c1234defg567/wal/14/" + standby_host: "acid-minimal-cluster.default" + # standby_port: "5432" diff --git a/pkg/apis/acid.zalan.do/v1/const.go b/pkg/apis/acid.zalan.do/v1/const.go index 3cb1c1ade..4102ea3d3 100644 --- a/pkg/apis/acid.zalan.do/v1/const.go +++ b/pkg/apis/acid.zalan.do/v1/const.go @@ -1,6 +1,6 @@ package v1 -// ClusterStatusUnknown etc : status of a Postgres cluster known to the operator +// ClusterStatusUnknown etc : status of a Postgres cluster known to the operator const ( ClusterStatusUnknown = "" ClusterStatusCreating = "Creating" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 83e7273e4..54aea7c1b 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1,1596 +1,46 @@ package v1 import ( - acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" - "github.com/zalando/postgres-operator/pkg/util" + _ "embed" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" ) // CRDResource* define names necesssary for the k8s CRD API const ( - PostgresCRDResourceKind = "postgresql" - PostgresCRDResourcePlural = "postgresqls" - PostgresCRDResouceName = PostgresCRDResourcePlural + "." + acidzalando.GroupName - PostgresCRDResourceShort = "pg" - - OperatorConfigCRDResouceKind = "OperatorConfiguration" - OperatorConfigCRDResourcePlural = "operatorconfigurations" - OperatorConfigCRDResourceName = OperatorConfigCRDResourcePlural + "." + acidzalando.GroupName - OperatorConfigCRDResourceShort = "opconfig" + PostgresCRDResourceKind = "postgresql" + OperatorConfigCRDResourceKind = "OperatorConfiguration" ) -// PostgresCRDResourceColumns definition of AdditionalPrinterColumns for postgresql CRD -var PostgresCRDResourceColumns = []apiextv1.CustomResourceColumnDefinition{ - { - Name: "Team", - Type: "string", - Description: "Team responsible for Postgres cluster", - JSONPath: ".spec.teamId", - }, - { - Name: "Version", - Type: "string", - Description: "PostgreSQL version", - JSONPath: ".spec.postgresql.version", - }, - { - Name: "Pods", - Type: "integer", - Description: "Number of Pods per Postgres cluster", - JSONPath: ".spec.numberOfInstances", - }, - { - Name: "Volume", - Type: "string", - Description: "Size of the bound volume", - JSONPath: ".spec.volume.size", - }, - { - Name: "CPU-Request", - Type: "string", - Description: "Requested CPU for Postgres containers", - JSONPath: ".spec.resources.requests.cpu", - }, - { - Name: "Memory-Request", - Type: "string", - Description: "Requested memory for Postgres containers", - JSONPath: ".spec.resources.requests.memory", - }, - { - Name: "Age", - Type: "date", - JSONPath: ".metadata.creationTimestamp", - }, - { - Name: "Status", - Type: "string", - Description: "Current sync status of postgresql resource", - JSONPath: ".status.PostgresClusterStatus", - }, -} - -// OperatorConfigCRDResourceColumns definition of AdditionalPrinterColumns for OperatorConfiguration CRD -var OperatorConfigCRDResourceColumns = []apiextv1.CustomResourceColumnDefinition{ - { - Name: "Image", - Type: "string", - Description: "Spilo image to be used for Pods", - JSONPath: ".configuration.docker_image", - }, - { - Name: "Cluster-Label", - Type: "string", - Description: "Label for K8s resources created by operator", - JSONPath: ".configuration.kubernetes.cluster_name_label", - }, - { - Name: "Service-Account", - Type: "string", - Description: "Name of service account to be used", - JSONPath: ".configuration.kubernetes.pod_service_account_name", - }, - { - Name: "Min-Instances", - Type: "integer", - Description: "Minimum number of instances per Postgres cluster", - JSONPath: ".configuration.min_instances", - }, - { - Name: "Age", - Type: "date", - JSONPath: ".metadata.creationTimestamp", - }, -} - -var min0 = 0.0 -var min1 = 1.0 -var minDisable = -1.0 - -// PostgresCRDResourceValidation to check applied manifest parameters -var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"kind", "apiVersion", "spec"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "kind": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"postgresql"`), - }, - }, - }, - "apiVersion": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"acid.zalan.do/v1"`), - }, - }, - }, - "spec": { - Type: "object", - Required: []string{"numberOfInstances", "teamId", "postgresql", "volume"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "additionalVolumes": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"name", "mountPath", "volumeSource"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "name": { - Type: "string", - }, - "mountPath": { - Type: "string", - }, - "targetContainers": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "volumeSource": { - Type: "object", - XPreserveUnknownFields: util.True(), - }, - "subPath": { - Type: "string", - }, - }, - }, - }, - }, - "allowedSourceRanges": { - Type: "array", - Nullable: true, - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - Pattern: "^(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\/(\\d|[1-2]\\d|3[0-2])$", - }, - }, - }, - "clone": { - Type: "object", - Required: []string{"cluster"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "cluster": { - Type: "string", - }, - "s3_endpoint": { - Type: "string", - }, - "s3_access_key_id": { - Type: "string", - }, - "s3_secret_access_key": { - Type: "string", - }, - "s3_force_path_style": { - Type: "boolean", - }, - "s3_wal_path": { - Type: "string", - }, - "timestamp": { - Type: "string", - Description: "Date-time format that specifies a timezone as an offset relative to UTC e.g. 1996-12-19T16:39:57-08:00", - Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", - }, - "uid": { - Type: "string", - Format: "uuid", - }, - }, - }, - "connectionPooler": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "dockerImage": { - Type: "string", - }, - "maxDBConnections": { - Type: "integer", - }, - "mode": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"session"`), - }, - { - Raw: []byte(`"transaction"`), - }, - }, - }, - "numberOfInstances": { - Type: "integer", - Minimum: &min1, - }, - "resources": { - Type: "object", - Required: []string{"requests", "limits"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "limits": { - Type: "object", - Required: []string{"cpu", "memory"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "cpu": { - Type: "string", - Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "memory": { - Type: "string", - Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - }, - }, - "requests": { - Type: "object", - Required: []string{"cpu", "memory"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "cpu": { - Type: "string", - Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "memory": { - Type: "string", - Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - }, - }, - }, - }, - "schema": { - Type: "string", - }, - "user": { - Type: "string", - }, - }, - }, - "databases": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - Description: "User names specified here as database owners must be declared in the users key of the spec key", - }, - }, - }, - "dockerImage": { - Type: "string", - }, - "enableConnectionPooler": { - Type: "boolean", - }, - "enableReplicaConnectionPooler": { - Type: "boolean", - }, - "enableLogicalBackup": { - Type: "boolean", - }, - "enableMasterLoadBalancer": { - Type: "boolean", - }, - "enableReplicaLoadBalancer": { - Type: "boolean", - }, - "enableShmVolume": { - Type: "boolean", - }, - "init_containers": { - Type: "array", - Description: "Deprecated", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - XPreserveUnknownFields: util.True(), - }, - }, - }, - "initContainers": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - XPreserveUnknownFields: util.True(), - }, - }, - }, - "logicalBackupSchedule": { - Type: "string", - Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", - }, - "maintenanceWindows": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - Pattern: "^\\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))-((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\\d):([0-5]?\\d)|(2[0-3]|[01]?\\d):([0-5]?\\d))\\ *$", - }, - }, - }, - "numberOfInstances": { - Type: "integer", - Minimum: &min0, - }, - "patroni": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "initdb": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "loop_wait": { - Type: "integer", - }, - "maximum_lag_on_failover": { - Type: "integer", - }, - "pg_hba": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "retry_timeout": { - Type: "integer", - }, - "slots": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - }, - }, - "synchronous_mode": { - Type: "boolean", - }, - "synchronous_mode_strict": { - Type: "boolean", - }, - "ttl": { - Type: "integer", - }, - }, - }, - "podAnnotations": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "pod_priority_class_name": { - Type: "string", - Description: "Deprecated", - }, - "podPriorityClassName": { - Type: "string", - }, - "postgresql": { - Type: "object", - Required: []string{"version"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "version": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"9.3"`), - }, - { - Raw: []byte(`"9.4"`), - }, - { - Raw: []byte(`"9.5"`), - }, - { - Raw: []byte(`"9.6"`), - }, - { - Raw: []byte(`"10"`), - }, - { - Raw: []byte(`"11"`), - }, - { - Raw: []byte(`"12"`), - }, - { - Raw: []byte(`"13"`), - }, - }, - }, - "parameters": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - }, - }, - "preparedDatabases": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "defaultUsers": { - Type: "boolean", - }, - "extensions": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "schemas": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "defaultUsers": { - Type: "boolean", - }, - "defaultRoles": { - Type: "boolean", - }, - }, - }, - }, - }, - }, - }, - }, - }, - "replicaLoadBalancer": { - Type: "boolean", - Description: "Deprecated", - }, - "resources": { - Type: "object", - Required: []string{"requests", "limits"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "limits": { - Type: "object", - Required: []string{"cpu", "memory"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "cpu": { - Type: "string", - Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "memory": { - Type: "string", - Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - }, - }, - "requests": { - Type: "object", - Required: []string{"cpu", "memory"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "cpu": { - Type: "string", - Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "memory": { - Type: "string", - Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - }, - }, - }, - }, - "schedulerName": { - Type: "string", - }, - "serviceAnnotations": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "sidecars": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - XPreserveUnknownFields: util.True(), - }, - }, - }, - "spiloRunAsUser": { - Type: "integer", - }, - "spiloRunAsGroup": { - Type: "integer", - }, - "spiloFSGroup": { - Type: "integer", - }, - "standby": { - Type: "object", - Required: []string{"s3_wal_path"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "s3_wal_path": { - Type: "string", - }, - }, - }, - "teamId": { - Type: "string", - }, - "tls": { - Type: "object", - Required: []string{"secretName"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "secretName": { - Type: "string", - }, - "certificateFile": { - Type: "string", - }, - "privateKeyFile": { - Type: "string", - }, - "caFile": { - Type: "string", - }, - "caSecretName": { - Type: "string", - }, - }, - }, - "nodeAffinity": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "preferredDuringSchedulingIgnoredDuringExecution": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"preference", "weight"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "preference": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "matchExpressions": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, - }, - }, - }, - }, - "matchFields": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, - }, - }, - }, - }, - }, - }, - "weight": { - Type: "integer", - Format: "int32", - }, - }, - }, - }, - }, - "requiredDuringSchedulingIgnoredDuringExecution": { - Type: "object", - Required: []string{"nodeSelectorTerms"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "nodeSelectorTerms": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "matchExpressions": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, - }, - }, - }, - }, - "matchFields": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - "tolerations": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"key", "operator", "effect"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "key": { - Type: "string", - }, - "operator": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"Equal"`), - }, - { - Raw: []byte(`"Exists"`), - }, - }, - }, - "value": { - Type: "string", - }, - "effect": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"NoExecute"`), - }, - { - Raw: []byte(`"NoSchedule"`), - }, - { - Raw: []byte(`"PreferNoSchedule"`), - }, - }, - }, - "tolerationSeconds": { - Type: "integer", - }, - }, - }, - }, - }, - "useLoadBalancer": { - Type: "boolean", - Description: "Deprecated", - }, - "users": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "array", - Description: "Role flags specified here must not contradict each other", - Nullable: true, - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"bypassrls"`), - }, - { - Raw: []byte(`"BYPASSRLS"`), - }, - { - Raw: []byte(`"nobypassrls"`), - }, - { - Raw: []byte(`"NOBYPASSRLS"`), - }, - { - Raw: []byte(`"createdb"`), - }, - { - Raw: []byte(`"CREATEDB"`), - }, - { - Raw: []byte(`"nocreatedb"`), - }, - { - Raw: []byte(`"NOCREATEDB"`), - }, - { - Raw: []byte(`"createrole"`), - }, - { - Raw: []byte(`"CREATEROLE"`), - }, - { - Raw: []byte(`"nocreaterole"`), - }, - { - Raw: []byte(`"NOCREATEROLE"`), - }, - { - Raw: []byte(`"inherit"`), - }, - { - Raw: []byte(`"INHERIT"`), - }, - { - Raw: []byte(`"noinherit"`), - }, - { - Raw: []byte(`"NOINHERIT"`), - }, - { - Raw: []byte(`"login"`), - }, - { - Raw: []byte(`"LOGIN"`), - }, - { - Raw: []byte(`"nologin"`), - }, - { - Raw: []byte(`"NOLOGIN"`), - }, - { - Raw: []byte(`"replication"`), - }, - { - Raw: []byte(`"REPLICATION"`), - }, - { - Raw: []byte(`"noreplication"`), - }, - { - Raw: []byte(`"NOREPLICATION"`), - }, - { - Raw: []byte(`"superuser"`), - }, - { - Raw: []byte(`"SUPERUSER"`), - }, - { - Raw: []byte(`"nosuperuser"`), - }, - { - Raw: []byte(`"NOSUPERUSER"`), - }, - }, - }, - }, - }, - }, - }, - "volume": { - Type: "object", - Required: []string{"size"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "iops": { - Type: "integer", - }, - "size": { - Type: "string", - Description: "Value must not be zero", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - "storageClass": { - Type: "string", - }, - "subPath": { - Type: "string", - }, - "throughput": { - Type: "integer", - }, - }, - }, - }, - }, - "status": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - }, - }, -} - -// OperatorConfigCRDResourceValidation to check applied manifest parameters -var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"kind", "apiVersion", "configuration"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "kind": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"OperatorConfiguration"`), - }, - }, - }, - "apiVersion": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"acid.zalan.do/v1"`), - }, - }, - }, - "configuration": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "docker_image": { - Type: "string", - }, - "enable_crd_validation": { - Type: "boolean", - }, - "enable_lazy_spilo_upgrade": { - Type: "boolean", - }, - "enable_shm_volume": { - Type: "boolean", - }, - "enable_spilo_wal_path_compat": { - Type: "boolean", - }, - "etcd_host": { - Type: "string", - }, - "kubernetes_use_configmaps": { - Type: "boolean", - }, - "max_instances": { - Type: "integer", - Description: "-1 = disabled", - Minimum: &minDisable, - }, - "min_instances": { - Type: "integer", - Description: "-1 = disabled", - Minimum: &minDisable, - }, - "resync_period": { - Type: "string", - }, - "repair_period": { - Type: "string", - }, - "set_memory_request_to_limit": { - Type: "boolean", - }, - "sidecar_docker_images": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "sidecars": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - XPreserveUnknownFields: util.True(), - }, - }, - }, - "workers": { - Type: "integer", - Minimum: &min1, - }, - "users": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "replication_username": { - Type: "string", - }, - "super_username": { - Type: "string", - }, - }, - }, - "major_version_upgrade": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "major_version_upgrade_mode": { - Type: "string", - }, - "minimal_major_version": { - Type: "string", - }, - "target_major_version": { - Type: "string", - }, - }, - }, - "kubernetes": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "additional_pod_capabilities": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "cluster_domain": { - Type: "string", - }, - "cluster_labels": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "cluster_name_label": { - Type: "string", - }, - "custom_pod_annotations": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "delete_annotation_date_key": { - Type: "string", - }, - "delete_annotation_name_key": { - Type: "string", - }, - "downscaler_annotations": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "enable_init_containers": { - Type: "boolean", - }, - "enable_pod_antiaffinity": { - Type: "boolean", - }, - "enable_pod_disruption_budget": { - Type: "boolean", - }, - "enable_sidecars": { - Type: "boolean", - }, - "infrastructure_roles_secret_name": { - Type: "string", - }, - "infrastructure_roles_secrets": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"secretname", "userkey", "passwordkey"}, - Properties: map[string]apiextv1.JSONSchemaProps{ - "secretname": { - Type: "string", - }, - "userkey": { - Type: "string", - }, - "passwordkey": { - Type: "string", - }, - "rolekey": { - Type: "string", - }, - "defaultuservalue": { - Type: "string", - }, - "defaultrolevalue": { - Type: "string", - }, - "details": { - Type: "string", - }, - "template": { - Type: "boolean", - }, - }, - }, - }, - }, - "inherited_annotations": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "inherited_labels": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "master_pod_move_timeout": { - Type: "string", - }, - "node_readiness_label": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "oauth_token_secret_name": { - Type: "string", - }, - "pdb_name_format": { - Type: "string", - }, - "pod_antiaffinity_topology_key": { - Type: "string", - }, - "pod_environment_configmap": { - Type: "string", - }, - "pod_environment_secret": { - Type: "string", - }, - "pod_management_policy": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"ordered_ready"`), - }, - { - Raw: []byte(`"parallel"`), - }, - }, - }, - "pod_priority_class_name": { - Type: "string", - }, - "pod_role_label": { - Type: "string", - }, - "pod_service_account_definition": { - Type: "string", - }, - "pod_service_account_name": { - Type: "string", - }, - "pod_service_account_role_binding_definition": { - Type: "string", - }, - "pod_terminate_grace_period": { - Type: "string", - }, - "secret_name_template": { - Type: "string", - }, - "spilo_runasuser": { - Type: "integer", - }, - "spilo_runasgroup": { - Type: "integer", - }, - "spilo_fsgroup": { - Type: "integer", - }, - "spilo_privileged": { - Type: "boolean", - }, - "spilo_allow_privilege_escalation": { - Type: "boolean", - }, - "storage_resize_mode": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"ebs"`), - }, - { - Raw: []byte(`"pvc"`), - }, - { - Raw: []byte(`"off"`), - }, - }, - }, - "toleration": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "watched_namespace": { - Type: "string", - }, - }, - }, - "postgres_pod_resources": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "default_cpu_limit": { - Type: "string", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "default_cpu_request": { - Type: "string", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "default_memory_limit": { - Type: "string", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - "default_memory_request": { - Type: "string", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - "min_cpu_limit": { - Type: "string", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "min_memory_limit": { - Type: "string", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - }, - }, - "timeouts": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "pod_label_wait_timeout": { - Type: "string", - }, - "pod_deletion_wait_timeout": { - Type: "string", - }, - "ready_wait_interval": { - Type: "string", - }, - "ready_wait_timeout": { - Type: "string", - }, - "resource_check_interval": { - Type: "string", - }, - "resource_check_timeout": { - Type: "string", - }, - }, - }, - "load_balancer": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "custom_service_annotations": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "db_hosted_zone": { - Type: "string", - }, - "enable_master_load_balancer": { - Type: "boolean", - }, - "enable_replica_load_balancer": { - Type: "boolean", - }, - "external_traffic_policy": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"Cluster"`), - }, - { - Raw: []byte(`"Local"`), - }, - }, - }, - "master_dns_name_format": { - Type: "string", - }, - "replica_dns_name_format": { - Type: "string", - }, - }, - }, - "aws_or_gcp": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "additional_secret_mount": { - Type: "string", - }, - "additional_secret_mount_path": { - Type: "string", - }, - "aws_region": { - Type: "string", - }, - "enable_ebs_gp3_migration": { - Type: "boolean", - }, - "enable_ebs_gp3_migration_max_size": { - Type: "integer", - }, - "gcp_credentials": { - Type: "string", - }, - "kube_iam_role": { - Type: "string", - }, - "log_s3_bucket": { - Type: "string", - }, - "wal_s3_bucket": { - Type: "string", - }, - }, - }, - "logical_backup": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "logical_backup_docker_image": { - Type: "string", - }, - "logical_backup_google_application_credentials": { - Type: "string", - }, - "logical_backup_job_prefix": { - Type: "string", - }, - "logical_backup_provider": { - Type: "string", - }, - "logical_backup_s3_access_key_id": { - Type: "string", - }, - "logical_backup_s3_bucket": { - Type: "string", - }, - "logical_backup_s3_endpoint": { - Type: "string", - }, - "logical_backup_s3_region": { - Type: "string", - }, - "logical_backup_s3_secret_access_key": { - Type: "string", - }, - "logical_backup_s3_sse": { - Type: "string", - }, - "logical_backup_schedule": { - Type: "string", - Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", - }, - }, - }, - "debug": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "debug_logging": { - Type: "boolean", - }, - "enable_database_access": { - Type: "boolean", - }, - }, - }, - "teams_api": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "enable_admin_role_for_users": { - Type: "boolean", - }, - "enable_postgres_team_crd": { - Type: "boolean", - }, - "enable_postgres_team_crd_superusers": { - Type: "boolean", - }, - "enable_team_member_deprecation": { - Type: "boolean", - }, - "enable_team_superuser": { - Type: "boolean", - }, - "enable_teams_api": { - Type: "boolean", - }, - "pam_configuration": { - Type: "string", - }, - "pam_role_name": { - Type: "string", - }, - "postgres_superuser_teams": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "protected_role_names": { - Type: "array", - Items: &apiextv1.JSONSchemaPropsOrArray{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "role_deletion_suffix": { - Type: "string", - }, - "team_admin_role": { - Type: "string", - }, - "team_api_role_configuration": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - "teams_api_url": { - Type: "string", - }, - }, - }, - "logging_rest_api": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "api_port": { - Type: "integer", - }, - "cluster_history_entries": { - Type: "integer", - }, - "ring_log_lines": { - Type: "integer", - }, - }, - }, - "scalyr": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "scalyr_api_key": { - Type: "string", - }, - "scalyr_cpu_limit": { - Type: "string", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "scalyr_cpu_request": { - Type: "string", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "scalyr_image": { - Type: "string", - }, - "scalyr_memory_limit": { - Type: "string", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - "scalyr_memory_request": { - Type: "string", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - "scalyr_server_url": { - Type: "string", - }, - }, - }, - "connection_pooler": { - Type: "object", - Properties: map[string]apiextv1.JSONSchemaProps{ - "connection_pooler_default_cpu_limit": { - Type: "string", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "connection_pooler_default_cpu_request": { - Type: "string", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", - }, - "connection_pooler_default_memory_limit": { - Type: "string", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - "connection_pooler_default_memory_request": { - Type: "string", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", - }, - "connection_pooler_image": { - Type: "string", - }, - "connection_pooler_max_db_connections": { - Type: "integer", - }, - "connection_pooler_mode": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"session"`), - }, - { - Raw: []byte(`"transaction"`), - }, - }, - }, - "connection_pooler_number_of_instances": { - Type: "integer", - Minimum: &min1, - }, - "connection_pooler_schema": { - Type: "string", - }, - "connection_pooler_user": { - Type: "string", - }, - }, - }, - }, - }, - "status": { - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - }, - }, -} - -func buildCRD(name, kind, plural, short string, columns []apiextv1.CustomResourceColumnDefinition, validation apiextv1.CustomResourceValidation) *apiextv1.CustomResourceDefinition { - return &apiextv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: apiextv1.CustomResourceDefinitionSpec{ - Group: SchemeGroupVersion.Group, - Names: apiextv1.CustomResourceDefinitionNames{ - Plural: plural, - ShortNames: []string{short}, - Kind: kind, - Categories: []string{"all"}, - }, - Scope: apiextv1.NamespaceScoped, - Versions: []apiextv1.CustomResourceDefinitionVersion{ - { - Name: SchemeGroupVersion.Version, - Served: true, - Storage: true, - Subresources: &apiextv1.CustomResourceSubresources{ - Status: &apiextv1.CustomResourceSubresourceStatus{}, - }, - AdditionalPrinterColumns: columns, - Schema: &validation, - }, - }, - }, - } -} +//go:embed postgresql.crd.yaml +var postgresqlCRDYAML []byte // PostgresCRD returns CustomResourceDefinition built from PostgresCRDResource -func PostgresCRD(enableValidation *bool) *apiextv1.CustomResourceDefinition { - postgresCRDvalidation := apiextv1.CustomResourceValidation{} - - if enableValidation != nil && *enableValidation { - postgresCRDvalidation = PostgresCRDResourceValidation +func PostgresCRD(crdCategories []string) (*apiextv1.CustomResourceDefinition, error) { + var crd apiextv1.CustomResourceDefinition + err := yaml.Unmarshal(postgresqlCRDYAML, &crd) + if err != nil { + return nil, err } - return buildCRD(PostgresCRDResouceName, - PostgresCRDResourceKind, - PostgresCRDResourcePlural, - PostgresCRDResourceShort, - PostgresCRDResourceColumns, - postgresCRDvalidation) + crd.Spec.Names.Categories = crdCategories + + return &crd, nil } -// ConfigurationCRD returns CustomResourceDefinition built from OperatorConfigCRDResource -func ConfigurationCRD(enableValidation *bool) *apiextv1.CustomResourceDefinition { - opconfigCRDvalidation := apiextv1.CustomResourceValidation{} +//go:embed operatorconfiguration.crd.yaml +var operatorConfigurationCRDYAML []byte - if enableValidation != nil && *enableValidation { - opconfigCRDvalidation = OperatorConfigCRDResourceValidation +// OperatorConfigurationCRD returns CustomResourceDefinition built from OperatorConfigurationCRDResource +func OperatorConfigurationCRD(crdCategories []string) (*apiextv1.CustomResourceDefinition, error) { + var crd apiextv1.CustomResourceDefinition + err := yaml.Unmarshal(operatorConfigurationCRDYAML, &crd) + if err != nil { + return nil, err } - return buildCRD(OperatorConfigCRDResourceName, - OperatorConfigCRDResouceKind, - OperatorConfigCRDResourcePlural, - OperatorConfigCRDResourceShort, - OperatorConfigCRDResourceColumns, - opconfigCRDvalidation) + crd.Spec.Names.Categories = crdCategories + + return &crd, nil } diff --git a/pkg/apis/acid.zalan.do/v1/marshal.go b/pkg/apis/acid.zalan.do/v1/marshal.go index 9521082fc..014214fed 100644 --- a/pkg/apis/acid.zalan.do/v1/marshal.go +++ b/pkg/apis/acid.zalan.do/v1/marshal.go @@ -31,7 +31,8 @@ func (m *MaintenanceWindow) UnmarshalJSON(data []byte) error { err error ) - parts := strings.Split(string(data[1:len(data)-1]), "-") + dataStr := strings.Trim(string(data), "\"") + parts := strings.Split(dataStr, "-") if len(parts) != 2 { return fmt.Errorf("incorrect maintenance window format") } @@ -81,7 +82,7 @@ func (ps *PostgresStatus) UnmarshalJSON(data []byte) error { if err != nil { metaErr := json.Unmarshal(data, &status) if metaErr != nil { - return fmt.Errorf("Could not parse status: %v; err %v", string(data), metaErr) + return fmt.Errorf("could not parse status: %v; err %v", string(data), metaErr) } tmp.PostgresClusterStatus = status } @@ -110,15 +111,9 @@ func (p *Postgresql) UnmarshalJSON(data []byte) error { } tmp2 := Postgresql(tmp) - if clusterName, err := extractClusterName(tmp2.ObjectMeta.Name, tmp2.Spec.TeamID); err != nil { - tmp2.Error = err.Error() - tmp2.Status = PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid} - } else if err := validateCloneClusterDescription(tmp2.Spec.Clone); err != nil { - + if err := validateCloneClusterDescription(tmp2.Spec.Clone); err != nil { tmp2.Error = err.Error() tmp2.Status.PostgresClusterStatus = ClusterStatusInvalid - } else { - tmp2.Spec.ClusterName = clusterName } *p = tmp2 diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index cf581431b..60793c45c 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -13,11 +13,17 @@ import ( ) // +genclient -// +genclient:onlyVerbs=get -// +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // OperatorConfiguration defines the specification for the OperatorConfiguration. +// +k8s:deepcopy-gen=true +// +kubebuilder:resource:categories=all,shortName=opconfig,scope=Namespaced +// +kubebuilder:printcolumn:name="Image",type=string,JSONPath=`.configuration.docker_image`,description="Spilo image to be used for Pods" +// +kubebuilder:printcolumn:name="Cluster-Label",type=string,JSONPath=`.configuration.kubernetes.cluster_name_label`,description="Label for K8s resources created by operator" +// +kubebuilder:printcolumn:name="Service-Account",type=string,JSONPath=`.configuration.kubernetes.pod_service_account_name`,description="Name of service account to be used" +// +kubebuilder:printcolumn:name="Min-Instances",type=integer,JSONPath=`.configuration.min_instances`,description="Minimum number of instances per Postgres cluster" +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`,description="Age of the OperatorConfiguration resource" +// +kubebuilder:subresource:status type OperatorConfiguration struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata"` @@ -37,211 +43,440 @@ type OperatorConfigurationList struct { // PostgresUsersConfiguration defines the system users of Postgres. type PostgresUsersConfiguration struct { - SuperUsername string `json:"super_username,omitempty"` - ReplicationUsername string `json:"replication_username,omitempty"` + // +kubebuilder:default=postgres + SuperUsername string `json:"super_username,omitempty"` + // +kubebuilder:default=standby + ReplicationUsername string `json:"replication_username,omitempty"` + AdditionalOwnerRoles []string `json:"additional_owner_roles,omitempty"` + EnablePasswordRotation bool `json:"enable_password_rotation,omitempty"` + // +kubebuilder:default=90 + PasswordRotationInterval uint32 `json:"password_rotation_interval,omitempty"` + // +kubebuilder:default=120 + PasswordRotationUserRetention uint32 `json:"password_rotation_user_retention,omitempty"` } // MajorVersionUpgradeConfiguration defines how to execute major version upgrades of Postgres. type MajorVersionUpgradeConfiguration struct { - MajorVersionUpgradeMode string `json:"major_version_upgrade_mode" default:"off"` // off - no actions, manual - manifest triggers action, full - manifest and minimal version violation trigger upgrade - MinimalMajorVersion string `json:"minimal_major_version" default:"9.5"` - TargetMajorVersion string `json:"target_major_version" default:"13"` + // +kubebuilder:validation:Enum=off;manual;full + // +kubebuilder:default=manual + MajorVersionUpgradeMode string `json:"major_version_upgrade_mode,omitempty"` // off - no actions, manual - manifest triggers action, full - manifest and minimal version violation trigger upgrade + MajorVersionUpgradeTeamAllowList []string `json:"major_version_upgrade_team_allow_list,omitempty"` + // +kubebuilder:default="14" + MinimalMajorVersion string `json:"minimal_major_version,omitempty"` + // +kubebuilder:default="18" + TargetMajorVersion string `json:"target_major_version,omitempty"` } // KubernetesMetaConfiguration defines k8s conf required for all Postgres clusters and the operator itself type KubernetesMetaConfiguration struct { + EnableOwnerReferences *bool `json:"enable_owner_references,omitempty"` + // +kubebuilder:default=postgres-pod PodServiceAccountName string `json:"pod_service_account_name,omitempty"` // TODO: change it to the proper json - PodServiceAccountDefinition string `json:"pod_service_account_definition,omitempty"` - PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` - PodTerminateGracePeriod Duration `json:"pod_terminate_grace_period,omitempty"` - SpiloPrivileged bool `json:"spilo_privileged,omitempty"` - SpiloAllowPrivilegeEscalation *bool `json:"spilo_allow_privilege_escalation,omitempty"` - SpiloRunAsUser *int64 `json:"spilo_runasuser,omitempty"` - SpiloRunAsGroup *int64 `json:"spilo_runasgroup,omitempty"` - SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` - AdditionalPodCapabilities []string `json:"additional_pod_capabilities,omitempty"` - WatchedNamespace string `json:"watched_namespace,omitempty"` - PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` - EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` - StorageResizeMode string `json:"storage_resize_mode,omitempty"` - EnableInitContainers *bool `json:"enable_init_containers,omitempty"` - EnableSidecars *bool `json:"enable_sidecars,omitempty"` - SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` - ClusterDomain string `json:"cluster_domain,omitempty"` - OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` - InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` - InfrastructureRolesDefs []*config.InfrastructureRole `json:"infrastructure_roles_secrets,omitempty"` - PodRoleLabel string `json:"pod_role_label,omitempty"` - ClusterLabels map[string]string `json:"cluster_labels,omitempty"` - InheritedLabels []string `json:"inherited_labels,omitempty"` - InheritedAnnotations []string `json:"inherited_annotations,omitempty"` - DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` - ClusterNameLabel string `json:"cluster_name_label,omitempty"` - DeleteAnnotationDateKey string `json:"delete_annotation_date_key,omitempty"` - DeleteAnnotationNameKey string `json:"delete_annotation_name_key,omitempty"` - NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` - CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` + PodServiceAccountDefinition string `json:"pod_service_account_definition,omitempty"` + PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` + // +kubebuilder:default="5m" + // Postgres pods are terminated forcefully after this timeout + PodTerminateGracePeriod Duration `json:"pod_terminate_grace_period,omitempty"` + // +optional + LivenessProbe *v1.Probe `json:"liveness_probe"` + SpiloPrivileged bool `json:"spilo_privileged,omitempty"` + // +kubebuilder:default=true + SpiloAllowPrivilegeEscalation *bool `json:"spilo_allow_privilege_escalation,omitempty"` + SpiloRunAsUser *int64 `json:"spilo_runasuser,omitempty"` + SpiloRunAsGroup *int64 `json:"spilo_runasgroup,omitempty"` + SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` + AdditionalPodCapabilities []string `json:"additional_pod_capabilities,omitempty"` + WatchedNamespace string `json:"watched_namespace,omitempty"` + // +kubebuilder:default="postgres-{cluster}-pdb" + // defines the template for PDB names + PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` + // +kubebuilder:default=true + PDBMasterLabelSelector *bool `json:"pdb_master_label_selector,omitempty"` + // +kubebuilder:default=true + EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` + // +kubebuilder:validation:Enum=ebs;mixed;pvc;off + // +kubebuilder:default=pvc + StorageResizeMode string `json:"storage_resize_mode,omitempty"` + // +kubebuilder:default=true + EnableInitContainers *bool `json:"enable_init_containers,omitempty"` + // +kubebuilder:default=true + EnableSidecars *bool `json:"enable_sidecars,omitempty"` + SharePgSocketWithSidecars *bool `json:"share_pgsocket_with_sidecars,omitempty"` + // +kubebuilder:default="{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + // template for database user secrets generated by the operator, + // here username contains the namespace in the format namespace.username + // if the user is in different namespace than cluster and cross namespace secrets + // are enabled via `enable_cross_namespace_secret` flag in the configuration. + SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` + // +kubebuilder:default="cluster.local" + ClusterDomain string `json:"cluster_domain,omitempty"` + // +kubebuilder:default=postgres-operator + // namespaced name of the secret containing the OAuth2 token to pass to the teams API + OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` + InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` + // +kubebuilder:validation:Type=array + // namespaced name of the secret containing infrastructure roles names and passwords + InfrastructureRolesDefs []*config.InfrastructureRole `json:"infrastructure_roles_secrets,omitempty"` + // +kubebuilder:default=spilo-role + PodRoleLabel string `json:"pod_role_label,omitempty"` + // +kubebuilder:default={application: spilo} + ClusterLabels map[string]string `json:"cluster_labels,omitempty"` + InheritedLabels []string `json:"inherited_labels,omitempty"` + InheritedAnnotations []string `json:"inherited_annotations,omitempty"` + DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` + IgnoredAnnotations []string `json:"ignored_annotations,omitempty"` + // +kubebuilder:default=cluster-name + ClusterNameLabel string `json:"cluster_name_label,omitempty"` + DeleteAnnotationDateKey string `json:"delete_annotation_date_key,omitempty"` + DeleteAnnotationNameKey string `json:"delete_annotation_name_key,omitempty"` + NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` + // +kubebuilder:validation:Enum=AND;OR + NodeReadinessLabelMerge string `json:"node_readiness_label_merge,omitempty"` + CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? - PodToleration map[string]string `json:"toleration,omitempty"` - PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"` - PodEnvironmentSecret string `json:"pod_environment_secret,omitempty"` - PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` - MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` - EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` - PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` - PodManagementPolicy string `json:"pod_management_policy,omitempty"` + PodToleration map[string]string `json:"toleration,omitempty"` + // namespaced name of the ConfigMap with environment variables to populate on every pod + PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"` + PodEnvironmentSecret string `json:"pod_environment_secret,omitempty"` + PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` + // +kubebuilder:default="20m" + // timeout for successful migration of master pods from unschedulable node + MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` + EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` + PodAntiAffinityPreferredDuringScheduling bool `json:"pod_antiaffinity_preferred_during_scheduling,omitempty"` + // +kubebuilder:default="kubernetes.io/hostname" + PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` + // +kubebuilder:validation:Enum=ordered_ready;parallel + // +kubebuilder:default=ordered_ready + PodManagementPolicy string `json:"pod_management_policy,omitempty"` + PersistentVolumeClaimRetentionPolicy map[string]string `json:"persistent_volume_claim_retention_policy,omitempty"` + + // +kubebuilder:default=true + EnableSecretsDeletion *bool `json:"enable_secrets_deletion,omitempty"` + // +kubebuilder:default=true + EnablePersistentVolumeClaimDeletion *bool `json:"enable_persistent_volume_claim_deletion,omitempty"` + EnableReadinessProbe bool `json:"enable_readiness_probe,omitempty"` + EnableCrossNamespaceSecret bool `json:"enable_cross_namespace_secret,omitempty"` + EnableFinalizers *bool `json:"enable_finalizers,omitempty"` } // PostgresPodResourcesDefaults defines the spec of default resources type PostgresPodResourcesDefaults struct { - DefaultCPURequest string `json:"default_cpu_request,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + DefaultCPURequest string `json:"default_cpu_request,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` DefaultMemoryRequest string `json:"default_memory_request,omitempty"` - DefaultCPULimit string `json:"default_cpu_limit,omitempty"` - DefaultMemoryLimit string `json:"default_memory_limit,omitempty"` - MinCPULimit string `json:"min_cpu_limit,omitempty"` - MinMemoryLimit string `json:"min_memory_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + DefaultCPULimit string `json:"default_cpu_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + DefaultMemoryLimit string `json:"default_memory_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + MinCPULimit string `json:"min_cpu_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + MinMemoryLimit string `json:"min_memory_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + MaxCPURequest string `json:"max_cpu_request,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + MaxMemoryRequest string `json:"max_memory_request,omitempty"` } // OperatorTimeouts defines the timeout of ResourceCheck, PodWait, ReadyWait type OperatorTimeouts struct { - ResourceCheckInterval Duration `json:"resource_check_interval,omitempty"` - ResourceCheckTimeout Duration `json:"resource_check_timeout,omitempty"` - PodLabelWaitTimeout Duration `json:"pod_label_wait_timeout,omitempty"` + // +kubebuilder:default="3s" + // interval to wait between consecutive attempts to check for some K8s resources + ResourceCheckInterval Duration `json:"resource_check_interval,omitempty"` + // +kubebuilder:default="10m" + // timeout when waiting for the presence of a certain K8s resource + ResourceCheckTimeout Duration `json:"resource_check_timeout,omitempty"` + // +kubebuilder:default="10m" + // timeout when waiting for pod role and cluster labels + PodLabelWaitTimeout Duration `json:"pod_label_wait_timeout,omitempty"` + // +kubebuilder:default="10m" + // timeout when waiting for the Postgres pods to be deleted PodDeletionWaitTimeout Duration `json:"pod_deletion_wait_timeout,omitempty"` - ReadyWaitInterval Duration `json:"ready_wait_interval,omitempty"` - ReadyWaitTimeout Duration `json:"ready_wait_timeout,omitempty"` + // +kubebuilder:default="4s" + // interval between consecutive attempts waiting for postgresql CRD to be created + ReadyWaitInterval Duration `json:"ready_wait_interval,omitempty"` + // +kubebuilder:default="30s" + // timeout for the complete postgres CRD creation + ReadyWaitTimeout Duration `json:"ready_wait_timeout,omitempty"` + // +kubebuilder:default="1s" + // interval between consecutive attempts of operator calling the Patroni API + PatroniAPICheckInterval Duration `json:"patroni_api_check_interval,omitempty"` + // +kubebuilder:default="5s" + // timeout when waiting for successful response from Patroni API + PatroniAPICheckTimeout Duration `json:"patroni_api_check_timeout,omitempty"` } // LoadBalancerConfiguration defines the LB configuration type LoadBalancerConfiguration struct { - DbHostedZone string `json:"db_hosted_zone,omitempty"` - EnableMasterLoadBalancer bool `json:"enable_master_load_balancer,omitempty"` - EnableReplicaLoadBalancer bool `json:"enable_replica_load_balancer,omitempty"` - CustomServiceAnnotations map[string]string `json:"custom_service_annotations,omitempty"` - MasterDNSNameFormat config.StringTemplate `json:"master_dns_name_format,omitempty"` - ReplicaDNSNameFormat config.StringTemplate `json:"replica_dns_name_format,omitempty"` - ExternalTrafficPolicy string `json:"external_traffic_policy" default:"Cluster"` + DbHostedZone string `json:"db_hosted_zone,omitempty"` + EnableMasterLoadBalancer bool `json:"enable_master_load_balancer,omitempty"` + EnableMasterPoolerLoadBalancer bool `json:"enable_master_pooler_load_balancer,omitempty"` + EnableReplicaLoadBalancer bool `json:"enable_replica_load_balancer,omitempty"` + EnableReplicaPoolerLoadBalancer bool `json:"enable_replica_pooler_load_balancer,omitempty"` + + // NodePort flags kept in LoadBalancerConfiguration because all the other parameters apply here too + + EnableMasterNodePort bool `json:"enable_master_node_port,omitempty"` + EnableMasterPoolerNodePort bool `json:"enable_master_pooler_node_port,omitempty"` + EnableReplicaNodePort bool `json:"enable_replica_node_port,omitempty"` + EnableReplicaPoolerNodePort bool `json:"enable_replica_pooler_node_port,omitempty"` + + CustomServiceAnnotations map[string]string `json:"custom_service_annotations,omitempty"` + // +kubebuilder:default="{cluster}.{namespace}.{hostedzone}" + // defines the DNS name string template for the master load balancer cluster + MasterDNSNameFormat config.StringTemplate `json:"master_dns_name_format,omitempty"` + // +kubebuilder:default="{cluster}.{team}.{hostedzone}" + // deprecated DNS template for master load balancer using team name + MasterLegacyDNSNameFormat config.StringTemplate `json:"master_legacy_dns_name_format,omitempty"` + // +kubebuilder:default="{cluster}-repl.{namespace}.{hostedzone}" + // defines the DNS name string template for the replica load balancer cluster + ReplicaDNSNameFormat config.StringTemplate `json:"replica_dns_name_format,omitempty"` + // +kubebuilder:default="{cluster}-repl.{team}.{hostedzone}" + // deprecated DNS template for replica load balancer using team name + ReplicaLegacyDNSNameFormat config.StringTemplate `json:"replica_legacy_dns_name_format,omitempty"` + // +kubebuilder:validation:Enum=Cluster;Local + // +kubebuilder:default=Cluster + ExternalTrafficPolicy string `json:"external_traffic_policy,omitempty"` } // AWSGCPConfiguration defines the configuration for AWS // TODO complete Google Cloud Platform (GCP) configuration type AWSGCPConfiguration struct { - WALES3Bucket string `json:"wal_s3_bucket,omitempty"` + WALES3Bucket string `json:"wal_s3_bucket,omitempty"` + // +kubebuilder:default=eu-central-1 AWSRegion string `json:"aws_region,omitempty"` WALGSBucket string `json:"wal_gs_bucket,omitempty"` GCPCredentials string `json:"gcp_credentials,omitempty"` + WALAZStorageAccount string `json:"wal_az_storage_account,omitempty"` LogS3Bucket string `json:"log_s3_bucket,omitempty"` KubeIAMRole string `json:"kube_iam_role,omitempty"` AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` - AdditionalSecretMountPath string `json:"additional_secret_mount_path" default:"/meta/credentials"` - EnableEBSGp3Migration bool `json:"enable_ebs_gp3_migration" default:"false"` - EnableEBSGp3MigrationMaxSize int64 `json:"enable_ebs_gp3_migration_max_size" default:"1000"` + AdditionalSecretMountPath string `json:"additional_secret_mount_path,omitempty"` + EnableEBSGp3Migration bool `json:"enable_ebs_gp3_migration,omitempty"` + EnableEBSGp3MigrationMaxSize int64 `json:"enable_ebs_gp3_migration_max_size,omitempty"` } // OperatorDebugConfiguration defines options for the debug mode type OperatorDebugConfiguration struct { - DebugLogging bool `json:"debug_logging,omitempty"` - EnableDBAccess bool `json:"enable_database_access,omitempty"` + // +kubebuilder:default=true + DebugLogging *bool `json:"debug_logging,omitempty"` + // +kubebuilder:default=true + EnableDBAccess *bool `json:"enable_database_access,omitempty"` } // TeamsAPIConfiguration defines the configuration of TeamsAPI type TeamsAPIConfiguration struct { - EnableTeamsAPI bool `json:"enable_teams_api,omitempty"` - TeamsAPIUrl string `json:"teams_api_url,omitempty"` - TeamAPIRoleConfiguration map[string]string `json:"team_api_role_configuration,omitempty"` - EnableTeamSuperuser bool `json:"enable_team_superuser,omitempty"` - EnableAdminRoleForUsers bool `json:"enable_admin_role_for_users,omitempty"` - TeamAdminRole string `json:"team_admin_role,omitempty"` - PamRoleName string `json:"pam_role_name,omitempty"` - PamConfiguration string `json:"pam_configuration,omitempty"` - ProtectedRoles []string `json:"protected_role_names,omitempty"` - PostgresSuperuserTeams []string `json:"postgres_superuser_teams,omitempty"` - EnablePostgresTeamCRD bool `json:"enable_postgres_team_crd,omitempty"` - EnablePostgresTeamCRDSuperusers bool `json:"enable_postgres_team_crd_superusers,omitempty"` - EnableTeamMemberDeprecation bool `json:"enable_team_member_deprecation,omitempty"` - RoleDeletionSuffix string `json:"role_deletion_suffix,omitempty"` + EnableTeamsAPI bool `json:"enable_teams_api,omitempty"` + // +kubebuilder:default="https://teams.example.com/api/" + TeamsAPIUrl string `json:"teams_api_url,omitempty"` + // +kubebuilder:default={log_statement: all} + TeamAPIRoleConfiguration map[string]string `json:"team_api_role_configuration,omitempty"` + EnableTeamSuperuser bool `json:"enable_team_superuser,omitempty"` + // +kubebuilder:default=true + EnableAdminRoleForUsers bool `json:"enable_admin_role_for_users,omitempty"` + // +kubebuilder:default=admin + TeamAdminRole string `json:"team_admin_role,omitempty"` + // +kubebuilder:default=zalandos + PamRoleName string `json:"pam_role_name,omitempty"` + // +kubebuilder:default="https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees" + PamConfiguration string `json:"pam_configuration,omitempty"` + // +kubebuilder:default="[\"admin\", \"cron_admin\"]" + ProtectedRoles []string `json:"protected_role_names,omitempty"` + PostgresSuperuserTeams []string `json:"postgres_superuser_teams,omitempty"` + // +kubebuilder:default=true + EnablePostgresTeamCRD bool `json:"enable_postgres_team_crd,omitempty"` + EnablePostgresTeamCRDSuperusers bool `json:"enable_postgres_team_crd_superusers,omitempty"` + EnableTeamMemberDeprecation bool `json:"enable_team_member_deprecation,omitempty"` + // +kubebuilder:default=_deleted + RoleDeletionSuffix string `json:"role_deletion_suffix,omitempty"` } // LoggingRESTAPIConfiguration defines Logging API conf type LoggingRESTAPIConfiguration struct { - APIPort int `json:"api_port,omitempty"` - RingLogLines int `json:"ring_log_lines,omitempty"` + // +kubebuilder:default=8080 + APIPort int `json:"api_port,omitempty"` + // +kubebuilder:default=100 + RingLogLines int `json:"ring_log_lines,omitempty"` + // +kubebuilder:default=1000 ClusterHistoryEntries int `json:"cluster_history_entries,omitempty"` } // ScalyrConfiguration defines the configuration for ScalyrAPI type ScalyrConfiguration struct { - ScalyrAPIKey string `json:"scalyr_api_key,omitempty"` - ScalyrImage string `json:"scalyr_image,omitempty"` - ScalyrServerURL string `json:"scalyr_server_url,omitempty"` - ScalyrCPURequest string `json:"scalyr_cpu_request,omitempty"` + ScalyrAPIKey string `json:"scalyr_api_key,omitempty"` + ScalyrImage string `json:"scalyr_image,omitempty"` + // +kubebuilder:default="https://upload.eu.scalyr.com" + ScalyrServerURL string `json:"scalyr_server_url,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + // +kubebuilder:default="100m" + ScalyrCPURequest string `json:"scalyr_cpu_request,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + // +kubebuilder:default="50Mi" ScalyrMemoryRequest string `json:"scalyr_memory_request,omitempty"` - ScalyrCPULimit string `json:"scalyr_cpu_limit,omitempty"` - ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + // +kubebuilder:default="1" + ScalyrCPULimit string `json:"scalyr_cpu_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + // +kubebuilder:default="500Mi" + ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` } // ConnectionPoolerConfiguration defines default configuration for connection pooler type ConnectionPoolerConfiguration struct { - NumberOfInstances *int32 `json:"connection_pooler_number_of_instances,omitempty"` - Schema string `json:"connection_pooler_schema,omitempty"` - User string `json:"connection_pooler_user,omitempty"` - Image string `json:"connection_pooler_image,omitempty"` - Mode string `json:"connection_pooler_mode,omitempty"` - MaxDBConnections *int32 `json:"connection_pooler_max_db_connections,omitempty"` - DefaultCPURequest string `json:"connection_pooler_default_cpu_request,omitempty"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=2 + NumberOfInstances *int32 `json:"connection_pooler_number_of_instances,omitempty"` + // +kubebuilder:default=pooler + Schema string `json:"connection_pooler_schema,omitempty"` + // +kubebuilder:default=pooler + User string `json:"connection_pooler_user,omitempty"` + // +kubebuilder:default="ghcr.io/zalando/postgres-operator/pgbouncer:latest" + Image string `json:"connection_pooler_image,omitempty"` + // +kubebuilder:validation:Enum=session;transaction + // +kubebuilder:default=transaction + Mode string `json:"connection_pooler_mode,omitempty"` + MaxDBConnections *int32 `json:"connection_pooler_max_db_connections,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + DefaultCPURequest string `json:"connection_pooler_default_cpu_request,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` DefaultMemoryRequest string `json:"connection_pooler_default_memory_request,omitempty"` - DefaultCPULimit string `json:"connection_pooler_default_cpu_limit,omitempty"` - DefaultMemoryLimit string `json:"connection_pooler_default_memory_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + DefaultCPULimit string `json:"connection_pooler_default_cpu_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + DefaultMemoryLimit string `json:"connection_pooler_default_memory_limit,omitempty"` } // OperatorLogicalBackupConfiguration defines configuration for logical backup type OperatorLogicalBackupConfiguration struct { - Schedule string `json:"logical_backup_schedule,omitempty"` - DockerImage string `json:"logical_backup_docker_image,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$` + // +kubebuilder:default="30 00 * * *" + Schedule string `json:"logical_backup_schedule,omitempty"` + // +kubebuilder:default="ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1" + DockerImage string `json:"logical_backup_docker_image,omitempty"` + // +kubebuilder:validation:Enum=az;gcs;s3 + // +kubebuilder:default=s3 BackupProvider string `json:"logical_backup_provider,omitempty"` + AzureStorageAccountName string `json:"logical_backup_azure_storage_account_name,omitempty"` + AzureStorageContainer string `json:"logical_backup_azure_storage_container,omitempty"` + AzureStorageAccountKey string `json:"logical_backup_azure_storage_account_key,omitempty"` S3Bucket string `json:"logical_backup_s3_bucket,omitempty"` + S3BucketPrefix string `json:"logical_backup_s3_bucket_prefix,omitempty"` S3Region string `json:"logical_backup_s3_region,omitempty"` S3Endpoint string `json:"logical_backup_s3_endpoint,omitempty"` S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` S3SSE string `json:"logical_backup_s3_sse,omitempty"` + RetentionTime string `json:"logical_backup_s3_retention_time,omitempty"` GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"` - JobPrefix string `json:"logical_backup_job_prefix,omitempty"` + // +kubebuilder:default=logical-backup- + JobPrefix string `json:"logical_backup_job_prefix,omitempty"` + CronjobEnvironmentSecret string `json:"logical_backup_cronjob_environment_secret,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + CPURequest string `json:"logical_backup_cpu_request,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + MemoryRequest string `json:"logical_backup_memory_request,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + CPULimit string `json:"logical_backup_cpu_limit,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + MemoryLimit string `json:"logical_backup_memory_limit,omitempty"` + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=3 + SuccessfulJobsHistoryLimit *int32 `json:"logical_backup_successful_jobs_history_limit,omitempty"` + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=3 + FailedJobsHistoryLimit *int32 `json:"logical_backup_failed_jobs_history_limit,omitempty"` + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=86400 + TTLSecondsAfterFinished *int32 `json:"logical_backup_ttl_seconds_after_finished,omitempty"` +} + +// PatroniConfiguration defines configuration for Patroni +type PatroniConfiguration struct { + FailsafeMode *bool `json:"enable_patroni_failsafe_mode,omitempty"` } // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { - EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` - EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` - EnablePgVersionEnvVar bool `json:"enable_pgversion_env_var,omitempty"` - EnableSpiloWalPathCompat bool `json:"enable_spilo_wal_path_compat,omitempty"` - EtcdHost string `json:"etcd_host,omitempty"` - KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` - DockerImage string `json:"docker_image,omitempty"` - Workers uint32 `json:"workers,omitempty"` - MinInstances int32 `json:"min_instances,omitempty"` - MaxInstances int32 `json:"max_instances,omitempty"` - ResyncPeriod Duration `json:"resync_period,omitempty"` - RepairPeriod Duration `json:"repair_period,omitempty"` - SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` - ShmVolume *bool `json:"enable_shm_volume,omitempty"` - SidecarImages map[string]string `json:"sidecar_docker_images,omitempty"` // deprecated in favour of SidecarContainers - SidecarContainers []v1.Container `json:"sidecars,omitempty"` - PostgresUsersConfiguration PostgresUsersConfiguration `json:"users"` - MajorVersionUpgrade MajorVersionUpgradeConfiguration `json:"major_version_upgrade"` - Kubernetes KubernetesMetaConfiguration `json:"kubernetes"` - PostgresPodResources PostgresPodResourcesDefaults `json:"postgres_pod_resources"` - Timeouts OperatorTimeouts `json:"timeouts"` - LoadBalancer LoadBalancerConfiguration `json:"load_balancer"` - AWSGCP AWSGCPConfiguration `json:"aws_or_gcp"` - OperatorDebug OperatorDebugConfiguration `json:"debug"` - TeamsAPI TeamsAPIConfiguration `json:"teams_api"` - LoggingRESTAPI LoggingRESTAPIConfiguration `json:"logging_rest_api"` - Scalyr ScalyrConfiguration `json:"scalyr"` - LogicalBackup OperatorLogicalBackupConfiguration `json:"logical_backup"` - ConnectionPooler ConnectionPoolerConfiguration `json:"connection_pooler"` + // +kubebuilder:default=true + EnableCRDRegistration *bool `json:"enable_crd_registration,omitempty"` + CRDCategories []string `json:"crd_categories,omitempty"` + EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` + // +kubebuilder:default=true + EnablePgVersionEnvVar bool `json:"enable_pgversion_env_var,omitempty"` + EnableSpiloWalPathCompat bool `json:"enable_spilo_wal_path_compat,omitempty"` + EnableTeamIdClusternamePrefix bool `json:"enable_team_id_clustername_prefix,omitempty"` + // +kubebuilder:default="" + EtcdHost string `json:"etcd_host,omitempty"` + // +kubebuilder:default=true + KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` + // +kubebuilder:default="ghcr.io/zalando/spilo-18:4.1-p1" + DockerImage string `json:"docker_image,omitempty"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=8 + Workers uint32 `json:"workers,omitempty"` + // +kubebuilder:default="30m" + // period between consecutive sync requests + ResyncPeriod Duration `json:"resync_period,omitempty"` + // +kubebuilder:default="5m" + // period between consecutive repair requests + RepairPeriod Duration `json:"repair_period,omitempty"` + // +kubebuilder:default=true + EnableMaintenanceWindows *bool `json:"enable_maintenance_windows,omitempty"` + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Type=array + MaintenanceWindows []MaintenanceWindow `json:"maintenance_windows,omitempty"` + SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` + // +kubebuilder:default=true + ShmVolume *bool `json:"enable_shm_volume,omitempty"` + SidecarImages map[string]string `json:"sidecar_docker_images,omitempty"` // deprecated in favour of SidecarContainers + // +kubebuilder:validation:XPreserveUnknownFields + // +kubebuilder:validation:Type=object + // +kubebuilder:validation:Schemaless + SidecarContainers []v1.Container `json:"sidecars,omitempty"` + // +optional + PostgresUsersConfiguration PostgresUsersConfiguration `json:"users"` + // +optional + MajorVersionUpgrade MajorVersionUpgradeConfiguration `json:"major_version_upgrade"` + // +optional + Kubernetes KubernetesMetaConfiguration `json:"kubernetes"` + // +optional + PostgresPodResources PostgresPodResourcesDefaults `json:"postgres_pod_resources"` + // +optional + Timeouts OperatorTimeouts `json:"timeouts"` + // +optional + LoadBalancer LoadBalancerConfiguration `json:"load_balancer"` + // +optional + AWSGCP AWSGCPConfiguration `json:"aws_or_gcp"` + // +optional + OperatorDebug OperatorDebugConfiguration `json:"debug"` + // +optional + TeamsAPI TeamsAPIConfiguration `json:"teams_api"` + // +optional + LoggingRESTAPI LoggingRESTAPIConfiguration `json:"logging_rest_api"` + // +optional + Scalyr ScalyrConfiguration `json:"scalyr"` + // +optional + LogicalBackup OperatorLogicalBackupConfiguration `json:"logical_backup"` + // +optional + ConnectionPooler ConnectionPoolerConfiguration `json:"connection_pooler"` + // +optional + Patroni PatroniConfiguration `json:"patroni"` + + // +kubebuilder:validation:Minimum=-1 + // +kubebuilder:default=-1 + // -1 = disabled + MinInstances int32 `json:"min_instances,omitempty"` + // +kubebuilder:validation:Minimum=-1 + // +kubebuilder:default=-1 + // -1 = disabled + MaxInstances int32 `json:"max_instances,omitempty"` + + IgnoreInstanceLimitsAnnotationKey string `json:"ignore_instance_limits_annotation_key,omitempty"` + IgnoreResourcesLimitsAnnotationKey string `json:"ignore_resources_limits_annotation_key,omitempty"` } -//Duration shortens this frequently used name +// Duration shortens this frequently used name type Duration time.Duration diff --git a/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml b/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml new file mode 100644 index 000000000..5f347f2ac --- /dev/null +++ b/pkg/apis/acid.zalan.do/v1/operatorconfiguration.crd.yaml @@ -0,0 +1,971 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: operatorconfigurations.acid.zalan.do +spec: + group: acid.zalan.do + names: + categories: + - all + kind: OperatorConfiguration + listKind: OperatorConfigurationList + plural: operatorconfigurations + shortNames: + - opconfig + singular: operatorconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Spilo image to be used for Pods + jsonPath: .configuration.docker_image + name: Image + type: string + - description: Label for K8s resources created by operator + jsonPath: .configuration.kubernetes.cluster_name_label + name: Cluster-Label + type: string + - description: Name of service account to be used + jsonPath: .configuration.kubernetes.pod_service_account_name + name: Service-Account + type: string + - description: Minimum number of instances per Postgres cluster + jsonPath: .configuration.min_instances + name: Min-Instances + type: integer + - description: Age of the OperatorConfiguration resource + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: OperatorConfiguration defines the specification for the OperatorConfiguration. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + configuration: + description: OperatorConfigurationData defines the operation config + properties: + aws_or_gcp: + description: AWSGCPConfiguration defines the configuration for AWS + properties: + additional_secret_mount: + type: string + additional_secret_mount_path: + type: string + aws_region: + default: eu-central-1 + type: string + enable_ebs_gp3_migration: + type: boolean + enable_ebs_gp3_migration_max_size: + format: int64 + type: integer + gcp_credentials: + type: string + kube_iam_role: + type: string + log_s3_bucket: + type: string + wal_az_storage_account: + type: string + wal_gs_bucket: + type: string + wal_s3_bucket: + type: string + type: object + connection_pooler: + description: ConnectionPoolerConfiguration defines default configuration + for connection pooler + properties: + connection_pooler_default_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + connection_pooler_default_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + connection_pooler_default_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + connection_pooler_default_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + connection_pooler_image: + default: ghcr.io/zalando/postgres-operator/pgbouncer:latest + type: string + connection_pooler_max_db_connections: + format: int32 + type: integer + connection_pooler_mode: + default: transaction + enum: + - session + - transaction + type: string + connection_pooler_number_of_instances: + default: 2 + format: int32 + minimum: 1 + type: integer + connection_pooler_schema: + default: pooler + type: string + connection_pooler_user: + default: pooler + type: string + type: object + crd_categories: + items: + type: string + type: array + debug: + description: OperatorDebugConfiguration defines options for the debug + mode + properties: + debug_logging: + default: true + type: boolean + enable_database_access: + default: true + type: boolean + type: object + docker_image: + default: ghcr.io/zalando/spilo-18:4.1-p1 + type: string + enable_crd_registration: + default: true + type: boolean + enable_lazy_spilo_upgrade: + type: boolean + enable_maintenance_windows: + default: true + type: boolean + enable_pgversion_env_var: + default: true + type: boolean + enable_shm_volume: + default: true + type: boolean + enable_spilo_wal_path_compat: + type: boolean + enable_team_id_clustername_prefix: + type: boolean + etcd_host: + default: "" + type: string + ignore_instance_limits_annotation_key: + type: string + ignore_resources_limits_annotation_key: + type: string + kubernetes: + description: KubernetesMetaConfiguration defines k8s conf required + for all Postgres clusters and the operator itself + properties: + additional_pod_capabilities: + items: + type: string + type: array + cluster_domain: + default: cluster.local + type: string + cluster_labels: + additionalProperties: + type: string + default: + application: spilo + type: object + cluster_name_label: + default: cluster-name + type: string + custom_pod_annotations: + additionalProperties: + type: string + type: object + delete_annotation_date_key: + type: string + delete_annotation_name_key: + type: string + downscaler_annotations: + items: + type: string + type: array + enable_cross_namespace_secret: + type: boolean + enable_finalizers: + type: boolean + enable_init_containers: + default: true + type: boolean + enable_owner_references: + type: boolean + enable_persistent_volume_claim_deletion: + default: true + type: boolean + enable_pod_antiaffinity: + type: boolean + enable_pod_disruption_budget: + default: true + type: boolean + enable_readiness_probe: + type: boolean + enable_secrets_deletion: + default: true + type: boolean + enable_sidecars: + default: true + type: boolean + ignored_annotations: + items: + type: string + type: array + infrastructure_roles_secret_name: + description: |- + NamespacedName comprises a resource name, with a mandatory namespace, + rendered as "/". Being a type captures intent and + helps make sure that UIDs, namespaced names and non-namespaced names + do not get conflated in code. For most use cases, namespace and name + will already have been format validated at the API entry point, so we + don't do that here. Where that's not the case (e.g. in testing), + consider using NamespacedNameOrDie() in testing.go in this package. + + from: https://github.com/kubernetes/apimachinery/blob/master/pkg/types/namespacedname.go + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + infrastructure_roles_secrets: + description: namespaced name of the secret containing infrastructure + roles names and passwords + items: + properties: + defaultrolevalue: + type: string + defaultuservalue: + type: string + details: + description: This field point out the detailed yaml definition + of the role, if exists + type: string + passwordkey: + type: string + rolekey: + type: string + secretname: + description: |- + Name of a secret which describes the role, and optionally name of a + configmap with an extra information + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + template: + type: boolean + userkey: + type: string + type: object + type: array + inherited_annotations: + items: + type: string + type: array + inherited_labels: + items: + type: string + type: array + liveness_probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + master_pod_move_timeout: + default: 20m + description: timeout for successful migration of master pods from + unschedulable node + format: int64 + type: integer + node_readiness_label: + additionalProperties: + type: string + type: object + node_readiness_label_merge: + enum: + - AND + - OR + type: string + oauth_token_secret_name: + default: postgres-operator + description: namespaced name of the secret containing the OAuth2 + token to pass to the teams API + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + pdb_master_label_selector: + default: true + type: boolean + pdb_name_format: + default: postgres-{cluster}-pdb + description: defines the template for PDB names + type: string + persistent_volume_claim_retention_policy: + additionalProperties: + type: string + type: object + pod_antiaffinity_preferred_during_scheduling: + type: boolean + pod_antiaffinity_topology_key: + default: kubernetes.io/hostname + type: string + pod_environment_configmap: + description: namespaced name of the ConfigMap with environment + variables to populate on every pod + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + pod_environment_secret: + type: string + pod_management_policy: + default: ordered_ready + enum: + - ordered_ready + - parallel + type: string + pod_priority_class_name: + type: string + pod_role_label: + default: spilo-role + type: string + pod_service_account_definition: + type: string + pod_service_account_name: + default: postgres-pod + type: string + pod_service_account_role_binding_definition: + type: string + pod_terminate_grace_period: + default: 5m + description: Postgres pods are terminated forcefully after this + timeout + format: int64 + type: integer + secret_name_template: + default: '{username}.{cluster}.credentials.{tprkind}.{tprgroup}' + description: |- + template for database user secrets generated by the operator, + here username contains the namespace in the format namespace.username + if the user is in different namespace than cluster and cross namespace secrets + are enabled via `enable_cross_namespace_secret` flag in the configuration. + type: string + share_pgsocket_with_sidecars: + type: boolean + spilo_allow_privilege_escalation: + default: true + type: boolean + spilo_fsgroup: + format: int64 + type: integer + spilo_privileged: + type: boolean + spilo_runasgroup: + format: int64 + type: integer + spilo_runasuser: + format: int64 + type: integer + storage_resize_mode: + default: pvc + enum: + - ebs + - mixed + - pvc + - "off" + type: string + toleration: + additionalProperties: + type: string + type: object + watched_namespace: + type: string + type: object + kubernetes_use_configmaps: + default: true + type: boolean + load_balancer: + description: LoadBalancerConfiguration defines the LB configuration + properties: + custom_service_annotations: + additionalProperties: + type: string + type: object + db_hosted_zone: + type: string + enable_master_load_balancer: + type: boolean + enable_master_node_port: + type: boolean + enable_master_pooler_load_balancer: + type: boolean + enable_master_pooler_node_port: + type: boolean + enable_replica_load_balancer: + type: boolean + enable_replica_node_port: + type: boolean + enable_replica_pooler_load_balancer: + type: boolean + enable_replica_pooler_node_port: + type: boolean + external_traffic_policy: + default: Cluster + enum: + - Cluster + - Local + type: string + master_dns_name_format: + default: '{cluster}.{namespace}.{hostedzone}' + description: defines the DNS name string template for the master + load balancer cluster + type: string + master_legacy_dns_name_format: + default: '{cluster}.{team}.{hostedzone}' + description: deprecated DNS template for master load balancer + using team name + type: string + replica_dns_name_format: + default: '{cluster}-repl.{namespace}.{hostedzone}' + description: defines the DNS name string template for the replica + load balancer cluster + type: string + replica_legacy_dns_name_format: + default: '{cluster}-repl.{team}.{hostedzone}' + description: deprecated DNS template for replica load balancer + using team name + type: string + type: object + logging_rest_api: + description: LoggingRESTAPIConfiguration defines Logging API conf + properties: + api_port: + default: 8080 + type: integer + cluster_history_entries: + default: 1000 + type: integer + ring_log_lines: + default: 100 + type: integer + type: object + logical_backup: + description: OperatorLogicalBackupConfiguration defines configuration + for logical backup + properties: + logical_backup_azure_storage_account_key: + type: string + logical_backup_azure_storage_account_name: + type: string + logical_backup_azure_storage_container: + type: string + logical_backup_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + logical_backup_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + logical_backup_cronjob_environment_secret: + type: string + logical_backup_docker_image: + default: ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1 + type: string + logical_backup_failed_jobs_history_limit: + default: 3 + format: int32 + minimum: 0 + type: integer + logical_backup_google_application_credentials: + type: string + logical_backup_job_prefix: + default: logical-backup- + type: string + logical_backup_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + logical_backup_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + logical_backup_provider: + default: s3 + enum: + - az + - gcs + - s3 + type: string + logical_backup_s3_access_key_id: + type: string + logical_backup_s3_bucket: + type: string + logical_backup_s3_bucket_prefix: + type: string + logical_backup_s3_endpoint: + type: string + logical_backup_s3_region: + type: string + logical_backup_s3_retention_time: + type: string + logical_backup_s3_secret_access_key: + type: string + logical_backup_s3_sse: + type: string + logical_backup_schedule: + default: 30 00 * * * + pattern: ^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$ + type: string + logical_backup_successful_jobs_history_limit: + default: 3 + format: int32 + minimum: 0 + type: integer + logical_backup_ttl_seconds_after_finished: + default: 86400 + format: int32 + minimum: 0 + type: integer + type: object + maintenance_windows: + type: array + major_version_upgrade: + description: MajorVersionUpgradeConfiguration defines how to execute + major version upgrades of Postgres. + properties: + major_version_upgrade_mode: + default: manual + enum: + - "off" + - manual + - full + type: string + major_version_upgrade_team_allow_list: + items: + type: string + type: array + minimal_major_version: + default: "14" + type: string + target_major_version: + default: "18" + type: string + type: object + max_instances: + default: -1 + description: -1 = disabled + format: int32 + minimum: -1 + type: integer + min_instances: + default: -1 + description: -1 = disabled + format: int32 + minimum: -1 + type: integer + patroni: + description: PatroniConfiguration defines configuration for Patroni + properties: + enable_patroni_failsafe_mode: + type: boolean + type: object + postgres_pod_resources: + description: PostgresPodResourcesDefaults defines the spec of default + resources + properties: + default_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + default_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + default_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + default_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + max_cpu_request: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + max_memory_request: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + min_cpu_limit: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + min_memory_limit: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + repair_period: + default: 5m + description: period between consecutive repair requests + format: int64 + type: integer + resync_period: + default: 30m + description: period between consecutive sync requests + format: int64 + type: integer + scalyr: + description: ScalyrConfiguration defines the configuration for ScalyrAPI + properties: + scalyr_api_key: + type: string + scalyr_cpu_limit: + default: "1" + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + scalyr_cpu_request: + default: 100m + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + scalyr_image: + type: string + scalyr_memory_limit: + default: 500Mi + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + scalyr_memory_request: + default: 50Mi + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + scalyr_server_url: + default: https://upload.eu.scalyr.com + type: string + type: object + set_memory_request_to_limit: + type: boolean + sidecar_docker_images: + additionalProperties: + type: string + type: object + sidecars: + type: object + x-kubernetes-preserve-unknown-fields: true + teams_api: + description: TeamsAPIConfiguration defines the configuration of TeamsAPI + properties: + enable_admin_role_for_users: + default: true + type: boolean + enable_postgres_team_crd: + default: true + type: boolean + enable_postgres_team_crd_superusers: + type: boolean + enable_team_member_deprecation: + type: boolean + enable_team_superuser: + type: boolean + enable_teams_api: + type: boolean + pam_configuration: + default: https://info.example.com/oauth2/tokeninfo?access_token= + uid realm=/employees + type: string + pam_role_name: + default: zalandos + type: string + postgres_superuser_teams: + items: + type: string + type: array + protected_role_names: + default: '["admin", "cron_admin"]' + items: + type: string + type: array + role_deletion_suffix: + default: _deleted + type: string + team_admin_role: + default: admin + type: string + team_api_role_configuration: + additionalProperties: + type: string + default: + log_statement: all + type: object + teams_api_url: + default: https://teams.example.com/api/ + type: string + type: object + timeouts: + description: OperatorTimeouts defines the timeout of ResourceCheck, + PodWait, ReadyWait + properties: + patroni_api_check_interval: + default: 1s + description: interval between consecutive attempts of operator + calling the Patroni API + format: int64 + type: integer + patroni_api_check_timeout: + default: 5s + description: timeout when waiting for successful response from + Patroni API + format: int64 + type: integer + pod_deletion_wait_timeout: + default: 10m + description: timeout when waiting for the Postgres pods to be + deleted + format: int64 + type: integer + pod_label_wait_timeout: + default: 10m + description: timeout when waiting for pod role and cluster labels + format: int64 + type: integer + ready_wait_interval: + default: 4s + description: interval between consecutive attempts waiting for + postgresql CRD to be created + format: int64 + type: integer + ready_wait_timeout: + default: 30s + description: timeout for the complete postgres CRD creation + format: int64 + type: integer + resource_check_interval: + default: 3s + description: interval to wait between consecutive attempts to + check for some K8s resources + format: int64 + type: integer + resource_check_timeout: + default: 10m + description: timeout when waiting for the presence of a certain + K8s resource + format: int64 + type: integer + type: object + users: + description: PostgresUsersConfiguration defines the system users of + Postgres. + properties: + additional_owner_roles: + items: + type: string + type: array + enable_password_rotation: + type: boolean + password_rotation_interval: + default: 90 + format: int32 + type: integer + password_rotation_user_retention: + default: 120 + format: int32 + type: integer + replication_username: + default: standby + type: string + super_username: + default: postgres + type: string + type: object + workers: + default: 8 + format: int32 + minimum: 1 + type: integer + type: object + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + required: + - configuration + - metadata + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/acid.zalan.do/v1/postgres_team_type.go b/pkg/apis/acid.zalan.do/v1/postgres_team_type.go index 5697c193e..ffedaef57 100644 --- a/pkg/apis/acid.zalan.do/v1/postgres_team_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgres_team_type.go @@ -8,18 +8,33 @@ import ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // PostgresTeam defines Custom Resource Definition Object for team management. +// +k8s:deepcopy-gen=true +// +kubebuilder:resource:shortName=pgteam,categories=all +// +kubebuilder:subresource:status type PostgresTeam struct { metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` + metav1.ObjectMeta `json:"metadata"` Spec PostgresTeamSpec `json:"spec"` } +// List of users who will also be added to the Postgres cluster. +type Users []string + +// List of teams whose members will also be added to the Postgres cluster. +type Teams []string + +// List of teams to become Postgres superusers +type SuperUserTeams []string + // PostgresTeamSpec defines the specification for the PostgresTeam TPR. type PostgresTeamSpec struct { - AdditionalSuperuserTeams map[string][]string `json:"additionalSuperuserTeams,omitempty"` - AdditionalTeams map[string][]string `json:"additionalTeams,omitempty"` - AdditionalMembers map[string][]string `json:"additionalMembers,omitempty"` + // Map for teamId and associated additional superuser teams + AdditionalSuperuserTeams map[string]SuperUserTeams `json:"additionalSuperuserTeams,omitempty"` + // Map for teamId and associated additional teams + AdditionalTeams map[string]Teams `json:"additionalTeams,omitempty"` + // Map for teamId and associated additional users + AdditionalMembers map[string]Users `json:"additionalMembers,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml b/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml new file mode 100644 index 000000000..9b855eb68 --- /dev/null +++ b/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml @@ -0,0 +1,4836 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: postgresqls.acid.zalan.do +spec: + group: acid.zalan.do + names: + categories: + - all + kind: postgresql + listKind: postgresqlList + plural: postgresqls + shortNames: + - pg + singular: postgresql + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Team responsible for Postgres cluster + jsonPath: .spec.teamId + name: Team + type: string + - description: PostgreSQL version + jsonPath: .spec.postgresql.version + name: Version + type: string + - description: Number of Pods per Postgres cluster + jsonPath: .spec.numberOfInstances + name: Pods + type: integer + - description: Size of the bound volume + jsonPath: .spec.volume.size + name: Volume + type: string + - description: Requested CPU for Postgres containers + jsonPath: .spec.resources.requests.cpu + name: CPU-Request + type: string + - description: Requested memory for Postgres containers + jsonPath: .spec.resources.requests.memory + name: Memory-Request + type: string + - description: Age of the PostgreSQL cluster + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Current sync status of postgresql resource + jsonPath: .status.PostgresClusterStatus + name: Status + type: string + name: v1 + schema: + openAPIV3Schema: + description: Postgresql defines PostgreSQL Custom Resource Definition Object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: PostgresSpec defines the specification for the PostgreSQL + TPR. + properties: + additionalVolumes: + items: + description: AdditionalVolume specs additional optional volumes + for statefulset + properties: + isSubPathExpr: + type: boolean + mountPath: + type: string + name: + type: string + subPath: + type: string + targetContainers: + items: + type: string + nullable: true + type: array + volumeSource: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - mountPath + - name + - volumeSource + type: object + type: array + allowedSourceRanges: + description: load balancers' source ranges are the same for master + and replica services + items: + pattern: ^((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:))\/(12[0-8]|1[01][0-9]|[1-9]?[0-9]))$ + type: string + nullable: true + type: array + clone: + description: CloneDescription describes which cluster the new should + clone and up to which point in time + properties: + cluster: + type: string + s3_access_key_id: + type: string + s3_endpoint: + type: string + s3_force_path_style: + type: boolean + s3_secret_access_key: + type: string + s3_wal_path: + type: string + timestamp: + description: |- + The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + Example: 1996-12-19T16:39:57-08:00 + Note: this field requires a timezone + pattern: ^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$ + type: string + uid: + format: uuid + type: string + required: + - cluster + type: object + connectionPooler: + description: |- + ConnectionPooler Options for connection pooler + + pgbouncer-large (with higher resources) or odyssey-small (with smaller + resources) + Type string `json:"type,omitempty"` + + makes sense to expose. E.g. pool size (min/max boundaries), max client + connections etc. + properties: + dockerImage: + type: string + maxDBConnections: + format: int32 + type: integer + mode: + enum: + - session + - transaction + type: string + numberOfInstances: + format: int32 + minimum: 1 + type: integer + resources: + description: Resources describes requests and limits for the cluster + resouces. + properties: + limits: + description: ResourceDescription describes CPU and memory + resources defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + requests: + description: ResourceDescription describes CPU and memory + resources defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + type: object + schema: + type: string + user: + type: string + type: object + databases: + additionalProperties: + type: string + description: |- + Note: usernames specified here as database owners must be declared + in the users key of the spec key. + type: object + dockerImage: + type: string + enableConnectionPooler: + type: boolean + enableLogicalBackup: + type: boolean + enableMasterLoadBalancer: + description: |- + vars that enable load balancers are pointers because it is important to know if any of them is omitted from the Postgres manifest + in that case the var evaluates to nil and the value is taken from the operator config + type: boolean + enableMasterNodePort: + description: |- + vars to enable and configure nodeport services + set ports to 0 or nil to let kubernetes decide which port to use + overrides loadbalancer configuration + type: boolean + enableMasterPoolerLoadBalancer: + type: boolean + enableMasterPoolerNodePort: + type: boolean + enableReplicaConnectionPooler: + type: boolean + enableReplicaLoadBalancer: + type: boolean + enableReplicaNodePort: + type: boolean + enableReplicaPoolerLoadBalancer: + type: boolean + enableReplicaPoolerNodePort: + type: boolean + enableShmVolume: + type: boolean + env: + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + init_containers: + description: deprecated + items: + description: A single application container that you want to run + within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set + of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a + single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource resize + policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + initContainers: + items: + description: A single application container that you want to run + within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source may consist of any printable ASCII characters except '='. + When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set + of ConfigMaps or Secrets + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: |- + Optional text to prepend to the name of each environment variable. + May consist of any printable ASCII characters except '='. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in + the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to + perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container + should sleep. + properties: + seconds: + description: Seconds is the number of seconds to + sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + description: |- + StopSignal defines which signal will be sent to a container when it is being stopped. + If not specified, the default is defined by the container runtime in use. + StopSignal can only be set for Pods with a non-empty .spec.os.name + type: string + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a + single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource resize + policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This overrides the pod-level restart policy. When this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Additionally, setting the RestartPolicy as "Always" for the init container will + have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + restartPolicyRules: + description: |- + Represents a list of rules to be checked to determine if the + container should be restarted on exit. The rules are evaluated in + order. Once a rule matches a container exit condition, the remaining + rules are ignored. If no rule matches the container exit condition, + the Container-level restart policy determines the whether the container + is restarted or not. Constraints on the rules: + - At most 20 rules are allowed. + - Rules can have the same action. + - Identical rules are not forbidden in validations. + When rules are specified, container MUST set RestartPolicy explicitly + even it if matches the Pod's RestartPolicy. + items: + description: ContainerRestartRule describes how a container + exit is handled. + properties: + action: + description: |- + Specifies the action taken on a container exit if the requirements + are satisfied. The only possible value is "Restart" to restart the + container. + type: string + exitCodes: + description: Represents the exit codes to check on container + exits. + properties: + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Possible values are: + - In: the requirement is satisfied if the container exit code is in the + set of specified values. + - NotIn: the requirement is satisfied if the container exit code is + not in the set of specified values. + type: string + values: + description: |- + Specifies the set of values to check for container exit codes. + At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the + container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + livenessProbe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes a custom header to be + used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + logicalBackupRetention: + type: string + logicalBackupSchedule: + pattern: ^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$ + type: string + maintenanceWindows: + items: + pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$' + type: string + type: array + masterNodePort: + format: int32 + type: integer + masterPoolerNodePort: + format: int32 + type: integer + masterServiceAnnotations: + additionalProperties: + type: string + description: MasterServiceAnnotations takes precedence over ServiceAnnotations + for master role if not empty + type: object + nodeAffinity: + description: Node affinity is a group of node affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + numberOfInstances: + format: int32 + minimum: 0 + type: integer + patroni: + description: Patroni contains Patroni-specific configuration + properties: + failsafe_mode: + type: boolean + initdb: + additionalProperties: + type: string + type: object + loop_wait: + format: int32 + type: integer + maximum_lag_on_failover: + format: int64 + type: integer + pg_hba: + items: + type: string + type: array + retry_timeout: + format: int32 + type: integer + slots: + additionalProperties: + additionalProperties: + type: string + type: object + type: object + synchronous_mode: + type: boolean + synchronous_mode_strict: + type: boolean + synchronous_node_count: + format: int32 + type: integer + ttl: + format: int32 + type: integer + type: object + pod_priority_class_name: + description: deprecated + type: string + podAnnotations: + additionalProperties: + type: string + type: object + podPriorityClassName: + type: string + postgresql: + description: PostgresqlParam describes PostgreSQL version and pairs + of configuration parameter name - values. + properties: + parameters: + additionalProperties: + type: string + type: object + version: + enum: + - "14" + - "15" + - "16" + - "17" + - "18" + type: string + required: + - version + type: object + preparedDatabases: + additionalProperties: + description: PreparedDatabase describes elements to be bootstrapped + properties: + defaultUsers: + type: boolean + extensions: + additionalProperties: + type: string + type: object + schemas: + additionalProperties: + description: PreparedSchema describes elements to be bootstrapped + per schema + properties: + defaultRoles: + type: boolean + defaultUsers: + type: boolean + type: object + type: object + secretNamespace: + type: string + type: object + type: object + replicaLoadBalancer: + description: deprecated + type: boolean + replicaNodePort: + format: int32 + type: integer + replicaPoolerNodePort: + format: int32 + type: integer + replicaServiceAnnotations: + additionalProperties: + type: string + description: ReplicaServiceAnnotations takes precedence over ServiceAnnotations + for replica role if not empty + type: object + resources: + description: Resources describes requests and limits for the cluster + resouces. + properties: + limits: + description: ResourceDescription describes CPU and memory resources + defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + requests: + description: ResourceDescription describes CPU and memory resources + defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + type: object + schedulerName: + type: string + serviceAnnotations: + additionalProperties: + type: string + type: object + sidecars: + items: + description: Sidecar defines a container to be run in the same pod + as the Postgres container. + properties: + command: + items: + type: string + type: array + env: + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + type: string + name: + type: string + ports: + items: + description: ContainerPort represents a network port in a + single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + resources: + description: Resources describes requests and limits for the + cluster resouces. + properties: + limits: + description: ResourceDescription describes CPU and memory + resources defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + requests: + description: ResourceDescription describes CPU and memory + resources defined for a cluster. + properties: + cpu: + description: |- + Decimal natural followed by m, or decimal natural followed by + dot followed by up to three decimal digits. + + This is because the Kubernetes CPU resource has millis as the + maximum precision. The actual values are checked in code + because the regular expression would be huge and horrible and + not very helpful in validation error messages; this one checks + only the format of the given number. + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + + Note: the value specified here must not be zero or be lower + than the corresponding request. + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + hugepages-1Gi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + hugepages-2Mi: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + memory: + description: |- + You can express memory as a plain integer or as a fixed-point + integer using one of these suffixes: E, P, T, G, M, k. You can + also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + + https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + + Note: the value specified here must not be zero or be higher + than the corresponding limit. + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + type: object + type: object + type: object + type: array + spiloFSGroup: + format: int64 + type: integer + spiloRunAsGroup: + format: int64 + type: integer + spiloRunAsUser: + format: int64 + type: integer + standby: + anyOf: + - required: + - s3_wal_path + - required: + - gs_wal_path + - required: + - standby_host + not: + required: + - s3_wal_path + - gs_wal_path + description: |- + StandbyDescription contains remote primary config and/or s3/gs wal path. + standby_host can be specified alone or together with either s3_wal_path OR gs_wal_path (mutually exclusive). + At least one field must be specified. s3_wal_path and gs_wal_path are mutually exclusive. + properties: + gs_wal_path: + type: string + s3_wal_path: + type: string + standby_host: + type: string + standby_port: + type: string + standby_primary_slot_name: + type: string + type: object + streams: + items: + description: Stream defines properties for creating FabricEventStream + resources + properties: + applicationId: + type: string + batchSize: + format: int32 + type: integer + cpu: + pattern: ^(\d+m|\d+(\.\d{1,3})?)$ + type: string + database: + type: string + enableRecovery: + type: boolean + filter: + additionalProperties: + type: string + type: object + memory: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + tables: + additionalProperties: + description: StreamTable defines properties of outbox tables + for FabricEventStreams + properties: + eventType: + type: string + idColumn: + type: string + ignoreRecovery: + type: boolean + payloadColumn: + type: string + recoveryEventType: + type: string + required: + - eventType + type: object + type: object + required: + - applicationId + - database + - tables + type: object + type: array + teamId: + type: string + tls: + description: TLSDescription specs TLS properties + properties: + caFile: + type: string + caSecretName: + type: string + certificateFile: + type: string + privateKeyFile: + type: string + secretName: + type: string + required: + - secretName + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies how to spread matching + pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + useLoadBalancer: + description: |- + deprecated load balancer settings maintained for backward compatibility + see "Load balancers" operator docs + type: boolean + users: + additionalProperties: + description: UserFlags defines flags (such as superuser, nologin) + that could be assigned to individual users + items: + enum: + - bypassrls + - BYPASSRLS + - nobypassrls + - NOBYPASSRLS + - createdb + - CREATEDB + - nocreatedb + - NOCREATEDB + - createrole + - CREATEROLE + - nocreaterole + - NOCREATEROLE + - inherit + - INHERIT + - noinherit + - NOINHERIT + - login + - LOGIN + - nologin + - NOLOGIN + - replication + - REPLICATION + - noreplication + - NOREPLICATION + - superuser + - SUPERUSER + - nosuperuser + - NOSUPERUSER + type: string + type: array + type: object + usersIgnoringSecretRotation: + items: + type: string + nullable: true + type: array + usersWithInPlaceSecretRotation: + items: + type: string + nullable: true + type: array + usersWithSecretRotation: + items: + type: string + nullable: true + type: array + volume: + description: Volume describes a single volume in the manifest. + properties: + iops: + format: int64 + type: integer + isSubPathExpr: + type: boolean + selector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + size: + pattern: ^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$ + type: string + storageClass: + type: string + subPath: + type: string + throughput: + format: int64 + type: integer + type: + type: string + required: + - size + type: object + required: + - numberOfInstances + - postgresql + - teamId + - volume + type: object + status: + description: PostgresStatus contains status of the PostgreSQL cluster + (running, creation failed etc.) + properties: + PostgresClusterStatus: + type: string + required: + - PostgresClusterStatus + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 7346fb0e5..81efa42cc 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -11,13 +11,25 @@ import ( // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true // Postgresql defines PostgreSQL Custom Resource Definition Object. +// +kubebuilder:resource:categories=all,shortName=pg,scope=Namespaced +// +kubebuilder:printcolumn:name="Team",type=string,JSONPath=`.spec.teamId`,description="Team responsible for Postgres cluster" +// +kubebuilder:printcolumn:name="Version",type=string,JSONPath=`.spec.postgresql.version`,description="PostgreSQL version" +// +kubebuilder:printcolumn:name="Pods",type=integer,JSONPath=`.spec.numberOfInstances`,description="Number of Pods per Postgres cluster" +// +kubebuilder:printcolumn:name="Volume",type=string,JSONPath=`.spec.volume.size`,description="Size of the bound volume" +// +kubebuilder:printcolumn:name="CPU-Request",type=string,JSONPath=`.spec.resources.requests.cpu`,description="Requested CPU for Postgres containers" +// +kubebuilder:printcolumn:name="Memory-Request",type=string,JSONPath=`.spec.resources.requests.memory`,description="Requested memory for Postgres containers" +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`,description="Age of the PostgreSQL cluster" +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.PostgresClusterStatus`,description="Current sync status of postgresql resource" +// +kubebuilder:subresource:status type Postgresql struct { metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` + metav1.ObjectMeta `json:"metadata"` - Spec PostgresSpec `json:"spec"` + Spec PostgresSpec `json:"spec"` + // +optional Status PostgresStatus `json:"status"` Error string `json:"-"` } @@ -25,9 +37,10 @@ type Postgresql struct { // PostgresSpec defines the specification for the PostgreSQL TPR. type PostgresSpec struct { PostgresqlParam `json:"postgresql"` - Volume `json:"volume,omitempty"` - Patroni `json:"patroni,omitempty"` - Resources `json:"resources,omitempty"` + Volume `json:"volume"` + // +optional + Patroni `json:"patroni"` + *Resources `json:"resources,omitempty"` EnableConnectionPooler *bool `json:"enableConnectionPooler,omitempty"` EnableReplicaConnectionPooler *bool `json:"enableReplicaConnectionPooler,omitempty"` @@ -36,48 +49,91 @@ type PostgresSpec struct { TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` + // deprecated field storing cluster name without teamId prefix + ClusterName string `json:"-"` + SpiloRunAsUser *int64 `json:"spiloRunAsUser,omitempty"` SpiloRunAsGroup *int64 `json:"spiloRunAsGroup,omitempty"` SpiloFSGroup *int64 `json:"spiloFSGroup,omitempty"` // vars that enable load balancers are pointers because it is important to know if any of them is omitted from the Postgres manifest // in that case the var evaluates to nil and the value is taken from the operator config - EnableMasterLoadBalancer *bool `json:"enableMasterLoadBalancer,omitempty"` - EnableReplicaLoadBalancer *bool `json:"enableReplicaLoadBalancer,omitempty"` + EnableMasterLoadBalancer *bool `json:"enableMasterLoadBalancer,omitempty"` + EnableMasterPoolerLoadBalancer *bool `json:"enableMasterPoolerLoadBalancer,omitempty"` + EnableReplicaLoadBalancer *bool `json:"enableReplicaLoadBalancer,omitempty"` + EnableReplicaPoolerLoadBalancer *bool `json:"enableReplicaPoolerLoadBalancer,omitempty"` + + // vars to enable and configure nodeport services + // set ports to 0 or nil to let kubernetes decide which port to use + // overrides loadbalancer configuration + EnableMasterNodePort *bool `json:"enableMasterNodePort,omitempty"` + MasterNodePort *int32 `json:"masterNodePort,omitempty"` + EnableMasterPoolerNodePort *bool `json:"enableMasterPoolerNodePort,omitempty"` + MasterPoolerNodePort *int32 `json:"masterPoolerNodePort,omitempty"` + EnableReplicaNodePort *bool `json:"enableReplicaNodePort,omitempty"` + ReplicaNodePort *int32 `json:"replicaNodePort,omitempty"` + EnableReplicaPoolerNodePort *bool `json:"enableReplicaPoolerNodePort,omitempty"` + ReplicaPoolerNodePort *int32 `json:"replicaPoolerNodePort,omitempty"` // deprecated load balancer settings maintained for backward compatibility // see "Load balancers" operator docs - UseLoadBalancer *bool `json:"useLoadBalancer,omitempty"` + UseLoadBalancer *bool `json:"useLoadBalancer,omitempty"` + // deprecated ReplicaLoadBalancer *bool `json:"replicaLoadBalancer,omitempty"` // load balancers' source ranges are the same for master and replica services + // +nullable + // +kubebuilder:validation:items:Pattern=`^((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:))\/(12[0-8]|1[01][0-9]|[1-9]?[0-9]))$` + // +optional AllowedSourceRanges []string `json:"allowedSourceRanges"` - NumberOfInstances int32 `json:"numberOfInstances"` - Users map[string]UserFlags `json:"users,omitempty"` - MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` - Clone *CloneDescription `json:"clone,omitempty"` - ClusterName string `json:"-"` - Databases map[string]string `json:"databases,omitempty"` - PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` - SchedulerName *string `json:"schedulerName,omitempty"` - NodeAffinity *v1.NodeAffinity `json:"nodeAffinity,omitempty"` - Tolerations []v1.Toleration `json:"tolerations,omitempty"` - Sidecars []Sidecar `json:"sidecars,omitempty"` - InitContainers []v1.Container `json:"initContainers,omitempty"` - PodPriorityClassName string `json:"podPriorityClassName,omitempty"` - ShmVolume *bool `json:"enableShmVolume,omitempty"` - EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` - LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` - StandbyCluster *StandbyDescription `json:"standby,omitempty"` - PodAnnotations map[string]string `json:"podAnnotations,omitempty"` - ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` - TLS *TLSDescription `json:"tls,omitempty"` - AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` + Users map[string]UserFlags `json:"users,omitempty"` + // +nullable + UsersIgnoringSecretRotation []string `json:"usersIgnoringSecretRotation,omitempty"` + // +nullable + UsersWithSecretRotation []string `json:"usersWithSecretRotation,omitempty"` + // +nullable + UsersWithInPlaceSecretRotation []string `json:"usersWithInPlaceSecretRotation,omitempty"` - // deprecated json tags - InitContainersOld []v1.Container `json:"init_containers,omitempty"` - PodPriorityClassNameOld string `json:"pod_priority_class_name,omitempty"` + // +kubebuilder:validation:Minimum=0 + NumberOfInstances int32 `json:"numberOfInstances"` + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Type=array + MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` + Clone *CloneDescription `json:"clone,omitempty"` + // Note: usernames specified here as database owners must be declared + // in the users key of the spec key. + Databases map[string]string `json:"databases,omitempty"` + PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` + SchedulerName *string `json:"schedulerName,omitempty"` + NodeAffinity *v1.NodeAffinity `json:"nodeAffinity,omitempty"` + TopologySpreadConstraints []v1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + LivenessProbe *v1.Probe `json:"livenessProbe,omitempty"` + Sidecars []Sidecar `json:"sidecars,omitempty"` + InitContainers []v1.Container `json:"initContainers,omitempty"` + PodPriorityClassName string `json:"podPriorityClassName,omitempty"` + ShmVolume *bool `json:"enableShmVolume,omitempty"` + EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` + LogicalBackupRetention string `json:"logicalBackupRetention,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$` + LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` + StandbyCluster *StandbyDescription `json:"standby,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` + ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` + // MasterServiceAnnotations takes precedence over ServiceAnnotations for master role if not empty + MasterServiceAnnotations map[string]string `json:"masterServiceAnnotations,omitempty"` + // ReplicaServiceAnnotations takes precedence over ServiceAnnotations for replica role if not empty + ReplicaServiceAnnotations map[string]string `json:"replicaServiceAnnotations,omitempty"` + TLS *TLSDescription `json:"tls,omitempty"` + AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` + Streams []Stream `json:"streams,omitempty"` + Env []v1.EnvVar `json:"env,omitempty"` + + // deprecated + InitContainersOld []v1.Container `json:"init_containers,omitempty"` + // deprecated + PodPriorityClassNameOld string `json:"pod_priority_class_name,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -95,6 +151,7 @@ type PreparedDatabase struct { PreparedSchemas map[string]PreparedSchema `json:"schemas,omitempty"` DefaultUsers bool `json:"defaultUsers,omitempty" defaults:"false"` Extensions map[string]string `json:"extensions,omitempty"` + SecretNamespace string `json:"secretNamespace,omitempty"` } // PreparedSchema describes elements to be bootstrapped per schema @@ -105,47 +162,86 @@ type PreparedSchema struct { // MaintenanceWindow describes the time window when the operator is allowed to do maintenance on a cluster. type MaintenanceWindow struct { - Everyday bool - Weekday time.Weekday - StartTime metav1.Time // Start time - EndTime metav1.Time // End time + Everyday bool `json:"everyday,omitempty"` + Weekday time.Weekday `json:"weekday,omitempty"` + StartTime metav1.Time `json:"startTime"` + EndTime metav1.Time `json:"endTime"` } // Volume describes a single volume in the manifest. type Volume struct { - Size string `json:"size"` - StorageClass string `json:"storageClass,omitempty"` - SubPath string `json:"subPath,omitempty"` - Iops *int64 `json:"iops,omitempty"` - Throughput *int64 `json:"throughput,omitempty"` - VolumeType string `json:"type,omitempty"` + Selector *metav1.LabelSelector `json:"selector,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + Size string `json:"size"` + StorageClass string `json:"storageClass,omitempty"` + SubPath string `json:"subPath,omitempty"` + IsSubPathExpr *bool `json:"isSubPathExpr,omitempty"` + Iops *int64 `json:"iops,omitempty"` + Throughput *int64 `json:"throughput,omitempty"` + VolumeType string `json:"type,omitempty"` } // AdditionalVolume specs additional optional volumes for statefulset type AdditionalVolume struct { - Name string `json:"name"` - MountPath string `json:"mountPath"` - SubPath string `json:"subPath,omitempty"` - TargetContainers []string `json:"targetContainers"` - VolumeSource v1.VolumeSource `json:"volumeSource"` + Name string `json:"name"` + MountPath string `json:"mountPath"` + SubPath string `json:"subPath,omitempty"` + IsSubPathExpr *bool `json:"isSubPathExpr,omitempty"` + // +nullable + // +optional + TargetContainers []string `json:"targetContainers"` + // +kubebuilder:validation:XPreserveUnknownFields + // +kubebuilder:validation:Type=object + // +kubebuilder:validation:Schemaless + VolumeSource v1.VolumeSource `json:"volumeSource"` } // PostgresqlParam describes PostgreSQL version and pairs of configuration parameter name - values. type PostgresqlParam struct { + // +kubebuilder:validation:Enum="14";"15";"16";"17";"18" PgVersion string `json:"version"` Parameters map[string]string `json:"parameters,omitempty"` } // ResourceDescription describes CPU and memory resources defined for a cluster. type ResourceDescription struct { - CPU string `json:"cpu"` - Memory string `json:"memory"` + // Decimal natural followed by m, or decimal natural followed by + // dot followed by up to three decimal digits. + // + // This is because the Kubernetes CPU resource has millis as the + // maximum precision. The actual values are checked in code + // because the regular expression would be huge and horrible and + // not very helpful in validation error messages; this one checks + // only the format of the given number. + // + // https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu + // + // Note: the value specified here must not be zero or be lower + // than the corresponding request. + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + CPU *string `json:"cpu,omitempty"` + // You can express memory as a plain integer or as a fixed-point + // integer using one of these suffixes: E, P, T, G, M, k. You can + // also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki + // + // https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory + // + // Note: the value specified here must not be zero or be higher + // than the corresponding limit. + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + Memory *string `json:"memory,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + HugePages2Mi *string `json:"hugepages-2Mi,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + HugePages1Gi *string `json:"hugepages-1Gi,omitempty"` } // Resources describes requests and limits for the cluster resouces. type Resources struct { - ResourceRequests ResourceDescription `json:"requests,omitempty"` - ResourceLimits ResourceDescription `json:"limits,omitempty"` + // +optional + ResourceRequests ResourceDescription `json:"requests"` + // +optional + ResourceLimits ResourceDescription `json:"limits"` } // Patroni contains Patroni-specific configuration @@ -155,19 +251,28 @@ type Patroni struct { TTL uint32 `json:"ttl,omitempty"` LoopWait uint32 `json:"loop_wait,omitempty"` RetryTimeout uint32 `json:"retry_timeout,omitempty"` - MaximumLagOnFailover float32 `json:"maximum_lag_on_failover,omitempty"` // float32 because https://github.com/kubernetes/kubernetes/issues/30213 + MaximumLagOnFailover int64 `json:"maximum_lag_on_failover,omitempty"` Slots map[string]map[string]string `json:"slots,omitempty"` SynchronousMode bool `json:"synchronous_mode,omitempty"` SynchronousModeStrict bool `json:"synchronous_mode_strict,omitempty"` + SynchronousNodeCount uint32 `json:"synchronous_node_count,omitempty" defaults:"1"` + FailsafeMode *bool `json:"failsafe_mode,omitempty"` } -// StandbyDescription contains s3 wal path +// StandbyDescription contains remote primary config and/or s3/gs wal path. +// standby_host can be specified alone or together with either s3_wal_path OR gs_wal_path (mutually exclusive). +// At least one field must be specified. s3_wal_path and gs_wal_path are mutually exclusive. type StandbyDescription struct { - S3WalPath string `json:"s3_wal_path,omitempty"` + S3WalPath string `json:"s3_wal_path,omitempty"` + GSWalPath string `json:"gs_wal_path,omitempty"` + StandbyHost string `json:"standby_host,omitempty"` + StandbyPort string `json:"standby_port,omitempty"` + StandbyPrimarySlotName string `json:"standby_primary_slot_name,omitempty"` } // TLSDescription specs TLS properties type TLSDescription struct { + // +required SecretName string `json:"secretName,omitempty"` CertificateFile string `json:"certificateFile,omitempty"` PrivateKeyFile string `json:"privateKeyFile,omitempty"` @@ -177,8 +282,14 @@ type TLSDescription struct { // CloneDescription describes which cluster the new should clone and up to which point in time type CloneDescription struct { - ClusterName string `json:"cluster,omitempty"` - UID string `json:"uid,omitempty"` + // +required + ClusterName string `json:"cluster,omitempty"` + // +kubebuilder:validation:Format=uuid + UID string `json:"uid,omitempty"` + // The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + // Example: 1996-12-19T16:39:57-08:00 + // Note: this field requires a timezone + // +kubebuilder:validation:Pattern=`^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$` EndTimestamp string `json:"timestamp,omitempty"` S3WalPath string `json:"s3_wal_path,omitempty"` S3Endpoint string `json:"s3_endpoint,omitempty"` @@ -189,14 +300,17 @@ type CloneDescription struct { // Sidecar defines a container to be run in the same pod as the Postgres container. type Sidecar struct { - Resources `json:"resources,omitempty"` + *Resources `json:"resources,omitempty"` Name string `json:"name,omitempty"` DockerImage string `json:"image,omitempty"` Ports []v1.ContainerPort `json:"ports,omitempty"` Env []v1.EnvVar `json:"env,omitempty"` + Command []string `json:"command,omitempty"` } // UserFlags defines flags (such as superuser, nologin) that could be assigned to individual users +// +kubebuilder:validation:items:Enum=bypassrls;BYPASSRLS;nobypassrls;NOBYPASSRLS;createdb;CREATEDB;nocreatedb;NOCREATEDB;createrole;CREATEROLE;nocreaterole;NOCREATEROLE;inherit;INHERIT;noinherit;NOINHERIT;login;LOGIN;nologin;NOLOGIN;replication;REPLICATION;noreplication;NOREPLICATION;superuser;SUPERUSER;nosuperuser;NOSUPERUSER +// +nullable type UserFlags []string // PostgresStatus contains status of the PostgreSQL cluster (running, creation failed etc.) @@ -215,12 +329,37 @@ type PostgresStatus struct { // makes sense to expose. E.g. pool size (min/max boundaries), max client // connections etc. type ConnectionPooler struct { + // +kubebuilder:validation:Minimum=1 NumberOfInstances *int32 `json:"numberOfInstances,omitempty"` Schema string `json:"schema,omitempty"` User string `json:"user,omitempty"` - Mode string `json:"mode,omitempty"` - DockerImage string `json:"dockerImage,omitempty"` - MaxDBConnections *int32 `json:"maxDBConnections,omitempty"` + // +kubebuilder:validation:Enum=session;transaction + Mode string `json:"mode,omitempty"` + DockerImage string `json:"dockerImage,omitempty"` + MaxDBConnections *int32 `json:"maxDBConnections,omitempty"` - Resources `json:"resources,omitempty"` + *Resources `json:"resources,omitempty"` +} + +// Stream defines properties for creating FabricEventStream resources +type Stream struct { + ApplicationId string `json:"applicationId"` + Database string `json:"database"` + Tables map[string]StreamTable `json:"tables"` + Filter map[string]*string `json:"filter,omitempty"` + BatchSize *uint32 `json:"batchSize,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+m|\d+(\.\d{1,3})?)$` + CPU *string `json:"cpu,omitempty"` + // +kubebuilder:validation:Pattern=`^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$` + Memory *string `json:"memory,omitempty"` + EnableRecovery *bool `json:"enableRecovery,omitempty"` +} + +// StreamTable defines properties of outbox tables for FabricEventStreams +type StreamTable struct { + EventType string `json:"eventType"` + RecoveryEventType string `json:"recoveryEventType,omitempty"` + IgnoreRecovery *bool `json:"ignoreRecovery,omitempty"` + IdColumn *string `json:"idColumn,omitempty"` + PayloadColumn *string `json:"payloadColumn,omitempty"` } diff --git a/pkg/apis/acid.zalan.do/v1/util.go b/pkg/apis/acid.zalan.do/v1/util.go index a795ec685..719defe93 100644 --- a/pkg/apis/acid.zalan.do/v1/util.go +++ b/pkg/apis/acid.zalan.do/v1/util.go @@ -46,7 +46,7 @@ func parseWeekday(s string) (time.Weekday, error) { return time.Weekday(weekday), nil } -func extractClusterName(clusterName string, teamName string) (string, error) { +func ExtractClusterName(clusterName string, teamName string) (string, error) { teamNameLen := len(teamName) if len(clusterName) < teamNameLen+2 { return "", fmt.Errorf("cluster name must match {TEAM}-{NAME} format. Got cluster name '%v', team name '%v'", clusterName, teamName) diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index bf6875a82..857622581 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "reflect" + "regexp" "testing" "time" @@ -26,6 +27,10 @@ var parseTimeTests = []struct { {"expect error as minute is out of range", "23:69", metav1.Now(), errors.New(`parsing time "23:69": minute out of range`)}, } +func stringToPointer(str string) *string { + return &str +} + var parseWeekdayTests = []struct { about string in string @@ -87,6 +92,13 @@ var maintenanceWindows = []struct { StartTime: mustParseTime("10:00"), EndTime: mustParseTime("20:00"), }, nil}, + {"regular every day scenario", + []byte(`"05:00-07:00"`), + MaintenanceWindow{ + Everyday: true, + StartTime: mustParseTime("05:00"), + EndTime: mustParseTime("07:00"), + }, nil}, {"starts and ends at the same time", []byte(`"Mon:10:00-10:00"`), MaintenanceWindow{ @@ -119,6 +131,8 @@ var maintenanceWindows = []struct { {"expect error as weekday is empty", []byte(`":00:00-10:00"`), MaintenanceWindow{}, errors.New(`could not parse weekday: incorrect weekday`)}, {"expect error as maintenance window set seconds", []byte(`"Mon:00:00:00-10:00:00"`), MaintenanceWindow{}, errors.New(`incorrect maintenance window format`)}, {"expect error as 'To' time set seconds", []byte(`"Mon:00:00-00:00:00"`), MaintenanceWindow{}, errors.New("could not parse end time: incorrect time format")}, + // ideally, should be implemented + {"expect error as 'To' has a weekday", []byte(`"Mon:00:00-Fri:00:00"`), MaintenanceWindow{}, errors.New("could not parse end time: incorrect time format")}, {"expect error as 'To' time is missing", []byte(`"Mon:00:00"`), MaintenanceWindow{}, errors.New("incorrect maintenance window format")}} var postgresStatus = []struct { @@ -163,7 +177,7 @@ var unmarshalCluster = []struct { "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":"Invalid"}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":"Invalid"}`), err: nil}, { about: "example with /status subresource", @@ -184,7 +198,7 @@ var unmarshalCluster = []struct { "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), err: nil}, { about: "example with detailed input manifest and deprecated pod_priority_class_name -> podPriorityClassName", @@ -213,7 +227,7 @@ var unmarshalCluster = []struct { "127.0.0.1/32" ], "postgresql": { - "version": "9.6", + "version": "18", "parameters": { "shared_buffers": "32MB", "max_connections": "10", @@ -273,7 +287,7 @@ var unmarshalCluster = []struct { }, Spec: PostgresSpec{ PostgresqlParam: PostgresqlParam{ - PgVersion: "9.6", + PgVersion: "18", Parameters: map[string]string{ "shared_buffers": "32MB", "max_connections": "10", @@ -300,9 +314,9 @@ var unmarshalCluster = []struct { MaximumLagOnFailover: 33554432, Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, }, - Resources: Resources{ - ResourceRequests: ResourceDescription{CPU: "10m", Memory: "50Mi"}, - ResourceLimits: ResourceDescription{CPU: "300m", Memory: "3000Mi"}, + Resources: &Resources{ + ResourceRequests: ResourceDescription{CPU: stringToPointer("10m"), Memory: stringToPointer("50Mi")}, + ResourceLimits: ResourceDescription{CPU: stringToPointer("300m"), Memory: stringToPointer("3000Mi")}, }, TeamID: "acid", @@ -330,28 +344,10 @@ var unmarshalCluster = []struct { Clone: &CloneDescription{ ClusterName: "acid-batman", }, - ClusterName: "testcluster1", }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), - err: nil}, - { - about: "example with teamId set in input", - in: []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "teapot-testcluster1"}, "spec": {"teamId": "acid"}}`), - out: Postgresql{ - TypeMeta: metav1.TypeMeta{ - Kind: "Postgresql", - APIVersion: "acid.zalan.do/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "teapot-testcluster1", - }, - Spec: PostgresSpec{TeamID: "acid"}, - Status: PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid}, - Error: errors.New("name must match {TEAM}-{NAME} format").Error(), - }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null} ,"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"18","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, { about: "example with clone", @@ -369,11 +365,10 @@ var unmarshalCluster = []struct { Clone: &CloneDescription{ ClusterName: "team-batman", }, - ClusterName: "testcluster1", }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{"cluster":"team-batman"}},"status":{"PostgresClusterStatus":""}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{"cluster":"team-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, { about: "standby example", @@ -391,11 +386,10 @@ var unmarshalCluster = []struct { StandbyCluster: &StandbyDescription{ S3WalPath: "s3://custom/path/to/bucket/", }, - ClusterName: "testcluster1", }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"standby":{"s3_wal_path":"s3://custom/path/to/bucket/"}},"status":{"PostgresClusterStatus":""}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"standby":{"s3_wal_path":"s3://custom/path/to/bucket/"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, { about: "expect error on malformatted JSON", @@ -418,7 +412,7 @@ var postgresqlList = []struct { out PostgresqlList err error }{ - {"expect success", []byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace":"default","resourceVersion":"30446957","selfLink":"/apis/acid.zalan.do/v1/namespaces/default/postgresqls/acid-testcluster42","uid":"857cd208-33dc-11e7-b20a-0699041e4b03"},"spec":{"allowedSourceRanges":["185.85.220.0/22"],"numberOfInstances":1,"postgresql":{"version":"9.6"},"teamId":"acid","volume":{"size":"10Gi"}},"status":{"PostgresClusterStatus":"Running"}}],"kind":"List","metadata":{},"resourceVersion":"","selfLink":""}`), + {"expect success", []byte(`{"apiVersion":"v1","items":[{"apiVersion":"acid.zalan.do/v1","kind":"Postgresql","metadata":{"labels":{"team":"acid"},"name":"acid-testcluster42","namespace":"default","resourceVersion":"30446957","selfLink":"/apis/acid.zalan.do/v1/namespaces/default/postgresqls/acid-testcluster42","uid":"857cd208-33dc-11e7-b20a-0699041e4b03"},"spec":{"allowedSourceRanges":["185.85.220.0/22"],"numberOfInstances":1,"postgresql":{"version":"18"},"teamId":"acid","volume":{"size":"10Gi"}},"status":{"PostgresClusterStatus":"Running"}}],"kind":"List","metadata":{},"resourceVersion":"","selfLink":""}`), PostgresqlList{ TypeMeta: metav1.TypeMeta{ Kind: "List", @@ -439,7 +433,7 @@ var postgresqlList = []struct { }, Spec: PostgresSpec{ ClusterName: "testcluster42", - PostgresqlParam: PostgresqlParam{PgVersion: "9.6"}, + PostgresqlParam: PostgresqlParam{PgVersion: "18"}, Volume: Volume{Size: "10Gi"}, TeamID: "acid", AllowedSourceRanges: []string{"185.85.220.0/22"}, @@ -628,10 +622,10 @@ func TestServiceAnnotations(t *testing.T) { func TestClusterName(t *testing.T) { for _, tt := range clusterNames { t.Run(tt.about, func(t *testing.T) { - name, err := extractClusterName(tt.in, tt.inTeam) + name, err := ExtractClusterName(tt.in, tt.inTeam) if err != nil { if tt.err == nil || err.Error() != tt.err.Error() { - t.Errorf("extractClusterName expected error: %v, got: %v", tt.err, err) + t.Errorf("ExtractClusterName expected error: %v, got: %v", tt.err, err) } return } else if tt.err != nil { @@ -801,8 +795,6 @@ func TestPostgresListMeta(t *testing.T) { if a := tt.out.GetListMeta(); reflect.DeepEqual(a, tt.out.ListMeta) { t.Errorf("GetObjectMeta expected: %v, got: %v", tt.out.ListMeta, a) } - - return }) } } @@ -819,3 +811,47 @@ func TestPostgresqlClone(t *testing.T) { }) } } + +func TestAllowedSourceRangesPattern(t *testing.T) { + // pattern used in CRD validation for allowedSourceRanges + pattern := `^((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\/(\d|[1-2]\d|3[0-2])|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:))\/(12[0-8]|1[01][0-9]|[1-9]?[0-9]))$` + re := regexp.MustCompile(pattern) + + valid := []string{ + // IPv4 + "192.168.1.0/24", + "0.0.0.0/0", + "127.0.0.1/32", + "10.0.0.0/8", + "185.85.220.0/22", + // IPv6 + "fd01::/48", + "::1/128", + "::/0", + "2001:db8::/32", + "fe80::1/64", + "2001:0db8:85a3:0000:0000:8a2e:0370:7334/128", + } + + invalid := []string{ + "999.999.999.999/24", + "192.168.1.0/33", + "192.168.1.0", + "not-an-ip", + "fd01::/129", + "::gggg/64", + "", + } + + for _, cidr := range valid { + if !re.MatchString(cidr) { + t.Errorf("expected %q to match allowedSourceRanges pattern", cidr) + } + } + + for _, cidr := range invalid { + if re.MatchString(cidr) { + t.Errorf("expected %q NOT to match allowedSourceRanges pattern", cidr) + } + } +} diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 584a72143..7d18c2cf2 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -1,7 +1,8 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -29,6 +30,7 @@ package v1 import ( config "github.com/zalando/postgres-operator/pkg/util/config" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -51,6 +53,11 @@ func (in *AWSGCPConfiguration) DeepCopy() *AWSGCPConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AdditionalVolume) DeepCopyInto(out *AdditionalVolume) { *out = *in + if in.IsSubPathExpr != nil { + in, out := &in.IsSubPathExpr, &out.IsSubPathExpr + *out = new(bool) + **out = **in + } if in.TargetContainers != nil { in, out := &in.TargetContainers, &out.TargetContainers *out = make([]string, len(*in)) @@ -104,7 +111,11 @@ func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) { *out = new(int32) **out = **in } - out.Resources = in.Resources + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + (*in).DeepCopyInto(*out) + } return } @@ -147,6 +158,16 @@ func (in *ConnectionPoolerConfiguration) DeepCopy() *ConnectionPoolerConfigurati // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfiguration) { *out = *in + if in.EnableOwnerReferences != nil { + in, out := &in.EnableOwnerReferences, &out.EnableOwnerReferences + *out = new(bool) + **out = **in + } + if in.LivenessProbe != nil { + in, out := &in.LivenessProbe, &out.LivenessProbe + *out = new(corev1.Probe) + (*in).DeepCopyInto(*out) + } if in.SpiloAllowPrivilegeEscalation != nil { in, out := &in.SpiloAllowPrivilegeEscalation, &out.SpiloAllowPrivilegeEscalation *out = new(bool) @@ -172,6 +193,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = make([]string, len(*in)) copy(*out, *in) } + if in.PDBMasterLabelSelector != nil { + in, out := &in.PDBMasterLabelSelector, &out.PDBMasterLabelSelector + *out = new(bool) + **out = **in + } if in.EnablePodDisruptionBudget != nil { in, out := &in.EnablePodDisruptionBudget, &out.EnablePodDisruptionBudget *out = new(bool) @@ -187,6 +213,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = new(bool) **out = **in } + if in.SharePgSocketWithSidecars != nil { + in, out := &in.SharePgSocketWithSidecars, &out.SharePgSocketWithSidecars + *out = new(bool) + **out = **in + } out.OAuthTokenSecretName = in.OAuthTokenSecretName out.InfrastructureRolesSecretName = in.InfrastructureRolesSecretName if in.InfrastructureRolesDefs != nil { @@ -222,6 +253,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = make([]string, len(*in)) copy(*out, *in) } + if in.IgnoredAnnotations != nil { + in, out := &in.IgnoredAnnotations, &out.IgnoredAnnotations + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.NodeReadinessLabel != nil { in, out := &in.NodeReadinessLabel, &out.NodeReadinessLabel *out = make(map[string]string, len(*in)) @@ -244,6 +280,28 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura } } out.PodEnvironmentConfigMap = in.PodEnvironmentConfigMap + if in.PersistentVolumeClaimRetentionPolicy != nil { + in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EnableSecretsDeletion != nil { + in, out := &in.EnableSecretsDeletion, &out.EnableSecretsDeletion + *out = new(bool) + **out = **in + } + if in.EnablePersistentVolumeClaimDeletion != nil { + in, out := &in.EnablePersistentVolumeClaimDeletion, &out.EnablePersistentVolumeClaimDeletion + *out = new(bool) + **out = **in + } + if in.EnableFinalizers != nil { + in, out := &in.EnableFinalizers, &out.EnableFinalizers + *out = new(bool) + **out = **in + } return } @@ -317,6 +375,11 @@ func (in *MaintenanceWindow) DeepCopy() *MaintenanceWindow { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MajorVersionUpgradeConfiguration) DeepCopyInto(out *MajorVersionUpgradeConfiguration) { *out = *in + if in.MajorVersionUpgradeTeamAllowList != nil { + in, out := &in.MajorVersionUpgradeTeamAllowList, &out.MajorVersionUpgradeTeamAllowList + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -360,11 +423,28 @@ func (in *OperatorConfiguration) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData) { *out = *in - if in.EnableCRDValidation != nil { - in, out := &in.EnableCRDValidation, &out.EnableCRDValidation + if in.EnableCRDRegistration != nil { + in, out := &in.EnableCRDRegistration, &out.EnableCRDRegistration *out = new(bool) **out = **in } + if in.CRDCategories != nil { + in, out := &in.CRDCategories, &out.CRDCategories + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.EnableMaintenanceWindows != nil { + in, out := &in.EnableMaintenanceWindows, &out.EnableMaintenanceWindows + *out = new(bool) + **out = **in + } + if in.MaintenanceWindows != nil { + in, out := &in.MaintenanceWindows, &out.MaintenanceWindows + *out = make([]MaintenanceWindow, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.ShmVolume != nil { in, out := &in.ShmVolume, &out.ShmVolume *out = new(bool) @@ -384,19 +464,20 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData (*in)[i].DeepCopyInto(&(*out)[i]) } } - out.PostgresUsersConfiguration = in.PostgresUsersConfiguration - out.MajorVersionUpgrade = in.MajorVersionUpgrade + in.PostgresUsersConfiguration.DeepCopyInto(&out.PostgresUsersConfiguration) + in.MajorVersionUpgrade.DeepCopyInto(&out.MajorVersionUpgrade) in.Kubernetes.DeepCopyInto(&out.Kubernetes) out.PostgresPodResources = in.PostgresPodResources out.Timeouts = in.Timeouts in.LoadBalancer.DeepCopyInto(&out.LoadBalancer) out.AWSGCP = in.AWSGCP - out.OperatorDebug = in.OperatorDebug + in.OperatorDebug.DeepCopyInto(&out.OperatorDebug) in.TeamsAPI.DeepCopyInto(&out.TeamsAPI) out.LoggingRESTAPI = in.LoggingRESTAPI out.Scalyr = in.Scalyr - out.LogicalBackup = in.LogicalBackup + in.LogicalBackup.DeepCopyInto(&out.LogicalBackup) in.ConnectionPooler.DeepCopyInto(&out.ConnectionPooler) + in.Patroni.DeepCopyInto(&out.Patroni) return } @@ -446,6 +527,16 @@ func (in *OperatorConfigurationList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorDebugConfiguration) DeepCopyInto(out *OperatorDebugConfiguration) { *out = *in + if in.DebugLogging != nil { + in, out := &in.DebugLogging, &out.DebugLogging + *out = new(bool) + **out = **in + } + if in.EnableDBAccess != nil { + in, out := &in.EnableDBAccess, &out.EnableDBAccess + *out = new(bool) + **out = **in + } return } @@ -462,6 +553,21 @@ func (in *OperatorDebugConfiguration) DeepCopy() *OperatorDebugConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorLogicalBackupConfiguration) DeepCopyInto(out *OperatorLogicalBackupConfiguration) { *out = *in + if in.SuccessfulJobsHistoryLimit != nil { + in, out := &in.SuccessfulJobsHistoryLimit, &out.SuccessfulJobsHistoryLimit + *out = new(int32) + **out = **in + } + if in.FailedJobsHistoryLimit != nil { + in, out := &in.FailedJobsHistoryLimit, &out.FailedJobsHistoryLimit + *out = new(int32) + **out = **in + } + if in.TTLSecondsAfterFinished != nil { + in, out := &in.TTLSecondsAfterFinished, &out.TTLSecondsAfterFinished + *out = new(int32) + **out = **in + } return } @@ -523,6 +629,11 @@ func (in *Patroni) DeepCopyInto(out *Patroni) { (*out)[key] = outVal } } + if in.FailsafeMode != nil { + in, out := &in.FailsafeMode, &out.FailsafeMode + *out = new(bool) + **out = **in + } return } @@ -536,6 +647,27 @@ func (in *Patroni) DeepCopy() *Patroni { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PatroniConfiguration) DeepCopyInto(out *PatroniConfiguration) { + *out = *in + if in.FailsafeMode != nil { + in, out := &in.FailsafeMode, &out.FailsafeMode + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PatroniConfiguration. +func (in *PatroniConfiguration) DeepCopy() *PatroniConfiguration { + if in == nil { + return nil + } + out := new(PatroniConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PostgresPodResourcesDefaults) DeepCopyInto(out *PostgresPodResourcesDefaults) { *out = *in @@ -558,7 +690,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { in.PostgresqlParam.DeepCopyInto(&out.PostgresqlParam) in.Volume.DeepCopyInto(&out.Volume) in.Patroni.DeepCopyInto(&out.Patroni) - out.Resources = in.Resources + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + (*in).DeepCopyInto(*out) + } if in.EnableConnectionPooler != nil { in, out := &in.EnableConnectionPooler, &out.EnableConnectionPooler *out = new(bool) @@ -594,11 +730,61 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(bool) **out = **in } + if in.EnableMasterPoolerLoadBalancer != nil { + in, out := &in.EnableMasterPoolerLoadBalancer, &out.EnableMasterPoolerLoadBalancer + *out = new(bool) + **out = **in + } if in.EnableReplicaLoadBalancer != nil { in, out := &in.EnableReplicaLoadBalancer, &out.EnableReplicaLoadBalancer *out = new(bool) **out = **in } + if in.EnableReplicaPoolerLoadBalancer != nil { + in, out := &in.EnableReplicaPoolerLoadBalancer, &out.EnableReplicaPoolerLoadBalancer + *out = new(bool) + **out = **in + } + if in.EnableMasterNodePort != nil { + in, out := &in.EnableMasterNodePort, &out.EnableMasterNodePort + *out = new(bool) + **out = **in + } + if in.MasterNodePort != nil { + in, out := &in.MasterNodePort, &out.MasterNodePort + *out = new(int32) + **out = **in + } + if in.EnableMasterPoolerNodePort != nil { + in, out := &in.EnableMasterPoolerNodePort, &out.EnableMasterPoolerNodePort + *out = new(bool) + **out = **in + } + if in.MasterPoolerNodePort != nil { + in, out := &in.MasterPoolerNodePort, &out.MasterPoolerNodePort + *out = new(int32) + **out = **in + } + if in.EnableReplicaNodePort != nil { + in, out := &in.EnableReplicaNodePort, &out.EnableReplicaNodePort + *out = new(bool) + **out = **in + } + if in.ReplicaNodePort != nil { + in, out := &in.ReplicaNodePort, &out.ReplicaNodePort + *out = new(int32) + **out = **in + } + if in.EnableReplicaPoolerNodePort != nil { + in, out := &in.EnableReplicaPoolerNodePort, &out.EnableReplicaPoolerNodePort + *out = new(bool) + **out = **in + } + if in.ReplicaPoolerNodePort != nil { + in, out := &in.ReplicaPoolerNodePort, &out.ReplicaPoolerNodePort + *out = new(int32) + **out = **in + } if in.UseLoadBalancer != nil { in, out := &in.UseLoadBalancer, &out.UseLoadBalancer *out = new(bool) @@ -629,6 +815,21 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = outVal } } + if in.UsersIgnoringSecretRotation != nil { + in, out := &in.UsersIgnoringSecretRotation, &out.UsersIgnoringSecretRotation + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.UsersWithSecretRotation != nil { + in, out := &in.UsersWithSecretRotation, &out.UsersWithSecretRotation + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.UsersWithInPlaceSecretRotation != nil { + in, out := &in.UsersWithInPlaceSecretRotation, &out.UsersWithInPlaceSecretRotation + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.MaintenanceWindows != nil { in, out := &in.MaintenanceWindows, &out.MaintenanceWindows *out = make([]MaintenanceWindow, len(*in)) @@ -665,6 +866,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(corev1.NodeAffinity) (*in).DeepCopyInto(*out) } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]corev1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]corev1.Toleration, len(*in)) @@ -672,6 +880,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.LivenessProbe != nil { + in, out := &in.LivenessProbe, &out.LivenessProbe + *out = new(corev1.Probe) + (*in).DeepCopyInto(*out) + } if in.Sidecars != nil { in, out := &in.Sidecars, &out.Sidecars *out = make([]Sidecar, len(*in)) @@ -710,6 +923,20 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = val } } + if in.MasterServiceAnnotations != nil { + in, out := &in.MasterServiceAnnotations, &out.MasterServiceAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ReplicaServiceAnnotations != nil { + in, out := &in.ReplicaServiceAnnotations, &out.ReplicaServiceAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.TLS != nil { in, out := &in.TLS, &out.TLS *out = new(TLSDescription) @@ -722,6 +949,20 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Streams != nil { + in, out := &in.Streams, &out.Streams + *out = make([]Stream, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) @@ -823,14 +1064,14 @@ func (in *PostgresTeamSpec) DeepCopyInto(out *PostgresTeamSpec) { *out = *in if in.AdditionalSuperuserTeams != nil { in, out := &in.AdditionalSuperuserTeams, &out.AdditionalSuperuserTeams - *out = make(map[string][]string, len(*in)) + *out = make(map[string]SuperUserTeams, len(*in)) for key, val := range *in { var outVal []string if val == nil { (*out)[key] = nil } else { in, out := &val, &outVal - *out = make([]string, len(*in)) + *out = make(SuperUserTeams, len(*in)) copy(*out, *in) } (*out)[key] = outVal @@ -838,14 +1079,14 @@ func (in *PostgresTeamSpec) DeepCopyInto(out *PostgresTeamSpec) { } if in.AdditionalTeams != nil { in, out := &in.AdditionalTeams, &out.AdditionalTeams - *out = make(map[string][]string, len(*in)) + *out = make(map[string]Teams, len(*in)) for key, val := range *in { var outVal []string if val == nil { (*out)[key] = nil } else { in, out := &val, &outVal - *out = make([]string, len(*in)) + *out = make(Teams, len(*in)) copy(*out, *in) } (*out)[key] = outVal @@ -853,14 +1094,14 @@ func (in *PostgresTeamSpec) DeepCopyInto(out *PostgresTeamSpec) { } if in.AdditionalMembers != nil { in, out := &in.AdditionalMembers, &out.AdditionalMembers - *out = make(map[string][]string, len(*in)) + *out = make(map[string]Users, len(*in)) for key, val := range *in { var outVal []string if val == nil { (*out)[key] = nil } else { in, out := &val, &outVal - *out = make([]string, len(*in)) + *out = make(Users, len(*in)) copy(*out, *in) } (*out)[key] = outVal @@ -882,6 +1123,11 @@ func (in *PostgresTeamSpec) DeepCopy() *PostgresTeamSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PostgresUsersConfiguration) DeepCopyInto(out *PostgresUsersConfiguration) { *out = *in + if in.AdditionalOwnerRoles != nil { + in, out := &in.AdditionalOwnerRoles, &out.AdditionalOwnerRoles + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -1033,6 +1279,26 @@ func (in *PreparedSchema) DeepCopy() *PreparedSchema { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceDescription) DeepCopyInto(out *ResourceDescription) { *out = *in + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(string) + **out = **in + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + *out = new(string) + **out = **in + } + if in.HugePages2Mi != nil { + in, out := &in.HugePages2Mi, &out.HugePages2Mi + *out = new(string) + **out = **in + } + if in.HugePages1Gi != nil { + in, out := &in.HugePages1Gi, &out.HugePages1Gi + *out = new(string) + **out = **in + } return } @@ -1049,8 +1315,8 @@ func (in *ResourceDescription) DeepCopy() *ResourceDescription { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Resources) DeepCopyInto(out *Resources) { *out = *in - out.ResourceRequests = in.ResourceRequests - out.ResourceLimits = in.ResourceLimits + in.ResourceRequests.DeepCopyInto(&out.ResourceRequests) + in.ResourceLimits.DeepCopyInto(&out.ResourceLimits) return } @@ -1083,7 +1349,11 @@ func (in *ScalyrConfiguration) DeepCopy() *ScalyrConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Sidecar) DeepCopyInto(out *Sidecar) { *out = *in - out.Resources = in.Resources + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + (*in).DeepCopyInto(*out) + } if in.Ports != nil { in, out := &in.Ports, &out.Ports *out = make([]corev1.ContainerPort, len(*in)) @@ -1096,6 +1366,11 @@ func (in *Sidecar) DeepCopyInto(out *Sidecar) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -1125,6 +1400,115 @@ func (in *StandbyDescription) DeepCopy() *StandbyDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Stream) DeepCopyInto(out *Stream) { + *out = *in + if in.Tables != nil { + in, out := &in.Tables, &out.Tables + *out = make(map[string]StreamTable, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.Filter != nil { + in, out := &in.Filter, &out.Filter + *out = make(map[string]*string, len(*in)) + for key, val := range *in { + var outVal *string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(string) + **out = **in + } + (*out)[key] = outVal + } + } + if in.BatchSize != nil { + in, out := &in.BatchSize, &out.BatchSize + *out = new(uint32) + **out = **in + } + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(string) + **out = **in + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + *out = new(string) + **out = **in + } + if in.EnableRecovery != nil { + in, out := &in.EnableRecovery, &out.EnableRecovery + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Stream. +func (in *Stream) DeepCopy() *Stream { + if in == nil { + return nil + } + out := new(Stream) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StreamTable) DeepCopyInto(out *StreamTable) { + *out = *in + if in.IgnoreRecovery != nil { + in, out := &in.IgnoreRecovery, &out.IgnoreRecovery + *out = new(bool) + **out = **in + } + if in.IdColumn != nil { + in, out := &in.IdColumn, &out.IdColumn + *out = new(string) + **out = **in + } + if in.PayloadColumn != nil { + in, out := &in.PayloadColumn, &out.PayloadColumn + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StreamTable. +func (in *StreamTable) DeepCopy() *StreamTable { + if in == nil { + return nil + } + out := new(StreamTable) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in SuperUserTeams) DeepCopyInto(out *SuperUserTeams) { + { + in := &in + *out = make(SuperUserTeams, len(*in)) + copy(*out, *in) + return + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SuperUserTeams. +func (in SuperUserTeams) DeepCopy() SuperUserTeams { + if in == nil { + return nil + } + out := new(SuperUserTeams) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSDescription) DeepCopyInto(out *TLSDescription) { *out = *in @@ -1141,6 +1525,26 @@ func (in *TLSDescription) DeepCopy() *TLSDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Teams) DeepCopyInto(out *Teams) { + { + in := &in + *out = make(Teams, len(*in)) + copy(*out, *in) + return + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Teams. +func (in Teams) DeepCopy() Teams { + if in == nil { + return nil + } + out := new(Teams) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TeamsAPIConfiguration) DeepCopyInto(out *TeamsAPIConfiguration) { *out = *in @@ -1194,9 +1598,39 @@ func (in UserFlags) DeepCopy() UserFlags { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Users) DeepCopyInto(out *Users) { + { + in := &in + *out = make(Users, len(*in)) + copy(*out, *in) + return + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Users. +func (in Users) DeepCopy() Users { + if in == nil { + return nil + } + out := new(Users) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Volume) DeepCopyInto(out *Volume) { *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.IsSubPathExpr != nil { + in, out := &in.IsSubPathExpr, &out.IsSubPathExpr + *out = new(bool) + **out = **in + } if in.Iops != nil { in, out := &in.Iops, &out.Iops *out = new(int64) diff --git a/pkg/apis/zalando.org/register.go b/pkg/apis/zalando.org/register.go new file mode 100644 index 000000000..3dbd3f089 --- /dev/null +++ b/pkg/apis/zalando.org/register.go @@ -0,0 +1,6 @@ +package zalando + +const ( + // GroupName is the group name for the operator CRDs + GroupName = "zalando.org" +) diff --git a/pkg/apis/zalando.org/v1/fabriceventstream.go b/pkg/apis/zalando.org/v1/fabriceventstream.go new file mode 100644 index 000000000..cb2ccdef5 --- /dev/null +++ b/pkg/apis/zalando.org/v1/fabriceventstream.go @@ -0,0 +1,108 @@ +package v1 + +import ( + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// FabricEventStream defines FabricEventStream Custom Resource Definition Object. +// +k8s:deepcopy-gen=true +type FabricEventStream struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + + Spec FabricEventStreamSpec `json:"spec"` +} + +// FabricEventStreamSpec defines the specification for the FabricEventStream TPR. +// +k8s:deepcopy-gen=true +type FabricEventStreamSpec struct { + ApplicationId string `json:"applicationId"` + EventStreams []EventStream `json:"eventStreams"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// FabricEventStreamList defines a list of FabricEventStreams . +// +k8s:deepcopy-gen=true +type FabricEventStreamList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []FabricEventStream `json:"items"` +} + +// EventStream defines the source, flow and sink of the event stream +// +k8s:deepcopy-gen=true +type EventStream struct { + EventStreamFlow EventStreamFlow `json:"flow"` + EventStreamSink EventStreamSink `json:"sink"` + EventStreamSource EventStreamSource `json:"source"` + EventStreamRecovery EventStreamRecovery `json:"recovery"` +} + +// EventStreamFlow defines the flow characteristics of the event stream +// +k8s:deepcopy-gen=true +type EventStreamFlow struct { + Type string `json:"type"` + PayloadColumn *string `json:"payloadColumn,omitempty"` +} + +// EventStreamSink defines the target of the event stream +// +k8s:deepcopy-gen=true +type EventStreamSink struct { + Type string `json:"type"` + EventType string `json:"eventType,omitempty"` + MaxBatchSize *uint32 `json:"maxBatchSize,omitempty"` +} + +// EventStreamRecovery defines the target of dead letter queue +// +k8s:deepcopy-gen=true +type EventStreamRecovery struct { + Type string `json:"type"` + Sink *EventStreamSink `json:"sink"` +} + +// EventStreamSource defines the source of the event stream and connection for FES operator +// +k8s:deepcopy-gen=true +type EventStreamSource struct { + Type string `json:"type"` + Schema string `json:"schema,omitempty" defaults:"public"` + EventStreamTable EventStreamTable `json:"table"` + Filter *string `json:"filter,omitempty"` + Connection Connection `json:"jdbcConnection"` +} + +// EventStreamTable defines the name and ID column to be used for streaming +// +k8s:deepcopy-gen=true +type EventStreamTable struct { + Name string `json:"name"` + IDColumn *string `json:"idColumn,omitempty"` +} + +// Connection to be used for allowing the FES operator to connect to a database +// +k8s:deepcopy-gen=true +type Connection struct { + Url string `json:"jdbcUrl"` + SlotName string `json:"slotName"` + PluginType string `json:"pluginType,omitempty"` + PublicationName *string `json:"publicationName,omitempty"` + DBAuth DBAuth `json:"databaseAuthentication"` +} + +// DBAuth specifies the credentials to be used for connecting with the database +// +k8s:deepcopy-gen=true +type DBAuth struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + UserKey string `json:"userKey,omitempty"` + PasswordKey string `json:"passwordKey,omitempty"` +} + +type Slot struct { + Slot map[string]string `json:"slot"` + Publication map[string]acidv1.StreamTable `json:"publication"` +} diff --git a/pkg/apis/zalando.org/v1/register.go b/pkg/apis/zalando.org/v1/register.go new file mode 100644 index 000000000..33a2c718b --- /dev/null +++ b/pkg/apis/zalando.org/v1/register.go @@ -0,0 +1,44 @@ +package v1 + +import ( + "github.com/zalando/postgres-operator/pkg/apis/zalando.org" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" +) + +// APIVersion of the `fabriceventstream` CRD +const ( + APIVersion = "v1" +) + +var ( + schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = schemeBuilder.AddToScheme +) + +func init() { + err := AddToScheme(scheme.Scheme) + if err != nil { + panic(err) + } +} + +// SchemeGroupVersion is the group version used to register these objects. +var SchemeGroupVersion = schema.GroupVersion{Group: zalando.GroupName, Version: APIVersion} + +// Resource takes an unqualified resource and returns a Group-qualified GroupResource. +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +// addKnownTypes adds the set of types defined in this package to the supplied scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &FabricEventStream{}, + &FabricEventStreamList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go new file mode 100644 index 000000000..f7c555bcb --- /dev/null +++ b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go @@ -0,0 +1,280 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connection) DeepCopyInto(out *Connection) { + *out = *in + if in.PublicationName != nil { + in, out := &in.PublicationName, &out.PublicationName + *out = new(string) + **out = **in + } + out.DBAuth = in.DBAuth + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection. +func (in *Connection) DeepCopy() *Connection { + if in == nil { + return nil + } + out := new(Connection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DBAuth) DeepCopyInto(out *DBAuth) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DBAuth. +func (in *DBAuth) DeepCopy() *DBAuth { + if in == nil { + return nil + } + out := new(DBAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStream) DeepCopyInto(out *EventStream) { + *out = *in + in.EventStreamFlow.DeepCopyInto(&out.EventStreamFlow) + in.EventStreamSink.DeepCopyInto(&out.EventStreamSink) + in.EventStreamSource.DeepCopyInto(&out.EventStreamSource) + in.EventStreamRecovery.DeepCopyInto(&out.EventStreamRecovery) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EventStream. +func (in *EventStream) DeepCopy() *EventStream { + if in == nil { + return nil + } + out := new(EventStream) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamFlow) DeepCopyInto(out *EventStreamFlow) { + *out = *in + if in.PayloadColumn != nil { + in, out := &in.PayloadColumn, &out.PayloadColumn + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EventStreamFlow. +func (in *EventStreamFlow) DeepCopy() *EventStreamFlow { + if in == nil { + return nil + } + out := new(EventStreamFlow) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamRecovery) DeepCopyInto(out *EventStreamRecovery) { + *out = *in + if in.Sink != nil { + in, out := &in.Sink, &out.Sink + *out = new(EventStreamSink) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EventStreamRecovery. +func (in *EventStreamRecovery) DeepCopy() *EventStreamRecovery { + if in == nil { + return nil + } + out := new(EventStreamRecovery) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSink) DeepCopyInto(out *EventStreamSink) { + *out = *in + if in.MaxBatchSize != nil { + in, out := &in.MaxBatchSize, &out.MaxBatchSize + *out = new(uint32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EventStreamSink. +func (in *EventStreamSink) DeepCopy() *EventStreamSink { + if in == nil { + return nil + } + out := new(EventStreamSink) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSource) DeepCopyInto(out *EventStreamSource) { + *out = *in + in.EventStreamTable.DeepCopyInto(&out.EventStreamTable) + if in.Filter != nil { + in, out := &in.Filter, &out.Filter + *out = new(string) + **out = **in + } + in.Connection.DeepCopyInto(&out.Connection) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EventStreamSource. +func (in *EventStreamSource) DeepCopy() *EventStreamSource { + if in == nil { + return nil + } + out := new(EventStreamSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamTable) DeepCopyInto(out *EventStreamTable) { + *out = *in + if in.IDColumn != nil { + in, out := &in.IDColumn, &out.IDColumn + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EventStreamTable. +func (in *EventStreamTable) DeepCopy() *EventStreamTable { + if in == nil { + return nil + } + out := new(EventStreamTable) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FabricEventStream) DeepCopyInto(out *FabricEventStream) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStream. +func (in *FabricEventStream) DeepCopy() *FabricEventStream { + if in == nil { + return nil + } + out := new(FabricEventStream) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FabricEventStream) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FabricEventStreamList) DeepCopyInto(out *FabricEventStreamList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FabricEventStream, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamList. +func (in *FabricEventStreamList) DeepCopy() *FabricEventStreamList { + if in == nil { + return nil + } + out := new(FabricEventStreamList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FabricEventStreamList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FabricEventStreamSpec) DeepCopyInto(out *FabricEventStreamSpec) { + *out = *in + if in.EventStreams != nil { + in, out := &in.EventStreams, &out.EventStreams + *out = make([]EventStream, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamSpec. +func (in *FabricEventStreamSpec) DeepCopy() *FabricEventStreamSpec { + if in == nil { + return nil + } + out := new(FabricEventStreamSpec) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index c0fa1f349..97e389970 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -12,7 +12,6 @@ import ( "time" "github.com/sirupsen/logrus" - "github.com/zalando/postgres-operator/pkg/cluster" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" @@ -31,9 +30,9 @@ type controllerInformer interface { GetOperatorConfig() *config.Config GetStatus() *spec.ControllerStatus TeamClusterList() map[string][]spec.NamespacedName - ClusterStatus(team, namespace, cluster string) (*cluster.ClusterStatus, error) - ClusterLogs(team, namespace, cluster string) ([]*spec.LogEntry, error) - ClusterHistory(team, namespace, cluster string) ([]*spec.Diff, error) + ClusterStatus(namespace, cluster string) (*cluster.ClusterStatus, error) + ClusterLogs(namespace, cluster string) ([]*spec.LogEntry, error) + ClusterHistory(namespace, cluster string) ([]*spec.Diff, error) ClusterDatabasesMap() map[string][]string WorkerLogs(workerID uint32) ([]*spec.LogEntry, error) ListQueue(workerID uint32) (*spec.QueueDump, error) @@ -55,9 +54,9 @@ const ( ) var ( - clusterStatusRe = fmt.Sprintf(`^/clusters/%s/%s/%s/?$`, teamRe, namespaceRe, clusterRe) - clusterLogsRe = fmt.Sprintf(`^/clusters/%s/%s/%s/logs/?$`, teamRe, namespaceRe, clusterRe) - clusterHistoryRe = fmt.Sprintf(`^/clusters/%s/%s/%s/history/?$`, teamRe, namespaceRe, clusterRe) + clusterStatusRe = fmt.Sprintf(`^/clusters/%s/%s/?$`, namespaceRe, clusterRe) + clusterLogsRe = fmt.Sprintf(`^/clusters/%s/%s/logs/?$`, namespaceRe, clusterRe) + clusterHistoryRe = fmt.Sprintf(`^/clusters/%s/%s/history/?$`, namespaceRe, clusterRe) teamURLRe = fmt.Sprintf(`^/clusters/%s/?$`, teamRe) clusterStatusURL = regexp.MustCompile(clusterStatusRe) @@ -87,6 +86,7 @@ func New(controller controllerInformer, port int, logger *logrus.Logger) *Server mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) mux.Handle("/status/", http.HandlerFunc(s.controllerStatus)) + mux.Handle("/readyz/", http.HandlerFunc(s.controllerReady)) mux.Handle("/config/", http.HandlerFunc(s.operatorConfig)) mux.HandleFunc("/clusters/", s.clusters) @@ -155,6 +155,10 @@ func (s *Server) controllerStatus(w http.ResponseWriter, req *http.Request) { s.respond(s.controller.GetStatus(), nil, w) } +func (s *Server) controllerReady(w http.ResponseWriter, req *http.Request) { + s.respond("OK", nil, w) +} + func (s *Server) operatorConfig(w http.ResponseWriter, req *http.Request) { s.respond(map[string]interface{}{ "controller": s.controller.GetConfig(), @@ -170,7 +174,7 @@ func (s *Server) clusters(w http.ResponseWriter, req *http.Request) { if matches := util.FindNamedStringSubmatch(clusterStatusURL, req.URL.Path); matches != nil { namespace := matches["namespace"] - resp, err = s.controller.ClusterStatus(matches["team"], namespace, matches["cluster"]) + resp, err = s.controller.ClusterStatus(namespace, matches["cluster"]) } else if matches := util.FindNamedStringSubmatch(teamURL, req.URL.Path); matches != nil { teamClusters := s.controller.TeamClusterList() clusters, found := teamClusters[matches["team"]] @@ -181,21 +185,21 @@ func (s *Server) clusters(w http.ResponseWriter, req *http.Request) { clusterNames := make([]string, 0) for _, cluster := range clusters { - clusterNames = append(clusterNames, cluster.Name[len(matches["team"])+1:]) + clusterNames = append(clusterNames, cluster.Name) } resp, err = clusterNames, nil } else if matches := util.FindNamedStringSubmatch(clusterLogsURL, req.URL.Path); matches != nil { namespace := matches["namespace"] - resp, err = s.controller.ClusterLogs(matches["team"], namespace, matches["cluster"]) + resp, err = s.controller.ClusterLogs(namespace, matches["cluster"]) } else if matches := util.FindNamedStringSubmatch(clusterHistoryURL, req.URL.Path); matches != nil { namespace := matches["namespace"] - resp, err = s.controller.ClusterHistory(matches["team"], namespace, matches["cluster"]) + resp, err = s.controller.ClusterHistory(namespace, matches["cluster"]) } else if req.URL.Path == clustersURL { clusterNamesPerTeam := make(map[string][]string) for team, clusters := range s.controller.TeamClusterList() { for _, cluster := range clusters { - clusterNamesPerTeam[team] = append(clusterNamesPerTeam[team], cluster.Name[len(team)+1:]) + clusterNamesPerTeam[team] = append(clusterNamesPerTeam[team], cluster.Name) } } resp, err = clusterNamesPerTeam, nil diff --git a/pkg/apiserver/apiserver_test.go b/pkg/apiserver/apiserver_test.go index fb6484d03..36571667c 100644 --- a/pkg/apiserver/apiserver_test.go +++ b/pkg/apiserver/apiserver_test.go @@ -5,9 +5,9 @@ import ( ) const ( - clusterStatusTest = "/clusters/test-id/test_namespace/testcluster/" - clusterStatusNumericTest = "/clusters/test-id-1/test_namespace/testcluster/" - clusterLogsTest = "/clusters/test-id/test_namespace/testcluster/logs/" + clusterStatusTest = "/clusters/test-namespace/testcluster/" + clusterStatusNumericTest = "/clusters/test-namespace-1/testcluster/" + clusterLogsTest = "/clusters/test-namespace/testcluster/logs/" teamTest = "/clusters/test-id/" ) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index ff474884c..eed4ee933 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -5,6 +5,7 @@ package cluster import ( "context" "database/sql" + "encoding/json" "fmt" "reflect" "regexp" @@ -14,6 +15,7 @@ import ( "github.com/sirupsen/logrus" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" "github.com/zalando/postgres-operator/pkg/spec" @@ -27,9 +29,11 @@ import ( "github.com/zalando/postgres-operator/pkg/util/users" "github.com/zalando/postgres-operator/pkg/util/volumes" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" @@ -42,7 +46,8 @@ var ( alphaNumericRegexp = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9]*$") databaseNameRegexp = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") userRegexp = regexp.MustCompile(`^[a-z0-9]([-_a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-_a-z0-9]*[a-z0-9])?)*$`) - patroniObjectSuffixes = []string{"config", "failover", "sync"} + patroniObjectSuffixes = []string{"leader", "config", "sync", "failover"} + finalizerName = "postgres-operator.acid.zalan.do" ) // Config contains operator-wide clients and configuration used from a cluster. TODO: remove struct duplication. @@ -56,13 +61,18 @@ type Config struct { } type kubeResources struct { - Services map[PostgresRole]*v1.Service - Endpoints map[PostgresRole]*v1.Endpoints - Secrets map[types.UID]*v1.Secret - Statefulset *appsv1.StatefulSet - PodDisruptionBudget *policybeta1.PodDisruptionBudget - //Pods are treated separately - //PVCs are treated separately + Services map[PostgresRole]*v1.Service + Endpoints map[PostgresRole]*v1.Endpoints + PatroniEndpoints map[string]*v1.Endpoints + PatroniConfigMaps map[string]*v1.ConfigMap + Secrets map[types.UID]*v1.Secret + Statefulset *appsv1.StatefulSet + VolumeClaims map[types.UID]*v1.PersistentVolumeClaim + PrimaryPodDisruptionBudget *policyv1.PodDisruptionBudget + CriticalOpPodDisruptionBudget *policyv1.PodDisruptionBudget + LogicalBackupJob *batchv1.CronJob + Streams map[string]*zalandov1.FabricEventStream + // Pods are treated separately } // Cluster describes postgresql cluster @@ -82,11 +92,13 @@ type Cluster struct { mu sync.Mutex userSyncStrategy spec.UserSyncer deleteOptions metav1.DeleteOptions + podEventsStore cache.Store podEventsQueue *cache.FIFO + replicationSlots map[string]interface{} teamsAPIClient teams.Interface oauthTokenGetter OAuthTokenGetter - KubeClient k8sutil.KubernetesClient //TODO: move clients to the better place? + KubeClient k8sutil.KubernetesClient // TODO: move clients to the better place? currentProcess Process processMu sync.RWMutex // protects the current operation for reporting, no need to hold the master mutex specMu sync.RWMutex // protects the spec for reporting, no need to hold the master mutex @@ -97,27 +109,37 @@ type Cluster struct { } type compareStatefulsetResult struct { - match bool - replace bool - rollingUpdate bool - reasons []string + match bool + replace bool + rollingUpdate bool + reasons []string + deletedPodAnnotations []string +} + +type compareLogicalBackupJobResult struct { + match bool + reasons []string + deletedPodAnnotations []string } // New creates a new cluster. This function should be called from a controller. func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgresql, logger *logrus.Entry, eventRecorder record.EventRecorder) *Cluster { deletePropagationPolicy := metav1.DeletePropagationOrphan - podEventsQueue := cache.NewFIFO(func(obj interface{}) (string, error) { + keyFn := func(obj interface{}) (string, error) { e, ok := obj.(PodEvent) if !ok { - return "", fmt.Errorf("could not cast to PodEvent") + return "", fmt.Errorf("could not cast to pod event") } return fmt.Sprintf("%s-%s", e.PodName, e.ResourceVersion), nil - }) + } + podEventsStore := cache.NewStore(keyFn) + podEventsQueue := cache.NewFIFO(keyFn) + passwordEncryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"] if !ok { - passwordEncryption = "md5" + passwordEncryption = "scram-sha-256" } cluster := &Cluster{ @@ -127,16 +149,25 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres systemUsers: make(map[string]spec.PgUser), podSubscribers: make(map[spec.NamespacedName]chan PodEvent), kubeResources: kubeResources{ - Secrets: make(map[types.UID]*v1.Secret), - Services: make(map[PostgresRole]*v1.Service), - Endpoints: make(map[PostgresRole]*v1.Endpoints)}, + Secrets: make(map[types.UID]*v1.Secret), + Services: make(map[PostgresRole]*v1.Service), + Endpoints: make(map[PostgresRole]*v1.Endpoints), + PatroniEndpoints: make(map[string]*v1.Endpoints), + PatroniConfigMaps: make(map[string]*v1.ConfigMap), + VolumeClaims: make(map[types.UID]*v1.PersistentVolumeClaim), + Streams: make(map[string]*zalandov1.FabricEventStream), + }, userSyncStrategy: users.DefaultUserSyncStrategy{ - PasswordEncryption: passwordEncryption, - RoleDeletionSuffix: cfg.OpConfig.RoleDeletionSuffix}, + PasswordEncryption: passwordEncryption, + RoleDeletionSuffix: cfg.OpConfig.RoleDeletionSuffix, + AdditionalOwnerRoles: cfg.OpConfig.AdditionalOwnerRoles, + }, deleteOptions: metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}, + podEventsStore: podEventsStore, podEventsQueue: podEventsQueue, KubeClient: kubeClient, currentMajorVersion: 0, + replicationSlots: make(map[string]interface{}), } cluster.logger = logger.WithField("pkg", "cluster").WithField("cluster-name", cluster.clusterName()) cluster.teamsAPIClient = teams.NewTeamsAPI(cfg.OpConfig.TeamsAPIUrl, logger) @@ -147,7 +178,6 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres cluster.EBSVolumes = make(map[string]volumes.VolumeProperties) if cfg.OpConfig.StorageResizeMode != "pvc" || cfg.OpConfig.EnableEBSGp3Migration { cluster.VolumeResizer = &volumes.EBSVolumeResizer{AWSRegion: cfg.OpConfig.AWSRegion} - } return cluster @@ -224,53 +254,84 @@ func (c *Cluster) initUsers() error { } if err := c.initHumanUsers(); err != nil { + // remember all cached users in c.pgUsers + for cachedUserName, cachedUser := range c.pgUsersCache { + c.pgUsers[cachedUserName] = cachedUser + } return fmt.Errorf("could not init human users: %v", err) } + c.initAdditionalOwnerRoles() + return nil } // Create creates the new kubernetes objects associated with the cluster. -func (c *Cluster) Create() error { +func (c *Cluster) Create() (err error) { c.mu.Lock() defer c.mu.Unlock() - var ( - err error - service *v1.Service - ep *v1.Endpoints - ss *appsv1.StatefulSet + var ( + pgCreateStatus *acidv1.Postgresql + service *v1.Service + ep *v1.Endpoints + ss *appsv1.StatefulSet ) defer func() { - if err == nil { - c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusRunning) //TODO: are you sure it's running? - } else { - c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusAddFailed) + currentStatus := c.Status.DeepCopy() + pg := c.Postgresql.DeepCopy() + pg.Status.PostgresClusterStatus = acidv1.ClusterStatusRunning + + if err != nil { + c.logger.Warningf("cluster created failed: %v", err) + pg.Status.PostgresClusterStatus = acidv1.ClusterStatusAddFailed + } + + if !equality.Semantic.DeepEqual(currentStatus, pg.Status) { + pgUpdatedStatus, err := c.KubeClient.SetPostgresCRDStatus(c.clusterName(), pg) + if err != nil { + c.logger.Warningf("could not set cluster status: %v", err) + return + } + c.setSpec(pgUpdatedStatus) } }() - c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusCreating) - c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Create", "Started creation of new cluster resources") + pg := c.Postgresql.DeepCopy() + pg.Status.PostgresClusterStatus = acidv1.ClusterStatusCreating - if err = c.enforceMinResourceLimits(&c.Spec); err != nil { - return fmt.Errorf("could not enforce minimum resource limits: %v", err) + pgCreateStatus, err = c.KubeClient.SetPostgresCRDStatus(c.clusterName(), pg) + if err != nil { + return fmt.Errorf("could not set cluster status: %v", err) } + c.setSpec(pgCreateStatus) + + if c.OpConfig.EnableFinalizers != nil && *c.OpConfig.EnableFinalizers { + if err = c.addFinalizer(); err != nil { + return fmt.Errorf("could not add finalizer: %v", err) + } + } + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Create", "Started creation of new cluster resources") for _, role := range []PostgresRole{Master, Replica} { - if c.Endpoints[role] != nil { - return fmt.Errorf("%s endpoint already exists in the cluster", role) - } - if role == Master { - // replica endpoint will be created by the replica service. Master endpoint needs to be created by us, - // since the corresponding master service does not define any selectors. - ep, err = c.createEndpoint(role) - if err != nil { - return fmt.Errorf("could not create %s endpoint: %v", role, err) + // if kubernetes_use_configmaps is set Patroni will create configmaps + // otherwise it will use endpoints + if !c.patroniKubernetesUseConfigMaps() { + if c.Endpoints[role] != nil { + return fmt.Errorf("%s endpoint already exists in the cluster", role) + } + if role == Master { + // replica endpoint will be created by the replica service. Master endpoint needs to be created by us, + // since the corresponding master service does not define any selectors. + ep, err = c.createEndpoint(role) + if err != nil { + return fmt.Errorf("could not create %s endpoint: %v", role, err) + } + c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) } - c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) } if c.Services[role] != nil { @@ -295,14 +356,10 @@ func (c *Cluster) Create() error { c.logger.Infof("secrets have been successfully created") c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Secrets", "The secrets have been successfully created") - if c.PodDisruptionBudget != nil { - return fmt.Errorf("pod disruption budget already exists in the cluster") + if err = c.createPodDisruptionBudgets(); err != nil { + return fmt.Errorf("could not create pod disruption budgets: %v", err) } - pdb, err := c.createPodDisruptionBudget() - if err != nil { - return fmt.Errorf("could not create pod disruption budget: %v", err) - } - c.logger.Infof("pod disruption budget %q has been successfully created", util.NameFromMeta(pdb.ObjectMeta)) + c.logger.Info("pod disruption budgets have been successfully created") if c.Statefulset != nil { return fmt.Errorf("statefulset already exists in the cluster") @@ -323,9 +380,19 @@ func (c *Cluster) Create() error { c.logger.Infof("pods are ready") c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "StatefulSet", "Pods are ready") + // sync volume may already transition volumes to gp3, if iops/throughput or type is specified + if err = c.syncVolumes(); err != nil { + return err + } + + // sync resources created by Patroni + if err = c.syncPatroniResources(); err != nil { + c.logger.Warnf("Patroni resources not yet synced: %v", err) + } + // create database objects unless we are running without pods or disabled // that feature explicitly - if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || c.Spec.StandbyCluster != nil) { + if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || isStandbyCluster(&c.Spec)) { c.logger.Infof("Create roles") if err = c.createRoles(); err != nil { return fmt.Errorf("could not create users: %v", err) @@ -348,10 +415,6 @@ func (c *Cluster) Create() error { c.logger.Info("a k8s cron job for logical backup has been successfully created") } - if err := c.listResources(); err != nil { - c.logger.Errorf("could not list resources: %v", err) - } - // Create connection pooler deployment and services if necessary. Since we // need to perform some operations with the database itself (e.g. install // lookup function), do it as the last step, when everything is available. @@ -360,27 +423,70 @@ func (c *Cluster) Create() error { // something fails, report warning c.createConnectionPooler(c.installLookupFunction) + // remember slots to detect deletion from manifest + for slotName, desiredSlot := range c.Spec.Patroni.Slots { + c.replicationSlots[slotName] = desiredSlot + } + + if len(c.Spec.Streams) > 0 { + // creating streams requires syncing the statefulset first + err = c.syncStatefulSet() + if err != nil { + return fmt.Errorf("could not sync statefulset: %v", err) + } + if err = c.syncStreams(); err != nil { + c.logger.Errorf("could not create streams: %v", err) + } + } + + if err := c.listResources(); err != nil { + c.logger.Errorf("could not list resources: %v", err) + } + return nil } func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compareStatefulsetResult { + deletedPodAnnotations := []string{} reasons := make([]string, 0) var match, needsRollUpdate, needsReplace bool match = true - //TODO: improve me + // TODO: improve me if *c.Statefulset.Spec.Replicas != *statefulSet.Spec.Replicas { match = false reasons = append(reasons, "new statefulset's number of replicas does not match the current one") } - if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) { + if !reflect.DeepEqual(c.Statefulset.OwnerReferences, statefulSet.OwnerReferences) { match = false needsReplace = true - reasons = append(reasons, "new statefulset's annotations do not match the current one") + reasons = append(reasons, "new statefulset's ownerReferences do not match") + } + if changed, reason := c.compareAnnotations(c.Statefulset.Annotations, statefulSet.Annotations, nil); changed { + match = false + needsReplace = true + reasons = append(reasons, "new statefulset's annotations do not match: "+reason) + } + if c.Statefulset.Spec.PodManagementPolicy != statefulSet.Spec.PodManagementPolicy { + match = false + needsReplace = true + reasons = append(reasons, "new statefulset's pod management policy do not match") } - needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons) - needsRollUpdate, reasons = c.compareContainers("containers", c.Statefulset.Spec.Template.Spec.Containers, statefulSet.Spec.Template.Spec.Containers, needsRollUpdate, reasons) + if c.Statefulset.Spec.PersistentVolumeClaimRetentionPolicy == nil { + c.Statefulset.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + } + } + if !reflect.DeepEqual(c.Statefulset.Spec.PersistentVolumeClaimRetentionPolicy, statefulSet.Spec.PersistentVolumeClaimRetentionPolicy) { + match = false + needsReplace = true + reasons = append(reasons, "new statefulset's persistent volume claim retention policy do not match") + } + + needsRollUpdate, reasons = c.compareContainers("statefulset initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons) + needsRollUpdate, reasons = c.compareContainers("statefulset containers", c.Statefulset.Spec.Template.Spec.Containers, statefulSet.Spec.Template.Spec.Containers, needsRollUpdate, reasons) if len(c.Statefulset.Spec.Template.Spec.Containers) == 0 { c.logger.Warningf("statefulset %q has no container", util.NameFromMeta(c.Statefulset.ObjectMeta)) @@ -405,6 +511,16 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa needsRollUpdate = true reasons = append(reasons, "new statefulset's pod affinity does not match the current one") } + if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.TopologySpreadConstraints, statefulSet.Spec.Template.Spec.TopologySpreadConstraints) { + needsReplace = true + needsRollUpdate = true + reasons = append(reasons, "new statefulset's pod topologySpreadConstraints does not match the current one") + } + if len(c.Statefulset.Spec.Template.Spec.Tolerations) != len(statefulSet.Spec.Template.Spec.Tolerations) { + needsReplace = true + needsRollUpdate = true + reasons = append(reasons, "new statefulset's pod tolerations does not match the current one") + } // Some generated fields like creationTimestamp make it not possible to use DeepCompare on Spec.Template.ObjectMeta if !reflect.DeepEqual(c.Statefulset.Spec.Template.Labels, statefulSet.Spec.Template.Labels) { @@ -425,14 +541,12 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa } } - if !reflect.DeepEqual(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations) { + if changed, reason := c.compareAnnotations(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations, &deletedPodAnnotations); changed { match = false needsReplace = true - needsRollUpdate = true - reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one") + reasons = append(reasons, "new statefulset's pod template metadata annotations does not match "+reason) } if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) { - match = false needsReplace = true needsRollUpdate = true reasons = append(reasons, "new statefulset's pod template security context in spec does not match the current one") @@ -440,23 +554,24 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa if len(c.Statefulset.Spec.VolumeClaimTemplates) != len(statefulSet.Spec.VolumeClaimTemplates) { needsReplace = true reasons = append(reasons, "new statefulset's volumeClaimTemplates contains different number of volumes to the old one") - } - for i := 0; i < len(c.Statefulset.Spec.VolumeClaimTemplates); i++ { - name := c.Statefulset.Spec.VolumeClaimTemplates[i].Name - // Some generated fields like creationTimestamp make it not possible to use DeepCompare on ObjectMeta - if name != statefulSet.Spec.VolumeClaimTemplates[i].Name { - needsReplace = true - reasons = append(reasons, fmt.Sprintf("new statefulset's name for volume %d does not match the current one", i)) - continue - } - if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Annotations, statefulSet.Spec.VolumeClaimTemplates[i].Annotations) { - needsReplace = true - reasons = append(reasons, fmt.Sprintf("new statefulset's annotations for volume %q does not match the current one", name)) - } - if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Spec, statefulSet.Spec.VolumeClaimTemplates[i].Spec) { + } else { + for i := 0; i < len(c.Statefulset.Spec.VolumeClaimTemplates); i++ { name := c.Statefulset.Spec.VolumeClaimTemplates[i].Name - needsReplace = true - reasons = append(reasons, fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %q does not match the current one", name)) + // Some generated fields like creationTimestamp make it not possible to use DeepCompare on ObjectMeta + if name != statefulSet.Spec.VolumeClaimTemplates[i].Name { + needsReplace = true + reasons = append(reasons, fmt.Sprintf("new statefulset's name for volume %d does not match the current one", i)) + continue + } + if changed, reason := c.compareAnnotations(c.Statefulset.Spec.VolumeClaimTemplates[i].Annotations, statefulSet.Spec.VolumeClaimTemplates[i].Annotations, nil); changed { + needsReplace = true + reasons = append(reasons, fmt.Sprintf("new statefulset's annotations for volume %q do not match the current ones: %s", name, reason)) + } + if !reflect.DeepEqual(c.Statefulset.Spec.VolumeClaimTemplates[i].Spec, statefulSet.Spec.VolumeClaimTemplates[i].Spec) { + name := c.Statefulset.Spec.VolumeClaimTemplates[i].Name + needsReplace = true + reasons = append(reasons, fmt.Sprintf("new statefulset's volumeClaimTemplates specification for volume %q does not match the current one", name)) + } } } @@ -468,7 +583,6 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa // we assume any change in priority happens by rolling out a new priority class // changing the priority value in an existing class is not supproted if c.Statefulset.Spec.Template.Spec.PriorityClassName != statefulSet.Spec.Template.Spec.PriorityClassName { - match = false needsReplace = true needsRollUpdate = true reasons = append(reasons, "new statefulset's pod priority class in spec does not match the current one") @@ -487,7 +601,7 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa match = false } - return &compareStatefulsetResult{match: match, reasons: reasons, rollingUpdate: needsRollUpdate, replace: needsReplace} + return &compareStatefulsetResult{match: match, reasons: reasons, rollingUpdate: needsRollUpdate, replace: needsReplace, deletedPodAnnotations: deletedPodAnnotations} } type containerCondition func(a, b v1.Container) bool @@ -508,28 +622,32 @@ func newCheck(msg string, cond containerCondition) containerCheck { func (c *Cluster) compareContainers(description string, setA, setB []v1.Container, needsRollUpdate bool, reasons []string) (bool, []string) { if len(setA) != len(setB) { - return true, append(reasons, fmt.Sprintf("new statefulset %s's length does not match the current ones", description)) + return true, append(reasons, fmt.Sprintf("new %s's length does not match the current ones", description)) } checks := []containerCheck{ - newCheck("new statefulset %s's %s (index %d) name does not match the current one", + newCheck("new %s's %s (index %d) name does not match the current one", func(a, b v1.Container) bool { return a.Name != b.Name }), - newCheck("new statefulset %s's %s (index %d) ports do not match the current one", - func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Ports, b.Ports) }), - newCheck("new statefulset %s's %s (index %d) resources do not match the current ones", + newCheck("new %s's %s (index %d) readiness probe does not match the current one", + func(a, b v1.Container) bool { return !reflect.DeepEqual(a.ReadinessProbe, b.ReadinessProbe) }), + newCheck("new %s's %s (index %d) liveness probe does not match the current one", + func(a, b v1.Container) bool { return !reflect.DeepEqual(a.LivenessProbe, b.LivenessProbe) }), + newCheck("new %s's %s (index %d) ports do not match the current one", + func(a, b v1.Container) bool { return !comparePorts(a.Ports, b.Ports) }), + newCheck("new %s's %s (index %d) resources do not match the current ones", func(a, b v1.Container) bool { return !compareResources(&a.Resources, &b.Resources) }), - newCheck("new statefulset %s's %s (index %d) environment does not match the current one", - func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Env, b.Env) }), - newCheck("new statefulset %s's %s (index %d) environment sources do not match the current one", + newCheck("new %s's %s (index %d) environment does not match the current one", + func(a, b v1.Container) bool { return !compareEnv(a.Env, b.Env) }), + newCheck("new %s's %s (index %d) environment sources do not match the current one", func(a, b v1.Container) bool { return !reflect.DeepEqual(a.EnvFrom, b.EnvFrom) }), - newCheck("new statefulset %s's %s (index %d) security context does not match the current one", + newCheck("new %s's %s (index %d) security context does not match the current one", func(a, b v1.Container) bool { return !reflect.DeepEqual(a.SecurityContext, b.SecurityContext) }), - newCheck("new statefulset %s's %s (index %d) volume mounts do not match the current one", - func(a, b v1.Container) bool { return !reflect.DeepEqual(a.VolumeMounts, b.VolumeMounts) }), + newCheck("new %s's %s (index %d) volume mounts do not match the current one", + func(a, b v1.Container) bool { return !compareVolumeMounts(a.VolumeMounts, b.VolumeMounts) }), } if !c.OpConfig.EnableLazySpiloUpgrade { - checks = append(checks, newCheck("new statefulset %s's %s (index %d) image does not match the current one", + checks = append(checks, newCheck("new %s's %s (index %d) image does not match the current one", func(a, b v1.Container) bool { return a.Image != b.Image })) } @@ -573,68 +691,377 @@ func compareResourcesAssumeFirstNotNil(a *v1.ResourceRequirements, b *v1.Resourc } } return true - } -func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { - - var ( - isSmaller bool - err error - ) - - // setting limits too low can cause unnecessary evictions / OOM kills - minCPULimit := c.OpConfig.MinCPULimit - minMemoryLimit := c.OpConfig.MinMemoryLimit - - cpuLimit := spec.Resources.ResourceLimits.CPU - if cpuLimit != "" { - isSmaller, err = util.IsSmallerQuantity(cpuLimit, minCPULimit) - if err != nil { - return fmt.Errorf("could not compare defined CPU limit %s with configured minimum value %s: %v", cpuLimit, minCPULimit, err) +func compareEnv(a, b []v1.EnvVar) bool { + if len(a) != len(b) { + return false + } + var equal bool + for _, enva := range a { + hasmatch := false + for _, envb := range b { + if enva.Name == envb.Name { + hasmatch = true + if enva.Name == "SPILO_CONFIGURATION" { + equal = compareSpiloConfiguration(enva.Value, envb.Value) + } else { + if enva.Value == "" && envb.Value == "" { + equal = reflect.DeepEqual(enva.ValueFrom, envb.ValueFrom) + } else { + equal = (enva.Value == envb.Value) + } + } + if !equal { + return false + } + } } - if isSmaller { - c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be increased", cpuLimit, minCPULimit) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit) - spec.Resources.ResourceLimits.CPU = minCPULimit + if !hasmatch { + return false + } + } + return true +} + +func compareSpiloConfiguration(configa, configb string) bool { + var oa, ob spiloConfiguration + + var err error + err = json.Unmarshal([]byte(configa), &oa) + if err != nil { + return false + } + oa.Bootstrap.DCS = patroniDCS{} + err = json.Unmarshal([]byte(configb), &ob) + if err != nil { + return false + } + ob.Bootstrap.DCS = patroniDCS{} + return reflect.DeepEqual(oa, ob) +} + +func areProtocolsEqual(a, b v1.Protocol) bool { + return a == b || + (a == "" && b == v1.ProtocolTCP) || + (a == v1.ProtocolTCP && b == "") +} + +func comparePorts(a, b []v1.ContainerPort) bool { + if len(a) != len(b) { + return false + } + + areContainerPortsEqual := func(a, b v1.ContainerPort) bool { + return a.Name == b.Name && + a.HostPort == b.HostPort && + areProtocolsEqual(a.Protocol, b.Protocol) && + a.HostIP == b.HostIP + } + + findByPortValue := func(portSpecs []v1.ContainerPort, port int32) (v1.ContainerPort, bool) { + for _, portSpec := range portSpecs { + if portSpec.ContainerPort == port { + return portSpec, true + } + } + return v1.ContainerPort{}, false + } + + for _, portA := range a { + portB, found := findByPortValue(b, portA.ContainerPort) + if !found { + return false + } + if !areContainerPortsEqual(portA, portB) { + return false } } - memoryLimit := spec.Resources.ResourceLimits.Memory - if memoryLimit != "" { - isSmaller, err = util.IsSmallerQuantity(memoryLimit, minMemoryLimit) - if err != nil { - return fmt.Errorf("could not compare defined memory limit %s with configured minimum value %s: %v", memoryLimit, minMemoryLimit, err) - } - if isSmaller { - c.logger.Warningf("defined memory limit %s is below required minimum %s and will be increased", memoryLimit, minMemoryLimit) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit) - spec.Resources.ResourceLimits.Memory = minMemoryLimit + return true +} + +func compareVolumeMounts(old, new []v1.VolumeMount) bool { + if len(old) != len(new) { + return false + } + for _, mount := range old { + if !volumeMountExists(mount, new) { + return false } } + return true +} + +func volumeMountExists(mount v1.VolumeMount, mounts []v1.VolumeMount) bool { + for _, m := range mounts { + if reflect.DeepEqual(mount, m) { + return true + } + } + return false +} + +func (c *Cluster) compareAnnotations(old, new map[string]string, removedList *[]string) (bool, string) { + reason := "" + ignoredAnnotations := make(map[string]bool) + for _, ignore := range c.OpConfig.IgnoredAnnotations { + ignoredAnnotations[ignore] = true + } + + for key := range old { + if _, ok := ignoredAnnotations[key]; ok { + continue + } + if _, ok := new[key]; !ok { + reason += fmt.Sprintf(" Removed %q.", key) + if removedList != nil { + *removedList = append(*removedList, key) + } + } + } + + for key := range new { + if _, ok := ignoredAnnotations[key]; ok { + continue + } + v, ok := old[key] + if !ok { + reason += fmt.Sprintf(" Added %q with value %q.", key, new[key]) + } else if v != new[key] { + reason += fmt.Sprintf(" %q changed from %q to %q.", key, v, new[key]) + } + } + + return reason != "", reason +} + +func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) { + if old.Spec.Type != new.Spec.Type { + return false, fmt.Sprintf("new service's type %q does not match the current one %q", + new.Spec.Type, old.Spec.Type) + } + + oldSourceRanges := old.Spec.LoadBalancerSourceRanges + newSourceRanges := new.Spec.LoadBalancerSourceRanges + + /* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */ + if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) { + if !util.IsEqualIgnoreOrder(oldSourceRanges, newSourceRanges) { + return false, "new service's LoadBalancerSourceRange does not match the current one" + } + } + + if !reflect.DeepEqual(old.ObjectMeta.OwnerReferences, new.ObjectMeta.OwnerReferences) { + return false, "new service's owner references do not match the current ones" + } + + if !reflect.DeepEqual(old.Spec.Selector, new.Spec.Selector) { + return false, "new service's selector does not match the current one" + } + + if old.Spec.ExternalTrafficPolicy != new.Spec.ExternalTrafficPolicy { + return false, "new service's ExternalTrafficPolicy does not match the current one" + } + + if len(old.Spec.Ports) > 0 && len(new.Spec.Ports) > 0 { + // we need to check whether the new port is not zero (=user-defined) + // and only overwrite if it is + if new.Spec.Ports[0].NodePort != 0 && old.Spec.Ports[0].NodePort != new.Spec.Ports[0].NodePort { + return false, "new service's NodePort does not match the current one" + } + } + + return true, "" +} + +func (c *Cluster) compareLogicalBackupJob(cur, new *batchv1.CronJob) *compareLogicalBackupJobResult { + deletedPodAnnotations := []string{} + reasons := make([]string, 0) + match := true + + if cur.Spec.Schedule != new.Spec.Schedule { + match = false + reasons = append(reasons, fmt.Sprintf("new job's schedule %q does not match the current one %q", new.Spec.Schedule, cur.Spec.Schedule)) + } + + newImage := new.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image + curImage := cur.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image + if newImage != curImage { + match = false + reasons = append(reasons, fmt.Sprintf("new job's image %q does not match the current one %q", newImage, curImage)) + } + + newPodAnnotation := new.Spec.JobTemplate.Spec.Template.Annotations + curPodAnnotation := cur.Spec.JobTemplate.Spec.Template.Annotations + if changed, reason := c.compareAnnotations(curPodAnnotation, newPodAnnotation, &deletedPodAnnotations); changed { + match = false + reasons = append(reasons, fmt.Sprint("new job's pod template metadata annotations do not match "+reason)) + } + + newPgVersion := getPgVersion(new) + curPgVersion := getPgVersion(cur) + if newPgVersion != curPgVersion { + match = false + reasons = append(reasons, fmt.Sprintf("new job's env PG_VERSION %q does not match the current one %q", newPgVersion, curPgVersion)) + } + + if !reflect.DeepEqual(cur.Labels, new.Labels) { + match = false + reasons = append(reasons, "new job's labels do not match the current ones") + } + + if !reflect.DeepEqual(cur.Spec.JobTemplate.Labels, new.Spec.JobTemplate.Labels) { + match = false + reasons = append(reasons, "new job's template labels do not match the current ones") + } + + needsReplace := false + contReasons := make([]string, 0) + needsReplace, contReasons = c.compareContainers("cronjob container", cur.Spec.JobTemplate.Spec.Template.Spec.Containers, new.Spec.JobTemplate.Spec.Template.Spec.Containers, needsReplace, contReasons) + if needsReplace { + match = false + reasons = append(reasons, fmt.Sprintf("logical backup container specs do not match: %v", strings.Join(contReasons, `', '`))) + } + + if !reflect.DeepEqual(cur.Spec.SuccessfulJobsHistoryLimit, new.Spec.SuccessfulJobsHistoryLimit) { + match = false + reasons = append(reasons, fmt.Sprintf("new job's successfulJobsHistoryLimit %v does not match the current one %v", new.Spec.SuccessfulJobsHistoryLimit, cur.Spec.SuccessfulJobsHistoryLimit)) + } + + if !reflect.DeepEqual(cur.Spec.FailedJobsHistoryLimit, new.Spec.FailedJobsHistoryLimit) { + match = false + reasons = append(reasons, fmt.Sprintf("new job's failedJobsHistoryLimit %v does not match the current one %v", new.Spec.FailedJobsHistoryLimit, cur.Spec.FailedJobsHistoryLimit)) + } + + if !reflect.DeepEqual(cur.Spec.JobTemplate.Spec.TTLSecondsAfterFinished, new.Spec.JobTemplate.Spec.TTLSecondsAfterFinished) { + match = false + reasons = append(reasons, fmt.Sprintf("new job's TTLSecondsAfterFinished %v does not match the current one %v", new.Spec.JobTemplate.Spec.TTLSecondsAfterFinished, cur.Spec.JobTemplate.Spec.TTLSecondsAfterFinished)) + } + + return &compareLogicalBackupJobResult{match: match, reasons: reasons, deletedPodAnnotations: deletedPodAnnotations} +} + +func (c *Cluster) comparePodDisruptionBudget(cur, new *policyv1.PodDisruptionBudget) (bool, string) { + // TODO: improve comparison + if !reflect.DeepEqual(new.Spec, cur.Spec) { + return false, "new PDB's spec does not match the current one" + } + if !reflect.DeepEqual(new.ObjectMeta.OwnerReferences, cur.ObjectMeta.OwnerReferences) { + return false, "new PDB's owner references do not match the current ones" + } + if changed, reason := c.compareAnnotations(cur.Annotations, new.Annotations, nil); changed { + return false, "new PDB's annotations do not match the current ones:" + reason + } + return true, "" +} + +func getPgVersion(cronJob *batchv1.CronJob) string { + envs := cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env + for _, env := range envs { + if env.Name == "PG_VERSION" { + return env.Value + } + } + return "" +} + +// addFinalizer patches the postgresql CR to add finalizer +func (c *Cluster) addFinalizer() error { + if c.hasFinalizer() { + return nil + } + + c.logger.Infof("adding finalizer %s", finalizerName) + finalizers := append(c.ObjectMeta.Finalizers, finalizerName) + newSpec, err := c.KubeClient.SetFinalizer(c.clusterName(), c.DeepCopy(), finalizers) + if err != nil { + return fmt.Errorf("error adding finalizer: %v", err) + } + + // update the spec, maintaining the new resourceVersion + c.setSpec(newSpec) return nil } +// removeFinalizer patches postgresql CR to remove finalizer +func (c *Cluster) removeFinalizer() error { + if !c.hasFinalizer() { + return nil + } + + c.logger.Infof("removing finalizer %s", finalizerName) + + // Fetch the latest version of the object to avoid resourceVersion conflicts + clusterName := c.clusterName() + latestPg, err := c.KubeClient.PostgresqlsGetter.Postgresqls(clusterName.Namespace).Get( + context.TODO(), clusterName.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error fetching latest postgresql for finalizer removal: %v", err) + } + + finalizers := util.RemoveString(latestPg.ObjectMeta.Finalizers, finalizerName) + newSpec, err := c.KubeClient.SetFinalizer(clusterName, latestPg, finalizers) + if err != nil { + return fmt.Errorf("error removing finalizer: %v", err) + } + + // update the spec, maintaining the new resourceVersion. + c.setSpec(newSpec) + + return nil +} + +// hasFinalizer checks if finalizer is currently set or not +func (c *Cluster) hasFinalizer() bool { + for _, finalizer := range c.ObjectMeta.Finalizers { + if finalizer == finalizerName { + return true + } + } + return false +} + // Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object // (i.e. service) is treated as an error // logical backup cron jobs are an exception: a user-initiated Update can enable a logical backup job // for a cluster that had no such job before. In this case a missing job is not an error. func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed := false - syncStatefulSet := false + userInitFailed := false c.mu.Lock() defer c.mu.Unlock() - c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusUpdating) + newSpec.Status.PostgresClusterStatus = acidv1.ClusterStatusUpdating + + newSpec, err := c.KubeClient.SetPostgresCRDStatus(c.clusterName(), newSpec) + if err != nil { + return fmt.Errorf("could not set cluster status to updating: %w", err) + } + + if !c.isInMaintenanceWindow(newSpec.Spec.MaintenanceWindows) { + // do not apply any major version related changes yet + newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion + } c.setSpec(newSpec) defer func() { + currentStatus := newSpec.Status.DeepCopy() + newSpec.Status.PostgresClusterStatus = acidv1.ClusterStatusRunning + if updateFailed { - c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusUpdateFailed) - } else { - c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusRunning) + newSpec.Status.PostgresClusterStatus = acidv1.ClusterStatusUpdateFailed + } + + if !equality.Semantic.DeepEqual(currentStatus, newSpec.Status) { + pgUpdatedStatus, err := c.KubeClient.SetPostgresCRDStatus(c.clusterName(), newSpec) + if err != nil { + c.logger.Warningf("could not set cluster status: %v", err) + return + } + c.setSpec(pgUpdatedStatus) } }() @@ -643,7 +1070,6 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { if IsBiggerPostgresVersion(oldSpec.Spec.PostgresqlParam.PgVersion, c.GetDesiredMajorVersion()) { c.logger.Infof("postgresql version increased (%s -> %s), depending on config manual upgrade needed", oldSpec.Spec.PostgresqlParam.PgVersion, newSpec.Spec.PostgresqlParam.PgVersion) - syncStatefulSet = true } else { c.logger.Infof("postgresql major version unchanged or smaller, no changes needed") // sticking with old version, this will also advance GetDesiredVersion next time. @@ -651,93 +1077,95 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } // Service - if !reflect.DeepEqual(c.generateService(Master, &oldSpec.Spec), c.generateService(Master, &newSpec.Spec)) || - !reflect.DeepEqual(c.generateService(Replica, &oldSpec.Spec), c.generateService(Replica, &newSpec.Spec)) { - if err := c.syncServices(); err != nil { - c.logger.Errorf("could not sync services: %v", err) - updateFailed = true - } + if err := c.syncServices(); err != nil { + c.logger.Errorf("could not sync services: %v", err) + updateFailed = true } - // connection pooler needs one system user created, which is done in - // initUsers. Check if it needs to be called. - sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) && - reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) - needConnectionPooler := needMasterConnectionPoolerWorker(&newSpec.Spec) || - needReplicaConnectionPoolerWorker(&newSpec.Spec) - if !sameUsers || needConnectionPooler { - c.logger.Debugf("initialize users") - if err := c.initUsers(); err != nil { - c.logger.Errorf("could not init users: %v", err) - updateFailed = true - } - - c.logger.Debugf("syncing secrets") - - //TODO: mind the secrets of the deleted/new users - if err := c.syncSecrets(); err != nil { - c.logger.Errorf("could not sync secrets: %v", err) - updateFailed = true - } + // Patroni service and endpoints / config maps + if err := c.syncPatroniResources(); err != nil { + c.logger.Errorf("could not sync services: %v", err) + updateFailed = true } - // Volume - if c.OpConfig.StorageResizeMode != "off" { - c.syncVolumes() - } else { - c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.") - } - - // Statefulset + // Users func() { - if err := c.enforceMinResourceLimits(&c.Spec); err != nil { - c.logger.Errorf("could not sync resources: %v", err) - updateFailed = true - return - } + // check if users need to be synced during update + sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) && + reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) + sameRotatedUsers := reflect.DeepEqual(oldSpec.Spec.UsersWithSecretRotation, newSpec.Spec.UsersWithSecretRotation) && + reflect.DeepEqual(oldSpec.Spec.UsersWithInPlaceSecretRotation, newSpec.Spec.UsersWithInPlaceSecretRotation) - oldSs, err := c.generateStatefulSet(&oldSpec.Spec) - if err != nil { - c.logger.Errorf("could not generate old statefulset spec: %v", err) - updateFailed = true - return - } + // connection pooler needs one system user created who is initialized in initUsers + // only when disabled in oldSpec and enabled in newSpec + needPoolerUser := c.needConnectionPoolerUser(&oldSpec.Spec, &newSpec.Spec) - // update newSpec to for latter comparison with oldSpec - c.enforceMinResourceLimits(&newSpec.Spec) + // streams new replication user created who is initialized in initUsers + // only when streams were not specified in oldSpec but in newSpec + needStreamUser := len(oldSpec.Spec.Streams) == 0 && len(newSpec.Spec.Streams) > 0 - newSs, err := c.generateStatefulSet(&newSpec.Spec) - if err != nil { - c.logger.Errorf("could not generate new statefulset spec: %v", err) - updateFailed = true - return + initUsers := !sameUsers || !sameRotatedUsers || needPoolerUser || needStreamUser + + // if inherited annotations differ secrets have to be synced on update + newAnnotations := c.annotationsSet(nil) + oldAnnotations := make(map[string]string) + for _, secret := range c.Secrets { + oldAnnotations = secret.ObjectMeta.Annotations + break } - if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) { - c.logger.Debugf("syncing statefulsets") - syncStatefulSet = false - // TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet - if err := c.syncStatefulSet(); err != nil { - c.logger.Errorf("could not sync statefulsets: %v", err) + annotationsChanged, _ := c.compareAnnotations(oldAnnotations, newAnnotations, nil) + + if initUsers || annotationsChanged { + c.logger.Debug("initialize users") + if err := c.initUsers(); err != nil { + c.logger.Errorf("could not init users - skipping sync of secrets and databases: %v", err) + userInitFailed = true + updateFailed = true + return + } + + c.logger.Debug("syncing secrets") + // TODO: mind the secrets of the deleted/new users + if err := c.syncSecrets(); err != nil { + c.logger.Errorf("could not sync secrets: %v", err) updateFailed = true } } }() - // pod disruption budget - if oldSpec.Spec.NumberOfInstances != newSpec.Spec.NumberOfInstances { - c.logger.Debug("syncing pod disruption budgets") - if err := c.syncPodDisruptionBudget(true); err != nil { - c.logger.Errorf("could not sync pod disruption budget: %v", err) + // Volume + if c.OpConfig.StorageResizeMode != "off" { + c.syncVolumes() + } else { + c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume size sync.") + } + + // Statefulset + func() { + if err := c.syncStatefulSet(); err != nil { + c.logger.Errorf("could not sync statefulsets: %v", err) updateFailed = true } + }() + + // add or remove standby_cluster section from Patroni config depending on changes in standby section + if !reflect.DeepEqual(oldSpec.Spec.StandbyCluster, newSpec.Spec.StandbyCluster) { + if err := c.syncStandbyClusterConfiguration(); err != nil { + return fmt.Errorf("could not set StandbyCluster configuration options: %v", err) + } + } + + // pod disruption budgets + if err := c.syncPodDisruptionBudgets(true); err != nil { + c.logger.Errorf("could not sync pod disruption budgets: %v", err) + updateFailed = true } // logical backup job func() { - // create if it did not exist if !oldSpec.Spec.EnableLogicalBackup && newSpec.Spec.EnableLogicalBackup { - c.logger.Debugf("creating backup cron job") + c.logger.Debug("creating backup cron job") if err := c.createLogicalBackupJob(); err != nil { c.logger.Errorf("could not create a k8s cron job for logical backups: %v", err) updateFailed = true @@ -747,7 +1175,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // delete if no longer needed if oldSpec.Spec.EnableLogicalBackup && !newSpec.Spec.EnableLogicalBackup { - c.logger.Debugf("deleting backup cron job") + c.logger.Debug("deleting backup cron job") if err := c.deleteLogicalBackupJob(); err != nil { c.logger.Errorf("could not delete a k8s cron job for logical backups: %v", err) updateFailed = true @@ -756,22 +1184,17 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } - // apply schedule changes - // this is the only parameter of logical backups a user can overwrite in the cluster manifest - if (oldSpec.Spec.EnableLogicalBackup && newSpec.Spec.EnableLogicalBackup) && - (newSpec.Spec.LogicalBackupSchedule != oldSpec.Spec.LogicalBackupSchedule) { - c.logger.Debugf("updating schedule of the backup cron job") + if oldSpec.Spec.EnableLogicalBackup && newSpec.Spec.EnableLogicalBackup { if err := c.syncLogicalBackupJob(); err != nil { c.logger.Errorf("could not sync logical backup jobs: %v", err) updateFailed = true } } - }() // Roles and Databases - if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || c.Spec.StandbyCluster != nil) { - c.logger.Debugf("syncing roles") + if !userInitFailed && !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || c.Spec.StandbyCluster != nil) { + c.logger.Debug("syncing roles") if err := c.syncRoles(); err != nil { c.logger.Errorf("could not sync roles: %v", err) updateFailed = true @@ -798,12 +1221,20 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // need to process. In the future we may want to do this more careful and // check which databases we need to process, but even repeating the whole // installation process should be good enough. - if _, err := c.syncConnectionPooler(oldSpec, newSpec, c.installLookupFunction); err != nil { c.logger.Errorf("could not sync connection pooler: %v", err) updateFailed = true } + // streams + if len(newSpec.Spec.Streams) > 0 || len(oldSpec.Spec.Streams) != len(newSpec.Spec.Streams) { + c.logger.Debug("syncing streams") + if err := c.syncStreams(); err != nil { + c.logger.Errorf("could not sync streams: %v", err) + updateFailed = true + } + } + if !updateFailed { // Major version upgrade must only fire after success of earlier operations and should stay last if err := c.majorVersionUpgrade(); err != nil { @@ -834,44 +1265,68 @@ func syncResources(a, b *v1.ResourceRequirements) bool { // DCS, reuses the master's endpoint to store the leader related metadata. If we remove the endpoint // before the pods, it will be re-created by the current master pod and will remain, obstructing the // creation of the new cluster with the same name. Therefore, the endpoints should be deleted last. -func (c *Cluster) Delete() { +func (c *Cluster) Delete() error { + anyErrors := false c.mu.Lock() defer c.mu.Unlock() - c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Delete", "Started deletion of new cluster resources") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Delete", "Started deletion of cluster resources") + + if err := c.deleteStreams(); err != nil { + anyErrors = true + c.logger.Warningf("could not delete event streams: %v", err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Delete", "could not delete event streams: %v", err) + } // delete the backup job before the stateful set of the cluster to prevent connections to non-existing pods // deleting the cron job also removes pods and batch jobs it created if err := c.deleteLogicalBackupJob(); err != nil { + anyErrors = true c.logger.Warningf("could not remove the logical backup k8s cron job; %v", err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Delete", "could not remove the logical backup k8s cron job; %v", err) } if err := c.deleteStatefulSet(); err != nil { + anyErrors = true c.logger.Warningf("could not delete statefulset: %v", err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Delete", "could not delete statefulset: %v", err) } - if err := c.deleteSecrets(); err != nil { - c.logger.Warningf("could not delete secrets: %v", err) + if c.OpConfig.EnableSecretsDeletion != nil && *c.OpConfig.EnableSecretsDeletion { + if err := c.deleteSecrets(); err != nil { + anyErrors = true + c.logger.Warningf("could not delete secrets: %v", err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Delete", "could not delete secrets: %v", err) + } + } else { + c.logger.Info("not deleting secrets because disabled in configuration") } - if err := c.deletePodDisruptionBudget(); err != nil { - c.logger.Warningf("could not delete pod disruption budget: %v", err) + if err := c.deletePodDisruptionBudgets(); err != nil { + anyErrors = true + c.logger.Warningf("could not delete pod disruption budgets: %v", err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Delete", "could not delete pod disruption budgets: %v", err) } for _, role := range []PostgresRole{Master, Replica} { - if !c.patroniKubernetesUseConfigMaps() { if err := c.deleteEndpoint(role); err != nil { + anyErrors = true c.logger.Warningf("could not delete %s endpoint: %v", role, err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Delete", "could not delete %s endpoint: %v", role, err) } } if err := c.deleteService(role); err != nil { + anyErrors = true c.logger.Warningf("could not delete %s service: %v", role, err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Delete", "could not delete %s service: %v", role, err) } } - if err := c.deletePatroniClusterObjects(); err != nil { - c.logger.Warningf("could not remove leftover patroni objects; %v", err) + if err := c.deletePatroniResources(); err != nil { + anyErrors = true + c.logger.Warningf("could not delete all Patroni resources: %v", err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Delete", "could not delete all Patroni resources: %v", err) } // Delete connection pooler objects anyway, even if it's not mentioned in the @@ -879,39 +1334,61 @@ func (c *Cluster) Delete() { // wrong for _, role := range [2]PostgresRole{Master, Replica} { if err := c.deleteConnectionPooler(role); err != nil { + anyErrors = true c.logger.Warningf("could not remove connection pooler: %v", err) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Delete", "could not remove connection pooler: %v", err) } } + // If we are done deleting our various resources we remove the finalizer to let K8S finally delete the Postgres CR + if anyErrors { + c.eventRecorder.Event(c.GetReference(), v1.EventTypeWarning, "Delete", "some resources could be successfully deleted yet") + return fmt.Errorf("some error(s) occurred when deleting resources, NOT removing finalizer yet") + } + if err := c.removeFinalizer(); err != nil { + return fmt.Errorf("done cleaning up, but error when removing finalizer: %v", err) + } + + return nil } -//NeedsRepair returns true if the cluster should be included in the repair scan (based on its in-memory status). +// NeedsRepair returns true if the cluster should be included in the repair scan (based on its in-memory status). func (c *Cluster) NeedsRepair() (bool, acidv1.PostgresStatus) { c.specMu.RLock() defer c.specMu.RUnlock() return !c.Status.Success(), c.Status - } // ReceivePodEvent is called back by the controller in order to add the cluster's pod event to the queue. func (c *Cluster) ReceivePodEvent(event PodEvent) { + if err := c.podEventsStore.Add(event); err != nil { + c.logger.Errorf("error when receiving pod event for lookup: %v", err) + } if err := c.podEventsQueue.Add(event); err != nil { c.logger.Errorf("error when receiving pod events: %v", err) } } -func (c *Cluster) processPodEvent(obj interface{}) error { +func (c *Cluster) processPodEvent(obj interface{}, isInInitialList bool) error { event, ok := obj.(PodEvent) if !ok { return fmt.Errorf("could not cast to PodEvent") } + // can only take lock when (un)registerPodSubscriber is finshed c.podSubscribersMu.RLock() subscriber, ok := c.podSubscribers[spec.NamespacedName(event.PodName)] - c.podSubscribersMu.RUnlock() if ok { - subscriber <- event + select { + case subscriber <- event: + default: + // ending up here when there is no receiver on the channel (i.e. waitForPodLabel finished) + // avoids blocking channel: https://gobyexample.com/non-blocking-channel-operations + } } + // hold lock for the time of processing the event to avoid race condition + // with unregisterPodSubscriber closing the channel (see #1876) + c.podSubscribersMu.RUnlock() return nil } @@ -927,7 +1404,19 @@ func (c *Cluster) processPodEventQueue(stopCh <-chan struct{}) { case <-stopCh: return default: - if _, err := c.podEventsQueue.Pop(cache.PopProcessFunc(c.processPodEvent)); err != nil { + _, err := c.podEventsQueue.Pop(cache.PopProcessFunc(func(obj interface{}, isInInitialList bool) error { + event, ok := obj.(PodEvent) + if !ok { + c.logger.Errorf("could not cast to pod event") + return nil // skip event to keep processing + } + c.processPodEvent(event, isInInitialList) + if err := c.podEventsStore.Delete(obj); err != nil { + c.logger.Errorf("failed to delete key from lookup store: %v", err) + } + return nil + })) + if err != nil { c.logger.Errorf("error when processing pod event queue %v", err) } } @@ -940,59 +1429,57 @@ func (c *Cluster) initSystemUsers() { // secrets, therefore, setting flags like SUPERUSER or REPLICATION // is not necessary here c.systemUsers[constants.SuperuserKeyName] = spec.PgUser{ - Origin: spec.RoleOriginSystem, - Name: c.OpConfig.SuperUsername, - Password: util.RandomPassword(constants.PasswordLength), + Origin: spec.RoleOriginSystem, + Name: c.OpConfig.SuperUsername, + Namespace: c.Namespace, + Password: util.RandomPassword(constants.PasswordLength), } c.systemUsers[constants.ReplicationUserKeyName] = spec.PgUser{ - Origin: spec.RoleOriginSystem, - Name: c.OpConfig.ReplicationUsername, - Password: util.RandomPassword(constants.PasswordLength), + Origin: spec.RoleOriginSystem, + Name: c.OpConfig.ReplicationUsername, + Namespace: c.Namespace, + Flags: []string{constants.RoleFlagLogin}, + Password: util.RandomPassword(constants.PasswordLength), } - // Connection pooler user is an exception, if requested it's going to be - // created by operator as a normal pgUser + // Connection pooler user is an exception + // if requested it's going to be created by operator if needConnectionPooler(&c.Spec) { - // initialize empty connection pooler if not done yet - if c.Spec.ConnectionPooler == nil { - c.Spec.ConnectionPooler = &acidv1.ConnectionPooler{} - } - - // Using superuser as pooler user is not a good idea. First of all it's - // not going to be synced correctly with the current implementation, - // and second it's a bad practice. - username := c.OpConfig.ConnectionPooler.User - - isSuperUser := c.Spec.ConnectionPooler.User == c.OpConfig.SuperUsername - isProtectedUser := c.shouldAvoidProtectedOrSystemRole( - c.Spec.ConnectionPooler.User, "connection pool role") - - if !isSuperUser && !isProtectedUser { - username = util.Coalesce( - c.Spec.ConnectionPooler.User, - c.OpConfig.ConnectionPooler.User) - } + username := c.poolerUser(&c.Spec) // connection pooler application should be able to login with this role connectionPoolerUser := spec.PgUser{ - Origin: spec.RoleConnectionPooler, - Name: username, - Flags: []string{constants.RoleFlagLogin}, - Password: util.RandomPassword(constants.PasswordLength), - } - - if _, exists := c.pgUsers[username]; !exists { - c.pgUsers[username] = connectionPoolerUser + Origin: spec.RoleOriginConnectionPooler, + Name: username, + Namespace: c.Namespace, + Flags: []string{constants.RoleFlagLogin}, + Password: util.RandomPassword(constants.PasswordLength), } if _, exists := c.systemUsers[constants.ConnectionPoolerUserKeyName]; !exists { c.systemUsers[constants.ConnectionPoolerUserKeyName] = connectionPoolerUser } } + + // replication users for event streams are another exception + // the operator will create one replication user for all streams + if len(c.Spec.Streams) > 0 { + username := fmt.Sprintf("%s%s", constants.EventStreamSourceSlotPrefix, constants.UserRoleNameSuffix) + streamUser := spec.PgUser{ + Origin: spec.RoleOriginStream, + Name: username, + Namespace: c.Namespace, + Flags: []string{constants.RoleFlagLogin, constants.RoleFlagReplication}, + Password: util.RandomPassword(constants.PasswordLength), + } + + if _, exists := c.systemUsers[constants.EventStreamUserKeyName]; !exists { + c.systemUsers[constants.EventStreamUserKeyName] = streamUser + } + } } func (c *Cluster) initPreparedDatabaseRoles() error { - if c.Spec.PreparedDatabases != nil && len(c.Spec.PreparedDatabases) == 0 { // TODO: add option to disable creating such a default DB c.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{strings.Replace(c.Name, "-", "_", -1): {}} } @@ -1004,9 +1491,9 @@ func (c *Cluster) initPreparedDatabaseRoles() error { constants.WriterRoleNameSuffix: constants.ReaderRoleNameSuffix, } defaultUsers := map[string]string{ - constants.OwnerRoleNameSuffix + constants.UserRoleNameSuffix: constants.OwnerRoleNameSuffix, - constants.ReaderRoleNameSuffix + constants.UserRoleNameSuffix: constants.ReaderRoleNameSuffix, - constants.WriterRoleNameSuffix + constants.UserRoleNameSuffix: constants.WriterRoleNameSuffix, + fmt.Sprintf("%s%s", constants.OwnerRoleNameSuffix, constants.UserRoleNameSuffix): constants.OwnerRoleNameSuffix, + fmt.Sprintf("%s%s", constants.ReaderRoleNameSuffix, constants.UserRoleNameSuffix): constants.ReaderRoleNameSuffix, + fmt.Sprintf("%s%s", constants.WriterRoleNameSuffix, constants.UserRoleNameSuffix): constants.WriterRoleNameSuffix, } for preparedDbName, preparedDB := range c.Spec.PreparedDatabases { @@ -1016,18 +1503,18 @@ func (c *Cluster) initPreparedDatabaseRoles() error { preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} } - var searchPath strings.Builder - searchPath.WriteString(constants.DefaultSearchPath) + searchPathArr := []string{constants.DefaultSearchPath} for preparedSchemaName := range preparedSchemas { - searchPath.WriteString(", " + preparedSchemaName) + searchPathArr = append(searchPathArr, fmt.Sprintf("%q", preparedSchemaName)) } + searchPath := strings.Join(searchPathArr, ", ") // default roles per database - if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName, searchPath.String()); err != nil { + if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName, searchPath, preparedDB.SecretNamespace); err != nil { return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) } if preparedDB.DefaultUsers { - if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName, searchPath.String()); err != nil { + if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName, searchPath, preparedDB.SecretNamespace); err != nil { return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) } } @@ -1038,14 +1525,16 @@ func (c *Cluster) initPreparedDatabaseRoles() error { if err := c.initDefaultRoles(defaultRoles, preparedDbName+constants.OwnerRoleNameSuffix, preparedDbName+"_"+preparedSchemaName, - constants.DefaultSearchPath+", "+preparedSchemaName); err != nil { + fmt.Sprintf("%s, %q", constants.DefaultSearchPath, preparedSchemaName), + preparedDB.SecretNamespace); err != nil { return fmt.Errorf("could not initialize default roles for database schema %s: %v", preparedSchemaName, err) } if preparedSchema.DefaultUsers { if err := c.initDefaultRoles(defaultUsers, preparedDbName+constants.OwnerRoleNameSuffix, preparedDbName+"_"+preparedSchemaName, - constants.DefaultSearchPath+", "+preparedSchemaName); err != nil { + fmt.Sprintf("%s, %q", constants.DefaultSearchPath, preparedSchemaName), + preparedDB.SecretNamespace); err != nil { return fmt.Errorf("could not initialize default users for database schema %s: %v", preparedSchemaName, err) } } @@ -1055,11 +1544,18 @@ func (c *Cluster) initPreparedDatabaseRoles() error { return nil } -func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix string, searchPath string) error { - +func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix, searchPath, secretNamespace string) error { for defaultRole, inherits := range defaultRoles { - - roleName := prefix + defaultRole + namespace := c.Namespace + // if namespaced secrets are allowed + if secretNamespace != "" { + if c.Config.OpConfig.EnableCrossNamespaceSecret { + namespace = secretNamespace + } else { + c.logger.Warn("secretNamespace ignored because enable_cross_namespace_secret set to false. Creating secrets in cluster namespace.") + } + } + roleName := fmt.Sprintf("%s%s", prefix, defaultRole) flags := []string{constants.RoleFlagNoLogin} if defaultRole[len(defaultRole)-5:] == constants.UserRoleNameSuffix { @@ -1072,20 +1568,24 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix } adminRole := "" + isOwner := false if strings.Contains(defaultRole, constants.OwnerRoleNameSuffix) { adminRole = admin + isOwner = true } else { - adminRole = prefix + constants.OwnerRoleNameSuffix + adminRole = fmt.Sprintf("%s%s", prefix, constants.OwnerRoleNameSuffix) } newRole := spec.PgUser{ Origin: spec.RoleOriginBootstrap, Name: roleName, + Namespace: namespace, Password: util.RandomPassword(constants.PasswordLength), Flags: flags, MemberOf: memberOf, Parameters: map[string]string{"search_path": searchPath}, AdminRole: adminRole, + IsDbOwner: isOwner, } if currentRole, present := c.pgUsers[roleName]; present { c.pgUsers[roleName] = c.resolveNameConflict(¤tRole, &newRole) @@ -1105,6 +1605,25 @@ func (c *Cluster) initRobotUsers() error { if c.shouldAvoidProtectedOrSystemRole(username, "manifest robot role") { continue } + namespace := c.Namespace + + // check if role is specified as database owner + isOwner := false + for _, owner := range c.Spec.Databases { + if username == owner { + isOwner = true + } + } + + // if namespaced secrets are allowed + if c.Config.OpConfig.EnableCrossNamespaceSecret { + if strings.Contains(username, ".") { + splits := strings.Split(username, ".") + namespace = splits[0] + c.logger.Warningf("enable_cross_namespace_secret is set. Database role name contains the respective namespace i.e. %s is the created user", username) + } + } + flags, err := normalizeUserFlags(userFlags) if err != nil { return fmt.Errorf("invalid flags for user %q: %v", username, err) @@ -1116,9 +1635,11 @@ func (c *Cluster) initRobotUsers() error { newRole := spec.PgUser{ Origin: spec.RoleOriginManifest, Name: username, + Namespace: namespace, Password: util.RandomPassword(constants.PasswordLength), Flags: flags, AdminRole: adminRole, + IsDbOwner: isOwner, } if currentRole, present := c.pgUsers[username]; present { c.pgUsers[username] = c.resolveNameConflict(¤tRole, &newRole) @@ -1129,9 +1650,22 @@ func (c *Cluster) initRobotUsers() error { return nil } +func (c *Cluster) initAdditionalOwnerRoles() { + if len(c.OpConfig.AdditionalOwnerRoles) == 0 { + return + } + + // fetch database owners and assign additional owner roles + for username, pgUser := range c.pgUsers { + if pgUser.IsDbOwner { + pgUser.MemberOf = append(pgUser.MemberOf, c.OpConfig.AdditionalOwnerRoles...) + c.pgUsers[username] = pgUser + } + } +} + func (c *Cluster) initTeamMembers(teamID string, isPostgresSuperuserTeam bool) error { teamMembers, err := c.getTeamMembers(teamID) - if err != nil { return fmt.Errorf("could not get list of team members for team %q: %v", teamID, err) } @@ -1170,7 +1704,6 @@ func (c *Cluster) initTeamMembers(teamID string, isPostgresSuperuserTeam bool) e } func (c *Cluster) initHumanUsers() error { - var clusterIsOwnedBySuperuserTeam bool superuserTeams := []string{} @@ -1233,6 +1766,7 @@ func (c *Cluster) initInfrastructureRoles() error { return fmt.Errorf("invalid flags for user '%v': %v", username, err) } newRole.Flags = flags + newRole.Namespace = c.Namespace if currentRole, present := c.pgUsers[username]; present { c.pgUsers[username] = c.resolveNameConflict(¤tRole, &newRole) @@ -1278,73 +1812,118 @@ func (c *Cluster) GetCurrentProcess() Process { // GetStatus provides status of the cluster func (c *Cluster) GetStatus() *ClusterStatus { - return &ClusterStatus{ - Cluster: c.Spec.ClusterName, - Team: c.Spec.TeamID, - Status: c.Status, - Spec: c.Spec, - - MasterService: c.GetServiceMaster(), - ReplicaService: c.GetServiceReplica(), - MasterEndpoint: c.GetEndpointMaster(), - ReplicaEndpoint: c.GetEndpointReplica(), - StatefulSet: c.GetStatefulSet(), - PodDisruptionBudget: c.GetPodDisruptionBudget(), - CurrentProcess: c.GetCurrentProcess(), + status := &ClusterStatus{ + Cluster: c.Name, + Namespace: c.Namespace, + Team: c.Spec.TeamID, + Status: c.Status, + Spec: c.Spec, + MasterService: c.GetServiceMaster(), + ReplicaService: c.GetServiceReplica(), + StatefulSet: c.GetStatefulSet(), + PrimaryPodDisruptionBudget: c.GetPrimaryPodDisruptionBudget(), + CriticalOpPodDisruptionBudget: c.GetCriticalOpPodDisruptionBudget(), + CurrentProcess: c.GetCurrentProcess(), Error: fmt.Errorf("error: %s", c.Error), } + + if !c.patroniKubernetesUseConfigMaps() { + status.MasterEndpoint = c.GetEndpointMaster() + status.ReplicaEndpoint = c.GetEndpointReplica() + } + + return status +} + +func (c *Cluster) GetSwitchoverSchedule() string { + now := time.Now().UTC() + return c.getSwitchoverScheduleAtTime(now) +} + +func (c *Cluster) getSwitchoverScheduleAtTime(now time.Time) string { + var possibleSwitchover, schedule time.Time + + maintenanceWindows := c.Spec.MaintenanceWindows + if len(maintenanceWindows) == 0 { + maintenanceWindows = make([]acidv1.MaintenanceWindow, 0, len(c.OpConfig.MaintenanceWindows)) + for _, windowStr := range c.OpConfig.MaintenanceWindows { + var window acidv1.MaintenanceWindow + if err := window.UnmarshalJSON([]byte(windowStr)); err != nil { + c.logger.Errorf("could not parse default maintenance window %q: %v", windowStr, err) + continue + } + maintenanceWindows = append(maintenanceWindows, window) + } + } + + for _, window := range maintenanceWindows { + // in the best case it is possible today + possibleSwitchover = time.Date(now.Year(), now.Month(), now.Day(), window.StartTime.Hour(), window.StartTime.Minute(), 0, 0, time.UTC) + if window.Everyday { + if now.After(possibleSwitchover) { + // we are already past the time for today, try tomorrow + possibleSwitchover = possibleSwitchover.AddDate(0, 0, 1) + } + } else { + if now.Weekday() != window.Weekday { + // get closest possible time for this window + possibleSwitchover = possibleSwitchover.AddDate(0, 0, int((7+window.Weekday-now.Weekday())%7)) + } else if now.After(possibleSwitchover) { + // we are already past the time for today, try next week + possibleSwitchover = possibleSwitchover.AddDate(0, 0, 7) + } + } + + if (schedule.Equal(time.Time{})) || possibleSwitchover.Before(schedule) { + schedule = possibleSwitchover + } + } + + if schedule.IsZero() { + return "" + } + + return schedule.Format("2006-01-02T15:04+00") } // Switchover does a switchover (via Patroni) to a candidate pod -func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName) error { - +func (c *Cluster) Switchover(curMaster *v1.Pod, candidate spec.NamespacedName, scheduled bool) error { var err error - c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switching over from %q to %q", curMaster.Name, candidate) - var wg sync.WaitGroup - - podLabelErr := make(chan error) stopCh := make(chan struct{}) + ch := c.registerPodSubscriber(candidate) + defer c.unregisterPodSubscriber(candidate) + defer close(stopCh) - wg.Add(1) + var scheduled_at string + if scheduled { + scheduled_at = c.GetSwitchoverSchedule() + } else { + c.logger.Debugf("switching over from %q to %q", curMaster.Name, candidate) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switching over from %q to %q", curMaster.Name, candidate) + scheduled_at = "" + } - go func() { - defer wg.Done() - ch := c.registerPodSubscriber(candidate) - defer c.unregisterPodSubscriber(candidate) - - role := Master - - select { - case <-stopCh: - case podLabelErr <- func() (err2 error) { - _, err2 = c.waitForPodLabel(ch, stopCh, &role) - return - }(): + if err = c.patroni.Switchover(curMaster, candidate.Name, scheduled_at); err == nil { + if scheduled { + c.logger.Infof("switchover from %q to %q is scheduled at %s", curMaster.Name, candidate, scheduled_at) + return nil } - }() - - if err = c.patroni.Switchover(curMaster, candidate.Name); err == nil { c.logger.Debugf("successfully switched over from %q to %q", curMaster.Name, candidate) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Successfully switched over from %q to %q", curMaster.Name, candidate) - if err = <-podLabelErr; err != nil { + _, err = c.waitForPodLabel(ch, stopCh, nil) + if err != nil { err = fmt.Errorf("could not get master pod label: %v", err) } } else { + if scheduled { + return fmt.Errorf("could not schedule switchover: %v", err) + } err = fmt.Errorf("could not switch over from %q to %q: %v", curMaster.Name, candidate, err) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Switchover", "Switchover from %q to %q FAILED: %v", curMaster.Name, candidate, err) } - // signal the role label waiting goroutine to close the shop and go home - close(stopCh) - // wait until the goroutine terminates, since unregisterPodSubscriber - // must be called before the outer return; otherwise we risk subscribing to the same pod twice. - wg.Wait() - // close the label waiting channel no sooner than the waiting goroutine terminates. - close(podLabelErr) - return err } @@ -1357,94 +1936,3 @@ func (c *Cluster) Lock() { func (c *Cluster) Unlock() { c.mu.Unlock() } - -type simpleActionWithResult func() error - -type clusterObjectGet func(name string) (spec.NamespacedName, error) - -type clusterObjectDelete func(name string) error - -func (c *Cluster) deletePatroniClusterObjects() error { - // TODO: figure out how to remove leftover patroni objects in other cases - var actionsList []simpleActionWithResult - - if !c.patroniUsesKubernetes() { - c.logger.Infof("not cleaning up Etcd Patroni objects on cluster delete") - } - - if !c.patroniKubernetesUseConfigMaps() { - actionsList = append(actionsList, c.deletePatroniClusterEndpoints) - } - actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps) - - c.logger.Debugf("removing leftover Patroni objects (endpoints / services and configmaps)") - for _, deleter := range actionsList { - if err := deleter(); err != nil { - return err - } - } - return nil -} - -func (c *Cluster) deleteClusterObject( - get clusterObjectGet, - del clusterObjectDelete, - objType string) error { - for _, suffix := range patroniObjectSuffixes { - name := fmt.Sprintf("%s-%s", c.Name, suffix) - - if namespacedName, err := get(name); err == nil { - c.logger.Debugf("deleting Patroni cluster object %q with name %q", - objType, namespacedName) - - if err = del(name); err != nil { - return fmt.Errorf("could not delete Patroni cluster object %q with name %q: %v", - objType, namespacedName, err) - } - - } else if !k8sutil.ResourceNotFound(err) { - return fmt.Errorf("could not fetch Patroni Endpoint %q: %v", - namespacedName, err) - } - } - return nil -} - -func (c *Cluster) deletePatroniClusterServices() error { - get := func(name string) (spec.NamespacedName, error) { - svc, err := c.KubeClient.Services(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) - return util.NameFromMeta(svc.ObjectMeta), err - } - - deleteServiceFn := func(name string) error { - return c.KubeClient.Services(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) - } - - return c.deleteClusterObject(get, deleteServiceFn, "service") -} - -func (c *Cluster) deletePatroniClusterEndpoints() error { - get := func(name string) (spec.NamespacedName, error) { - ep, err := c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) - return util.NameFromMeta(ep.ObjectMeta), err - } - - deleteEndpointFn := func(name string) error { - return c.KubeClient.Endpoints(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) - } - - return c.deleteClusterObject(get, deleteEndpointFn, "endpoint") -} - -func (c *Cluster) deletePatroniClusterConfigMaps() error { - get := func(name string) (spec.NamespacedName, error) { - cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) - return util.NameFromMeta(cm.ObjectMeta), err - } - - deleteConfigMapFn := func(name string) error { - return c.KubeClient.ConfigMaps(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) - } - - return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") -} diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 1f6510e65..95a445ff3 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -1,56 +1,173 @@ package cluster import ( + "context" "fmt" + "net/http" "reflect" + "strings" "testing" + "time" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "github.com/zalando/postgres-operator/pkg/util/patroni" "github.com/zalando/postgres-operator/pkg/util/teams" + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/record" ) const ( superUserName = "postgres" replicationUserName = "standby" + poolerUserName = "pooler" + adminUserName = "admin" + exampleSpiloConfig = `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 scram-sha-256","host all all 0.0.0.0/0 scram-sha-256"]},"bootstrap":{"initdb":[{"auth-host":"scram-sha-256"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}` + spiloConfigDiff = `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 scram-sha-256","host all all 0.0.0.0/0 scram-sha-256"]},"bootstrap":{"initdb":[{"auth-host":"scram-sha-256"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"dcs":{"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}` ) var logger = logrus.New().WithField("test", "cluster") -var eventRecorder = record.NewFakeRecorder(1) + +// eventRecorder needs buffer for TestCreate which emit events for +// 1 cluster, primary endpoint, 2 services, the secrets, the statefulset and pods being ready +var eventRecorder = record.NewFakeRecorder(7) var cl = New( Config{ OpConfig: config.Config{ PodManagementPolicy: "ordered_ready", - ProtectedRoles: []string{"admin"}, + ProtectedRoles: []string{adminUserName, "cron_admin", "part_man"}, Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + AdditionalOwnerRoles: []string{"cron_admin", "part_man"}, }, Resources: config.Resources{ DownscalerAnnotations: []string{"downscaler/*"}, }, + ConnectionPooler: config.ConnectionPooler{ + User: poolerUserName, + }, }, }, k8sutil.NewMockKubernetesClient(), - acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test", Annotations: map[string]string{"downscaler/downtime_replicas": "0"}}}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: "acid-test", + Namespace: "test", + Annotations: map[string]string{"downscaler/downtime_replicas": "0"}, + }, + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: util.True(), + Streams: []acidv1.Stream{ + { + ApplicationId: "test-app", + Database: "test_db", + Tables: map[string]acidv1.StreamTable{ + "test_table": { + EventType: "test-app.test", + }, + }, + }, + }, + }, + }, logger, eventRecorder, ) +func TestCreate(t *testing.T) { + clientSet := fake.NewSimpleClientset() + acidClientSet := fakeacidv1.NewSimpleClientset() + clusterName := "cluster-with-finalizer" + clusterNamespace := "test" + + client := k8sutil.KubernetesClient{ + DeploymentsGetter: clientSet.AppsV1(), + CronJobsGetter: clientSet.BatchV1(), + EndpointsGetter: clientSet.CoreV1(), + PersistentVolumeClaimsGetter: clientSet.CoreV1(), + PodDisruptionBudgetsGetter: clientSet.PolicyV1(), + PodsGetter: clientSet.CoreV1(), + PostgresqlsGetter: acidClientSet.AcidV1(), + ServicesGetter: clientSet.CoreV1(), + SecretsGetter: clientSet.CoreV1(), + StatefulSetsGetter: clientSet.AppsV1(), + } + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: clusterNamespace, + }, + Spec: acidv1.PostgresSpec{ + EnableLogicalBackup: true, + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-0", clusterName), + Namespace: clusterNamespace, + Labels: map[string]string{ + "application": "spilo", + "cluster-name": clusterName, + "spilo-role": "master", + }, + }, + } + + // manually create resources which must be found by further API calls and are not created by cluster.Create() + client.Postgresqls(clusterNamespace).Create(context.TODO(), &pg, metav1.CreateOptions{}) + client.Pods(clusterNamespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) + + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + ResourceCheckInterval: time.Duration(3), + ResourceCheckTimeout: time.Duration(10), + }, + EnableFinalizers: util.True(), + }, + }, client, pg, logger, eventRecorder) + + err := cluster.Create() + assert.NoError(t, err) + + if !cluster.hasFinalizer() { + t.Errorf("%s - expected finalizer not found on cluster", t.Name()) + } +} + func TestStatefulSetAnnotations(t *testing.T) { - testName := "CheckStatefulsetAnnotations" spec := acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ - ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, - ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, }, Volume: acidv1.Volume{ Size: "1G", @@ -58,75 +175,173 @@ func TestStatefulSetAnnotations(t *testing.T) { } ss, err := cl.generateStatefulSet(&spec) if err != nil { - t.Errorf("in %s no statefulset created %v", testName, err) + t.Errorf("in %s no statefulset created %v", t.Name(), err) } if ss != nil { annotation := ss.ObjectMeta.GetAnnotations() if _, ok := annotation["downscaler/downtime_replicas"]; !ok { - t.Errorf("in %s respective annotation not found on sts", testName) + t.Errorf("in %s respective annotation not found on sts", t.Name()) } } +} +func TestStatefulSetUpdateWithEnv(t *testing.T) { + oldSpec := &acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + } + oldSS, err := cl.generateStatefulSet(oldSpec) + if err != nil { + t.Errorf("in %s no StatefulSet created %v", t.Name(), err) + } + + newSpec := oldSpec.DeepCopy() + newSS, err := cl.generateStatefulSet(newSpec) + if err != nil { + t.Errorf("in %s no StatefulSet created %v", t.Name(), err) + } + + if !reflect.DeepEqual(oldSS, newSS) { + t.Errorf("in %s StatefulSet's must be equal", t.Name()) + } + + newSpec.Env = []v1.EnvVar{ + { + Name: "CUSTOM_ENV_VARIABLE", + Value: "data", + }, + } + newSS, err = cl.generateStatefulSet(newSpec) + if err != nil { + t.Errorf("in %s no StatefulSet created %v", t.Name(), err) + } + + if reflect.DeepEqual(oldSS, newSS) { + t.Errorf("in %s StatefulSet's must be not equal", t.Name()) + } } func TestInitRobotUsers(t *testing.T) { - testName := "TestInitRobotUsers" tests := []struct { + testCase string manifestUsers map[string]acidv1.UserFlags infraRoles map[string]spec.PgUser result map[string]spec.PgUser err error }{ { + testCase: "manifest user called like infrastructure role - latter should take percedence", manifestUsers: map[string]acidv1.UserFlags{"foo": {"superuser", "createdb"}}, - infraRoles: map[string]spec.PgUser{"foo": {Origin: spec.RoleOriginInfrastructure, Name: "foo", Password: "bar"}}, - result: map[string]spec.PgUser{"foo": {Origin: spec.RoleOriginInfrastructure, Name: "foo", Password: "bar"}}, + infraRoles: map[string]spec.PgUser{"foo": {Origin: spec.RoleOriginInfrastructure, Name: "foo", Namespace: cl.Namespace, Password: "bar"}}, + result: map[string]spec.PgUser{"foo": {Origin: spec.RoleOriginInfrastructure, Name: "foo", Namespace: cl.Namespace, Password: "bar"}}, err: nil, }, { + testCase: "manifest user with forbidden characters", manifestUsers: map[string]acidv1.UserFlags{"!fooBar": {"superuser", "createdb"}}, err: fmt.Errorf(`invalid username: "!fooBar"`), }, { + testCase: "manifest user with unknown privileges (should be catched by CRD, too)", manifestUsers: map[string]acidv1.UserFlags{"foobar": {"!superuser", "createdb"}}, err: fmt.Errorf(`invalid flags for user "foobar": ` + `user flag "!superuser" is not alphanumeric`), }, { + testCase: "manifest user with unknown privileges - part 2 (should be catched by CRD, too)", manifestUsers: map[string]acidv1.UserFlags{"foobar": {"superuser1", "createdb"}}, err: fmt.Errorf(`invalid flags for user "foobar": ` + `user flag "SUPERUSER1" is not valid`), }, { + testCase: "manifest user with conflicting flags", manifestUsers: map[string]acidv1.UserFlags{"foobar": {"inherit", "noinherit"}}, err: fmt.Errorf(`invalid flags for user "foobar": ` + `conflicting user flags: "NOINHERIT" and "INHERIT"`), }, { - manifestUsers: map[string]acidv1.UserFlags{"admin": {"superuser"}, superUserName: {"createdb"}}, + testCase: "manifest user called like Spilo system users", + manifestUsers: map[string]acidv1.UserFlags{superUserName: {"createdb"}, replicationUserName: {"replication"}}, + infraRoles: map[string]spec.PgUser{}, + result: map[string]spec.PgUser{}, + err: nil, + }, + { + testCase: "manifest user called like protected user name", + manifestUsers: map[string]acidv1.UserFlags{adminUserName: {"superuser"}}, + infraRoles: map[string]spec.PgUser{}, + result: map[string]spec.PgUser{}, + err: nil, + }, + { + testCase: "manifest user called like pooler system user", + manifestUsers: map[string]acidv1.UserFlags{poolerUserName: {}}, + infraRoles: map[string]spec.PgUser{}, + result: map[string]spec.PgUser{}, + err: nil, + }, + { + testCase: "manifest user called like stream system user", + manifestUsers: map[string]acidv1.UserFlags{"fes_user": {"replication"}}, infraRoles: map[string]spec.PgUser{}, result: map[string]spec.PgUser{}, err: nil, }, } + cl.initSystemUsers() for _, tt := range tests { cl.Spec.Users = tt.manifestUsers cl.pgUsers = tt.infraRoles if err := cl.initRobotUsers(); err != nil { if tt.err == nil { - t.Errorf("%s got an unexpected error: %v", testName, err) + t.Errorf("%s - %s: got an unexpected error: %v", tt.testCase, t.Name(), err) } if err.Error() != tt.err.Error() { - t.Errorf("%s expected error %v, got %v", testName, tt.err, err) + t.Errorf("%s - %s: expected error %v, got %v", tt.testCase, t.Name(), tt.err, err) } } else { if !reflect.DeepEqual(cl.pgUsers, tt.result) { - t.Errorf("%s expected: %#v, got %#v", testName, tt.result, cl.pgUsers) + t.Errorf("%s - %s: expected: %#v, got %#v", tt.testCase, t.Name(), tt.result, cl.pgUsers) } } } } +func TestInitAdditionalOwnerRoles(t *testing.T) { + manifestUsers := map[string]acidv1.UserFlags{"foo_owner": {}, "bar_owner": {}, "app_user": {}} + expectedUsers := map[string]spec.PgUser{ + "foo_owner": {Origin: spec.RoleOriginManifest, Name: "foo_owner", Namespace: cl.Namespace, Password: "f123", Flags: []string{"LOGIN"}, IsDbOwner: true, MemberOf: []string{"cron_admin", "part_man"}}, + "bar_owner": {Origin: spec.RoleOriginManifest, Name: "bar_owner", Namespace: cl.Namespace, Password: "b123", Flags: []string{"LOGIN"}, IsDbOwner: true, MemberOf: []string{"cron_admin", "part_man"}}, + "app_user": {Origin: spec.RoleOriginManifest, Name: "app_user", Namespace: cl.Namespace, Password: "a123", Flags: []string{"LOGIN"}, IsDbOwner: false}, + } + + cl.Spec.Databases = map[string]string{"foo_db": "foo_owner", "bar_db": "bar_owner"} + cl.Spec.Users = manifestUsers + + // this should set IsDbOwner field for manifest users + if err := cl.initRobotUsers(); err != nil { + t.Errorf("%s could not init manifest users", t.Name()) + } + + // now assign additional roles to owners + cl.initAdditionalOwnerRoles() + + // update passwords to compare with result + for username, existingPgUser := range cl.pgUsers { + expectedPgUser := expectedUsers[username] + if !util.IsEqualIgnoreOrder(expectedPgUser.MemberOf, existingPgUser.MemberOf) { + t.Errorf("%s unexpected membership of user %q: expected member of %#v, got member of %#v", + t.Name(), username, expectedPgUser.MemberOf, existingPgUser.MemberOf) + } + } +} + type mockOAuthTokenGetter struct { } @@ -138,8 +353,15 @@ type mockTeamsAPIClient struct { members []string } -func (m *mockTeamsAPIClient) TeamInfo(teamID, token string) (tm *teams.Team, err error) { - return &teams.Team{Members: m.members}, nil +func (m *mockTeamsAPIClient) TeamInfo(teamID, token string) (tm *teams.Team, statusCode int, err error) { + if len(m.members) > 0 { + return &teams.Team{Members: m.members}, http.StatusOK, nil + } + + // when members are not set handle this as an error for this mock API + // makes it easier to test behavior when teams API is unavailable + return nil, http.StatusInternalServerError, + fmt.Errorf("mocked %d error of mock Teams API for team %q", http.StatusInternalServerError, teamID) } func (m *mockTeamsAPIClient) setMembers(members []string) { @@ -148,48 +370,67 @@ func (m *mockTeamsAPIClient) setMembers(members []string) { // Test adding a member of a product team owning a particular DB cluster func TestInitHumanUsers(t *testing.T) { - var mockTeamsAPI mockTeamsAPIClient cl.oauthTokenGetter = &mockOAuthTokenGetter{} cl.teamsAPIClient = &mockTeamsAPI - testName := "TestInitHumanUsers" // members of a product team are granted superuser rights for DBs of their team cl.OpConfig.EnableTeamSuperuser = true - cl.OpConfig.EnableTeamsAPI = true + cl.OpConfig.EnableTeamMemberDeprecation = true cl.OpConfig.PamRoleName = "zalandos" cl.Spec.TeamID = "test" + cl.Spec.Users = map[string]acidv1.UserFlags{"bar": []string{}} tests := []struct { existingRoles map[string]spec.PgUser teamRoles []string result map[string]spec.PgUser + err error }{ { existingRoles: map[string]spec.PgUser{"foo": {Name: "foo", Origin: spec.RoleOriginTeamsAPI, - Flags: []string{"NOLOGIN"}}, "bar": {Name: "bar", Flags: []string{"NOLOGIN"}}}, + Flags: []string{"LOGIN"}}, "bar": {Name: "bar", Flags: []string{"LOGIN"}}}, teamRoles: []string{"foo"}, result: map[string]spec.PgUser{"foo": {Name: "foo", Origin: spec.RoleOriginTeamsAPI, MemberOf: []string{cl.OpConfig.PamRoleName}, Flags: []string{"LOGIN", "SUPERUSER"}}, - "bar": {Name: "bar", Flags: []string{"NOLOGIN"}}}, + "bar": {Name: "bar", Flags: []string{"LOGIN"}}}, + err: fmt.Errorf("could not init human users: cannot initialize members for team %q who owns the Postgres cluster: could not get list of team members for team %q: could not get team info for team %q: mocked %d error of mock Teams API for team %q", + cl.Spec.TeamID, cl.Spec.TeamID, cl.Spec.TeamID, http.StatusInternalServerError, cl.Spec.TeamID), }, { existingRoles: map[string]spec.PgUser{}, - teamRoles: []string{"admin", replicationUserName}, + teamRoles: []string{adminUserName, replicationUserName}, result: map[string]spec.PgUser{}, + err: nil, }, } for _, tt := range tests { + // set pgUsers so that initUsers sets up pgUsersCache with team roles + cl.pgUsers = tt.existingRoles + + // initUsers calls initHumanUsers which should fail + // because no members are set for mocked teams API + if err := cl.initUsers(); err != nil { + // check that at least team roles are remembered in c.pgUsers + if len(cl.pgUsers) < len(tt.teamRoles) { + t.Errorf("%s unexpected size of pgUsers: expected at least %d, got %d", t.Name(), len(tt.teamRoles), len(cl.pgUsers)) + } + if err.Error() != tt.err.Error() { + t.Errorf("%s expected error %v, got %v", t.Name(), err, tt.err) + } + } + + // set pgUsers again to test initHumanUsers with working teams API cl.pgUsers = tt.existingRoles mockTeamsAPI.setMembers(tt.teamRoles) if err := cl.initHumanUsers(); err != nil { - t.Errorf("%s got an unexpected error %v", testName, err) + t.Errorf("%s got an unexpected error %v", t.Name(), err) } if !reflect.DeepEqual(cl.pgUsers, tt.result) { - t.Errorf("%s expects %#v, got %#v", testName, tt.result, cl.pgUsers) + t.Errorf("%s expects %#v, got %#v", t.Name(), tt.result, cl.pgUsers) } } } @@ -204,25 +445,25 @@ type mockTeamsAPIClientMultipleTeams struct { teams []mockTeam } -func (m *mockTeamsAPIClientMultipleTeams) TeamInfo(teamID, token string) (tm *teams.Team, err error) { +func (m *mockTeamsAPIClientMultipleTeams) TeamInfo(teamID, token string) (tm *teams.Team, statusCode int, err error) { for _, team := range m.teams { if team.teamID == teamID { - return &teams.Team{Members: team.members}, nil + return &teams.Team{Members: team.members}, http.StatusOK, nil } } - // should not be reached if a slice with teams is populated correctly - return nil, nil + // when given teamId is not found in teams return StatusNotFound + // the operator should only return a warning in this case and not error out (#1842) + return nil, http.StatusNotFound, + fmt.Errorf("mocked %d error of mock Teams API for team %q", http.StatusNotFound, teamID) } // Test adding members of maintenance teams that get superuser rights for all PG databases func TestInitHumanUsersWithSuperuserTeams(t *testing.T) { - var mockTeamsAPI mockTeamsAPIClientMultipleTeams cl.oauthTokenGetter = &mockOAuthTokenGetter{} cl.teamsAPIClient = &mockTeamsAPI cl.OpConfig.EnableTeamSuperuser = false - testName := "TestInitHumanUsersWithSuperuserTeams" cl.OpConfig.EnableTeamsAPI = true cl.OpConfig.PamRoleName = "zalandos" @@ -313,6 +554,16 @@ func TestInitHumanUsersWithSuperuserTeams(t *testing.T) { "postgres_superuser": userA, }, }, + // case 4: the team does not exist which should not return an error + { + ownerTeam: "acid", + existingRoles: map[string]spec.PgUser{}, + superuserTeams: []string{"postgres_superusers"}, + teams: []mockTeam{teamA, teamB, teamTest}, + result: map[string]spec.PgUser{ + "postgres_superuser": userA, + }, + }, } for _, tt := range tests { @@ -324,17 +575,16 @@ func TestInitHumanUsersWithSuperuserTeams(t *testing.T) { cl.OpConfig.PostgresSuperuserTeams = tt.superuserTeams if err := cl.initHumanUsers(); err != nil { - t.Errorf("%s got an unexpected error %v", testName, err) + t.Errorf("%s got an unexpected error %v", t.Name(), err) } if !reflect.DeepEqual(cl.pgUsers, tt.result) { - t.Errorf("%s expects %#v, got %#v", testName, tt.result, cl.pgUsers) + t.Errorf("%s expects %#v, got %#v", t.Name(), tt.result, cl.pgUsers) } } } func TestPodAnnotations(t *testing.T) { - testName := "TestPodAnnotations" tests := []struct { subTest string operator map[string]string @@ -381,13 +631,13 @@ func TestPodAnnotations(t *testing.T) { for k, v := range annotations { if observed, expected := v, tt.merged[k]; observed != expected { t.Errorf("%v expects annotation value %v for key %v, but found %v", - testName+"/"+tt.subTest, expected, observed, k) + t.Name()+"/"+tt.subTest, expected, observed, k) } } for k, v := range tt.merged { if observed, expected := annotations[k], v; observed != expected { t.Errorf("%v expects annotation value %v for key %v, but found %v", - testName+"/"+tt.subTest, expected, observed, k) + t.Name()+"/"+tt.subTest, expected, observed, k) } } } @@ -403,8 +653,11 @@ func TestServiceAnnotations(t *testing.T) { enableMasterLoadBalancerOC bool enableReplicaLoadBalancerSpec *bool enableReplicaLoadBalancerOC bool + enableTeamIdClusterPrefix bool operatorAnnotations map[string]string - clusterAnnotations map[string]string + serviceAnnotations map[string]string + masterServiceAnnotations map[string]string + replicaServiceAnnotations map[string]string expect map[string]string }{ //MASTER @@ -413,8 +666,9 @@ func TestServiceAnnotations(t *testing.T) { role: "master", enableMasterLoadBalancerSpec: &disabled, enableMasterLoadBalancerOC: false, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: make(map[string]string), }, { @@ -422,11 +676,11 @@ func TestServiceAnnotations(t *testing.T) { role: "master", enableMasterLoadBalancerSpec: &enabled, enableMasterLoadBalancerOC: false, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg.test.db.example.com,test-stg.acid.db.example.com", }, }, { @@ -434,30 +688,31 @@ func TestServiceAnnotations(t *testing.T) { role: "master", enableMasterLoadBalancerSpec: &disabled, enableMasterLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: make(map[string]string), }, { about: "Master with no annotations and EnableMasterLoadBalancer defined only on operator config", role: "master", enableMasterLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg.test.db.example.com,test-stg.acid.db.example.com", }, }, { about: "Master with cluster annotations and load balancer enabled", role: "master", enableMasterLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: map[string]string{"foo": "bar"}, + serviceAnnotations: map[string]string{"foo": "bar"}, expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg.test.db.example.com,test-stg.acid.db.example.com", "foo": "bar", }, }, @@ -466,19 +721,20 @@ func TestServiceAnnotations(t *testing.T) { role: "master", enableMasterLoadBalancerSpec: &disabled, enableMasterLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: map[string]string{"foo": "bar"}, + serviceAnnotations: map[string]string{"foo": "bar"}, expect: map[string]string{"foo": "bar"}, }, { about: "Master with operator annotations and load balancer enabled", role: "master", enableMasterLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: map[string]string{"foo": "bar"}, - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg.test.db.example.com,test-stg.acid.db.example.com", "foo": "bar", }, }, @@ -486,12 +742,13 @@ func TestServiceAnnotations(t *testing.T) { about: "Master with operator annotations override default annotations", role: "master", enableMasterLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: map[string]string{ "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", }, - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg.test.db.example.com,test-stg.acid.db.example.com", "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", }, }, @@ -499,12 +756,13 @@ func TestServiceAnnotations(t *testing.T) { about: "Master with cluster annotations override default annotations", role: "master", enableMasterLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: map[string]string{ + serviceAnnotations: map[string]string{ "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", }, expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg.test.db.example.com,test-stg.acid.db.example.com", "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", }, }, @@ -512,26 +770,43 @@ func TestServiceAnnotations(t *testing.T) { about: "Master with cluster annotations do not override external-dns annotations", role: "master", enableMasterLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: map[string]string{ + serviceAnnotations: map[string]string{ "external-dns.alpha.kubernetes.io/hostname": "wrong.external-dns-name.example.com", }, expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg.test.db.example.com,test-stg.acid.db.example.com", }, }, { - about: "Master with operator annotations do not override external-dns annotations", + about: "Master with cluster name teamId prefix enabled", role: "master", enableMasterLoadBalancerOC: true, - clusterAnnotations: make(map[string]string), - operatorAnnotations: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "wrong.external-dns-name.example.com", + enableTeamIdClusterPrefix: true, + serviceAnnotations: make(map[string]string), + operatorAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg.test.db.example.com,test-stg.acid.db.example.com", + }, + }, + { + about: "Master with master service annotations override service annotations", + role: "master", + enableMasterLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, + operatorAnnotations: make(map[string]string), + serviceAnnotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + masterServiceAnnotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "2000", }, expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg.test.db.example.com,test-stg.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "2000", }, }, // REPLICA @@ -540,8 +815,9 @@ func TestServiceAnnotations(t *testing.T) { role: "replica", enableReplicaLoadBalancerSpec: &disabled, enableReplicaLoadBalancerOC: false, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: make(map[string]string), }, { @@ -549,11 +825,11 @@ func TestServiceAnnotations(t *testing.T) { role: "replica", enableReplicaLoadBalancerSpec: &enabled, enableReplicaLoadBalancerOC: false, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg-repl.test.db.example.com,test-stg-repl.acid.db.example.com", }, }, { @@ -561,30 +837,31 @@ func TestServiceAnnotations(t *testing.T) { role: "replica", enableReplicaLoadBalancerSpec: &disabled, enableReplicaLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: make(map[string]string), }, { about: "Replica with no annotations and EnableReplicaLoadBalancer defined only on operator config", role: "replica", enableReplicaLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg-repl.test.db.example.com,test-stg-repl.acid.db.example.com", }, }, { about: "Replica with cluster annotations and load balancer enabled", role: "replica", enableReplicaLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: map[string]string{"foo": "bar"}, + serviceAnnotations: map[string]string{"foo": "bar"}, expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg-repl.test.db.example.com,test-stg-repl.acid.db.example.com", "foo": "bar", }, }, @@ -593,19 +870,20 @@ func TestServiceAnnotations(t *testing.T) { role: "replica", enableReplicaLoadBalancerSpec: &disabled, enableReplicaLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: map[string]string{"foo": "bar"}, + serviceAnnotations: map[string]string{"foo": "bar"}, expect: map[string]string{"foo": "bar"}, }, { about: "Replica with operator annotations and load balancer enabled", role: "replica", enableReplicaLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: map[string]string{"foo": "bar"}, - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg-repl.test.db.example.com,test-stg-repl.acid.db.example.com", "foo": "bar", }, }, @@ -613,12 +891,13 @@ func TestServiceAnnotations(t *testing.T) { about: "Replica with operator annotations override default annotations", role: "replica", enableReplicaLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: map[string]string{ "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", }, - clusterAnnotations: make(map[string]string), + serviceAnnotations: make(map[string]string), expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg-repl.test.db.example.com,test-stg-repl.acid.db.example.com", "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", }, }, @@ -626,12 +905,13 @@ func TestServiceAnnotations(t *testing.T) { about: "Replica with cluster annotations override default annotations", role: "replica", enableReplicaLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: map[string]string{ + serviceAnnotations: map[string]string{ "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", }, expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg-repl.test.db.example.com,test-stg-repl.acid.db.example.com", "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", }, }, @@ -639,26 +919,43 @@ func TestServiceAnnotations(t *testing.T) { about: "Replica with cluster annotations do not override external-dns annotations", role: "replica", enableReplicaLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, operatorAnnotations: make(map[string]string), - clusterAnnotations: map[string]string{ + serviceAnnotations: map[string]string{ "external-dns.alpha.kubernetes.io/hostname": "wrong.external-dns-name.example.com", }, expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg-repl.test.db.example.com,test-stg-repl.acid.db.example.com", }, }, { - about: "Replica with operator annotations do not override external-dns annotations", + about: "Replica with cluster name teamId prefix enabled", role: "replica", enableReplicaLoadBalancerOC: true, - clusterAnnotations: make(map[string]string), - operatorAnnotations: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "wrong.external-dns-name.example.com", + enableTeamIdClusterPrefix: true, + serviceAnnotations: make(map[string]string), + operatorAnnotations: make(map[string]string), + expect: map[string]string{ + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg-repl.test.db.example.com,test-stg-repl.acid.db.example.com", + }, + }, + { + about: "Replica with replica service annotations override service annotations", + role: "replica", + enableReplicaLoadBalancerOC: true, + enableTeamIdClusterPrefix: false, + operatorAnnotations: make(map[string]string), + serviceAnnotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "1800", + }, + replicaServiceAnnotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "2000", }, expect: map[string]string{ - "external-dns.alpha.kubernetes.io/hostname": "test-repl.acid.db.example.com", - "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600", + "external-dns.alpha.kubernetes.io/hostname": "acid-test-stg-repl.test.db.example.com,test-stg-repl.acid.db.example.com", + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip", + "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "2000", }, }, // COMMON @@ -666,32 +963,40 @@ func TestServiceAnnotations(t *testing.T) { about: "cluster annotations append to operator annotations", role: "replica", enableReplicaLoadBalancerOC: false, + enableTeamIdClusterPrefix: false, operatorAnnotations: map[string]string{"foo": "bar"}, - clusterAnnotations: map[string]string{"post": "gres"}, + serviceAnnotations: map[string]string{"post": "gres"}, expect: map[string]string{"foo": "bar", "post": "gres"}, }, { about: "cluster annotations override operator annotations", role: "replica", enableReplicaLoadBalancerOC: false, + enableTeamIdClusterPrefix: false, operatorAnnotations: map[string]string{"foo": "bar", "post": "gres"}, - clusterAnnotations: map[string]string{"post": "greSQL"}, + serviceAnnotations: map[string]string{"post": "greSQL"}, expect: map[string]string{"foo": "bar", "post": "greSQL"}, }, } for _, tt := range tests { t.Run(tt.about, func(t *testing.T) { + cl.OpConfig.EnableTeamIdClusternamePrefix = tt.enableTeamIdClusterPrefix + cl.OpConfig.CustomServiceAnnotations = tt.operatorAnnotations cl.OpConfig.EnableMasterLoadBalancer = tt.enableMasterLoadBalancerOC cl.OpConfig.EnableReplicaLoadBalancer = tt.enableReplicaLoadBalancerOC - cl.OpConfig.MasterDNSNameFormat = "{cluster}.{team}.{hostedzone}" - cl.OpConfig.ReplicaDNSNameFormat = "{cluster}-repl.{team}.{hostedzone}" + cl.OpConfig.MasterDNSNameFormat = "{cluster}-stg.{namespace}.{hostedzone}" + cl.OpConfig.MasterLegacyDNSNameFormat = "{cluster}-stg.{team}.{hostedzone}" + cl.OpConfig.ReplicaDNSNameFormat = "{cluster}-stg-repl.{namespace}.{hostedzone}" + cl.OpConfig.ReplicaLegacyDNSNameFormat = "{cluster}-stg-repl.{team}.{hostedzone}" cl.OpConfig.DbHostedZone = "db.example.com" - cl.Postgresql.Spec.ClusterName = "test" + cl.Postgresql.Spec.ClusterName = "" cl.Postgresql.Spec.TeamID = "acid" - cl.Postgresql.Spec.ServiceAnnotations = tt.clusterAnnotations + cl.Postgresql.Spec.ServiceAnnotations = tt.serviceAnnotations + cl.Postgresql.Spec.MasterServiceAnnotations = tt.masterServiceAnnotations + cl.Postgresql.Spec.ReplicaServiceAnnotations = tt.replicaServiceAnnotations cl.Postgresql.Spec.EnableMasterLoadBalancer = tt.enableMasterLoadBalancerSpec cl.Postgresql.Spec.EnableReplicaLoadBalancer = tt.enableReplicaLoadBalancerSpec @@ -710,70 +1015,99 @@ func TestServiceAnnotations(t *testing.T) { } func TestInitSystemUsers(t *testing.T) { - testName := "Test system users initialization" + // reset system users, pooler and stream section + cl.systemUsers = make(map[string]spec.PgUser) + cl.Spec.EnableConnectionPooler = boolToPointer(false) + cl.Spec.Streams = []acidv1.Stream{} - // default cluster without connection pooler + // default cluster without connection pooler and event streams cl.initSystemUsers() if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; exist { - t.Errorf("%s, connection pooler user is present", testName) + t.Errorf("%s, connection pooler user is present", t.Name()) + } + if _, exist := cl.systemUsers[constants.EventStreamUserKeyName]; exist { + t.Errorf("%s, stream user is present", t.Name()) } // cluster with connection pooler cl.Spec.EnableConnectionPooler = boolToPointer(true) cl.initSystemUsers() if _, exist := cl.systemUsers[constants.ConnectionPoolerUserKeyName]; !exist { - t.Errorf("%s, connection pooler user is not present", testName) + t.Errorf("%s, connection pooler user is not present", t.Name()) } // superuser is not allowed as connection pool user cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{ - User: "postgres", + User: superUserName, } - cl.OpConfig.SuperUsername = "postgres" - cl.OpConfig.ConnectionPooler.User = "pooler" + cl.OpConfig.SuperUsername = superUserName + cl.OpConfig.ConnectionPooler.User = poolerUserName cl.initSystemUsers() - if _, exist := cl.pgUsers["pooler"]; !exist { - t.Errorf("%s, Superuser is not allowed to be a connection pool user", testName) + if _, exist := cl.systemUsers[poolerUserName]; !exist { + t.Errorf("%s, Superuser is not allowed to be a connection pool user", t.Name()) } // neither protected users are - delete(cl.pgUsers, "pooler") + delete(cl.systemUsers, poolerUserName) cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{ - User: "admin", + User: adminUserName, } - cl.OpConfig.ProtectedRoles = []string{"admin"} + cl.OpConfig.ProtectedRoles = []string{adminUserName} cl.initSystemUsers() - if _, exist := cl.pgUsers["pooler"]; !exist { - t.Errorf("%s, Protected user are not allowed to be a connection pool user", testName) + if _, exist := cl.systemUsers[poolerUserName]; !exist { + t.Errorf("%s, Protected user are not allowed to be a connection pool user", t.Name()) } - delete(cl.pgUsers, "pooler") + delete(cl.systemUsers, poolerUserName) cl.Spec.ConnectionPooler = &acidv1.ConnectionPooler{ - User: "standby", + User: replicationUserName, } cl.initSystemUsers() - if _, exist := cl.pgUsers["pooler"]; !exist { - t.Errorf("%s, System users are not allowed to be a connection pool user", testName) + if _, exist := cl.systemUsers[poolerUserName]; !exist { + t.Errorf("%s, System users are not allowed to be a connection pool user", t.Name()) + } + + // using stream user in manifest but no streams defined should be treated like normal robot user + streamUser := fmt.Sprintf("%s%s", constants.EventStreamSourceSlotPrefix, constants.UserRoleNameSuffix) + cl.Spec.Users = map[string]acidv1.UserFlags{streamUser: []string{}} + cl.initSystemUsers() + if _, exist := cl.systemUsers[constants.EventStreamUserKeyName]; exist { + t.Errorf("%s, stream user is present", t.Name()) + } + + // cluster with streams + cl.Spec.Streams = []acidv1.Stream{ + { + ApplicationId: "test-app", + Database: "test_db", + Tables: map[string]acidv1.StreamTable{ + "test_table": { + EventType: "test-app.test", + }, + }, + }, + } + cl.initSystemUsers() + if _, exist := cl.systemUsers[constants.EventStreamUserKeyName]; !exist { + t.Errorf("%s, stream user is not present", t.Name()) } } func TestPreparedDatabases(t *testing.T) { - testName := "TestDefaultPreparedDatabase" - cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{} cl.initPreparedDatabaseRoles() for _, role := range []string{"acid_test_owner", "acid_test_reader", "acid_test_writer", "acid_test_data_owner", "acid_test_data_reader", "acid_test_data_writer"} { if _, exist := cl.pgUsers[role]; !exist { - t.Errorf("%s, default role %q for prepared database not present", testName, role) + t.Errorf("%s, default role %q for prepared database not present", t.Name(), role) } } - testName = "TestPreparedDatabaseWithSchema" + testName := "TestPreparedDatabaseWithSchema" cl.Spec.PreparedDatabases = map[string]acidv1.PreparedDatabase{ "foo": { @@ -807,7 +1141,7 @@ func TestPreparedDatabases(t *testing.T) { subTest: "Test admin role of owner", role: "foo_owner", memberOf: "", - admin: "admin", + admin: adminUserName, }, { subTest: "Test writer is a member of reader", @@ -845,3 +1179,1088 @@ func TestPreparedDatabases(t *testing.T) { } } } + +func TestCompareSpiloConfiguration(t *testing.T) { + testCases := []struct { + Config string + ExpectedResult bool + }{ + { + `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 scram-sha-256","host all all 0.0.0.0/0 scram-sha-256"]},"bootstrap":{"initdb":[{"auth-host":"scram-sha-256"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`, + true, + }, + { + `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 scram-sha-256","host all all 0.0.0.0/0 scram-sha-256"]},"bootstrap":{"initdb":[{"auth-host":"scram-sha-256"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"200","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`, + true, + }, + { + `{}`, + false, + }, + { + `invalidjson`, + false, + }, + } + refCase := testCases[0] + for _, testCase := range testCases { + if result := compareSpiloConfiguration(refCase.Config, testCase.Config); result != testCase.ExpectedResult { + t.Errorf("expected %v got %v", testCase.ExpectedResult, result) + } + } +} + +func TestCompareEnv(t *testing.T) { + testCases := []struct { + Envs []v1.EnvVar + ExpectedResult bool + }{ + { + Envs: []v1.EnvVar{ + { + Name: "VARIABLE1", + Value: "value1", + }, + { + Name: "VARIABLE2", + Value: "value2", + }, + { + Name: "VARIABLE3", + Value: "value3", + }, + { + Name: "SPILO_CONFIGURATION", + Value: exampleSpiloConfig, + }, + }, + ExpectedResult: true, + }, + { + Envs: []v1.EnvVar{ + { + Name: "VARIABLE1", + Value: "value1", + }, + { + Name: "VARIABLE2", + Value: "value2", + }, + { + Name: "VARIABLE3", + Value: "value3", + }, + { + Name: "SPILO_CONFIGURATION", + Value: spiloConfigDiff, + }, + }, + ExpectedResult: true, + }, + { + Envs: []v1.EnvVar{ + { + Name: "VARIABLE4", + Value: "value4", + }, + { + Name: "VARIABLE2", + Value: "value2", + }, + { + Name: "VARIABLE3", + Value: "value3", + }, + { + Name: "SPILO_CONFIGURATION", + Value: exampleSpiloConfig, + }, + }, + ExpectedResult: false, + }, + { + Envs: []v1.EnvVar{ + { + Name: "VARIABLE1", + Value: "value1", + }, + { + Name: "VARIABLE2", + Value: "value2", + }, + { + Name: "VARIABLE3", + Value: "value3", + }, + { + Name: "VARIABLE4", + Value: "value4", + }, + { + Name: "SPILO_CONFIGURATION", + Value: exampleSpiloConfig, + }, + }, + ExpectedResult: false, + }, + { + Envs: []v1.EnvVar{ + { + Name: "VARIABLE1", + Value: "value1", + }, + { + Name: "VARIABLE2", + Value: "value2", + }, + { + Name: "SPILO_CONFIGURATION", + Value: exampleSpiloConfig, + }, + }, + ExpectedResult: false, + }, + } + refCase := testCases[0] + for _, testCase := range testCases { + if result := compareEnv(refCase.Envs, testCase.Envs); result != testCase.ExpectedResult { + t.Errorf("expected %v got %v", testCase.ExpectedResult, result) + } + } +} + +func newService( + annotations map[string]string, + svcType v1.ServiceType, + sourceRanges []string, + selector map[string]string, + policy v1.ServiceExternalTrafficPolicyType, + nodePort *int32) *v1.Service { + svc := &v1.Service{ + Spec: v1.ServiceSpec{ + Selector: selector, + Type: svcType, + LoadBalancerSourceRanges: sourceRanges, + ExternalTrafficPolicy: policy, + }, + } + svc.Annotations = annotations + + if nodePort != nil { + svc.Spec.Ports = []v1.ServicePort{ + { + Name: "port", + NodePort: *nodePort, + }, + } + } + + return svc +} + +func TestCompareServices(t *testing.T) { + cluster := Cluster{ + Config: Config{ + OpConfig: config.Config{ + Resources: config.Resources{ + IgnoredAnnotations: []string{ + "k8s.v1.cni.cncf.io/network-status", + }, + }, + }, + }, + } + + defaultPolicy := v1.ServiceExternalTrafficPolicyTypeCluster + + serviceWithOwnerReference := newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, + defaultPolicy, + nil, + ) + + ownerRef := metav1.OwnerReference{ + APIVersion: "acid.zalan.do/v1", + Controller: boolToPointer(true), + Kind: "Postgresql", + Name: "clstr", + } + + serviceWithOwnerReference.ObjectMeta.OwnerReferences = append(serviceWithOwnerReference.ObjectMeta.OwnerReferences, ownerRef) + + portZero := int32(0) + portNotZero := int32(1337) + + tests := []struct { + about string + current *v1.Service + new *v1.Service + reason string + match bool + }{ + { + about: "two equal services", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy, nil), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy, nil), + match: true, + }, + { + about: "services differ on service type", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy, nil), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy, nil), + match: false, + reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`, + }, + { + about: "services differ on lb source ranges", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy, nil), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeLoadBalancer, + []string{"185.249.56.0/22"}, + nil, defaultPolicy, nil), + match: false, + reason: `new service's LoadBalancerSourceRange does not match the current one`, + }, + { + about: "new service doesn't have lb source ranges", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy, nil), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeLoadBalancer, + []string{}, + nil, defaultPolicy, nil), + match: false, + reason: `new service's LoadBalancerSourceRange does not match the current one`, + }, + { + about: "new service doesn't have owner references", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}, + nil, defaultPolicy, nil), + new: serviceWithOwnerReference, + match: false, + }, + { + about: "new service has a label selector", + current: newService( + map[string]string{}, + v1.ServiceTypeClusterIP, + []string{}, + nil, defaultPolicy, nil), + new: newService( + map[string]string{}, + v1.ServiceTypeClusterIP, + []string{}, + map[string]string{"cluster-name": "clstr", "spilo-role": "master"}, defaultPolicy, nil), + match: false, + }, + { + about: "services differ on external traffic policy", + current: newService( + map[string]string{}, + v1.ServiceTypeClusterIP, + []string{}, + nil, defaultPolicy, nil), + new: newService( + map[string]string{}, + v1.ServiceTypeClusterIP, + []string{}, + nil, v1.ServiceExternalTrafficPolicyTypeLocal, nil), + match: false, + }, + { + about: "services differ on node port", + current: newService( + map[string]string{}, + v1.ServiceTypeNodePort, + []string{}, + nil, defaultPolicy, &portZero), + new: newService( + map[string]string{}, + v1.ServiceTypeNodePort, + []string{}, + nil, defaultPolicy, &portNotZero), + match: false, + }, + { + about: "services do not differ on node port when requesting 0", + current: newService( + map[string]string{}, + v1.ServiceTypeNodePort, + []string{}, + nil, defaultPolicy, &portNotZero), + new: newService( + map[string]string{}, + v1.ServiceTypeNodePort, + []string{}, + nil, defaultPolicy, &portZero), + match: true, + }, + } + + for _, tt := range tests { + t.Run(tt.about, func(t *testing.T) { + match, reason := cluster.compareServices(tt.current, tt.new) + if match && !tt.match { + t.Logf("match=%v current=%v, old=%v reason=%s", match, tt.current.Annotations, tt.new.Annotations, reason) + t.Errorf("%s - expected services to do not match: %q and %q", t.Name(), tt.current, tt.new) + } + if !match && tt.match { + t.Errorf("%s - expected services to be the same: %q and %q", t.Name(), tt.current, tt.new) + } + if !match && !tt.match { + if !strings.HasPrefix(reason, tt.reason) { + t.Errorf("%s - expected reason prefix %s, found %s", t.Name(), tt.reason, reason) + } + } + }) + } +} + +var ( + defaultSuccessfulJobsHistoryLimit = int32(3) + defaultFailedJobsHistoryLimit = int32(3) + defaultTTLSecondsAfterFinished = int32(86400) +) + +func newCronJob(image, schedule string, vars []v1.EnvVar, mounts []v1.VolumeMount) *batchv1.CronJob { + cron := &batchv1.CronJob{ + Spec: batchv1.CronJobSpec{ + Schedule: schedule, + SuccessfulJobsHistoryLimit: &defaultSuccessfulJobsHistoryLimit, + FailedJobsHistoryLimit: &defaultFailedJobsHistoryLimit, + JobTemplate: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + TTLSecondsAfterFinished: &defaultTTLSecondsAfterFinished, + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "logical-backup", + Image: image, + Env: vars, + Ports: []v1.ContainerPort{ + { + ContainerPort: patroni.ApiPort, + Protocol: v1.ProtocolTCP, + }, + { + ContainerPort: pgPort, + Protocol: v1.ProtocolTCP, + }, + { + ContainerPort: operatorPort, + Protocol: v1.ProtocolTCP, + }, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: nil, + Privileged: util.False(), + ReadOnlyRootFilesystem: util.False(), + Capabilities: nil, + }, + VolumeMounts: mounts, + }, + }, + }, + }, + }, + }, + }, + } + return cron +} + +func TestCompareLogicalBackupJob(t *testing.T) { + + img1 := "ghcr.io/zalando/postgres-operator/logical-backup:v1.14.0" + img2 := "ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1" + + clientSet := fake.NewSimpleClientset() + acidClientSet := fakeacidv1.NewSimpleClientset() + namespace := "default" + + client := k8sutil.KubernetesClient{ + CronJobsGetter: clientSet.BatchV1(), + PostgresqlsGetter: acidClientSet.AcidV1(), + } + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: "acid-cron-cluster", + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Volume: acidv1.Volume{ + Size: "1Gi", + }, + EnableLogicalBackup: true, + LogicalBackupSchedule: "0 0 * * *", + LogicalBackupRetention: "3 months", + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + LogicalBackup: config.LogicalBackup{ + LogicalBackupSchedule: "30 00 * * *", + LogicalBackupDockerImage: img1, + LogicalBackupJobPrefix: "logical-backup-", + LogicalBackupCPURequest: "100m", + LogicalBackupCPULimit: "100m", + LogicalBackupMemoryRequest: "100Mi", + LogicalBackupMemoryLimit: "100Mi", + LogicalBackupProvider: "s3", + LogicalBackupS3Bucket: "testBucket", + LogicalBackupS3BucketPrefix: "spilo", + LogicalBackupS3Region: "eu-central-1", + LogicalBackupS3Endpoint: "https://s3.amazonaws.com", + LogicalBackupS3AccessKeyID: "access", + LogicalBackupS3SecretAccessKey: "secret", + LogicalBackupS3SSE: "aws:kms", + LogicalBackupS3RetentionTime: "3 months", + LogicalBackupCronjobEnvironmentSecret: "", + }, + }, + }, client, pg, logger, eventRecorder) + + desiredCronJob, err := cluster.generateLogicalBackupJob() + if err != nil { + t.Errorf("Could not generate logical backup job with error: %v", err) + } + + err = cluster.createLogicalBackupJob() + if err != nil { + t.Errorf("Could not create logical backup job with error: %v", err) + } + + currentCronJob, err := cluster.KubeClient.CronJobs(namespace).Get(context.TODO(), cluster.getLogicalBackupJobName(), metav1.GetOptions{}) + if err != nil { + t.Errorf("Could not create logical backup job with error: %v", err) + } + + tests := []struct { + about string + cronjob *batchv1.CronJob + match bool + reason string + }{ + { + about: "two equal cronjobs", + cronjob: newCronJob(img1, "0 0 * * *", []v1.EnvVar{}, []v1.VolumeMount{}), + match: true, + }, + { + about: "two cronjobs with different image", + cronjob: newCronJob(img2, "0 0 * * *", []v1.EnvVar{}, []v1.VolumeMount{}), + match: false, + reason: fmt.Sprintf("new job's image %q does not match the current one %q", img2, img1), + }, + { + about: "two cronjobs with different schedule", + cronjob: newCronJob(img1, "0 * * * *", []v1.EnvVar{}, []v1.VolumeMount{}), + match: false, + reason: fmt.Sprintf("new job's schedule %q does not match the current one %q", "0 * * * *", "0 0 * * *"), + }, + { + about: "two cronjobs with empty and nil volume mounts", + cronjob: newCronJob(img1, "0 0 * * *", []v1.EnvVar{}, nil), + match: true, + }, + { + about: "two cronjobs with different environment variables", + cronjob: newCronJob(img1, "0 0 * * *", []v1.EnvVar{{Name: "LOGICAL_BACKUP_S3_BUCKET_PREFIX", Value: "logical-backup"}}, []v1.VolumeMount{}), + match: false, + reason: "logical backup container specs do not match: new cronjob container's logical-backup (index 0) environment does not match the current one", + }, + } + + for _, tt := range tests { + t.Run(tt.about, func(t *testing.T) { + desiredCronJob.Spec.Schedule = tt.cronjob.Spec.Schedule + desiredCronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image = tt.cronjob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image + desiredCronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].VolumeMounts = tt.cronjob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].VolumeMounts + + for _, testEnv := range tt.cronjob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env { + for i, env := range desiredCronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env { + if env.Name == testEnv.Name { + desiredCronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env[i] = testEnv + } + } + } + + cmp := cluster.compareLogicalBackupJob(currentCronJob, desiredCronJob) + if cmp.match != tt.match { + t.Errorf("%s - unexpected match result %t when comparing cronjobs %#v and %#v", t.Name(), cmp.match, currentCronJob, desiredCronJob) + } else if !cmp.match { + found := false + for _, reason := range cmp.reasons { + if strings.HasPrefix(reason, tt.reason) { + found = true + break + } + found = false + } + if !found { + t.Errorf("%s - expected reason prefix %s, not found in %#v", t.Name(), tt.reason, cmp.reasons) + } + } + }) + } +} + +func TestCrossNamespacedSecrets(t *testing.T) { + testName := "test secrets in different namespace" + clientSet := fake.NewSimpleClientset() + acidClientSet := fakeacidv1.NewSimpleClientset() + namespace := "default" + + client := k8sutil.KubernetesClient{ + StatefulSetsGetter: clientSet.AppsV1(), + ServicesGetter: clientSet.CoreV1(), + DeploymentsGetter: clientSet.AppsV1(), + PostgresqlsGetter: acidClientSet.AcidV1(), + SecretsGetter: clientSet.CoreV1(), + } + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: "acid-fake-cluster", + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Volume: acidv1.Volume{ + Size: "1Gi", + }, + Users: map[string]acidv1.UserFlags{ + "appspace.db_user": {}, + "db_user": {}, + }, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(1), + }, + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + EnableCrossNamespaceSecret: true, + }, + }, client, pg, logger, eventRecorder) + + userNamespaceMap := map[string]string{ + cluster.Namespace: "db_user", + "appspace": "appspace.db_user", + } + + err := cluster.initRobotUsers() + if err != nil { + t.Errorf("Could not create secret for namespaced users with error: %s", err) + } + + for _, u := range cluster.pgUsers { + if u.Name != userNamespaceMap[u.Namespace] { + t.Errorf("%s: Could not create namespaced user in its correct namespaces for user %s in namespace %s", testName, u.Name, u.Namespace) + } + } +} + +func TestValidUsernames(t *testing.T) { + testName := "test username validity" + + invalidUsernames := []string{"_", ".", ".user", "appspace.", "user_", "_user", "-user", "user-", ",", "-", ",user", "user,", "namespace,user"} + validUsernames := []string{"user", "appspace.user", "appspace.dot.user", "user_name", "app_space.user_name"} + for _, username := range invalidUsernames { + if isValidUsername(username) { + t.Errorf("%s Invalid username is allowed: %s", testName, username) + } + } + for _, username := range validUsernames { + if !isValidUsername(username) { + t.Errorf("%s Valid username is not allowed: %s", testName, username) + } + } +} + +func TestComparePorts(t *testing.T) { + testCases := []struct { + name string + setA []v1.ContainerPort + setB []v1.ContainerPort + expected bool + }{ + { + name: "different ports", + setA: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + + setB: []v1.ContainerPort{ + { + Name: "http", + ContainerPort: 80, + Protocol: v1.ProtocolTCP, + }, + }, + expected: false, + }, + { + name: "no difference", + setA: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + setB: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + expected: true, + }, + { + name: "same ports, different order", + setA: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + { + Name: "http", + ContainerPort: 80, + Protocol: v1.ProtocolTCP, + }, + }, + setB: []v1.ContainerPort{ + { + Name: "http", + ContainerPort: 80, + Protocol: v1.ProtocolTCP, + }, + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + expected: true, + }, + { + name: "same ports, but one with default protocol", + setA: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + setB: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + }, + }, + expected: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + got := comparePorts(testCase.setA, testCase.setB) + assert.Equal(t, testCase.expected, got) + }) + } +} + +func TestCompareVolumeMounts(t *testing.T) { + testCases := []struct { + name string + mountsA []v1.VolumeMount + mountsB []v1.VolumeMount + expected bool + }{ + { + name: "empty vs nil", + mountsA: []v1.VolumeMount{}, + mountsB: nil, + expected: true, + }, + { + name: "both empty", + mountsA: []v1.VolumeMount{}, + mountsB: []v1.VolumeMount{}, + expected: true, + }, + { + name: "same mounts", + mountsA: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + }, + mountsB: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + }, + expected: true, + }, + { + name: "different mounts", + mountsA: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPathExpr: "$(POD_NAME)", + }, + }, + mountsB: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + }, + expected: false, + }, + { + name: "one equal mount one different", + mountsA: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + { + Name: "poddata", + ReadOnly: false, + MountPath: "/poddata", + SubPathExpr: "$(POD_NAME)", + }, + }, + mountsB: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + { + Name: "etc", + ReadOnly: true, + MountPath: "/etc", + }, + }, + expected: false, + }, + { + name: "same mounts, different order", + mountsA: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + { + Name: "etc", + ReadOnly: true, + MountPath: "/etc", + }, + }, + mountsB: []v1.VolumeMount{ + { + Name: "etc", + ReadOnly: true, + MountPath: "/etc", + }, + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + }, + expected: true, + }, + { + name: "new mounts added", + mountsA: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + }, + mountsB: []v1.VolumeMount{ + { + Name: "etc", + ReadOnly: true, + MountPath: "/etc", + }, + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + }, + expected: false, + }, + { + name: "one mount removed", + mountsA: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + { + Name: "etc", + ReadOnly: true, + MountPath: "/etc", + }, + }, + mountsB: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + SubPath: "subdir", + }, + }, + expected: false, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + got := compareVolumeMounts(tt.mountsA, tt.mountsB) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestGetSwitchoverSchedule(t *testing.T) { + now, _ := time.Parse(time.RFC3339, "2025-11-11T12:35:00Z") + + futureTimeStart := now.Add(1 * time.Hour) + futureWindowTimeStart := futureTimeStart.Format("15:04") + futureWindowTimeEnd := now.Add(2 * time.Hour).Format("15:04") + pastTimeStart := now.Add(-2 * time.Hour) + pastWindowTimeStart := pastTimeStart.Format("15:04") + pastWindowTimeEnd := now.Add(-1 * time.Hour).Format("15:04") + + defaultWindowStr := fmt.Sprintf("%s-%s", futureWindowTimeStart, futureWindowTimeEnd) + + tests := []struct { + name string + windows []acidv1.MaintenanceWindow + defaultWindows []string + expected string + }{ + { + name: "everyday maintenance windows is later today", + windows: []acidv1.MaintenanceWindow{ + { + Everyday: true, + StartTime: mustParseTime(futureWindowTimeStart), + EndTime: mustParseTime(futureWindowTimeEnd), + }, + }, + expected: futureTimeStart.Format("2006-01-02T15:04+00"), + }, + { + name: "everyday maintenance window is tomorrow", + windows: []acidv1.MaintenanceWindow{ + { + Everyday: true, + StartTime: mustParseTime(pastWindowTimeStart), + EndTime: mustParseTime(pastWindowTimeEnd), + }, + }, + expected: pastTimeStart.AddDate(0, 0, 1).Format("2006-01-02T15:04+00"), + }, + { + name: "weekday maintenance windows is later today", + windows: []acidv1.MaintenanceWindow{ + { + Weekday: now.Weekday(), + StartTime: mustParseTime(futureWindowTimeStart), + EndTime: mustParseTime(futureWindowTimeEnd), + }, + }, + expected: futureTimeStart.Format("2006-01-02T15:04+00"), + }, + { + name: "weekday maintenance windows is passed for today", + windows: []acidv1.MaintenanceWindow{ + { + Weekday: now.Weekday(), + StartTime: mustParseTime(pastWindowTimeStart), + EndTime: mustParseTime(pastWindowTimeEnd), + }, + }, + expected: pastTimeStart.AddDate(0, 0, 7).Format("2006-01-02T15:04+00"), + }, + { + name: "choose the earliest window", + windows: []acidv1.MaintenanceWindow{ + { + Weekday: now.AddDate(0, 0, 2).Weekday(), + StartTime: mustParseTime(futureWindowTimeStart), + EndTime: mustParseTime(futureWindowTimeEnd), + }, + { + Everyday: true, + StartTime: mustParseTime(pastWindowTimeStart), + EndTime: mustParseTime(pastWindowTimeEnd), + }, + }, + expected: pastTimeStart.AddDate(0, 0, 1).Format("2006-01-02T15:04+00"), + }, + { + name: "fallback to operator default window when spec is empty", + windows: []acidv1.MaintenanceWindow{}, + defaultWindows: []string{defaultWindowStr}, + expected: futureTimeStart.Format("2006-01-02T15:04+00"), + }, + { + name: "no windows defined returns empty string", + windows: []acidv1.MaintenanceWindow{}, + defaultWindows: nil, + expected: "", + }, + { + name: "choose the earliest window from multiple in spec", + windows: []acidv1.MaintenanceWindow{ + { + Weekday: now.AddDate(0, 0, 2).Weekday(), + StartTime: mustParseTime(futureWindowTimeStart), + EndTime: mustParseTime(futureWindowTimeEnd), + }, + { + Weekday: now.AddDate(0, 0, 1).Weekday(), + StartTime: mustParseTime(pastWindowTimeStart), + EndTime: mustParseTime(pastWindowTimeEnd), + }, + }, + expected: pastTimeStart.AddDate(0, 0, 1).Format("2006-01-02T15:04+00"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cluster.Spec.MaintenanceWindows = tt.windows + cluster.OpConfig.MaintenanceWindows = tt.defaultWindows + schedule := cluster.getSwitchoverScheduleAtTime(now) + if schedule != tt.expected { + t.Errorf("Expected GetSwitchoverSchedule to return %s, returned: %s", tt.expected, schedule) + } + }) + } +} diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 97cbc16c1..74e04c936 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -2,8 +2,11 @@ package cluster import ( "context" + "encoding/json" "fmt" + "reflect" "strings" + "time" "github.com/r3labs/diff" "github.com/sirupsen/logrus" @@ -20,10 +23,15 @@ import ( "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "github.com/zalando/postgres-operator/pkg/util/retryutil" ) +var poolerRunAsUser = int64(100) +var poolerRunAsGroup = int64(101) + // ConnectionPoolerObjects K8s objects that are belong to connection pooler type ConnectionPoolerObjects struct { + AuthSecret *v1.Secret Deployment *appsv1.Deployment Service *v1.Service Name string @@ -42,9 +50,9 @@ type ConnectionPoolerObjects struct { } func (c *Cluster) connectionPoolerName(role PostgresRole) string { - name := c.Name + "-pooler" + name := fmt.Sprintf("%s-%s", c.Name, constants.ConnectionPoolerResourceSuffix) if role == Replica { - name = name + "-repl" + name = fmt.Sprintf("%s-%s", name, "repl") } return name } @@ -60,7 +68,7 @@ func needMasterConnectionPooler(spec *acidv1.PostgresSpec) bool { } func needMasterConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { - return (nil != spec.EnableConnectionPooler && *spec.EnableConnectionPooler) || + return (spec.EnableConnectionPooler != nil && *spec.EnableConnectionPooler) || (spec.ConnectionPooler != nil && spec.EnableConnectionPooler == nil) } @@ -73,27 +81,67 @@ func needReplicaConnectionPoolerWorker(spec *acidv1.PostgresSpec) bool { *spec.EnableReplicaConnectionPooler } +func (c *Cluster) needConnectionPoolerUser(oldSpec, newSpec *acidv1.PostgresSpec) bool { + // return true if pooler is needed AND was not disabled before OR user name differs + return (needMasterConnectionPoolerWorker(newSpec) || needReplicaConnectionPoolerWorker(newSpec)) && + ((!needMasterConnectionPoolerWorker(oldSpec) && + !needReplicaConnectionPoolerWorker(oldSpec)) || + c.poolerUser(oldSpec) != c.poolerUser(newSpec)) +} + +func (c *Cluster) poolerUser(spec *acidv1.PostgresSpec) string { + connectionPoolerSpec := spec.ConnectionPooler + if connectionPoolerSpec == nil { + connectionPoolerSpec = &acidv1.ConnectionPooler{} + } + // Using superuser as pooler user is not a good idea. First of all it's + // not going to be synced correctly with the current implementation, + // and second it's a bad practice. + username := c.OpConfig.ConnectionPooler.User + + isSuperUser := connectionPoolerSpec.User == c.OpConfig.SuperUsername + isProtectedUser := c.shouldAvoidProtectedOrSystemRole( + connectionPoolerSpec.User, "connection pool role") + + if !isSuperUser && !isProtectedUser { + username = util.Coalesce( + connectionPoolerSpec.User, + c.OpConfig.ConnectionPooler.User) + } + + return username +} + +// when listing pooler k8s objects +func (c *Cluster) poolerLabelsSet(addExtraLabels bool) labels.Set { + poolerLabels := c.labelsSet(addExtraLabels) + + // TODO should be config values + poolerLabels["application"] = "db-connection-pooler" + + return poolerLabels +} + // Return connection pooler labels selector, which should from one point of view // inherit most of the labels from the cluster itself, but at the same time // have e.g. different `application` label, so that recreatePod operation will // not interfere with it (it lists all the pods via labels, and if there would // be no difference, it will recreate also pooler pods). func (c *Cluster) connectionPoolerLabels(role PostgresRole, addExtraLabels bool) *metav1.LabelSelector { - poolerLabels := c.labelsSet(addExtraLabels) + poolerLabelsSet := c.poolerLabelsSet(addExtraLabels) // TODO should be config values - poolerLabels["application"] = "db-connection-pooler" - poolerLabels["connection-pooler"] = c.connectionPoolerName(role) + poolerLabelsSet["connection-pooler"] = c.connectionPoolerName(role) if addExtraLabels { extraLabels := map[string]string{} extraLabels[c.OpConfig.PodRoleLabel] = string(role) - poolerLabels = labels.Merge(poolerLabels, extraLabels) + poolerLabelsSet = labels.Merge(poolerLabelsSet, extraLabels) } return &metav1.LabelSelector{ - MatchLabels: poolerLabels, + MatchLabels: poolerLabelsSet, MatchExpressions: nil, } } @@ -114,37 +162,76 @@ func (c *Cluster) createConnectionPooler(LookupFunction InstallFunction) (SyncRe c.setProcessName("creating connection pooler") //this is essentially sync with nil as oldSpec - if reason, err := c.syncConnectionPooler(nil, &c.Postgresql, LookupFunction); err != nil { + if reason, err := c.syncConnectionPooler(&acidv1.Postgresql{}, &c.Postgresql, LookupFunction); err != nil { return reason, err } return reason, nil } -// +func (c *Cluster) generateUserlist() string { + var sb strings.Builder + + poolerAdminUser := c.systemUsers[constants.ConnectionPoolerUserKeyName] + fmt.Fprintf(&sb, "\"%s\" \"%s\"\n", poolerAdminUser.Name, poolerAdminUser.Password) + + for roleName, infraRole := range c.InfrastructureRoles { + if infraRole.Password != "" { + fmt.Fprintf(&sb, "\"%s\" \"%s\"\n", roleName, infraRole.Password) + } + } + + return sb.String() +} + +func (c *Cluster) generateConnectionPoolerAuthSecret(connectionPooler *ConnectionPoolerObjects) *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Labels: c.connectionPoolerLabels(connectionPooler.Role, true).MatchLabels, + Name: fmt.Sprintf("%s-u", connectionPooler.Name), + Namespace: connectionPooler.Namespace, + Annotations: c.annotationsSet(nil), + OwnerReferences: c.ownerReferences(), + }, + Type: v1.SecretTypeOpaque, + // Secret data must be bytes. Kubernetes handles the encoding. + StringData: map[string]string{ + "userlist.txt": c.generateUserlist(), + }, + } +} + // Generate pool size related environment variables. // // MAX_DB_CONN would specify the global maximum for connections to a target -// database. +// +// database. // // MAX_CLIENT_CONN is not configurable at the moment, just set it high enough. // // DEFAULT_SIZE is a pool size per db/user (having in mind the use case when -// most of the queries coming through a connection pooler are from the same -// user to the same db). In case if we want to spin up more connection pooler -// instances, take this into account and maintain the same number of -// connections. +// +// most of the queries coming through a connection pooler are from the same +// user to the same db). In case if we want to spin up more connection pooler +// instances, take this into account and maintain the same number of +// connections. // // MIN_SIZE is a pool's minimal size, to prevent situation when sudden workload -// have to wait for spinning up a new connections. +// +// have to wait for spinning up a new connections. // // RESERVE_SIZE is how many additional connections to allow for a pooler. + func (c *Cluster) getConnectionPoolerEnvVars() []v1.EnvVar { spec := &c.Spec + connectionPoolerSpec := spec.ConnectionPooler + if connectionPoolerSpec == nil { + connectionPoolerSpec = &acidv1.ConnectionPooler{} + } effectiveMode := util.Coalesce( - spec.ConnectionPooler.Mode, + connectionPoolerSpec.Mode, c.OpConfig.ConnectionPooler.Mode) - numberOfInstances := spec.ConnectionPooler.NumberOfInstances + numberOfInstances := connectionPoolerSpec.NumberOfInstances if numberOfInstances == nil { numberOfInstances = util.CoalesceInt32( c.OpConfig.ConnectionPooler.NumberOfInstances, @@ -152,7 +239,7 @@ func (c *Cluster) getConnectionPoolerEnvVars() []v1.EnvVar { } effectiveMaxDBConn := util.CoalesceInt32( - spec.ConnectionPooler.MaxDBConnections, + connectionPoolerSpec.MaxDBConnections, c.OpConfig.ConnectionPooler.MaxDBConnections) if effectiveMaxDBConn == nil { @@ -205,26 +292,31 @@ func (c *Cluster) getConnectionPoolerEnvVars() []v1.EnvVar { func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( *v1.PodTemplateSpec, error) { spec := &c.Spec + connectionPoolerSpec := spec.ConnectionPooler + if connectionPoolerSpec == nil { + connectionPoolerSpec = &acidv1.ConnectionPooler{} + } gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) - resources, err := generateResourceRequirements( - spec.ConnectionPooler.Resources, - makeDefaultConnectionPoolerResources(&c.OpConfig)) - - effectiveDockerImage := util.Coalesce( - spec.ConnectionPooler.DockerImage, - c.OpConfig.ConnectionPooler.Image) - - effectiveSchema := util.Coalesce( - spec.ConnectionPooler.Schema, - c.OpConfig.ConnectionPooler.Schema) + resources, err := c.generateResourceRequirements( + connectionPoolerSpec.Resources, + makeDefaultConnectionPoolerResources(&c.OpConfig), + connectionPoolerContainer) if err != nil { return nil, fmt.Errorf("could not generate resource requirements: %v", err) } + effectiveDockerImage := util.Coalesce( + connectionPoolerSpec.DockerImage, + c.OpConfig.ConnectionPooler.Image) + + effectiveSchema := util.Coalesce( + connectionPoolerSpec.Schema, + c.OpConfig.ConnectionPooler.Schema) + secretSelector := func(key string) *v1.SecretKeySelector { effectiveUser := util.Coalesce( - spec.ConnectionPooler.User, + connectionPoolerSpec.User, c.OpConfig.ConnectionPooler.User) return &v1.SecretKeySelector{ @@ -242,7 +334,7 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( }, { Name: "PGPORT", - Value: c.servicePort(role), + Value: fmt.Sprint(c.servicePort(role)), }, { Name: "PGUSER", @@ -265,6 +357,18 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( } envVars = append(envVars, c.getConnectionPoolerEnvVars()...) + infraRolesList := make([]string, 0) + for infraRoleName := range c.InfrastructureRoles { + infraRolesList = append(infraRolesList, infraRoleName) + } + + if len(infraRolesList) > 0 { + envVars = append(envVars, v1.EnvVar{ + Name: "INFRASTRUCTURE_ROLES", + Value: strings.Join(infraRolesList, ","), + }) + } + poolerContainer := v1.Container{ Name: connectionPoolerContainer, Image: effectiveDockerImage, @@ -276,9 +380,8 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( Protocol: v1.ProtocolTCP, }, }, - Env: envVars, ReadinessProbe: &v1.Probe{ - Handler: v1.Handler{ + ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.IntOrString{IntVal: pgPort}, }, @@ -289,6 +392,76 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( }, } + var poolerVolumes []v1.Volume + var volumeMounts []v1.VolumeMount + + // mount secret volume with userlist.txt for pgBouncer to authenticate users + poolerVolumes = append(poolerVolumes, v1.Volume{ + Name: fmt.Sprintf("%s-u", c.connectionPoolerName(role)), + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: fmt.Sprintf("%s-u", c.connectionPoolerName(role)), + }, + }, + }) + volumeMounts = append(volumeMounts, v1.VolumeMount{ + Name: fmt.Sprintf("%s-u", c.connectionPoolerName(role)), + MountPath: "/etc/pgbouncer/userlist.txt", + SubPath: "userlist.txt", + ReadOnly: true, + }) + + // If the cluster has custom TLS certificates configured, we do the following: + // 1. Add environment variables to tell pgBouncer where to find the TLS certificates + // 2. Reference the secret in a volume + // 3. Mount the volume to the container at /tls + if spec.TLS != nil && spec.TLS.SecretName != "" { + getPoolerTLSEnv := func(k string) string { + keyName := "" + switch k { + case "tls.crt": + keyName = "CONNECTION_POOLER_CLIENT_TLS_CRT" + case "tls.key": + keyName = "CONNECTION_POOLER_CLIENT_TLS_KEY" + case "tls.ca": + keyName = "CONNECTION_POOLER_CLIENT_CA_FILE" + default: + panic(fmt.Sprintf("TLS env key for pooler unknown %s", k)) + } + + return keyName + } + tlsEnv, tlsVolumes := generateTlsMounts(spec, getPoolerTLSEnv) + envVars = append(envVars, tlsEnv...) + for _, vol := range tlsVolumes { + poolerVolumes = append(poolerVolumes, v1.Volume{ + Name: vol.Name, + VolumeSource: vol.VolumeSource, + }) + volumeMounts = append(volumeMounts, v1.VolumeMount{ + Name: vol.Name, + MountPath: vol.MountPath, + }) + } + } + + poolerContainer.Env = envVars + poolerContainer.VolumeMounts = volumeMounts + tolerationsSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration) + securityContext := v1.PodSecurityContext{} + + // determine the User, Group and FSGroup for the pooler pod + securityContext.RunAsUser = &poolerRunAsUser + securityContext.RunAsGroup = &poolerRunAsGroup + + effectiveFSGroup := c.OpConfig.Resources.SpiloFSGroup + if spec.SpiloFSGroup != nil { + effectiveFSGroup = spec.SpiloFSGroup + } + if effectiveFSGroup != nil { + securityContext.FSGroup = effectiveFSGroup + } + podTemplate := &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: c.connectionPoolerLabels(role, true).MatchLabels, @@ -298,12 +471,27 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( Spec: v1.PodSpec{ TerminationGracePeriodSeconds: &gracePeriod, Containers: []v1.Container{poolerContainer}, - // TODO: add tolerations to scheduler pooler on the same node - // as database - //Tolerations: *tolerationsSpec, + Tolerations: tolerationsSpec, + Volumes: poolerVolumes, + SecurityContext: &securityContext, + ServiceAccountName: c.OpConfig.PodServiceAccountName, }, } + nodeAffinity := c.nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity) + if c.OpConfig.EnablePodAntiAffinity { + labelsSet := labels.Set(c.connectionPoolerLabels(role, false).MatchLabels) + podTemplate.Spec.Affinity = podAffinity( + labelsSet, + c.OpConfig.PodAntiAffinityTopologyKey, + nodeAffinity, + c.OpConfig.PodAntiAffinityPreferredDuringScheduling, + true, + ) + } else if nodeAffinity != nil { + podTemplate.Spec.Affinity = nodeAffinity + } + return podTemplate, nil } @@ -317,12 +505,13 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio // default values, initialize it to an empty structure. It could be done // anywhere, but here is the earliest common entry point between sync and // create code, so init here. - if spec.ConnectionPooler == nil { - spec.ConnectionPooler = &acidv1.ConnectionPooler{} + connectionPoolerSpec := spec.ConnectionPooler + if connectionPoolerSpec == nil { + connectionPoolerSpec = &acidv1.ConnectionPooler{} } podTemplate, err := c.generateConnectionPoolerPodTemplate(connectionPooler.Role) - numberOfInstances := spec.ConnectionPooler.NumberOfInstances + numberOfInstances := connectionPoolerSpec.NumberOfInstances if numberOfInstances == nil { numberOfInstances = util.CoalesceInt32( c.OpConfig.ConnectionPooler.NumberOfInstances, @@ -330,7 +519,7 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio } if *numberOfInstances < constants.ConnectionPoolerMinInstances { - msg := "Adjusted number of connection pooler instances from %d to %d" + msg := "adjusted number of connection pooler instances from %d to %d" c.logger.Warningf(msg, *numberOfInstances, constants.ConnectionPoolerMinInstances) *numberOfInstances = constants.ConnectionPoolerMinInstances @@ -365,38 +554,34 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio } func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPoolerObjects) *v1.Service { - spec := &c.Spec - // there are two ways to enable connection pooler, either to specify a - // connectionPooler section or enableConnectionPooler. In the second case - // spec.connectionPooler will be nil, so to make it easier to calculate - // default values, initialize it to an empty structure. It could be done - // anywhere, but here is the earliest common entry point between sync and - // create code, so init here. - if spec.ConnectionPooler == nil { - spec.ConnectionPooler = &acidv1.ConnectionPooler{} - } - + poolerRole := connectionPooler.Role serviceSpec := v1.ServiceSpec{ Ports: []v1.ServicePort{ { Name: connectionPooler.Name, Port: pgPort, - TargetPort: intstr.IntOrString{StrVal: c.servicePort(connectionPooler.Role)}, + TargetPort: intstr.IntOrString{IntVal: c.servicePort(poolerRole)}, }, }, Type: v1.ServiceTypeClusterIP, Selector: map[string]string{ - "connection-pooler": c.connectionPoolerName(connectionPooler.Role), + "connection-pooler": c.connectionPoolerName(poolerRole), }, } + if ok, port := c.shouldCreateNodePortForPoolerService(poolerRole, spec); ok { + c.configureNodePortService(&serviceSpec, port) + } else if c.shouldCreateLoadBalancerForPoolerService(poolerRole, spec) { + c.configureLoadBalanceService(&serviceSpec, spec.AllowedSourceRanges) + } + service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: connectionPooler.Name, Namespace: connectionPooler.Namespace, Labels: c.connectionPoolerLabels(connectionPooler.Role, false).MatchLabels, - Annotations: c.annotationsSet(c.generateServiceAnnotations(connectionPooler.Role, spec)), + Annotations: c.annotationsSet(c.generatePoolerServiceAnnotations(poolerRole, spec)), // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" // propagation policy, which means that it's deletion will not @@ -411,14 +596,100 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo return service } -//delete connection pooler +func (c *Cluster) generatePoolerServiceAnnotations(role PostgresRole, spec *acidv1.PostgresSpec) map[string]string { + var dnsString string + annotations := c.getCustomServiceAnnotations(role, spec) + + nodePort, _ := c.shouldCreateNodePortForPoolerService(role, spec) + + if !nodePort && c.shouldCreateLoadBalancerForPoolerService(role, spec) { + // -repl suffix will be added by replicaDNSName + clusterNameWithPoolerSuffix := c.connectionPoolerName(Master) + if role == Master { + dnsString = c.masterDNSName(clusterNameWithPoolerSuffix) + } else { + dnsString = c.replicaDNSName(clusterNameWithPoolerSuffix) + } + annotations[constants.ZalandoDNSNameAnnotation] = dnsString + } + + if len(annotations) == 0 { + return nil + } + + return annotations +} + +func (c *Cluster) shouldCreateLoadBalancerForPoolerService(role PostgresRole, spec *acidv1.PostgresSpec) bool { + + switch role { + + case Replica: + // if the value is explicitly set in a Postgresql manifest, follow this setting + if spec.EnableReplicaPoolerLoadBalancer != nil { + return *spec.EnableReplicaPoolerLoadBalancer + } + // otherwise, follow the operator configuration + return c.OpConfig.EnableReplicaPoolerLoadBalancer + + case Master: + if spec.EnableMasterPoolerLoadBalancer != nil { + return *spec.EnableMasterPoolerLoadBalancer + } + return c.OpConfig.EnableMasterPoolerLoadBalancer + + default: + panic(fmt.Sprintf("Unknown role %v", role)) + } +} + +func (c *Cluster) shouldCreateNodePortForPoolerService(role PostgresRole, spec *acidv1.PostgresSpec) (bool, int32) { + switch role { + case Replica: + // if the value is explicitly set in a Postgresql manifest, follow this setting + if spec.EnableReplicaPoolerNodePort != nil { + port := int32(0) + if spec.ReplicaPoolerNodePort != nil { + port = *spec.ReplicaPoolerNodePort + } + + return *spec.EnableReplicaPoolerNodePort, port + } + + // otherwise, follow the operator configuration + return c.OpConfig.EnableReplicaPoolerNodePort, 0 + case Master: + if spec.EnableMasterPoolerNodePort != nil { + port := int32(0) + if spec.MasterPoolerNodePort != nil { + port = *spec.MasterPoolerNodePort + } + + return *spec.EnableMasterPoolerNodePort, port + } + + return c.OpConfig.EnableMasterPoolerNodePort, 0 + default: + panic(fmt.Sprintf("Unknown role %v", role)) + } +} + +func (c *Cluster) listPoolerPods(listOptions metav1.ListOptions) ([]v1.Pod, error) { + pods, err := c.KubeClient.Pods(c.Namespace).List(context.TODO(), listOptions) + if err != nil { + return nil, fmt.Errorf("could not get list of pooler pods: %v", err) + } + return pods.Items, nil +} + +// delete connection pooler func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) { c.logger.Infof("deleting connection pooler spilo-role=%s", role) // Lack of connection pooler objects is not a fatal error, just log it if // it was present before in the manifest if c.ConnectionPooler[role] == nil || role == "" { - c.logger.Debugf("no connection pooler to delete") + c.logger.Debug("no connection pooler to delete") return nil } @@ -438,7 +709,7 @@ func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) { Delete(context.TODO(), deployment.Name, options) if k8sutil.ResourceNotFound(err) { - c.logger.Debugf("connection pooler deployment was already deleted") + c.logger.Debugf("connection pooler deployment %s for role %s has already been deleted", deployment.Name, role) } else if err != nil { return fmt.Errorf("could not delete connection pooler deployment: %v", err) } @@ -449,7 +720,7 @@ func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) { // Repeat the same for the service object service := c.ConnectionPooler[role].Service if service == nil { - c.logger.Debugf("no connection pooler service object to delete") + c.logger.Debug("no connection pooler service object to delete") } else { err = c.KubeClient. @@ -457,7 +728,7 @@ func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) { Delete(context.TODO(), service.Name, options) if k8sutil.ResourceNotFound(err) { - c.logger.Debugf("connection pooler service was already deleted") + c.logger.Debugf("connection pooler service %s for role %s has already been already deleted", service.Name, role) } else if err != nil { return fmt.Errorf("could not delete connection pooler service: %v", err) } @@ -465,12 +736,31 @@ func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) { c.logger.Infof("connection pooler service %s has been deleted for role %s", service.Name, role) } + // Repeat the same for the auth secret + authSecret := c.ConnectionPooler[role].AuthSecret + if authSecret == nil { + c.logger.Debug("no connection pooler auth secret to delete") + } else { + err := c.KubeClient. + Secrets(c.Namespace). + Delete(context.TODO(), authSecret.Name, metav1.DeleteOptions{}) + + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("connection pooler auth secret %s for role %s has already been deleted", authSecret.Name, role) + } else if err != nil { + return fmt.Errorf("could not delete connection pooler auth secret: %v", err) + } + + c.logger.Infof("connection pooler auth secret %s has been deleted for role %s", authSecret.Name, role) + } + + c.ConnectionPooler[role].AuthSecret = nil c.ConnectionPooler[role].Deployment = nil c.ConnectionPooler[role].Service = nil return nil } -//delete connection pooler +// delete connection pooler secret func (c *Cluster) deleteConnectionPoolerSecret() (err error) { // Repeat the same for the secret object secretName := c.credentialSecretName(c.OpConfig.ConnectionPooler.User) @@ -482,20 +772,29 @@ func (c *Cluster) deleteConnectionPoolerSecret() (err error) { if err != nil { c.logger.Debugf("could not get connection pooler secret %s: %v", secretName, err) } else { - if err = c.deleteSecret(secret.UID, *secret); err != nil { + if err = c.deleteSecret(secret.UID); err != nil { return fmt.Errorf("could not delete pooler secret: %v", err) } } + return nil } // Perform actual patching of a connection pooler deployment, assuming that all // the check were already done before. -func updateConnectionPoolerDeployment(KubeClient k8sutil.KubernetesClient, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { +func updateConnectionPoolerDeployment(KubeClient k8sutil.KubernetesClient, newDeployment *appsv1.Deployment, doUpdate bool) (*appsv1.Deployment, error) { if newDeployment == nil { return nil, fmt.Errorf("there is no connection pooler in the cluster") } + if doUpdate { + updatedDeployment, err := KubeClient.Deployments(newDeployment.Namespace).Update(context.TODO(), newDeployment, metav1.UpdateOptions{}) + if err != nil { + return nil, fmt.Errorf("could not update pooler deployment to match desired state: %v", err) + } + return updatedDeployment, nil + } + patchData, err := specPatch(newDeployment.Spec) if err != nil { return nil, fmt.Errorf("could not form patch for the connection pooler deployment: %v", err) @@ -519,8 +818,8 @@ func updateConnectionPoolerDeployment(KubeClient k8sutil.KubernetesClient, newDe return deployment, nil } -//updateConnectionPoolerAnnotations updates the annotations of connection pooler deployment -func updateConnectionPoolerAnnotations(KubeClient k8sutil.KubernetesClient, deployment *appsv1.Deployment, annotations map[string]string) (*appsv1.Deployment, error) { +// patchConnectionPoolerAnnotations updates the annotations of connection pooler deployment +func patchConnectionPoolerAnnotations(KubeClient k8sutil.KubernetesClient, deployment *appsv1.Deployment, annotations map[string]string) (*appsv1.Deployment, error) { patchData, err := metaAnnotationsPatch(annotations) if err != nil { return nil, fmt.Errorf("could not form patch for the connection pooler deployment metadata: %v", err) @@ -579,11 +878,12 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1. if spec == nil { spec = &acidv1.ConnectionPooler{} } + if spec.NumberOfInstances == nil && *deployment.Spec.Replicas != *config.NumberOfInstances { sync = true - msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)", + msg := fmt.Sprintf("numberOfInstances is different (having %d, required %d)", *deployment.Spec.Replicas, *config.NumberOfInstances) reasons = append(reasons, msg) } @@ -592,13 +892,14 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1. poolerContainer.Image != config.Image { sync = true - msg := fmt.Sprintf("DockerImage is different (having %s, required %s)", + msg := fmt.Sprintf("dockerImage is different (having %s, required %s)", poolerContainer.Image, config.Image) reasons = append(reasons, msg) } - expectedResources, err := generateResourceRequirements(spec.Resources, - makeDefaultConnectionPoolerResources(&Config.OpConfig)) + expectedResources, err := c.generateResourceRequirements(spec.Resources, + makeDefaultConnectionPoolerResources(&Config.OpConfig), + connectionPoolerContainer) // An error to generate expected resources means something is not quite // right, but for the purpose of robustness do not panic here, just report @@ -606,7 +907,7 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1. // updates for new resource values). if err == nil && syncResources(&poolerContainer.Resources, expectedResources) { sync = true - msg := fmt.Sprintf("Resources are different (having %+v, required %+v)", + msg := fmt.Sprintf("resources are different (having %+v, required %+v)", poolerContainer.Resources, expectedResources) reasons = append(reasons, msg) } @@ -648,12 +949,12 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1. func makeDefaultConnectionPoolerResources(config *config.Config) acidv1.Resources { defaultRequests := acidv1.ResourceDescription{ - CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPURequest, - Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest, + CPU: &config.ConnectionPooler.ConnectionPoolerDefaultCPURequest, + Memory: &config.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest, } defaultLimits := acidv1.ResourceDescription{ - CPU: config.ConnectionPooler.ConnectionPoolerDefaultCPULimit, - Memory: config.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit, + CPU: &config.ConnectionPooler.ConnectionPoolerDefaultCPULimit, + Memory: &config.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit, } return acidv1.Resources{ @@ -664,12 +965,14 @@ func makeDefaultConnectionPoolerResources(config *config.Config) acidv1.Resource func logPoolerEssentials(log *logrus.Entry, oldSpec, newSpec *acidv1.Postgresql) { var v []string - var input []*bool + + newMasterConnectionPoolerEnabled := needMasterConnectionPoolerWorker(&newSpec.Spec) if oldSpec == nil { - input = []*bool{nil, nil, newSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler} + input = []*bool{nil, nil, &newMasterConnectionPoolerEnabled, newSpec.Spec.EnableReplicaConnectionPooler} } else { - input = []*bool{oldSpec.Spec.EnableConnectionPooler, oldSpec.Spec.EnableReplicaConnectionPooler, newSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler} + oldMasterConnectionPoolerEnabled := needMasterConnectionPoolerWorker(&oldSpec.Spec) + input = []*bool{&oldMasterConnectionPoolerEnabled, oldSpec.Spec.EnableReplicaConnectionPooler, &newMasterConnectionPoolerEnabled, newSpec.Spec.EnableReplicaConnectionPooler} } for _, b := range input { @@ -680,46 +983,14 @@ func logPoolerEssentials(log *logrus.Entry, oldSpec, newSpec *acidv1.Postgresql) } } - log.Debugf("syncing connection pooler from (%v, %v) to (%v, %v)", v[0], v[1], v[2], v[3]) + log.Debugf("syncing connection pooler (master, replica) from (%v, %v) to (%v, %v)", v[0], v[1], v[2], v[3]) } func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, LookupFunction InstallFunction) (SyncReason, error) { var reason SyncReason var err error - var newNeedConnectionPooler, oldNeedConnectionPooler bool - oldNeedConnectionPooler = false - - if oldSpec == nil { - oldSpec = &acidv1.Postgresql{ - Spec: acidv1.PostgresSpec{ - ConnectionPooler: &acidv1.ConnectionPooler{}, - }, - } - } - - needSync, _ := needSyncConnectionPoolerSpecs(oldSpec.Spec.ConnectionPooler, newSpec.Spec.ConnectionPooler, c.logger) - masterChanges, err := diff.Diff(oldSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableConnectionPooler) - if err != nil { - c.logger.Error("Error in getting diff of master connection pooler changes") - } - replicaChanges, err := diff.Diff(oldSpec.Spec.EnableReplicaConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler) - if err != nil { - c.logger.Error("Error in getting diff of replica connection pooler changes") - } - - // skip pooler sync only - // 1. if there is no diff in spec, AND - // 2. if connection pooler is already there and is also required as per newSpec - // - // Handling the case when connectionPooler is not there but it is required - // as per spec, hence do not skip syncing in that case, even though there - // is no diff in specs - if (!needSync && len(masterChanges) <= 0 && len(replicaChanges) <= 0) && - (c.ConnectionPooler != nil && (needConnectionPooler(&newSpec.Spec))) { - c.logger.Debugln("syncing pooler is not required") - return nil, nil - } + var connectionPoolerNeeded bool logPoolerEssentials(c.logger, oldSpec, newSpec) @@ -727,15 +998,9 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look for _, role := range [2]PostgresRole{Master, Replica} { if role == Master { - newNeedConnectionPooler = needMasterConnectionPoolerWorker(&newSpec.Spec) - if oldSpec != nil { - oldNeedConnectionPooler = needMasterConnectionPoolerWorker(&oldSpec.Spec) - } + connectionPoolerNeeded = needMasterConnectionPoolerWorker(&newSpec.Spec) } else { - newNeedConnectionPooler = needReplicaConnectionPoolerWorker(&newSpec.Spec) - if oldSpec != nil { - oldNeedConnectionPooler = needReplicaConnectionPoolerWorker(&oldSpec.Spec) - } + connectionPoolerNeeded = needReplicaConnectionPoolerWorker(&newSpec.Spec) } // if the call is via createConnectionPooler, then it is required to initialize @@ -755,24 +1020,23 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look } } - if newNeedConnectionPooler { + if connectionPoolerNeeded { // Try to sync in any case. If we didn't needed connection pooler before, // it means we want to create it. If it was already present, still sync // since it could happen that there is no difference in specs, and all // the resources are remembered, but the deployment was manually deleted // in between - // in this case also do not forget to install lookup function as for - // creating cluster - if !oldNeedConnectionPooler || !c.ConnectionPooler[role].LookupFunction { - newConnectionPooler := newSpec.Spec.ConnectionPooler - + // in this case also do not forget to install lookup function + // skip installation in standby clusters, since they are read-only + if !c.ConnectionPooler[role].LookupFunction && !isStandbyCluster(&newSpec.Spec) { + connectionPooler := c.Spec.ConnectionPooler specSchema := "" specUser := "" - if newConnectionPooler != nil { - specSchema = newConnectionPooler.Schema - specUser = newConnectionPooler.User + if connectionPooler != nil { + specSchema = connectionPooler.Schema + specUser = connectionPooler.User } schema := util.Coalesce( @@ -783,9 +1047,10 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look specUser, c.OpConfig.ConnectionPooler.User) - if err = LookupFunction(schema, user, role); err != nil { + if err = LookupFunction(schema, user); err != nil { return NoSync, err } + c.ConnectionPooler[role].LookupFunction = true } if reason, err = c.syncConnectionPoolerWorker(oldSpec, newSpec, role); err != nil { @@ -804,8 +1069,8 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look } } } - if !needMasterConnectionPoolerWorker(&newSpec.Spec) && - !needReplicaConnectionPoolerWorker(&newSpec.Spec) { + if (needMasterConnectionPoolerWorker(&oldSpec.Spec) || needReplicaConnectionPoolerWorker(&oldSpec.Spec)) && + !needMasterConnectionPoolerWorker(&newSpec.Spec) && !needReplicaConnectionPoolerWorker(&newSpec.Spec) { if err = c.deleteConnectionPoolerSecret(); err != nil { c.logger.Warningf("could not remove connection pooler secret: %v", err) } @@ -821,31 +1086,70 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql, role PostgresRole) ( SyncReason, error) { - deployment, err := c.KubeClient. + var ( + deployment *appsv1.Deployment + newDeployment *appsv1.Deployment + pods []v1.Pod + service *v1.Service + newService *v1.Service + authSecret *v1.Secret + newAuthSecret *v1.Secret + err error + ) + + updatedPodAnnotations := map[string]*string{} + syncReason := make([]string, 0) + + // create extra secret for connection pooler authentication + newAuthSecret = c.generateConnectionPoolerAuthSecret(c.ConnectionPooler[role]) + if authSecret, err = c.KubeClient.Secrets(c.Namespace).Get(context.TODO(), fmt.Sprintf("%s-u", c.connectionPoolerName(role)), metav1.GetOptions{}); err == nil { + c.ConnectionPooler[role].AuthSecret = authSecret + // make sure existing annotations are preserved + newAuthSecret.Annotations = c.annotationsSet(authSecret.Annotations) + authSecret, err = c.KubeClient.Secrets(authSecret.Namespace).Update(context.TODO(), newAuthSecret, metav1.UpdateOptions{}) + if err != nil { + return NoSync, fmt.Errorf("could not update connection pooler auth secret: %v", err) + } + c.ConnectionPooler[role].AuthSecret = authSecret + } else if !k8sutil.ResourceNotFound(err) { + return NoSync, fmt.Errorf("could not get auth secret for connection pooler to sync: %v", err) + } + + if k8sutil.ResourceNotFound(err) { + c.logger.Warningf("auth secret %s for connection pooler is not found, create it", fmt.Sprintf("%s-u", c.connectionPoolerName(role))) + authSecret, err = c.KubeClient. + Secrets(newAuthSecret.Namespace). + Create(context.TODO(), newAuthSecret, metav1.CreateOptions{}) + + if err != nil { + return NoSync, err + } + c.ConnectionPooler[role].AuthSecret = authSecret + } + + // next the pooler deployment + deployment, err = c.KubeClient. Deployments(c.Namespace). Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { - msg := "deployment %s for connection pooler synchronization is not found, create it" - c.logger.Warningf(msg, c.connectionPoolerName(role)) + c.logger.Warningf("deployment %s for connection pooler synchronization is not found, create it", c.connectionPoolerName(role)) - deploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) + newDeployment, err = c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) if err != nil { - msg = "could not generate deployment for connection pooler: %v" - return NoSync, fmt.Errorf(msg, err) + return NoSync, fmt.Errorf("could not generate deployment for connection pooler: %v", err) } - deployment, err := c.KubeClient. - Deployments(deploymentSpec.Namespace). - Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) + deployment, err = c.KubeClient. + Deployments(newDeployment.Namespace). + Create(context.TODO(), newDeployment, metav1.CreateOptions{}) if err != nil { return NoSync, err } c.ConnectionPooler[role].Deployment = deployment } else if err != nil { - msg := "could not get connection pooler deployment to sync: %v" - return NoSync, fmt.Errorf(msg, err) + return NoSync, fmt.Errorf("could not get connection pooler deployment to sync: %v", err) } else { c.ConnectionPooler[role].Deployment = deployment // actual synchronization @@ -870,71 +1174,143 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql newConnectionPooler = &acidv1.ConnectionPooler{} } - c.logger.Infof("old: %+v, new %+v", oldConnectionPooler, newConnectionPooler) - - var specSync bool + var specSync, updateDeployment bool var specReason []string + if !reflect.DeepEqual(deployment.ObjectMeta.OwnerReferences, c.ownerReferences()) { + c.logger.Info("new connection pooler owner references do not match the current ones") + updateDeployment = true + } + if oldSpec != nil { specSync, specReason = needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler, c.logger) + syncReason = append(syncReason, specReason...) + } + + newPodAnnotations := c.annotationsSet(c.generatePodAnnotations(&c.Spec)) + deletedPodAnnotations := []string{} + if changed, reason := c.compareAnnotations(deployment.Spec.Template.Annotations, newPodAnnotations, &deletedPodAnnotations); changed { + specSync = true + syncReason = append(syncReason, []string{"new connection pooler's pod template annotations do not match the current ones: " + reason}...) + + for _, anno := range deletedPodAnnotations { + updatedPodAnnotations[anno] = nil + } + templateMetadataReq := map[string]map[string]map[string]map[string]map[string]*string{ + "spec": {"template": {"metadata": {"annotations": updatedPodAnnotations}}}} + patch, err := json.Marshal(templateMetadataReq) + if err != nil { + return nil, fmt.Errorf("could not marshal ObjectMeta for %s connection pooler's pod template: %v", role, err) + } + deployment, err = c.KubeClient.Deployments(c.Namespace).Patch(context.TODO(), + deployment.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "") + if err != nil { + c.logger.Errorf("failed to patch %s connection pooler's pod template: %v", role, err) + return nil, err + } + + deployment.Spec.Template.Annotations = newPodAnnotations } defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(&c.Config, newConnectionPooler, deployment) - reason := append(specReason, defaultsReason...) + syncReason = append(syncReason, defaultsReason...) - if specSync || defaultsSync { - c.logger.Infof("Update connection pooler deployment %s, reason: %+v", - c.connectionPoolerName(role), reason) - newDeploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) + if specSync || defaultsSync || updateDeployment { + c.logger.Infof("update connection pooler deployment %s, reason: %+v", + c.connectionPoolerName(role), syncReason) + newDeployment, err = c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) if err != nil { - msg := "could not generate deployment for connection pooler: %v" - return reason, fmt.Errorf(msg, err) + return syncReason, fmt.Errorf("could not generate deployment for connection pooler: %v", err) } - deployment, err := updateConnectionPoolerDeployment(c.KubeClient, - newDeploymentSpec) + deployment, err = updateConnectionPoolerDeployment(c.KubeClient, newDeployment, updateDeployment) if err != nil { - return reason, err + return syncReason, err + } + c.ConnectionPooler[role].Deployment = deployment + } + + newAnnotations := c.AnnotationsToPropagate(c.annotationsSet(nil)) // including the downscaling annotations + if changed, _ := c.compareAnnotations(deployment.Annotations, newAnnotations, nil); changed { + deployment, err = patchConnectionPoolerAnnotations(c.KubeClient, deployment, newAnnotations) + if err != nil { + return nil, err } c.ConnectionPooler[role].Deployment = deployment } } - newAnnotations := c.AnnotationsToPropagate(c.annotationsSet(c.ConnectionPooler[role].Deployment.Annotations)) - if newAnnotations != nil { - deployment, err = updateConnectionPoolerAnnotations(c.KubeClient, c.ConnectionPooler[role].Deployment, newAnnotations) - if err != nil { - return nil, err + // check if pooler pods must be replaced due to secret update + listOptions := metav1.ListOptions{ + LabelSelector: labels.Set(c.connectionPoolerLabels(role, true).MatchLabels).String(), + } + pods, err = c.listPoolerPods(listOptions) + if err != nil { + return nil, err + } + for i, pod := range pods { + if c.getRollingUpdateFlagFromPod(&pod) { + podName := util.NameFromMeta(pods[i].ObjectMeta) + err := retryutil.Retry(1*time.Second, 5*time.Second, + func() (bool, error) { + err2 := c.KubeClient.Pods(podName.Namespace).Delete( + context.TODO(), + podName.Name, + c.deleteOptions) + if err2 != nil { + return false, err2 + } + return true, nil + }) + if err != nil { + return nil, fmt.Errorf("could not delete pooler pod: %v", err) + } + } else if changed, _ := c.compareAnnotations(pod.Annotations, deployment.Spec.Template.Annotations, nil); changed { + metadataReq := map[string]map[string]map[string]*string{"metadata": {}} + + for anno, val := range deployment.Spec.Template.Annotations { + updatedPodAnnotations[anno] = &val + } + metadataReq["metadata"]["annotations"] = updatedPodAnnotations + patch, err := json.Marshal(metadataReq) + if err != nil { + return nil, fmt.Errorf("could not marshal ObjectMeta for %s connection pooler's pods: %v", role, err) + } + _, err = c.KubeClient.Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return nil, fmt.Errorf("could not patch annotations for %s connection pooler's pod %q: %v", role, pod.Name, err) + } } - c.ConnectionPooler[role].Deployment = deployment } - service, err := c.KubeClient. - Services(c.Namespace). - Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}) - - if err != nil && k8sutil.ResourceNotFound(err) { - msg := "Service %s for connection pooler synchronization is not found, create it" - c.logger.Warningf(msg, c.connectionPoolerName(role)) - - serviceSpec := c.generateConnectionPoolerService(c.ConnectionPooler[role]) - service, err := c.KubeClient. - Services(serviceSpec.Namespace). - Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) - + if service, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil { + c.ConnectionPooler[role].Service = service + desiredSvc := c.generateConnectionPoolerService(c.ConnectionPooler[role]) + newService, err = c.updateService(role, service, desiredSvc) if err != nil { - return NoSync, err + return syncReason, fmt.Errorf("could not update %s service to match desired state: %v", role, err) } - c.ConnectionPooler[role].Service = service - - } else if err != nil { - msg := "could not get connection pooler service to sync: %v" - return NoSync, fmt.Errorf(msg, err) - } else { - // Service updates are not supported and probably not that useful anyway - c.ConnectionPooler[role].Service = service + c.ConnectionPooler[role].Service = newService + return NoSync, nil } + if !k8sutil.ResourceNotFound(err) { + return NoSync, fmt.Errorf("could not get connection pooler service to sync: %v", err) + } + + c.ConnectionPooler[role].Service = nil + c.logger.Warningf("service %s for connection pooler synchronization is not found, create it", c.connectionPoolerName(role)) + + serviceSpec := c.generateConnectionPoolerService(c.ConnectionPooler[role]) + newService, err = c.KubeClient. + Services(serviceSpec.Namespace). + Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) + + if err != nil { + return NoSync, err + } + c.ConnectionPooler[role].Service = newService + return NoSync, nil } diff --git a/pkg/cluster/connection_pooler_test.go b/pkg/cluster/connection_pooler_test.go index 280adb101..1b41cbb02 100644 --- a/pkg/cluster/connection_pooler_test.go +++ b/pkg/cluster/connection_pooler_test.go @@ -1,7 +1,7 @@ package cluster import ( - "errors" + "context" "fmt" "strings" "testing" @@ -11,6 +11,7 @@ import ( fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" appsv1 "k8s.io/api/apps/v1" @@ -19,7 +20,21 @@ import ( "k8s.io/client-go/kubernetes/fake" ) -func mockInstallLookupFunction(schema string, user string, role PostgresRole) error { +func newFakeK8sPoolerTestClient() (k8sutil.KubernetesClient, *fake.Clientset) { + acidClientSet := fakeacidv1.NewSimpleClientset() + clientSet := fake.NewSimpleClientset() + + return k8sutil.KubernetesClient{ + PodsGetter: clientSet.CoreV1(), + PostgresqlsGetter: acidClientSet.AcidV1(), + StatefulSetsGetter: clientSet.AppsV1(), + DeploymentsGetter: clientSet.AppsV1(), + ServicesGetter: clientSet.CoreV1(), + SecretsGetter: clientSet.CoreV1(), + }, clientSet +} + +func mockInstallLookupFunction(schema string, user string) error { return nil } @@ -27,10 +42,6 @@ func boolToPointer(value bool) *bool { return &value } -func int32ToPointer(value int32) *int32 { - return &value -} - func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error { for _, role := range [2]PostgresRole{Master, Replica} { @@ -267,6 +278,7 @@ func TestConnectionPoolerCreateDeletion(t *testing.T) { client := k8sutil.KubernetesClient{ StatefulSetsGetter: clientSet.AppsV1(), ServicesGetter: clientSet.CoreV1(), + PodsGetter: clientSet.CoreV1(), DeploymentsGetter: clientSet.AppsV1(), PostgresqlsGetter: acidClientSet.AcidV1(), SecretsGetter: clientSet.CoreV1(), @@ -294,7 +306,7 @@ func TestConnectionPoolerCreateDeletion(t *testing.T) { ConnectionPoolerDefaultCPULimit: "100m", ConnectionPoolerDefaultMemoryRequest: "100Mi", ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + NumberOfInstances: k8sutil.Int32ToPointer(1), }, PodManagementPolicy: "ordered_ready", Resources: config.Resources{ @@ -376,6 +388,7 @@ func TestConnectionPoolerSync(t *testing.T) { client := k8sutil.KubernetesClient{ StatefulSetsGetter: clientSet.AppsV1(), ServicesGetter: clientSet.CoreV1(), + PodsGetter: clientSet.CoreV1(), DeploymentsGetter: clientSet.AppsV1(), PostgresqlsGetter: acidClientSet.AcidV1(), SecretsGetter: clientSet.CoreV1(), @@ -401,7 +414,7 @@ func TestConnectionPoolerSync(t *testing.T) { ConnectionPoolerDefaultCPULimit: "100m", ConnectionPoolerDefaultMemoryRequest: "100Mi", ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + NumberOfInstances: k8sutil.Int32ToPointer(1), }, PodManagementPolicy: "ordered_ready", Resources: config.Resources{ @@ -639,7 +652,7 @@ func TestConnectionPoolerSync(t *testing.T) { for _, tt := range tests { tt.cluster.OpConfig.ConnectionPooler.Image = tt.defaultImage tt.cluster.OpConfig.ConnectionPooler.NumberOfInstances = - int32ToPointer(tt.defaultInstances) + k8sutil.Int32ToPointer(tt.defaultInstances) t.Logf("running test for %s [%s]", testName, tt.subTest) @@ -663,8 +676,9 @@ func TestConnectionPoolerPodSpec(t *testing.T) { SuperUsername: superUserName, ReplicationUsername: replicationUserName, }, + PodServiceAccountName: "postgres-pod", ConnectionPooler: config.ConnectionPooler{ - MaxDBConnections: int32ToPointer(60), + MaxDBConnections: k8sutil.Int32ToPointer(60), ConnectionPoolerDefaultCPURequest: "100m", ConnectionPoolerDefaultCPULimit: "100m", ConnectionPoolerDefaultMemoryRequest: "100Mi", @@ -697,38 +711,42 @@ func TestConnectionPoolerPodSpec(t *testing.T) { noCheck := func(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { return nil } tests := []struct { - subTest string - spec *acidv1.PostgresSpec - expected error - cluster *Cluster - check func(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error + subTest string + spec *acidv1.PostgresSpec + cluster *Cluster + check func(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error }{ { subTest: "default configuration", spec: &acidv1.PostgresSpec{ ConnectionPooler: &acidv1.ConnectionPooler{}, }, - expected: nil, - cluster: cluster, - check: noCheck, + cluster: cluster, + check: noCheck, + }, + { + subTest: "pooler uses pod service account", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + cluster: cluster, + check: testServiceAccount, }, { subTest: "no default resources", spec: &acidv1.PostgresSpec{ ConnectionPooler: &acidv1.ConnectionPooler{}, }, - expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), - cluster: clusterNoDefaultRes, - check: noCheck, + cluster: clusterNoDefaultRes, + check: noCheck, }, { subTest: "default resources are set", spec: &acidv1.PostgresSpec{ ConnectionPooler: &acidv1.ConnectionPooler{}, }, - expected: nil, - cluster: cluster, - check: testResources, + cluster: cluster, + check: testResources, }, { subTest: "labels for service", @@ -736,30 +754,23 @@ func TestConnectionPoolerPodSpec(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, EnableReplicaConnectionPooler: boolToPointer(true), }, - expected: nil, - cluster: cluster, - check: testLabels, + cluster: cluster, + check: testLabels, }, { subTest: "required envs", spec: &acidv1.PostgresSpec{ ConnectionPooler: &acidv1.ConnectionPooler{}, }, - expected: nil, - cluster: cluster, - check: testEnvs, + cluster: cluster, + check: testEnvs, }, } for _, role := range [2]PostgresRole{Master, Replica} { for _, tt := range tests { - podSpec, err := tt.cluster.generateConnectionPoolerPodTemplate(role) + podSpec, _ := tt.cluster.generateConnectionPoolerPodTemplate(role) - if err != tt.expected && err.Error() != tt.expected.Error() { - t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", - testName, tt.subTest, err, tt.expected) - } - - err = tt.check(cluster, podSpec, role) + err := tt.check(cluster, podSpec, role) if err != nil { t.Errorf("%s [%s]: Pod spec is incorrect, %+v", testName, tt.subTest, err) @@ -793,6 +804,7 @@ func TestConnectionPoolerDeploymentSpec(t *testing.T) { } cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{ Master: { + AuthSecret: nil, Deployment: nil, Service: nil, LookupFunction: true, @@ -859,6 +871,17 @@ func TestConnectionPoolerDeploymentSpec(t *testing.T) { } } +func testServiceAccount(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { + poolerServiceAccount := podSpec.Spec.ServiceAccountName + + if poolerServiceAccount != cluster.OpConfig.PodServiceAccountName { + return fmt.Errorf("Pooler service account does not match, got %+v, expected %+v", + poolerServiceAccount, cluster.OpConfig.PodServiceAccountName) + } + + return nil +} + func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] if cpuReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest { @@ -921,6 +944,126 @@ func testServiceSelector(cluster *Cluster, service *v1.Service, role PostgresRol return nil } +func TestPoolerTLS(t *testing.T) { + client, _ := newFakeK8sPoolerTestClient() + clusterName := "acid-test-cluster" + namespace := "default" + tlsSecretName := "my-secret" + spiloFSGroup := int64(103) + defaultMode := int32(0640) + mountPath := "/tls" + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + EnableConnectionPooler: util.True(), + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + TLS: &acidv1.TLSDescription{ + SecretName: tlsSecretName, CAFile: "ca.crt"}, + AdditionalVolumes: []acidv1.AdditionalVolume{ + { + Name: tlsSecretName, + MountPath: mountPath, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: tlsSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + }, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + SpiloFSGroup: &spiloFSGroup, + }, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + }, + PodServiceAccountName: "postgres-pod", + }, + }, client, pg, logger, eventRecorder) + + // create a statefulset + _, err := cluster.createStatefulSet() + assert.NoError(t, err) + + // create pooler resources + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{} + cluster.ConnectionPooler[Master] = &ConnectionPoolerObjects{ + AuthSecret: nil, + Deployment: nil, + Service: nil, + Name: cluster.connectionPoolerName(Master), + ClusterName: clusterName, + Namespace: namespace, + LookupFunction: false, + Role: Master, + } + + _, err = cluster.syncConnectionPoolerWorker(nil, &pg, Master) + assert.NoError(t, err) + + deploy, err := client.Deployments(namespace).Get(context.TODO(), cluster.connectionPoolerName(Master), metav1.GetOptions{}) + assert.NoError(t, err) + + fsGroup := int64(103) + assert.Equal(t, &fsGroup, deploy.Spec.Template.Spec.SecurityContext.FSGroup, "has a default FSGroup assigned") + + assert.Equal(t, "postgres-pod", deploy.Spec.Template.Spec.ServiceAccountName, "need to add a service account name") + + volume := v1.Volume{ + Name: "my-secret", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "my-secret", + DefaultMode: &defaultMode, + }, + }, + } + assert.Contains(t, deploy.Spec.Template.Spec.Volumes, volume, "the pod gets a secret volume") + + poolerContainer := deploy.Spec.Template.Spec.Containers[constants.ConnectionPoolerContainer] + assert.Contains(t, poolerContainer.VolumeMounts, v1.VolumeMount{ + MountPath: "/tls", + Name: "my-secret", + }, "the volume gets mounted in /tls") + + assert.Contains(t, poolerContainer.Env, v1.EnvVar{Name: "CONNECTION_POOLER_CLIENT_TLS_CRT", Value: "/tls/tls.crt"}) + assert.Contains(t, poolerContainer.Env, v1.EnvVar{Name: "CONNECTION_POOLER_CLIENT_TLS_KEY", Value: "/tls/tls.key"}) + assert.Contains(t, poolerContainer.Env, v1.EnvVar{Name: "CONNECTION_POOLER_CLIENT_CA_FILE", Value: "/tls/ca.crt"}) +} + func TestConnectionPoolerServiceSpec(t *testing.T) { testName := "Test connection pooler service spec generation" var cluster = New( @@ -937,6 +1080,9 @@ func TestConnectionPoolerServiceSpec(t *testing.T) { ConnectionPoolerDefaultMemoryRequest: "100Mi", ConnectionPoolerDefaultMemoryLimit: "100Mi", }, + Resources: config.Resources{ + EnableOwnerReferences: util.True(), + }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) cluster.Statefulset = &appsv1.StatefulSet{ @@ -946,12 +1092,14 @@ func TestConnectionPoolerServiceSpec(t *testing.T) { } cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{ Master: { + AuthSecret: nil, Deployment: nil, Service: nil, LookupFunction: false, Role: Master, }, Replica: { + AuthSecret: nil, Deployment: nil, Service: nil, LookupFunction: false, @@ -1006,3 +1154,181 @@ func TestConnectionPoolerServiceSpec(t *testing.T) { } } } + +func TestConnectionPoolerServiceType(t *testing.T) { + testName := "Test connection pooler service type selection" + + cluster := New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + }, + Resources: config.Resources{ + EnableOwnerReferences: util.True(), + }, + }, + }, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{}, + logger, + eventRecorder, + ) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{ + Master: { + Deployment: nil, + Service: nil, + LookupFunction: false, + Role: Master, + }, + Replica: { + Deployment: nil, + Service: nil, + LookupFunction: false, + Role: Replica, + }, + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + cluster *Cluster + expectedType map[PostgresRole]v1.ServiceType + }{ + { + subTest: "default configuration -> ClusterIP for both", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + }, + cluster: cluster, + expectedType: map[PostgresRole]v1.ServiceType{ + Master: v1.ServiceTypeClusterIP, + Replica: v1.ServiceTypeClusterIP, + }, + }, + { + subTest: "LoadBalancer for both roles", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableMasterPoolerLoadBalancer: boolToPointer(true), + EnableReplicaPoolerLoadBalancer: boolToPointer(true), + }, + cluster: cluster, + expectedType: map[PostgresRole]v1.ServiceType{ + Master: v1.ServiceTypeLoadBalancer, + Replica: v1.ServiceTypeLoadBalancer, + }, + }, + { + subTest: "LoadBalancer for master", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableMasterPoolerLoadBalancer: boolToPointer(true), + }, + cluster: cluster, + expectedType: map[PostgresRole]v1.ServiceType{ + Master: v1.ServiceTypeLoadBalancer, + Replica: v1.ServiceTypeClusterIP, + }, + }, + { + subTest: "LoadBalancer for replica", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaPoolerLoadBalancer: boolToPointer(true), + }, + cluster: cluster, + expectedType: map[PostgresRole]v1.ServiceType{ + Master: v1.ServiceTypeClusterIP, + Replica: v1.ServiceTypeLoadBalancer, + }, + }, + { + subTest: "NodePort for both roles", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableMasterPoolerNodePort: boolToPointer(true), + EnableReplicaPoolerNodePort: boolToPointer(true), + }, + cluster: cluster, + expectedType: map[PostgresRole]v1.ServiceType{ + Master: v1.ServiceTypeNodePort, + Replica: v1.ServiceTypeNodePort, + }, + }, + { + subTest: "NodePort for master", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableMasterPoolerNodePort: boolToPointer(true), + }, + cluster: cluster, + expectedType: map[PostgresRole]v1.ServiceType{ + Master: v1.ServiceTypeNodePort, + Replica: v1.ServiceTypeClusterIP, + }, + }, + { + subTest: "NodePort for replica", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableReplicaPoolerNodePort: boolToPointer(true), + }, + cluster: cluster, + expectedType: map[PostgresRole]v1.ServiceType{ + Master: v1.ServiceTypeClusterIP, + Replica: v1.ServiceTypeNodePort, + }, + }, + { + subTest: "NodePort overrides LoadBalancer for both roles", + spec: &acidv1.PostgresSpec{ + ConnectionPooler: &acidv1.ConnectionPooler{}, + EnableMasterPoolerLoadBalancer: boolToPointer(true), + EnableReplicaPoolerLoadBalancer: boolToPointer(true), + EnableMasterPoolerNodePort: boolToPointer(true), + EnableReplicaPoolerNodePort: boolToPointer(true), + }, + cluster: cluster, + expectedType: map[PostgresRole]v1.ServiceType{ + Master: v1.ServiceTypeNodePort, + Replica: v1.ServiceTypeNodePort, + }, + }, + } + + roles := []PostgresRole{Master, Replica} + + for _, tt := range tests { + tt.cluster.Spec = *tt.spec + + for _, role := range roles { + svc := tt.cluster.generateConnectionPoolerService(tt.cluster.ConnectionPooler[role]) + + expected, ok := tt.expectedType[role] + if !ok { + t.Fatalf("%s [%s]: missing expectedType for role %v", testName, tt.subTest, role) + } + + if svc.Spec.Type != expected { + t.Errorf("%s [%s] role=%s: service Type is incorrect, got %s, expected %s", + testName, tt.subTest, role, svc.Spec.Type, expected) + } + } + } +} diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 829c2e5c7..56b5f3638 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -14,18 +14,25 @@ import ( "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/retryutil" + "github.com/zalando/postgres-operator/pkg/util/users" ) const ( getUserSQL = `SELECT a.rolname, COALESCE(a.rolpassword, ''), a.rolsuper, a.rolinherit, - a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig, - ARRAY(SELECT b.rolname - FROM pg_catalog.pg_auth_members m - JOIN pg_catalog.pg_authid b ON (m.roleid = b.oid) - WHERE m.member = a.oid) as memberof - FROM pg_catalog.pg_authid a LEFT JOIN pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid) - WHERE a.rolname = ANY($1) - ORDER BY 1;` + a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig, + ARRAY(SELECT b.rolname + FROM pg_catalog.pg_auth_members m + JOIN pg_catalog.pg_authid b ON (m.roleid = b.oid) + WHERE m.member = a.oid) as memberof + FROM pg_catalog.pg_authid a LEFT JOIN pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid) + WHERE a.rolname = ANY($1) + ORDER BY 1;` + + getUsersForRetention = `SELECT r.rolname, right(r.rolname, 6) AS roldatesuffix + FROM pg_roles r + JOIN unnest($1::text[]) AS u(name) ON r.rolname LIKE u.name || '%' + AND right(r.rolname, 6) ~ '^[0-9\.]+$' + ORDER BY 1;` getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` getSchemasSQL = `SELECT n.nspname AS dbschema FROM pg_catalog.pg_namespace n @@ -39,6 +46,16 @@ const ( createExtensionSQL = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"` alterExtensionSQL = `ALTER EXTENSION "%s" SET SCHEMA "%s"` + getPublicationsSQL = `SELECT p.pubname, COALESCE(string_agg(pt.schemaname || '.' || pt.tablename, ', ' ORDER BY pt.schemaname, pt.tablename), '') AS pubtables + FROM pg_publication p + LEFT JOIN pg_publication_tables pt ON pt.pubname = p.pubname + WHERE p.pubowner = 'postgres'::regrole + AND p.pubname LIKE 'fes_%' + GROUP BY p.pubname;` + createPublicationSQL = `CREATE PUBLICATION "%s" FOR TABLE %s WITH (publish = 'insert, update');` + alterPublicationSQL = `ALTER PUBLICATION "%s" SET TABLE %s;` + dropPublicationSQL = `DROP PUBLICATION "%s";` + globalDefaultPrivilegesSQL = `SET ROLE TO "%s"; ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO "%s","%s"; ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO "%s"; @@ -94,7 +111,7 @@ func (c *Cluster) pgConnectionString(dbname string) string { func (c *Cluster) databaseAccessDisabled() bool { if !c.OpConfig.EnableDBAccess { - c.logger.Debugf("database access is disabled") + c.logger.Debug("database access is disabled") } return !c.OpConfig.EnableDBAccess @@ -140,7 +157,9 @@ func (c *Cluster) initDbConnWithName(dbname string) error { return false, err2 } - return false, err + // Retry open connection until succeeded. + c.logger.Warningf("could not connect to Postgres database: %v", err) + return false, nil }) if finalerr != nil { @@ -189,7 +208,11 @@ func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUser } defer func() { if err2 := rows.Close(); err2 != nil { - err = fmt.Errorf("error when closing query cursor: %v", err2) + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } } }() @@ -217,7 +240,8 @@ func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUser parameters[fields[0]] = fields[1] } - if strings.HasSuffix(rolname, c.OpConfig.RoleDeletionSuffix) { + // consider NOLOGIN roles with deleted suffix as deprecated users + if strings.HasSuffix(rolname, c.OpConfig.RoleDeletionSuffix) && !rolcanlogin { roldeleted = true } @@ -227,6 +251,83 @@ func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUser return users, nil } +func findUsersFromRotation(rotatedUsers []string, db *sql.DB) (map[string]string, error) { + extraUsers := make(map[string]string, 0) + rows, err := db.Query(getUsersForRetention, pq.Array(rotatedUsers)) + if err != nil { + return nil, fmt.Errorf("query failed: %v", err) + } + defer func() { + if err2 := rows.Close(); err2 != nil { + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + } + }() + + for rows.Next() { + var ( + rolname, roldatesuffix string + ) + err := rows.Scan(&rolname, &roldatesuffix) + if err != nil { + return nil, fmt.Errorf("error when processing rows of deprecated users: %v", err) + } + extraUsers[rolname] = roldatesuffix + } + + return extraUsers, nil +} + +func (c *Cluster) cleanupRotatedUsers(rotatedUsers []string) error { + c.setProcessName("checking for rotated users to remove from the database due to configured retention") + + err := c.initDbConn() + if err != nil { + return fmt.Errorf("could not init db connection: %v", err) + } + defer func() { + if c.connectionIsClosed() { + return + } + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection after removing users exceeding configured retention interval: %v", err) + } + }() + + extraUsers, err := findUsersFromRotation(rotatedUsers, c.pgDb) + if err != nil { + return fmt.Errorf("error when querying for deprecated users from password rotation: %v", err) + } + + // make sure user retention policy aligns with rotation interval + retenionDays := c.OpConfig.PasswordRotationUserRetention + if retenionDays < 2*c.OpConfig.PasswordRotationInterval { + retenionDays = 2 * c.OpConfig.PasswordRotationInterval + c.logger.Warnf("user retention days too few compared to rotation interval %d - setting it to %d", c.OpConfig.PasswordRotationInterval, retenionDays) + } + retentionDate := time.Now().AddDate(0, 0, int(retenionDays)*-1) + + for rotatedUser, dateSuffix := range extraUsers { + userCreationDate, err := time.Parse(constants.RotationUserDateFormat, dateSuffix) + if err != nil { + c.logger.Errorf("could not parse creation date suffix of user %q: %v", rotatedUser, err) + continue + } + if retentionDate.After(userCreationDate) { + c.logger.Infof("dropping user %q due to configured days in password_rotation_user_retention", rotatedUser) + if err = users.DropPgUser(rotatedUser, c.pgDb); err != nil { + c.logger.Errorf("could not drop role %q: %v", rotatedUser, err) + continue + } + } + } + + return nil +} + // getDatabases returns the map of current databases with owners // The caller is responsible for opening and closing the database connection func (c *Cluster) getDatabases() (dbs map[string]string, err error) { @@ -351,10 +452,30 @@ func (c *Cluster) execCreateDatabaseSchema(databaseName, schemaName, dbOwner, sc } // set default privileges for schema + // the schemaOwner defines them for global database roles c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName) + + // if schemaOwner and dbOwner differ we know that _ default roles were created if schemaOwner != dbOwner { - c.execAlterSchemaDefaultPrivileges(schemaName, dbOwner, databaseName+"_"+schemaName) - c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner, databaseName+"_"+schemaName) + defaultUsers := c.Spec.PreparedDatabases[databaseName].PreparedSchemas[schemaName].DefaultUsers + + // define schema privileges of __owner_user for global roles, too + if defaultUsers { + c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner+constants.UserRoleNameSuffix, databaseName) + } + + // collect all possible owner roles and define default schema privileges + // for __reader/writer roles + owners := c.getOwnerRoles(databaseName, c.Spec.PreparedDatabases[databaseName].DefaultUsers) + owners = append(owners, c.getOwnerRoles(databaseName+"_"+schemaName, defaultUsers)...) + for _, owner := range owners { + c.execAlterSchemaDefaultPrivileges(schemaName, owner, databaseName+"_"+schemaName) + } + } else { + // define schema privileges of _owner_user for global roles, too + if c.Spec.PreparedDatabases[databaseName].DefaultUsers { + c.execAlterSchemaDefaultPrivileges(schemaName, schemaOwner+constants.UserRoleNameSuffix, databaseName) + } } return nil @@ -418,6 +539,15 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin return result } +func (c *Cluster) getOwnerRoles(dbObjPath string, withUser bool) (owners []string) { + owners = append(owners, dbObjPath+constants.OwnerRoleNameSuffix) + if withUser { + owners = append(owners, dbObjPath+constants.OwnerRoleNameSuffix+constants.UserRoleNameSuffix) + } + + return owners +} + // getExtension returns the list of current database extensions // The caller is responsible for opening and closing the database connection func (c *Cluster) getExtensions() (dbExtensions map[string]string, err error) { @@ -477,9 +607,79 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi return nil } +// getPublications returns the list of current database publications with tables +// The caller is responsible for opening and closing the database connection +func (c *Cluster) getPublications() (publications map[string]string, err error) { + var ( + rows *sql.Rows + ) + + if rows, err = c.pgDb.Query(getPublicationsSQL); err != nil { + return nil, fmt.Errorf("could not query database publications: %v", err) + } + + defer func() { + if err2 := rows.Close(); err2 != nil { + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + } + }() + + dbPublications := make(map[string]string) + + for rows.Next() { + var ( + dbPublication string + dbPublicationTables string + ) + + if err = rows.Scan(&dbPublication, &dbPublicationTables); err != nil { + return nil, fmt.Errorf("error when processing row: %v", err) + } + dbPublications[dbPublication] = dbPublicationTables + } + + return dbPublications, err +} + +func (c *Cluster) executeDropPublication(pubName string) error { + c.logger.Infof("dropping publication %q", pubName) + if _, err := c.pgDb.Exec(fmt.Sprintf(dropPublicationSQL, pubName)); err != nil { + return fmt.Errorf("could not execute drop publication: %v", err) + } + return nil +} + +// executeCreatePublication creates new publication for given tables +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeCreatePublication(pubName, tableList string) error { + return c.execCreateOrAlterPublication(pubName, tableList, createPublicationSQL, + "creating publication", "create publication") +} + +// executeAlterExtension changes the table list of the given publication. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeAlterPublication(pubName, tableList string) error { + return c.execCreateOrAlterPublication(pubName, tableList, alterPublicationSQL, + "changing publication", "alter publication tables") +} + +func (c *Cluster) execCreateOrAlterPublication(pubName, tableList, statement, doing, operation string) error { + + c.logger.Debugf("%s %q with table list %q", doing, pubName, tableList) + if _, err := c.pgDb.Exec(fmt.Sprintf(statement, pubName, tableList)); err != nil { + return fmt.Errorf("could not execute %s: %v", operation, err) + } + + return nil +} + // Creates a connection pool credentials lookup function in every database to // perform remote authentication. -func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string, role PostgresRole) error { +func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { var stmtBytes bytes.Buffer c.logger.Info("Installing lookup function") @@ -575,8 +775,8 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string, role Po c.logger.Infof("pooler lookup function installed into %s", dbname) } - if len(failedDatabases) == 0 { - c.ConnectionPooler[role].LookupFunction = true + if len(failedDatabases) > 0 { + return fmt.Errorf("could not install pooler lookup function in every specified databases") } return nil diff --git a/pkg/cluster/exec.go b/pkg/cluster/exec.go index 8b5089b4e..5605a70f6 100644 --- a/pkg/cluster/exec.go +++ b/pkg/cluster/exec.go @@ -15,7 +15,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/constants" ) -//ExecCommand executes arbitrary command inside the pod +// ExecCommand executes arbitrary command inside the pod func (c *Cluster) ExecCommand(podName *spec.NamespacedName, command ...string) (string, error) { c.setProcessName("executing command %q", strings.Join(command, " ")) @@ -59,7 +59,7 @@ func (c *Cluster) ExecCommand(podName *spec.NamespacedName, command ...string) ( return "", fmt.Errorf("failed to init executor: %v", err) } - err = exec.Stream(remotecommand.StreamOptions{ + err = exec.StreamWithContext(context.TODO(), remotecommand.StreamOptions{ Stdout: &execOut, Stderr: &execErr, Tty: false, diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index cb11170d6..306185e90 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -4,17 +4,23 @@ import ( "context" "encoding/json" "fmt" + "maps" "path" + "slices" "sort" "strings" + "github.com/pkg/errors" "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -24,26 +30,22 @@ import ( "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" - batchv1 "k8s.io/api/batch/v1" - batchv1beta1 "k8s.io/api/batch/v1beta1" - "k8s.io/apimachinery/pkg/labels" + "github.com/zalando/postgres-operator/pkg/util/patroni" + "github.com/zalando/postgres-operator/pkg/util/retryutil" ) const ( - pgBinariesLocationTemplate = "/usr/lib/postgresql/%v/bin" - patroniPGBinariesParameterName = "bin_dir" - patroniPGParametersParameterName = "parameters" - patroniPGHBAConfParameterName = "pg_hba" - localHost = "127.0.0.1/32" - connectionPoolerContainer = "connection-pooler" - pgPort = 5432 + pgBinariesLocationTemplate = "/usr/lib/postgresql/%v/bin" + patroniPGBinariesParameterName = "bin_dir" + patroniPGHBAConfParameterName = "pg_hba" + localHost = "127.0.0.1/32" + scalyrSidecarName = "scalyr-sidecar" + logicalBackupContainerName = "logical-backup" + connectionPoolerContainer = "connection-pooler" + pgPort = 5432 + operatorPort = 8080 ) -type pgUser struct { - Password string `json:"password"` - Options []string `json:"options"` -} - type patroniDCS struct { TTL uint32 `json:"ttl,omitempty"` LoopWait uint32 `json:"loop_wait,omitempty"` @@ -51,14 +53,15 @@ type patroniDCS struct { MaximumLagOnFailover float32 `json:"maximum_lag_on_failover,omitempty"` SynchronousMode bool `json:"synchronous_mode,omitempty"` SynchronousModeStrict bool `json:"synchronous_mode_strict,omitempty"` + SynchronousNodeCount uint32 `json:"synchronous_node_count,omitempty"` PGBootstrapConfiguration map[string]interface{} `json:"postgresql,omitempty"` Slots map[string]map[string]string `json:"slots,omitempty"` + FailsafeMode *bool `json:"failsafe_mode,omitempty"` } type pgBootstrap struct { - Initdb []interface{} `json:"initdb"` - Users map[string]pgUser `json:"users"` - DCS patroniDCS `json:"dcs,omitempty"` + Initdb []interface{} `json:"initdb"` + DCS patroniDCS `json:"dcs,omitempty"` } type spiloConfiguration struct { @@ -70,19 +73,13 @@ func (c *Cluster) statefulSetName() string { return c.Name } -func (c *Cluster) endpointName(role PostgresRole) string { - name := c.Name - if role == Replica { - name = name + "-repl" - } - - return name -} - func (c *Cluster) serviceName(role PostgresRole) string { name := c.Name - if role == Replica { - name = name + "-repl" + switch role { + case Replica: + name = fmt.Sprintf("%s-%s", name, "repl") + case Patroni: + name = fmt.Sprintf("%s-%s", name, "config") } return name @@ -95,36 +92,40 @@ func (c *Cluster) serviceAddress(role PostgresRole) string { return service.ObjectMeta.Name } - c.logger.Warningf("No service for role %s", role) - return "" + defaultAddress := c.serviceName(role) + c.logger.Warningf("No service for role %s - defaulting to %s", role, defaultAddress) + return defaultAddress } -func (c *Cluster) servicePort(role PostgresRole) string { +func (c *Cluster) servicePort(role PostgresRole) int32 { service, exist := c.Services[role] if exist { - return fmt.Sprint(service.Spec.Ports[0].Port) + return service.Spec.Ports[0].Port } - c.logger.Warningf("No service for role %s", role) - return "" + c.logger.Warningf("No service for role %s - defaulting to port %d", role, pgPort) + return pgPort } -func (c *Cluster) podDisruptionBudgetName() string { +func (c *Cluster) PrimaryPodDisruptionBudgetName() string { return c.OpConfig.PDBNameFormat.Format("cluster", c.Name) } -func (c *Cluster) makeDefaultResources() acidv1.Resources { +func (c *Cluster) criticalOpPodDisruptionBudgetName() string { + pdbTemplate := config.StringTemplate("postgres-{cluster}-critical-op-pdb") + return pdbTemplate.Format("cluster", c.Name) +} - config := c.OpConfig +func makeDefaultResources(config *config.Config) acidv1.Resources { defaultRequests := acidv1.ResourceDescription{ - CPU: config.Resources.DefaultCPURequest, - Memory: config.Resources.DefaultMemoryRequest, + CPU: &config.Resources.DefaultCPURequest, + Memory: &config.Resources.DefaultMemoryRequest, } defaultLimits := acidv1.ResourceDescription{ - CPU: config.Resources.DefaultCPULimit, - Memory: config.Resources.DefaultMemoryLimit, + CPU: &config.Resources.DefaultCPULimit, + Memory: &config.Resources.DefaultMemoryLimit, } return acidv1.Resources{ @@ -133,14 +134,198 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources { } } -func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) { +func makeLogicalBackupResources(config *config.Config) acidv1.Resources { + + logicalBackupResourceRequests := acidv1.ResourceDescription{ + CPU: &config.LogicalBackup.LogicalBackupCPURequest, + Memory: &config.LogicalBackup.LogicalBackupMemoryRequest, + } + logicalBackupResourceLimits := acidv1.ResourceDescription{ + CPU: &config.LogicalBackup.LogicalBackupCPULimit, + Memory: &config.LogicalBackup.LogicalBackupMemoryLimit, + } + + return acidv1.Resources{ + ResourceRequests: logicalBackupResourceRequests, + ResourceLimits: logicalBackupResourceLimits, + } +} + +func (c *Cluster) enforceMinResourceLimits(resources *v1.ResourceRequirements) error { + var ( + isSmaller bool + err error + msg string + ) + + // setting limits too low can cause unnecessary evictions / OOM kills + cpuLimit := resources.Limits[v1.ResourceCPU] + minCPULimit := c.OpConfig.MinCPULimit + if minCPULimit != "" { + isSmaller, err = util.IsSmallerQuantity(cpuLimit.String(), minCPULimit) + if err != nil { + return fmt.Errorf("could not compare defined CPU limit %s for %q container with configured minimum value %s: %v", + cpuLimit.String(), constants.PostgresContainerName, minCPULimit, err) + } + if isSmaller { + msg = fmt.Sprintf("defined CPU limit %s for %q container is below required minimum %s and will be increased", + cpuLimit.String(), constants.PostgresContainerName, minCPULimit) + c.logger.Warningf("%s", msg) + c.eventRecorder.Event(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", msg) + resources.Limits[v1.ResourceCPU], _ = resource.ParseQuantity(minCPULimit) + } + } + + memoryLimit := resources.Limits[v1.ResourceMemory] + minMemoryLimit := c.OpConfig.MinMemoryLimit + if minMemoryLimit != "" { + isSmaller, err = util.IsSmallerQuantity(memoryLimit.String(), minMemoryLimit) + if err != nil { + return fmt.Errorf("could not compare defined memory limit %s for %q container with configured minimum value %s: %v", + memoryLimit.String(), constants.PostgresContainerName, minMemoryLimit, err) + } + if isSmaller { + msg = fmt.Sprintf("defined memory limit %s for %q container is below required minimum %s and will be increased", + memoryLimit.String(), constants.PostgresContainerName, minMemoryLimit) + c.logger.Warningf("%s", msg) + c.eventRecorder.Event(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", msg) + resources.Limits[v1.ResourceMemory], _ = resource.ParseQuantity(minMemoryLimit) + } + } + + return nil +} + +func (c *Cluster) enforceMaxResourceRequests(resources *v1.ResourceRequirements) error { + var ( + err error + ) + + cpuRequest := resources.Requests[v1.ResourceCPU] + maxCPURequest := c.OpConfig.MaxCPURequest + maxCPU, err := util.MinResource(maxCPURequest, cpuRequest.String()) + if err != nil { + return fmt.Errorf("could not compare defined CPU request %s for %q container with configured maximum value %s: %v", + cpuRequest.String(), constants.PostgresContainerName, maxCPURequest, err) + } + if !maxCPU.IsZero() { + resources.Requests[v1.ResourceCPU] = maxCPU + } + + memoryRequest := resources.Requests[v1.ResourceMemory] + maxMemoryRequest := c.OpConfig.MaxMemoryRequest + maxMemory, err := util.MinResource(maxMemoryRequest, memoryRequest.String()) + if err != nil { + return fmt.Errorf("could not compare defined memory request %s for %q container with configured maximum value %s: %v", + memoryRequest.String(), constants.PostgresContainerName, maxMemoryRequest, err) + } + if !maxMemory.IsZero() { + resources.Requests[v1.ResourceMemory] = maxMemory + } + + return nil +} + +func setMemoryRequestToLimit(resources *v1.ResourceRequirements, containerName string, logger *logrus.Entry) { + + requests := resources.Requests[v1.ResourceMemory] + limits := resources.Limits[v1.ResourceMemory] + isSmaller := requests.Cmp(limits) == -1 + if isSmaller { + logger.Warningf("memory request of %s for %q container is increased to match memory limit of %s", + requests.String(), containerName, limits.String()) + resources.Requests[v1.ResourceMemory] = limits + } +} + +func matchLimitsWithRequestsIfSmaller(resources *v1.ResourceRequirements, containerName string, logger *logrus.Entry) { + requests := resources.Requests + limits := resources.Limits + requestCPU, cpuRequestsExists := requests[v1.ResourceCPU] + limitCPU, cpuLimitExists := limits[v1.ResourceCPU] + if cpuRequestsExists && cpuLimitExists && limitCPU.Cmp(requestCPU) == -1 { + logger.Warningf("CPU limit of %s for %q container is increased to match CPU requests of %s", limitCPU.String(), containerName, requestCPU.String()) + resources.Limits[v1.ResourceCPU] = requestCPU + } + + requestMemory, memoryRequestsExists := requests[v1.ResourceMemory] + limitMemory, memoryLimitExists := limits[v1.ResourceMemory] + if memoryRequestsExists && memoryLimitExists && limitMemory.Cmp(requestMemory) == -1 { + logger.Warningf("memory limit of %s for %q container is increased to match memory requests of %s", limitMemory.String(), containerName, requestMemory.String()) + resources.Limits[v1.ResourceMemory] = requestMemory + } +} + +func fillResourceList(spec acidv1.ResourceDescription, defaults acidv1.ResourceDescription) (v1.ResourceList, error) { var err error + requests := v1.ResourceList{} + emptyResourceExamples := []string{"", "0", "null"} - specRequests := resources.ResourceRequests - specLimits := resources.ResourceLimits + if spec.CPU != nil && !slices.Contains(emptyResourceExamples, *spec.CPU) { + requests[v1.ResourceCPU], err = resource.ParseQuantity(*spec.CPU) + if err != nil { + return nil, fmt.Errorf("could not parse CPU quantity: %v", err) + } + } else { + if defaults.CPU != nil && !slices.Contains(emptyResourceExamples, *defaults.CPU) { + requests[v1.ResourceCPU], err = resource.ParseQuantity(*defaults.CPU) + if err != nil { + return nil, fmt.Errorf("could not parse default CPU quantity: %v", err) + } + } + } + if spec.Memory != nil && !slices.Contains(emptyResourceExamples, *spec.Memory) { + requests[v1.ResourceMemory], err = resource.ParseQuantity(*spec.Memory) + if err != nil { + return nil, fmt.Errorf("could not parse memory quantity: %v", err) + } + } else { + if defaults.Memory != nil && !slices.Contains(emptyResourceExamples, *defaults.Memory) { + requests[v1.ResourceMemory], err = resource.ParseQuantity(*defaults.Memory) + if err != nil { + return nil, fmt.Errorf("could not parse default memory quantity: %v", err) + } + } + } + if spec.HugePages2Mi != nil { + requests[v1.ResourceHugePagesPrefix+"2Mi"], err = resource.ParseQuantity(*spec.HugePages2Mi) + if err != nil { + return nil, fmt.Errorf("could not parse hugepages-2Mi quantity: %v", err) + } + } + if spec.HugePages1Gi != nil { + requests[v1.ResourceHugePagesPrefix+"1Gi"], err = resource.ParseQuantity(*spec.HugePages1Gi) + if err != nil { + return nil, fmt.Errorf("could not parse hugepages-1Gi quantity: %v", err) + } + } + + return requests, nil +} + +func (c *Cluster) generateResourceRequirements( + resources *acidv1.Resources, + defaultResources acidv1.Resources, + containerName string) (*v1.ResourceRequirements, error) { + var err error + specRequests := acidv1.ResourceDescription{} + specLimits := acidv1.ResourceDescription{} result := v1.ResourceRequirements{} + enforceThresholds := true + resourcesLimitAnnotationKey := c.OpConfig.IgnoreResourcesLimitsAnnotationKey + if resourcesLimitAnnotationKey != "" { + if value, exists := c.ObjectMeta.Annotations[resourcesLimitAnnotationKey]; exists && value == "true" { + enforceThresholds = false + } + } + + if resources != nil { + specRequests = resources.ResourceRequests + specLimits = resources.ResourceLimits + } + result.Requests, err = fillResourceList(specRequests, defaultResources.ResourceRequests) if err != nil { return nil, fmt.Errorf("could not fill resource requests: %v", err) @@ -151,45 +336,37 @@ func generateResourceRequirements(resources acidv1.Resources, defaultResources a return nil, fmt.Errorf("could not fill resource limits: %v", err) } + // enforce minimum cpu and memory limits for Postgres containers only + if containerName == constants.PostgresContainerName && enforceThresholds { + if err = c.enforceMinResourceLimits(&result); err != nil { + return nil, fmt.Errorf("could not enforce minimum resource limits: %v", err) + } + } + + // make sure after reflecting default and enforcing min limit values we don't have requests > limits + matchLimitsWithRequestsIfSmaller(&result, containerName, c.logger) + + // vice versa set memory requests to limit if option is enabled + if c.OpConfig.SetMemoryRequestToLimit { + setMemoryRequestToLimit(&result, containerName, c.logger) + } + + // enforce maximum cpu and memory requests for Postgres containers only + if containerName == constants.PostgresContainerName && enforceThresholds { + if err = c.enforceMaxResourceRequests(&result); err != nil { + return nil, fmt.Errorf("could not enforce maximum resource requests: %v", err) + } + } + return &result, nil } -func fillResourceList(spec acidv1.ResourceDescription, defaults acidv1.ResourceDescription) (v1.ResourceList, error) { - var err error - requests := v1.ResourceList{} - - if spec.CPU != "" { - requests[v1.ResourceCPU], err = resource.ParseQuantity(spec.CPU) - if err != nil { - return nil, fmt.Errorf("could not parse CPU quantity: %v", err) - } - } else { - requests[v1.ResourceCPU], err = resource.ParseQuantity(defaults.CPU) - if err != nil { - return nil, fmt.Errorf("could not parse default CPU quantity: %v", err) - } - } - if spec.Memory != "" { - requests[v1.ResourceMemory], err = resource.ParseQuantity(spec.Memory) - if err != nil { - return nil, fmt.Errorf("could not parse memory quantity: %v", err) - } - } else { - requests[v1.ResourceMemory], err = resource.ParseQuantity(defaults.Memory) - if err != nil { - return nil, fmt.Errorf("could not parse default memory quantity: %v", err) - } - } - - return requests, nil -} - -func generateSpiloJSONConfiguration(pg *acidv1.PostgresqlParam, patroni *acidv1.Patroni, pamRoleName string, EnablePgVersionEnvVar bool, logger *logrus.Entry) (string, error) { +func generateSpiloJSONConfiguration(pg *acidv1.PostgresqlParam, patroni *acidv1.Patroni, opConfig *config.Config, logger *logrus.Entry) (string, error) { config := spiloConfiguration{} config.Bootstrap = pgBootstrap{} - config.Bootstrap.Initdb = []interface{}{map[string]string{"auth-host": "md5"}, + config.Bootstrap.Initdb = []interface{}{map[string]string{"auth-host": "scram-sha-256"}, map[string]string{"auth-local": "trust"}} initdbOptionNames := []string{} @@ -243,7 +420,7 @@ PatroniInitDBParams: } if patroni.MaximumLagOnFailover >= 0 { - config.Bootstrap.DCS.MaximumLagOnFailover = patroni.MaximumLagOnFailover + config.Bootstrap.DCS.MaximumLagOnFailover = float32(patroni.MaximumLagOnFailover) } if patroni.LoopWait != 0 { config.Bootstrap.DCS.LoopWait = patroni.LoopWait @@ -263,6 +440,14 @@ PatroniInitDBParams: if patroni.SynchronousModeStrict { config.Bootstrap.DCS.SynchronousModeStrict = patroni.SynchronousModeStrict } + if patroni.SynchronousNodeCount >= 1 { + config.Bootstrap.DCS.SynchronousNodeCount = patroni.SynchronousNodeCount + } + if patroni.FailsafeMode != nil { + config.Bootstrap.DCS.FailsafeMode = patroni.FailsafeMode + } else if opConfig.EnablePatroniFailsafeMode != nil { + config.Bootstrap.DCS.FailsafeMode = opConfig.EnablePatroniFailsafeMode + } config.PgLocalConfiguration = make(map[string]interface{}) @@ -270,18 +455,18 @@ PatroniInitDBParams: // setting postgresq.bin_dir in the SPILO_CONFIGURATION still works and takes precedence over PGVERSION // so we add postgresq.bin_dir only if PGVERSION is unused // see PR 222 in Spilo - if !EnablePgVersionEnvVar { + if !opConfig.EnablePgVersionEnvVar { config.PgLocalConfiguration[patroniPGBinariesParameterName] = fmt.Sprintf(pgBinariesLocationTemplate, pg.PgVersion) } if len(pg.Parameters) > 0 { local, bootstrap := getLocalAndBoostrapPostgreSQLParameters(pg.Parameters) if len(local) > 0 { - config.PgLocalConfiguration[patroniPGParametersParameterName] = local + config.PgLocalConfiguration[constants.PatroniPGParametersParameterName] = local } if len(bootstrap) > 0 { config.Bootstrap.DCS.PGBootstrapConfiguration = make(map[string]interface{}) - config.Bootstrap.DCS.PGBootstrapConfiguration[patroniPGParametersParameterName] = bootstrap + config.Bootstrap.DCS.PGBootstrapConfiguration[constants.PatroniPGParametersParameterName] = bootstrap } } // Patroni gives us a choice of writing pg_hba.conf to either the bootstrap section or to the local postgresql one. @@ -291,13 +476,6 @@ PatroniInitDBParams: config.PgLocalConfiguration[patroniPGHBAConfParameterName] = patroni.PgHba } - config.Bootstrap.Users = map[string]pgUser{ - pamRoleName: { - Password: "", - Options: []string{constants.RoleFlagCreateDB, constants.RoleFlagNoLogin}, - }, - } - res, err := json.Marshal(config) return string(res), err } @@ -328,7 +506,7 @@ func generateCapabilities(capabilities []string) *v1.Capabilities { return nil } -func nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAffinity) *v1.Affinity { +func (c *Cluster) nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAffinity) *v1.Affinity { if len(nodeReadinessLabel) == 0 && nodeAffinity == nil { return nil } @@ -353,8 +531,19 @@ func nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAff }, } } else { - nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{ - NodeSelectorTerms: append(nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, nodeReadinessSelectorTerm), + switch c.OpConfig.NodeReadinessLabelMerge { + case "OR": + manifestTerms := nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + manifestTerms = append(manifestTerms, nodeReadinessSelectorTerm) + nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{ + NodeSelectorTerms: manifestTerms, + } + case "AND": + for i, nodeSelectorTerm := range nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + manifestExpressions := nodeSelectorTerm.MatchExpressions + manifestExpressions = append(manifestExpressions, matchExpressions...) + nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i] = v1.NodeSelectorTerm{MatchExpressions: manifestExpressions} + } } } } @@ -364,17 +553,26 @@ func nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAff } } -func generatePodAffinity(labels labels.Set, topologyKey string, nodeAffinity *v1.Affinity) *v1.Affinity { - // generate pod anti-affinity to avoid multiple pods of the same Postgres cluster in the same topology , e.g. node - podAffinity := v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - TopologyKey: topologyKey, - }}, +func podAffinity( + labels labels.Set, + topologyKey string, + nodeAffinity *v1.Affinity, + preferredDuringScheduling bool, + anti bool) *v1.Affinity { + + var podAffinity v1.Affinity + + podAffinityTerm := v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, }, + TopologyKey: topologyKey, + } + + if anti { + podAffinity.PodAntiAffinity = generatePodAntiAffinity(podAffinityTerm, preferredDuringScheduling) + } else { + podAffinity.PodAffinity = generatePodAffinity(podAffinityTerm, preferredDuringScheduling) } if nodeAffinity != nil && nodeAffinity.NodeAffinity != nil { @@ -384,6 +582,43 @@ func generatePodAffinity(labels labels.Set, topologyKey string, nodeAffinity *v1 return &podAffinity } +func generatePodAffinity(podAffinityTerm v1.PodAffinityTerm, preferredDuringScheduling bool) *v1.PodAffinity { + podAffinity := &v1.PodAffinity{} + + if preferredDuringScheduling { + podAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []v1.WeightedPodAffinityTerm{{ + Weight: 1, + PodAffinityTerm: podAffinityTerm, + }} + } else { + podAffinity.RequiredDuringSchedulingIgnoredDuringExecution = []v1.PodAffinityTerm{podAffinityTerm} + } + + return podAffinity +} + +func generatePodAntiAffinity(podAffinityTerm v1.PodAffinityTerm, preferredDuringScheduling bool) *v1.PodAntiAffinity { + podAntiAffinity := &v1.PodAntiAffinity{} + + if preferredDuringScheduling { + podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []v1.WeightedPodAffinityTerm{{ + Weight: 1, + PodAffinityTerm: podAffinityTerm, + }} + } else { + podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = []v1.PodAffinityTerm{podAffinityTerm} + } + + return podAntiAffinity +} + +func generateTopologySpreadConstraints(labels labels.Set, topologySpreadConstraints []v1.TopologySpreadConstraint) []v1.TopologySpreadConstraint { + for _, topologySpreadConstraint := range topologySpreadConstraints { + topologySpreadConstraint.LabelSelector = &metav1.LabelSelector{MatchLabels: labels} + } + return topologySpreadConstraints +} + func tolerations(tolerationsSpec *[]v1.Toleration, podToleration map[string]string) []v1.Toleration { // allow to override tolerations by postgresql manifest if len(*tolerationsSpec) > 0 { @@ -412,23 +647,49 @@ func tolerations(tolerationsSpec *[]v1.Toleration, podToleration map[string]stri // Those parameters must go to the bootstrap/dcs/postgresql/parameters section. // See http://patroni.readthedocs.io/en/latest/dynamic_configuration.html. func isBootstrapOnlyParameter(param string) bool { - return param == "max_connections" || - param == "max_locks_per_transaction" || - param == "max_worker_processes" || - param == "max_prepared_transactions" || - param == "wal_level" || - param == "wal_log_hints" || - param == "track_commit_timestamp" + params := map[string]bool{ + "archive_command": false, + "shared_buffers": false, + "logging_collector": false, + "log_destination": false, + "log_directory": false, + "log_filename": false, + "log_file_mode": false, + "log_rotation_age": false, + "log_truncate_on_rotation": false, + "ssl": false, + "ssl_ca_file": false, + "ssl_crl_file": false, + "ssl_cert_file": false, + "ssl_key_file": false, + "shared_preload_libraries": false, + "bg_mon.listen_address": false, + "bg_mon.history_buckets": false, + "pg_stat_statements.track_utility": false, + "extwlist.extensions": false, + "extwlist.custom_path": false, + } + result, ok := params[param] + if !ok { + result = true + } + return result } func generateVolumeMounts(volume acidv1.Volume) []v1.VolumeMount { - return []v1.VolumeMount{ + volumeMount := []v1.VolumeMount{ { Name: constants.DataVolumeName, MountPath: constants.PostgresDataMount, //TODO: fetch from manifest - SubPath: volume.SubPath, }, } + + if volume.IsSubPathExpr != nil && *volume.IsSubPathExpr { + volumeMount[0].SubPathExpr = volume.SubPath + } else { + volumeMount[0].SubPath = volume.SubPath + } + return volumeMount } func generateContainer( @@ -448,15 +709,15 @@ func generateContainer( Resources: *resourceRequirements, Ports: []v1.ContainerPort{ { - ContainerPort: 8008, + ContainerPort: patroni.ApiPort, Protocol: v1.ProtocolTCP, }, { - ContainerPort: 5432, + ContainerPort: pgPort, Protocol: v1.ProtocolTCP, }, { - ContainerPort: 8080, + ContainerPort: operatorPort, Protocol: v1.ProtocolTCP, }, }, @@ -471,22 +732,20 @@ func generateContainer( } } -func generateSidecarContainers(sidecars []acidv1.Sidecar, - defaultResources acidv1.Resources, startIndex int, logger *logrus.Entry) ([]v1.Container, error) { +func (c *Cluster) generateSidecarContainers(sidecars []acidv1.Sidecar, + defaultResources acidv1.Resources, startIndex int) ([]v1.Container, error) { if len(sidecars) > 0 { result := make([]v1.Container, 0) for index, sidecar := range sidecars { + var resourcesSpec acidv1.Resources + if sidecar.Resources == nil { + resourcesSpec = acidv1.Resources{} + } else { + sidecar.Resources.DeepCopyInto(&resourcesSpec) + } - resources, err := generateResourceRequirements( - makeResources( - sidecar.Resources.ResourceRequests.CPU, - sidecar.Resources.ResourceRequests.Memory, - sidecar.Resources.ResourceLimits.CPU, - sidecar.Resources.ResourceLimits.Memory, - ), - defaultResources, - ) + resources, err := c.generateResourceRequirements(&resourcesSpec, defaultResources, sidecar.Name) if err != nil { return nil, err } @@ -500,7 +759,7 @@ func generateSidecarContainers(sidecars []acidv1.Sidecar, } // adds common fields to sidecars -func patchSidecarContainers(in []v1.Container, volumeMounts []v1.VolumeMount, superUserName string, credentialsSecretName string, logger *logrus.Entry) []v1.Container { +func patchSidecarContainers(in []v1.Container, volumeMounts []v1.VolumeMount, superUserName string, credentialsSecretName string) []v1.Container { result := []v1.Container{} for _, container := range in { @@ -540,8 +799,7 @@ func patchSidecarContainers(in []v1.Container, volumeMounts []v1.VolumeMount, su }, }, } - mergedEnv := append(env, container.Env...) - container.Env = deduplicateEnvVars(mergedEnv, container.Name, logger) + container.Env = appendEnvVars(env, container.Env...) result = append(result, container) } @@ -565,10 +823,9 @@ func (c *Cluster) generatePodTemplate( spiloContainer *v1.Container, initContainers []v1.Container, sidecarContainers []v1.Container, + sharePgSocketWithSidecars *bool, + topologySpreadConstraintsSpec []v1.TopologySpreadConstraint, tolerationsSpec *[]v1.Toleration, - spiloRunAsUser *int64, - spiloRunAsGroup *int64, - spiloFSGroup *int64, nodeAffinity *v1.Affinity, schedulerName *string, terminateGracePeriod int64, @@ -578,6 +835,7 @@ func (c *Cluster) generatePodTemplate( shmVolume *bool, podAntiAffinity bool, podAntiAffinityTopologyKey string, + podAntiAffinityPreferredDuringScheduling bool, additionalSecretMount string, additionalSecretMountPath string, additionalVolumes []acidv1.AdditionalVolume, @@ -586,18 +844,22 @@ func (c *Cluster) generatePodTemplate( terminateGracePeriodSeconds := terminateGracePeriod containers := []v1.Container{*spiloContainer} containers = append(containers, sidecarContainers...) - securityContext := v1.PodSecurityContext{} - - if spiloRunAsUser != nil { - securityContext.RunAsUser = spiloRunAsUser + securityContext := v1.PodSecurityContext{ + RunAsUser: c.OpConfig.Resources.SpiloRunAsUser, + RunAsGroup: c.OpConfig.Resources.SpiloRunAsGroup, + FSGroup: c.OpConfig.Resources.SpiloFSGroup, } - if spiloRunAsGroup != nil { - securityContext.RunAsGroup = spiloRunAsGroup + if c.Spec.SpiloRunAsUser != nil { + securityContext.RunAsUser = c.Spec.SpiloRunAsUser } - if spiloFSGroup != nil { - securityContext.FSGroup = spiloFSGroup + if c.Spec.SpiloRunAsGroup != nil { + securityContext.RunAsGroup = c.Spec.SpiloRunAsGroup + } + + if c.Spec.SpiloFSGroup != nil { + securityContext.FSGroup = c.Spec.SpiloFSGroup } podSpec := v1.PodSpec{ @@ -618,7 +880,13 @@ func (c *Cluster) generatePodTemplate( } if podAntiAffinity { - podSpec.Affinity = generatePodAffinity(labels, podAntiAffinityTopologyKey, nodeAffinity) + podSpec.Affinity = podAffinity( + labels, + podAntiAffinityTopologyKey, + nodeAffinity, + podAntiAffinityPreferredDuringScheduling, + true, + ) } else if nodeAffinity != nil { podSpec.Affinity = nodeAffinity } @@ -627,11 +895,17 @@ func (c *Cluster) generatePodTemplate( podSpec.PriorityClassName = priorityClassName } + podSpec.TopologySpreadConstraints = generateTopologySpreadConstraints(labels, topologySpreadConstraintsSpec) + + if sharePgSocketWithSidecars != nil && *sharePgSocketWithSidecars { + addVarRunVolume(&podSpec) + } + if additionalSecretMount != "" { addSecretVolume(&podSpec, additionalSecretMount, additionalSecretMountPath) } - if additionalVolumes != nil { + if len(additionalVolumes) > 0 { c.addAdditionalVolumes(&podSpec, additionalVolumes) } @@ -654,7 +928,13 @@ func (c *Cluster) generatePodTemplate( } // generatePodEnvVars generates environment variables for the Spilo Pod -func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration string, cloneDescription *acidv1.CloneDescription, standbyDescription *acidv1.StandbyDescription, customPodEnvVarsList []v1.EnvVar) []v1.EnvVar { +func (c *Cluster) generateSpiloPodEnvVars( + spec *acidv1.PostgresSpec, + uid types.UID, + spiloConfiguration string) ([]v1.EnvVar, error) { + + // hard-coded set of environment variables we need + // to guarantee core functionality of the operator envVars := []v1.EnvVar{ { Name: "SCOPE", @@ -729,6 +1009,11 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri Value: c.OpConfig.PamRoleName, }, } + + if c.OpConfig.EnableSpiloWalPathCompat { + envVars = append(envVars, v1.EnvVar{Name: "ENABLE_WAL_PATH_COMPAT", Value: "true"}) + } + if c.OpConfig.EnablePgVersionEnvVar { envVars = append(envVars, v1.EnvVar{Name: "PGVERSION", Value: c.GetDesiredMajorVersion()}) } @@ -744,6 +1029,9 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri if c.patroniUsesKubernetes() { envVars = append(envVars, v1.EnvVar{Name: "DCS_ENABLE_KUBERNETES_API", Value: "true"}) + if c.OpConfig.EnablePodDisruptionBudget != nil && *c.OpConfig.EnablePodDisruptionBudget { + envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_BOOTSTRAP_LABELS", Value: "{\"critical-operation\":\"true\"}"}) + } } else { envVars = append(envVars, v1.EnvVar{Name: "ETCD_HOST", Value: c.OpConfig.EtcdHost}) } @@ -752,75 +1040,90 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_USE_CONFIGMAPS", Value: "true"}) } - if cloneDescription != nil && cloneDescription.ClusterName != "" { - envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...) + // fetch cluster-specific variables that will override all subsequent global variables + if len(spec.Env) > 0 { + envVars = appendEnvVars(envVars, spec.Env...) } - if c.Spec.StandbyCluster != nil { - envVars = append(envVars, c.generateStandbyEnvironment(standbyDescription)...) + if spec.Clone != nil && spec.Clone.ClusterName != "" { + envVars = append(envVars, c.generateCloneEnvironment(spec.Clone)...) } - // add vars taken from pod_environment_configmap and pod_environment_secret first - // (to allow them to override the globals set in the operator config) - if len(customPodEnvVarsList) > 0 { - envVars = append(envVars, customPodEnvVarsList...) + if spec.StandbyCluster != nil { + envVars = append(envVars, c.generateStandbyEnvironment(spec.StandbyCluster)...) } + // fetch variables from custom environment Secret + // that will override all subsequent global variables + secretEnvVarsList, err := c.getPodEnvironmentSecretVariables() + if err != nil { + return nil, err + } + envVars = appendEnvVars(envVars, secretEnvVarsList...) + + // fetch variables from custom environment ConfigMap + // that will override all subsequent global variables + configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables() + if err != nil { + return nil, err + } + envVars = appendEnvVars(envVars, configMapEnvVarsList...) + + // global variables derived from operator configuration + opConfigEnvVars := make([]v1.EnvVar, 0) if c.OpConfig.WALES3Bucket != "" { - envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } if c.OpConfig.WALGSBucket != "" { - envVars = append(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + } + + if c.OpConfig.WALAZStorageAccount != "" { + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } if c.OpConfig.GCPCredentials != "" { - envVars = append(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) } if c.OpConfig.LogS3Bucket != "" { - envVars = append(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket}) - envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""}) } - return envVars + envVars = appendEnvVars(envVars, opConfigEnvVars...) + + return envVars, nil } -// deduplicateEnvVars makes sure there are no duplicate in the target envVar array. While Kubernetes already -// deduplicates variables defined in a container, it leaves the last definition in the list and this behavior is not -// well-documented, which means that the behavior can be reversed at some point (it may also start producing an error). -// Therefore, the merge is done by the operator, the entries that are ahead in the passed list take priority over those -// that are behind, and only the name is considered in order to eliminate duplicates. -func deduplicateEnvVars(input []v1.EnvVar, containerName string, logger *logrus.Entry) []v1.EnvVar { - result := make([]v1.EnvVar, 0) - names := make(map[string]int) - - for i, va := range input { - if names[va.Name] == 0 { - names[va.Name]++ - result = append(result, input[i]) - } else if names[va.Name] == 1 { - names[va.Name]++ - - // Some variables (those to configure the WAL_ and LOG_ shipping) may be overwritten, only log as info - if strings.HasPrefix(va.Name, "WAL_") || strings.HasPrefix(va.Name, "LOG_") { - logger.Infof("global variable %q has been overwritten by configmap/secret for container %q", - va.Name, containerName) - } else { - logger.Warningf("variable %q is defined in %q more than once, the subsequent definitions are ignored", - va.Name, containerName) - } +func appendEnvVars(envs []v1.EnvVar, appEnv ...v1.EnvVar) []v1.EnvVar { + collectedEnvs := envs + for _, env := range appEnv { + if !isEnvVarPresent(collectedEnvs, env.Name) { + collectedEnvs = append(collectedEnvs, env) } } - return result + return collectedEnvs } -// Return list of variables the pod recieved from the configured ConfigMap +func isEnvVarPresent(envs []v1.EnvVar, key string) bool { + for _, env := range envs { + if strings.EqualFold(env.Name, key) { + return true + } + } + return false +} + +// Return list of variables the pod received from the configured ConfigMap func (c *Cluster) getPodEnvironmentConfigMapVariables() ([]v1.EnvVar, error) { configMapPodEnvVarsList := make([]v1.EnvVar, 0) @@ -844,9 +1147,11 @@ func (c *Cluster) getPodEnvironmentConfigMapVariables() ([]v1.EnvVar, error) { return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) } } + for k, v := range cm.Data { configMapPodEnvVarsList = append(configMapPodEnvVarsList, v1.EnvVar{Name: k, Value: v}) } + sort.Slice(configMapPodEnvVarsList, func(i, j int) bool { return configMapPodEnvVarsList[i].Name < configMapPodEnvVarsList[j].Name }) return configMapPodEnvVarsList, nil } @@ -858,12 +1163,30 @@ func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) { return secretPodEnvVarsList, nil } - secret, err := c.KubeClient.Secrets(c.Namespace).Get( - context.TODO(), - c.OpConfig.PodEnvironmentSecret, - metav1.GetOptions{}) + secret := &v1.Secret{} + var notFoundErr error + err := retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, + func() (bool, error) { + var err error + secret, err = c.KubeClient.Secrets(c.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentSecret, + metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + notFoundErr = err + return false, nil + } + return false, err + } + return true, nil + }, + ) + if notFoundErr != nil && err != nil { + err = errors.Wrap(notFoundErr, err.Error()) + } if err != nil { - return nil, fmt.Errorf("could not read Secret PodEnvironmentSecretName: %v", err) + return nil, errors.Wrap(err, "could not read Secret PodEnvironmentSecretName") } for k := range secret.Data { @@ -878,9 +1201,41 @@ func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) { }}) } + sort.Slice(secretPodEnvVarsList, func(i, j int) bool { return secretPodEnvVarsList[i].Name < secretPodEnvVarsList[j].Name }) return secretPodEnvVarsList, nil } +// Return list of variables the cronjob received from the configured Secret +func (c *Cluster) getCronjobEnvironmentSecretVariables() ([]v1.EnvVar, error) { + secretCronjobEnvVarsList := make([]v1.EnvVar, 0) + + if c.OpConfig.LogicalBackupCronjobEnvironmentSecret == "" { + return secretCronjobEnvVarsList, nil + } + + secret, err := c.KubeClient.Secrets(c.Namespace).Get( + context.TODO(), + c.OpConfig.LogicalBackupCronjobEnvironmentSecret, + metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("could not read Secret CronjobEnvironmentSecretName: %v", err) + } + + for k := range secret.Data { + secretCronjobEnvVarsList = append(secretCronjobEnvVarsList, + v1.EnvVar{Name: k, ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.OpConfig.LogicalBackupCronjobEnvironmentSecret, + }, + Key: k, + }, + }}) + } + + return secretCronjobEnvVarsList, nil +} + func getSidecarContainer(sidecar acidv1.Sidecar, index int, resources *v1.ResourceRequirements) *v1.Container { name := sidecar.Name if name == "" { @@ -894,6 +1249,7 @@ func getSidecarContainer(sidecar acidv1.Sidecar, index int, resources *v1.Resour Resources: *resources, Env: sidecar.Env, Ports: sidecar.Ports, + Command: sidecar.Command, } } @@ -907,12 +1263,12 @@ func getBucketScopeSuffix(uid string) string { func makeResources(cpuRequest, memoryRequest, cpuLimit, memoryLimit string) acidv1.Resources { return acidv1.Resources{ ResourceRequests: acidv1.ResourceDescription{ - CPU: cpuRequest, - Memory: memoryRequest, + CPU: &cpuRequest, + Memory: &memoryRequest, }, ResourceLimits: acidv1.ResourceDescription{ - CPU: cpuLimit, - Memory: memoryLimit, + CPU: &cpuLimit, + Memory: &memoryLimit, }, } } @@ -926,6 +1282,36 @@ func extractPgVersionFromBinPath(binPath string, template string) (string, error return fmt.Sprintf("%v", pgVersion), nil } +func generateSpiloReadinessProbe() *v1.Probe { + return &v1.Probe{ + FailureThreshold: 3, + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/readiness", + Port: intstr.IntOrString{IntVal: patroni.ApiPort}, + Scheme: v1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 6, + PeriodSeconds: 10, + SuccessThreshold: 1, + TimeoutSeconds: 5, + } +} + +func generateSpiloLivenessProbe(probe, defaultProbe *v1.Probe) *v1.Probe { + + if probe != nil { + return probe + } + + if defaultProbe != nil { + return defaultProbe + } + + return nil +} + func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.StatefulSet, error) { var ( @@ -937,109 +1323,26 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef additionalVolumes = spec.AdditionalVolumes ) - // Improve me. Please. - if c.OpConfig.SetMemoryRequestToLimit { - - // controller adjusts the default memory request at operator startup - - request := spec.Resources.ResourceRequests.Memory - if request == "" { - request = c.OpConfig.Resources.DefaultMemoryRequest - } - - limit := spec.Resources.ResourceLimits.Memory - if limit == "" { - limit = c.OpConfig.Resources.DefaultMemoryLimit - } - - isSmaller, err := util.IsSmallerQuantity(request, limit) - if err != nil { - return nil, err - } - if isSmaller { - c.logger.Warningf("The memory request of %v for the Postgres container is increased to match the memory limit of %v.", request, limit) - spec.Resources.ResourceRequests.Memory = limit - - } - - // controller adjusts the Scalyr sidecar request at operator startup - // as this sidecar is managed separately - - // adjust sidecar containers defined for that particular cluster - for _, sidecar := range spec.Sidecars { - - // TODO #413 - sidecarRequest := sidecar.Resources.ResourceRequests.Memory - if request == "" { - request = c.OpConfig.Resources.DefaultMemoryRequest - } - - sidecarLimit := sidecar.Resources.ResourceLimits.Memory - if limit == "" { - limit = c.OpConfig.Resources.DefaultMemoryLimit - } - - isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit) - if err != nil { - return nil, err - } - if isSmaller { - c.logger.Warningf("The memory request of %v for the %v sidecar container is increased to match the memory limit of %v.", sidecar.Resources.ResourceRequests.Memory, sidecar.Name, sidecar.Resources.ResourceLimits.Memory) - sidecar.Resources.ResourceRequests.Memory = sidecar.Resources.ResourceLimits.Memory - } - } - - } - - defaultResources := c.makeDefaultResources() - - resourceRequirements, err := generateResourceRequirements(spec.Resources, defaultResources) + defaultResources := makeDefaultResources(&c.OpConfig) + resourceRequirements, err := c.generateResourceRequirements( + spec.Resources, defaultResources, constants.PostgresContainerName) if err != nil { return nil, fmt.Errorf("could not generate resource requirements: %v", err) } - if spec.InitContainers != nil && len(spec.InitContainers) > 0 { + if len(spec.InitContainers) > 0 { if c.OpConfig.EnableInitContainers != nil && !(*c.OpConfig.EnableInitContainers) { c.logger.Warningf("initContainers specified but disabled in configuration - next statefulset creation would fail") } initContainers = spec.InitContainers - } - - spiloCompathWalPathList := make([]v1.EnvVar, 0) - if c.OpConfig.EnableSpiloWalPathCompat { - spiloCompathWalPathList = append(spiloCompathWalPathList, - v1.EnvVar{ - Name: "ENABLE_WAL_PATH_COMPAT", - Value: "true", - }, - ) - } - - // fetch env vars from custom ConfigMap - configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables() - if err != nil { - return nil, err - } - - // fetch env vars from custom ConfigMap - secretEnvVarsList, err := c.getPodEnvironmentSecretVariables() - if err != nil { - return nil, err - } - - // concat all custom pod env vars and sort them - customPodEnvVarsList := append(spiloCompathWalPathList, configMapEnvVarsList...) - customPodEnvVarsList = append(customPodEnvVarsList, secretEnvVarsList...) - sort.Slice(customPodEnvVarsList, - func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name }) - - if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" { - return nil, fmt.Errorf("s3_wal_path is empty for standby cluster") + if err := c.validateContainers(initContainers); err != nil { + return nil, fmt.Errorf("invalid init containers: %v", err) + } } // backward compatible check for InitContainers if spec.InitContainersOld != nil { - msg := "Manifest parameter init_containers is deprecated." + msg := "manifest parameter init_containers is deprecated." if spec.InitContainers == nil { c.logger.Warningf("%s Consider using initContainers instead.", msg) spec.InitContainers = spec.InitContainersOld @@ -1050,7 +1353,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // backward compatible check for PodPriorityClassName if spec.PodPriorityClassNameOld != "" { - msg := "Manifest parameter pod_priority_class_name is deprecated." + msg := "manifest parameter pod_priority_class_name is deprecated." if spec.PodPriorityClassName == "" { c.logger.Warningf("%s Consider using podPriorityClassName instead.", msg) spec.PodPriorityClassName = spec.PodPriorityClassNameOld @@ -1059,119 +1362,73 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef } } - spiloConfiguration, err := generateSpiloJSONConfiguration(&spec.PostgresqlParam, &spec.Patroni, c.OpConfig.PamRoleName, c.OpConfig.EnablePgVersionEnvVar, c.logger) + spiloConfiguration, err := generateSpiloJSONConfiguration(&spec.PostgresqlParam, &spec.Patroni, &c.OpConfig, c.logger) if err != nil { return nil, fmt.Errorf("could not generate Spilo JSON configuration: %v", err) } // generate environment variables for the spilo container - spiloEnvVars := c.generateSpiloPodEnvVars( - c.Postgresql.GetUID(), - spiloConfiguration, - spec.Clone, - spec.StandbyCluster, - customPodEnvVarsList, - ) + spiloEnvVars, err := c.generateSpiloPodEnvVars(spec, c.Postgresql.GetUID(), spiloConfiguration) + if err != nil { + return nil, fmt.Errorf("could not generate Spilo env vars: %v", err) + } // pickup the docker image for the spilo container effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage) - // determine the User, Group and FSGroup for the spilo pod - effectiveRunAsUser := c.OpConfig.Resources.SpiloRunAsUser - if spec.SpiloRunAsUser != nil { - effectiveRunAsUser = spec.SpiloRunAsUser - } - - effectiveRunAsGroup := c.OpConfig.Resources.SpiloRunAsGroup - if spec.SpiloRunAsGroup != nil { - effectiveRunAsGroup = spec.SpiloRunAsGroup - } - - effectiveFSGroup := c.OpConfig.Resources.SpiloFSGroup - if spec.SpiloFSGroup != nil { - effectiveFSGroup = spec.SpiloFSGroup - } - volumeMounts := generateVolumeMounts(spec.Volume) // configure TLS with a custom secret volume if spec.TLS != nil && spec.TLS.SecretName != "" { - // this is combined with the FSGroup in the section above - // to give read access to the postgres user - defaultMode := int32(0640) - mountPath := "/tls" - additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{ - Name: spec.TLS.SecretName, - MountPath: mountPath, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: spec.TLS.SecretName, - DefaultMode: &defaultMode, - }, - }, - }) - - // use the same filenames as Secret resources by default - certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt") - privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key") - spiloEnvVars = append( - spiloEnvVars, - v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: certFile}, - v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: privateKeyFile}, - ) - - if spec.TLS.CAFile != "" { - // support scenario when the ca.crt resides in a different secret, diff path - mountPathCA := mountPath - if spec.TLS.CASecretName != "" { - mountPathCA = mountPath + "ca" + getSpiloTLSEnv := func(k string) string { + keyName := "" + switch k { + case "tls.crt": + keyName = "SSL_CERTIFICATE_FILE" + case "tls.key": + keyName = "SSL_PRIVATE_KEY_FILE" + case "tls.ca": + keyName = "SSL_CA_FILE" + default: + panic(fmt.Sprintf("TLS env key unknown %s", k)) } - caFile := ensurePath(spec.TLS.CAFile, mountPathCA, "") - spiloEnvVars = append( - spiloEnvVars, - v1.EnvVar{Name: "SSL_CA_FILE", Value: caFile}, - ) - - // the ca file from CASecretName secret takes priority - if spec.TLS.CASecretName != "" { - additionalVolumes = append(additionalVolumes, acidv1.AdditionalVolume{ - Name: spec.TLS.CASecretName, - MountPath: mountPathCA, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: spec.TLS.CASecretName, - DefaultMode: &defaultMode, - }, - }, - }) - } + return keyName } + tlsEnv, tlsVolumes := generateTlsMounts(spec, getSpiloTLSEnv) + for _, env := range tlsEnv { + spiloEnvVars = appendEnvVars(spiloEnvVars, env) + } + additionalVolumes = append(additionalVolumes, tlsVolumes...) } // generate the spilo container - c.logger.Debugf("Generating Spilo container, environment variables") - c.logger.Debugf("%v", spiloEnvVars) - spiloContainer := generateContainer(constants.PostgresContainerName, &effectiveDockerImage, resourceRequirements, - deduplicateEnvVars(spiloEnvVars, constants.PostgresContainerName, c.logger), + spiloEnvVars, volumeMounts, c.OpConfig.Resources.SpiloPrivileged, c.OpConfig.Resources.SpiloAllowPrivilegeEscalation, generateCapabilities(c.OpConfig.AdditionalPodCapabilities), ) + // Patroni responds 200 to probe only if it either owns the leader lock or postgres is running and DCS is accessible + if c.OpConfig.EnableReadinessProbe { + spiloContainer.ReadinessProbe = generateSpiloReadinessProbe() + } + + spiloContainer.LivenessProbe = generateSpiloLivenessProbe(spec.LivenessProbe, c.OpConfig.LivenessProbe) + // generate container specs for sidecars specified in the cluster manifest clusterSpecificSidecars := []v1.Container{} - if spec.Sidecars != nil && len(spec.Sidecars) > 0 { + if len(spec.Sidecars) > 0 { // warn if sidecars are defined, but globally disabled (does not apply to globally defined sidecars) if c.OpConfig.EnableSidecars != nil && !(*c.OpConfig.EnableSidecars) { c.logger.Warningf("sidecars specified but disabled in configuration - next statefulset creation would fail") } - if clusterSpecificSidecars, err = generateSidecarContainers(spec.Sidecars, defaultResources, 0, c.logger); err != nil { + if clusterSpecificSidecars, err = c.generateSidecarContainers(spec.Sidecars, defaultResources, 0); err != nil { return nil, fmt.Errorf("could not generate sidecar containers: %v", err) } } @@ -1182,7 +1439,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef for name, dockerImage := range c.OpConfig.SidecarImages { globalSidecarsByDockerImage = append(globalSidecarsByDockerImage, acidv1.Sidecar{Name: name, DockerImage: dockerImage}) } - if globalSidecarContainersByDockerImage, err = generateSidecarContainers(globalSidecarsByDockerImage, defaultResources, len(clusterSpecificSidecars), c.logger); err != nil { + if globalSidecarContainersByDockerImage, err = c.generateSidecarContainers(globalSidecarsByDockerImage, defaultResources, len(clusterSpecificSidecars)); err != nil { return nil, fmt.Errorf("could not generate sidecar containers: %v", err) } // make the resulting list reproducible @@ -1195,7 +1452,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // generate scalyr sidecar container var scalyrSidecars []v1.Container if scalyrSidecar, err := - generateScalyrSidecarSpec(c.Name, + c.generateScalyrSidecarSpec(c.Name, c.OpConfig.ScalyrAPIKey, c.OpConfig.ScalyrServerURL, c.OpConfig.ScalyrImage, @@ -1203,8 +1460,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.ScalyrMemoryRequest, c.OpConfig.ScalyrCPULimit, c.OpConfig.ScalyrMemoryLimit, - defaultResources, - c.logger); err != nil { + defaultResources); err != nil { return nil, fmt.Errorf("could not generate Scalyr sidecar: %v", err) } else { if scalyrSidecar != nil { @@ -1213,12 +1469,16 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef } sidecarContainers, conflicts := mergeContainers(clusterSpecificSidecars, c.Config.OpConfig.SidecarContainers, globalSidecarContainersByDockerImage, scalyrSidecars) - for containerName := range conflicts { - c.logger.Warningf("a sidecar is specified twice. Ignoring sidecar %q in favor of %q with high a precendence", + for _, containerName := range conflicts { + c.logger.Warningf("a sidecar is specified twice. Ignoring sidecar %q in favor of %q with high a precedence", containerName, containerName) } - sidecarContainers = patchSidecarContainers(sidecarContainers, volumeMounts, c.OpConfig.SuperUsername, c.credentialSecretName(c.OpConfig.SuperUsername), c.logger) + sidecarContainers = patchSidecarContainers(sidecarContainers, volumeMounts, c.OpConfig.SuperUsername, c.credentialSecretName(c.OpConfig.SuperUsername)) + + if err := c.validateContainers(sidecarContainers); err != nil { + return nil, fmt.Errorf("invalid sidecar containers: %v", err) + } tolerationSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration) effectivePodPriorityClassName := util.Coalesce(spec.PodPriorityClassName, c.OpConfig.PodPriorityClassName) @@ -1233,11 +1493,10 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef spiloContainer, initContainers, sidecarContainers, + c.OpConfig.SharePgSocketWithSidecars, + spec.TopologySpreadConstraints, &tolerationSpec, - effectiveRunAsUser, - effectiveRunAsGroup, - effectiveFSGroup, - nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity), + c.nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity), spec.SchedulerName, int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, @@ -1246,6 +1505,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef mountShmVolumeNeeded(c.OpConfig, spec), c.OpConfig.EnablePodAntiAffinity, c.OpConfig.PodAntiAffinityTopologyKey, + c.OpConfig.PodAntiAffinityPreferredDuringScheduling, c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, additionalVolumes) @@ -1254,11 +1514,12 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef return nil, fmt.Errorf("could not generate pod template: %v", err) } - if volumeClaimTemplate, err = generatePersistentVolumeClaimTemplate(spec.Volume.Size, - spec.Volume.StorageClass); err != nil { + if volumeClaimTemplate, err = c.generatePersistentVolumeClaimTemplate(spec.Volume.Size, + spec.Volume.StorageClass, spec.Volume.Selector); err != nil { return nil, fmt.Errorf("could not generate volume claim template: %v", err) } + // global minInstances and maxInstances settings can overwrite manifest numberOfInstances := c.getNumberOfInstances(spec) // the operator has domain-specific logic on how to do rolling updates of PG clusters @@ -1267,41 +1528,110 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef updateStrategy := appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType} var podManagementPolicy appsv1.PodManagementPolicyType - if c.OpConfig.PodManagementPolicy == "ordered_ready" { + switch c.OpConfig.PodManagementPolicy { + case "ordered_ready": podManagementPolicy = appsv1.OrderedReadyPodManagement - } else if c.OpConfig.PodManagementPolicy == "parallel" { + case "parallel": podManagementPolicy = appsv1.ParallelPodManagement - } else { + default: return nil, fmt.Errorf("could not set the pod management policy to the unknown value: %v", c.OpConfig.PodManagementPolicy) } + var persistentVolumeClaimRetentionPolicy appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy + if c.OpConfig.PersistentVolumeClaimRetentionPolicy["when_deleted"] == "delete" { + persistentVolumeClaimRetentionPolicy.WhenDeleted = appsv1.DeletePersistentVolumeClaimRetentionPolicyType + } else { + persistentVolumeClaimRetentionPolicy.WhenDeleted = appsv1.RetainPersistentVolumeClaimRetentionPolicyType + } + + if c.OpConfig.PersistentVolumeClaimRetentionPolicy["when_scaled"] == "delete" { + persistentVolumeClaimRetentionPolicy.WhenScaled = appsv1.DeletePersistentVolumeClaimRetentionPolicyType + } else { + persistentVolumeClaimRetentionPolicy.WhenScaled = appsv1.RetainPersistentVolumeClaimRetentionPolicyType + } + statefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: c.statefulSetName(), - Namespace: c.Namespace, - Labels: c.labelsSet(true), - Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)), + Name: c.statefulSetName(), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)), + OwnerReferences: c.ownerReferences(), }, Spec: appsv1.StatefulSetSpec{ - Replicas: &numberOfInstances, - Selector: c.labelsSelector(), - ServiceName: c.serviceName(Master), - Template: *podTemplate, - VolumeClaimTemplates: []v1.PersistentVolumeClaim{*volumeClaimTemplate}, - UpdateStrategy: updateStrategy, - PodManagementPolicy: podManagementPolicy, + Replicas: &numberOfInstances, + Selector: c.labelsSelector(), + ServiceName: c.serviceName(Master), + Template: *podTemplate, + VolumeClaimTemplates: []v1.PersistentVolumeClaim{*volumeClaimTemplate}, + UpdateStrategy: updateStrategy, + PodManagementPolicy: podManagementPolicy, + PersistentVolumeClaimRetentionPolicy: &persistentVolumeClaimRetentionPolicy, }, } return statefulSet, nil } +func generateTlsMounts(spec *acidv1.PostgresSpec, tlsEnv func(key string) string) ([]v1.EnvVar, []acidv1.AdditionalVolume) { + // this is combined with the FSGroup in the section above + // to give read access to the postgres user + defaultMode := int32(0640) + mountPath := "/tls" + env := make([]v1.EnvVar, 0) + volumes := make([]acidv1.AdditionalVolume, 0) + + volumes = append(volumes, acidv1.AdditionalVolume{ + Name: spec.TLS.SecretName, + MountPath: mountPath, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: spec.TLS.SecretName, + DefaultMode: &defaultMode, + }, + }, + }) + + // use the same filenames as Secret resources by default + certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt") + privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key") + env = append(env, v1.EnvVar{Name: tlsEnv("tls.crt"), Value: certFile}) + env = append(env, v1.EnvVar{Name: tlsEnv("tls.key"), Value: privateKeyFile}) + + if spec.TLS.CAFile != "" { + // support scenario when the ca.crt resides in a different secret, diff path + mountPathCA := mountPath + if spec.TLS.CASecretName != "" { + mountPathCA = mountPath + "ca" + } + + caFile := ensurePath(spec.TLS.CAFile, mountPathCA, "") + env = append(env, v1.EnvVar{Name: tlsEnv("tls.ca"), Value: caFile}) + + // the ca file from CASecretName secret takes priority + if spec.TLS.CASecretName != "" { + volumes = append(volumes, acidv1.AdditionalVolume{ + Name: spec.TLS.CASecretName, + MountPath: mountPathCA, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: spec.TLS.CASecretName, + DefaultMode: &defaultMode, + }, + }, + }) + } + } + + return env, volumes +} + func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]string { annotations := make(map[string]string) for k, v := range c.OpConfig.CustomPodAnnotations { annotations[k] = v } - if spec != nil || spec.PodAnnotations != nil { + if spec.PodAnnotations != nil { for k, v := range spec.PodAnnotations { annotations[k] = v } @@ -1314,12 +1644,12 @@ func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]s return annotations } -func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string, +func (c *Cluster) generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string, scalyrCPURequest string, scalyrMemoryRequest string, scalyrCPULimit string, scalyrMemoryLimit string, - defaultResources acidv1.Resources, logger *logrus.Entry) (*v1.Container, error) { + defaultResources acidv1.Resources) (*v1.Container, error) { if APIKey == "" || dockerImage == "" { if APIKey == "" && dockerImage != "" { - logger.Warning("Not running Scalyr sidecar: SCALYR_API_KEY must be defined") + c.logger.Warning("Not running Scalyr sidecar: SCALYR_API_KEY must be defined") } return nil, nil } @@ -1329,7 +1659,8 @@ func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage strin scalyrCPULimit, scalyrMemoryLimit, ) - resourceRequirementsScalyrSidecar, err := generateResourceRequirements(resourcesScalyrSidecar, defaultResources) + resourceRequirementsScalyrSidecar, err := c.generateResourceRequirements( + &resourcesScalyrSidecar, defaultResources, scalyrSidecarName) if err != nil { return nil, fmt.Errorf("invalid resources for Scalyr sidecar: %v", err) } @@ -1347,7 +1678,7 @@ func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage strin env = append(env, v1.EnvVar{Name: "SCALYR_SERVER_URL", Value: serverURL}) } return &v1.Container{ - Name: "scalyr-sidecar", + Name: scalyrSidecarName, Image: dockerImage, Env: env, ImagePullPolicy: v1.PullIfNotPresent, @@ -1358,10 +1689,17 @@ func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage strin func (c *Cluster) getNumberOfInstances(spec *acidv1.PostgresSpec) int32 { min := c.OpConfig.MinInstances max := c.OpConfig.MaxInstances + instanceLimitAnnotationKey := c.OpConfig.IgnoreInstanceLimitsAnnotationKey cur := spec.NumberOfInstances newcur := cur - if spec.StandbyCluster != nil { + if instanceLimitAnnotationKey != "" { + if value, exists := c.ObjectMeta.Annotations[instanceLimitAnnotationKey]; exists && value == "true" { + return cur + } + } + + if isStandbyCluster(spec) { if newcur == 1 { min = newcur max = newcur @@ -1417,6 +1755,28 @@ func addShmVolume(podSpec *v1.PodSpec) { podSpec.Volumes = volumes } +func addVarRunVolume(podSpec *v1.PodSpec) { + volumes := append(podSpec.Volumes, v1.Volume{ + Name: "postgresql-run", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: "Memory", + }, + }, + }) + + for i := range podSpec.Containers { + mounts := append(podSpec.Containers[i].VolumeMounts, + v1.VolumeMount{ + Name: constants.RunVolumeName, + MountPath: constants.RunVolumePath, + }) + podSpec.Containers[i].VolumeMounts = mounts + } + + podSpec.Volumes = volumes +} + func addSecretVolume(podSpec *v1.PodSpec, additionalSecretMount string, additionalSecretMountPath string) { volumes := append(podSpec.Volumes, v1.Volume{ Name: additionalSecretMount, @@ -1446,13 +1806,13 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec, mountPaths := map[string]acidv1.AdditionalVolume{} for i, additionalVolume := range additionalVolumes { if previousVolume, exist := mountPaths[additionalVolume.MountPath]; exist { - msg := "Volume %+v cannot be mounted to the same path as %+v" + msg := "volume %+v cannot be mounted to the same path as %+v" c.logger.Warningf(msg, additionalVolume, previousVolume) continue } if additionalVolume.MountPath == constants.PostgresDataMount { - msg := "Cannot mount volume on postgresql data directory, %+v" + msg := "cannot mount volume on postgresql data directory, %+v" c.logger.Warningf(msg, additionalVolume) continue } @@ -1465,7 +1825,7 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec, for _, target := range additionalVolume.TargetContainers { if target == "all" && len(additionalVolume.TargetContainers) != 1 { - msg := `Target containers could be either "all" or a list + msg := `target containers could be either "all" or a list of containers, mixing those is not allowed, %+v` c.logger.Warningf(msg, additionalVolume) continue @@ -1489,11 +1849,18 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec, for _, additionalVolume := range additionalVolumes { for _, target := range additionalVolume.TargetContainers { if podSpec.Containers[i].Name == target || target == "all" { - mounts = append(mounts, v1.VolumeMount{ + v := v1.VolumeMount{ Name: additionalVolume.Name, MountPath: additionalVolume.MountPath, - SubPath: additionalVolume.SubPath, - }) + } + + if additionalVolume.IsSubPathExpr != nil && *additionalVolume.IsSubPathExpr { + v.SubPathExpr = additionalVolume.SubPath + } else { + v.SubPath = additionalVolume.SubPath + } + + mounts = append(mounts, v) } } } @@ -1503,20 +1870,12 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec, podSpec.Volumes = volumes } -func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string) (*v1.PersistentVolumeClaim, error) { +func (c *Cluster) generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string, + volumeSelector *metav1.LabelSelector) (*v1.PersistentVolumeClaim, error) { var storageClassName *string - - metadata := metav1.ObjectMeta{ - Name: constants.DataVolumeName, - } if volumeStorageClass != "" { - // TODO: remove the old annotation, switching completely to the StorageClassName field. - metadata.Annotations = map[string]string{"volume.beta.kubernetes.io/storage-class": volumeStorageClass} storageClassName = &volumeStorageClass - } else { - metadata.Annotations = map[string]string{"volume.alpha.kubernetes.io/storage-class": "default"} - storageClassName = nil } quantity, err := resource.ParseQuantity(volumeSize) @@ -1526,16 +1885,21 @@ func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string volumeMode := v1.PersistentVolumeFilesystem volumeClaim := &v1.PersistentVolumeClaim{ - ObjectMeta: metadata, + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DataVolumeName, + Annotations: c.annotationsSet(nil), + Labels: c.labelsSet(true), + }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, - Resources: v1.ResourceRequirements{ + Resources: v1.VolumeResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceStorage: quantity, }, }, StorageClassName: storageClassName, VolumeMode: &volumeMode, + Selector: volumeSelector, }, } @@ -1543,18 +1907,17 @@ func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string } func (c *Cluster) generateUserSecrets() map[string]*v1.Secret { - secrets := make(map[string]*v1.Secret, len(c.pgUsers)) - namespace := c.Namespace + secrets := make(map[string]*v1.Secret, len(c.pgUsers)+len(c.systemUsers)) for username, pgUser := range c.pgUsers { //Skip users with no password i.e. human users (they'll be authenticated using pam) - secret := c.generateSingleUserSecret(namespace, pgUser) + secret := c.generateSingleUserSecret(pgUser) if secret != nil { secrets[username] = secret } } /* special case for the system user */ for _, systemUser := range c.systemUsers { - secret := c.generateSingleUserSecret(namespace, systemUser) + secret := c.generateSingleUserSecret(systemUser) if secret != nil { secrets[systemUser.Name] = secret } @@ -1563,7 +1926,7 @@ func (c *Cluster) generateUserSecrets() map[string]*v1.Secret { return secrets } -func (c *Cluster) generateSingleUserSecret(namespace string, pgUser spec.PgUser) *v1.Secret { +func (c *Cluster) generateSingleUserSecret(pgUser spec.PgUser) *v1.Secret { //Skip users with no password i.e. human users (they'll be authenticated using pam) if pgUser.Password == "" { if pgUser.Origin != spec.RoleOriginTeamsAPI { @@ -1587,12 +1950,21 @@ func (c *Cluster) generateSingleUserSecret(namespace string, pgUser spec.PgUser) lbls = c.connectionPoolerLabels("", false).MatchLabels } + // if secret lives in another namespace we cannot set ownerReferences + var ownerReferences []metav1.OwnerReference + if c.Config.OpConfig.EnableCrossNamespaceSecret && c.Postgresql.ObjectMeta.Namespace != pgUser.Namespace { + ownerReferences = nil + } else { + ownerReferences = c.ownerReferences() + } + secret := v1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: c.credentialSecretName(username), - Namespace: namespace, - Labels: lbls, - Annotations: c.annotationsSet(nil), + Name: c.credentialSecretName(username), + Namespace: pgUser.Namespace, + Labels: lbls, + Annotations: c.annotationsSet(nil), + OwnerReferences: ownerReferences, }, Type: v1.SecretTypeOpaque, Data: map[string][]byte{ @@ -1632,42 +2004,62 @@ func (c *Cluster) shouldCreateLoadBalancerForService(role PostgresRole, spec *ac } +func (c *Cluster) shouldCreateNodePortForService(role PostgresRole, spec *acidv1.PostgresSpec) (bool, int32) { + switch role { + case Replica: + // if the value is explicitly set in a Postgresql manifest, follow this setting + if spec.EnableReplicaNodePort != nil { + port := int32(0) + if spec.ReplicaNodePort != nil { + port = *spec.ReplicaNodePort + } + + return *spec.EnableReplicaNodePort, port + } + + // otherwise, follow the operator configuration + return c.OpConfig.EnableReplicaNodePort, 0 + case Master: + if spec.EnableMasterNodePort != nil { + port := int32(0) + if spec.MasterNodePort != nil { + port = *spec.MasterNodePort + } + + return *spec.EnableMasterNodePort, port + } + + return c.OpConfig.EnableMasterNodePort, 0 + default: + panic(fmt.Sprintf("Unknown role %v", role)) + } +} + func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) *v1.Service { serviceSpec := v1.ServiceSpec{ - Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}}, + Ports: []v1.ServicePort{{Name: "postgresql", Port: pgPort, TargetPort: intstr.IntOrString{IntVal: pgPort}}}, Type: v1.ServiceTypeClusterIP, } + // no selector for master, see https://github.com/zalando/postgres-operator/issues/340 + // if kubernetes_use_configmaps is set master service needs a selector if role == Replica || c.patroniKubernetesUseConfigMaps() { serviceSpec.Selector = c.roleLabelsSet(false, role) } - if c.shouldCreateLoadBalancerForService(role, spec) { - - // spec.AllowedSourceRanges evaluates to the empty slice of zero length - // when omitted or set to 'null'/empty sequence in the PG manifest - if len(spec.AllowedSourceRanges) > 0 { - serviceSpec.LoadBalancerSourceRanges = spec.AllowedSourceRanges - } else { - // safe default value: lock a load balancer only to the local address unless overridden explicitly - serviceSpec.LoadBalancerSourceRanges = []string{localHost} - } - - c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges) - serviceSpec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(c.OpConfig.ExternalTrafficPolicy) - serviceSpec.Type = v1.ServiceTypeLoadBalancer - } else if role == Replica { - // before PR #258, the replica service was only created if allocated a LB - // now we always create the service but warn if the LB is absent - c.logger.Debugf("No load balancer created for the replica service") + if ok, port := c.shouldCreateNodePortForService(role, spec); ok { + c.configureNodePortService(&serviceSpec, port) + } else if c.shouldCreateLoadBalancerForService(role, spec) { + c.configureLoadBalanceService(&serviceSpec, spec.AllowedSourceRanges) } service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: c.serviceName(role), - Namespace: c.Namespace, - Labels: c.roleLabelsSet(true, role), - Annotations: c.annotationsSet(c.generateServiceAnnotations(role, spec)), + Name: c.serviceName(role), + Namespace: c.Namespace, + Labels: c.roleLabelsSet(true, role), + Annotations: c.annotationsSet(c.generateServiceAnnotations(role, spec)), + OwnerReferences: c.ownerReferences(), }, Spec: serviceSpec, } @@ -1675,31 +2067,38 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) return service } +func (c *Cluster) configureLoadBalanceService(serviceSpec *v1.ServiceSpec, sourceRanges []string) { + // spec.AllowedSourceRanges evaluates to the empty slice of zero length + // when omitted or set to 'null'/empty sequence in the PG manifest + if len(sourceRanges) > 0 { + serviceSpec.LoadBalancerSourceRanges = sourceRanges + } else { + // safe default value: lock a load balancer only to the local address unless overridden explicitly + serviceSpec.LoadBalancerSourceRanges = []string{localHost} + } + + c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges) + serviceSpec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(c.OpConfig.ExternalTrafficPolicy) + serviceSpec.Type = v1.ServiceTypeLoadBalancer +} + +func (c *Cluster) configureNodePortService(serviceSpec *v1.ServiceSpec, port int32) { + serviceSpec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(c.OpConfig.ExternalTrafficPolicy) + serviceSpec.Type = v1.ServiceTypeNodePort + + if port != 0 && len(serviceSpec.Ports) > 0 { + serviceSpec.Ports[0].NodePort = port + } +} + func (c *Cluster) generateServiceAnnotations(role PostgresRole, spec *acidv1.PostgresSpec) map[string]string { - annotations := make(map[string]string) + annotations := c.getCustomServiceAnnotations(role, spec) - for k, v := range c.OpConfig.CustomServiceAnnotations { - annotations[k] = v - } - if spec != nil || spec.ServiceAnnotations != nil { - for k, v := range spec.ServiceAnnotations { - annotations[k] = v - } - } + nodePort, _ := c.shouldCreateNodePortForService(role, spec) - if c.shouldCreateLoadBalancerForService(role, spec) { - var dnsName string - if role == Master { - dnsName = c.masterDNSName() - } else { - dnsName = c.replicaDNSName() - } + if !nodePort && c.shouldCreateLoadBalancerForService(role, spec) { + dnsName := c.dnsName(role) - // Just set ELB Timeout annotation with default value, if it does not - // have a cutom value - if _, ok := annotations[constants.ElbTimeoutAnnotationName]; !ok { - annotations[constants.ElbTimeoutAnnotationName] = constants.ElbTimeoutAnnotationValue - } // External DNS name annotation is not customizable annotations[constants.ZalandoDNSNameAnnotation] = dnsName } @@ -1711,12 +2110,32 @@ func (c *Cluster) generateServiceAnnotations(role PostgresRole, spec *acidv1.Pos return annotations } +func (c *Cluster) getCustomServiceAnnotations(role PostgresRole, spec *acidv1.PostgresSpec) map[string]string { + annotations := make(map[string]string) + maps.Copy(annotations, c.OpConfig.CustomServiceAnnotations) + + if spec != nil { + maps.Copy(annotations, spec.ServiceAnnotations) + + switch role { + case Master: + maps.Copy(annotations, spec.MasterServiceAnnotations) + case Replica: + maps.Copy(annotations, spec.ReplicaServiceAnnotations) + } + } + + return annotations +} + func (c *Cluster) generateEndpoint(role PostgresRole, subsets []v1.EndpointSubset) *v1.Endpoints { endpoints := &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ - Name: c.endpointName(role), - Namespace: c.Namespace, - Labels: c.roleLabelsSet(true, role), + Name: c.serviceName(role), + Namespace: c.Namespace, + Annotations: c.annotationsSet(nil), + Labels: c.roleLabelsSet(true, role), + OwnerReferences: c.ownerReferences(), }, } if len(subsets) > 0 { @@ -1736,6 +2155,7 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) cluster := description.ClusterName result = append(result, v1.EnvVar{Name: "CLONE_SCOPE", Value: cluster}) if description.EndTimestamp == "" { + c.logger.Infof("cloning with basebackup from %s", cluster) // cloning with basebackup, make a connection string to the cluster to clone from host, port := c.getClusterServiceConnectionParameters(cluster) // TODO: make some/all of those constants @@ -1757,49 +2177,30 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) }, }) } else { - // cloning with S3, find out the bucket to clone - msg := "Clone from S3 bucket" - c.logger.Info(msg, description.S3WalPath) - + c.logger.Info("cloning from WAL location") if description.S3WalPath == "" { - msg := "Figure out which S3 bucket to use from env" - c.logger.Info(msg, description.S3WalPath) + c.logger.Info("no S3 WAL path defined - taking value from global config", description.S3WalPath) if c.OpConfig.WALES3Bucket != "" { - envs := []v1.EnvVar{ - { - Name: "CLONE_WAL_S3_BUCKET", - Value: c.OpConfig.WALES3Bucket, - }, - } - result = append(result, envs...) + c.logger.Debugf("found WALES3Bucket %s - will set CLONE_WAL_S3_BUCKET", c.OpConfig.WALES3Bucket) + result = append(result, v1.EnvVar{Name: "CLONE_WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) } else if c.OpConfig.WALGSBucket != "" { - envs := []v1.EnvVar{ - { - Name: "CLONE_WAL_GS_BUCKET", - Value: c.OpConfig.WALGSBucket, - }, - { - Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", - Value: c.OpConfig.GCPCredentials, - }, + c.logger.Debugf("found WALGSBucket %s - will set CLONE_WAL_GS_BUCKET", c.OpConfig.WALGSBucket) + result = append(result, v1.EnvVar{Name: "CLONE_WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) + if c.OpConfig.GCPCredentials != "" { + result = append(result, v1.EnvVar{Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) } - result = append(result, envs...) + } else if c.OpConfig.WALAZStorageAccount != "" { + c.logger.Debugf("found WALAZStorageAccount %s - will set CLONE_AZURE_STORAGE_ACCOUNT", c.OpConfig.WALAZStorageAccount) + result = append(result, v1.EnvVar{Name: "CLONE_AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount}) } else { - c.logger.Error("Cannot figure out S3 or GS bucket. Both are empty.") + c.logger.Error("cannot figure out S3 or GS bucket or AZ storage account. All options are empty in the config.") } - envs := []v1.EnvVar{ - { - Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", - Value: getBucketScopeSuffix(description.UID), - }, - } - - result = append(result, envs...) + // append suffix because WAL location name is not the whole path + result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID)}) } else { - msg := "Use custom parsed S3WalPath %s from the manifest" - c.logger.Warningf(msg, description.S3WalPath) + c.logger.Debugf("use S3WalPath %s from the manifest", description.S3WalPath) result = append(result, v1.EnvVar{ Name: "CLONE_WALE_S3_PREFIX", @@ -1841,44 +2242,105 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescription) []v1.EnvVar { result := make([]v1.EnvVar, 0) - if description.S3WalPath == "" { - return nil + if description.StandbyHost != "" { + c.logger.Info("standby cluster streaming from remote primary") + result = append(result, v1.EnvVar{ + Name: "STANDBY_HOST", + Value: description.StandbyHost, + }) + if description.StandbyPort != "" { + result = append(result, v1.EnvVar{ + Name: "STANDBY_PORT", + Value: description.StandbyPort, + }) + } + if description.StandbyPrimarySlotName != "" { + result = append(result, v1.EnvVar{ + Name: "STANDBY_PRIMARY_SLOT_NAME", + Value: description.StandbyPrimarySlotName, + }) + } } - // standby with S3, find out the bucket to setup standby - msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s " - c.logger.Infof(msg, description.S3WalPath) - result = append(result, v1.EnvVar{ - Name: "STANDBY_WALE_S3_PREFIX", - Value: description.S3WalPath, - }) - - result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) - result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + // WAL archive can be specified with or without standby_host + if description.S3WalPath != "" { + c.logger.Info("standby cluster using S3 WAL archive") + result = append(result, v1.EnvVar{ + Name: "STANDBY_WALE_S3_PREFIX", + Value: description.S3WalPath, + }) + result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) + result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + } else if description.GSWalPath != "" { + c.logger.Info("standby cluster using GCS WAL archive") + result = append(result, v1.EnvVar{ + Name: "STANDBY_WALE_GS_PREFIX", + Value: description.GSWalPath, + }) + result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) + result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + } return result } -func (c *Cluster) generatePodDisruptionBudget() *policybeta1.PodDisruptionBudget { +func (c *Cluster) generatePrimaryPodDisruptionBudget() *policyv1.PodDisruptionBudget { minAvailable := intstr.FromInt(1) pdbEnabled := c.OpConfig.EnablePodDisruptionBudget + pdbMasterLabelSelector := c.OpConfig.PDBMasterLabelSelector + + // if PodDisruptionBudget is disabled or if there are no DB pods, set the budget to 0. + if (pdbEnabled != nil && !(*pdbEnabled)) || c.Spec.NumberOfInstances <= 0 { + minAvailable = intstr.FromInt(0) + } + + // define label selector and add the master role selector if enabled + labels := c.labelsSet(false) + if pdbMasterLabelSelector == nil || *c.OpConfig.PDBMasterLabelSelector { + labels[c.OpConfig.PodRoleLabel] = string(Master) + } + + return &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.PrimaryPodDisruptionBudgetName(), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: c.annotationsSet(nil), + OwnerReferences: c.ownerReferences(), + }, + Spec: policyv1.PodDisruptionBudgetSpec{ + MinAvailable: &minAvailable, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + }, + } +} + +func (c *Cluster) generateCriticalOpPodDisruptionBudget() *policyv1.PodDisruptionBudget { + minAvailable := intstr.FromInt32(c.Spec.NumberOfInstances) + pdbEnabled := c.OpConfig.EnablePodDisruptionBudget // if PodDisruptionBudget is disabled or if there are no DB pods, set the budget to 0. if (pdbEnabled != nil && !(*pdbEnabled)) || c.Spec.NumberOfInstances <= 0 { minAvailable = intstr.FromInt(0) } - return &policybeta1.PodDisruptionBudget{ + labels := c.labelsSet(false) + labels["critical-operation"] = "true" + + return &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ - Name: c.podDisruptionBudgetName(), - Namespace: c.Namespace, - Labels: c.labelsSet(true), - Annotations: c.annotationsSet(nil), + Name: c.criticalOpPodDisruptionBudgetName(), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: c.annotationsSet(nil), + OwnerReferences: c.ownerReferences(), }, - Spec: policybeta1.PodDisruptionBudgetSpec{ + Spec: policyv1.PodDisruptionBudgetSpec{ MinAvailable: &minAvailable, Selector: &metav1.LabelSelector{ - MatchLabels: c.roleLabelsSet(false, Master), + MatchLabels: labels, }, }, } @@ -1889,11 +2351,11 @@ func (c *Cluster) generatePodDisruptionBudget() *policybeta1.PodDisruptionBudget // TODO: handle clusters in different namespaces func (c *Cluster) getClusterServiceConnectionParameters(clusterName string) (host string, port string) { host = clusterName - port = "5432" + port = fmt.Sprintf("%d", pgPort) return } -func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { +func (c *Cluster) generateLogicalBackupJob() (*batchv1.CronJob, error) { var ( err error @@ -1901,20 +2363,31 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { resourceRequirements *v1.ResourceRequirements ) + spec := &c.Spec + // NB: a cron job creates standard batch jobs according to schedule; these batch jobs manage pods and clean-up c.logger.Debug("Generating logical backup pod template") - // allocate for the backup pod the same amount of resources as for normal DB pods - defaultResources := c.makeDefaultResources() - resourceRequirements, err = generateResourceRequirements(c.Spec.Resources, defaultResources) + // allocate configured resources for logical backup pod + logicalBackupResources := makeLogicalBackupResources(&c.OpConfig) + // if not defined only default resources from spilo pods are used + resourceRequirements, err = c.generateResourceRequirements( + &logicalBackupResources, makeDefaultResources(&c.OpConfig), logicalBackupContainerName) + if err != nil { return nil, fmt.Errorf("could not generate resource requirements for logical backup pods: %v", err) } + secretEnvVarsList, err := c.getCronjobEnvironmentSecretVariables() + if err != nil { + return nil, err + } + envVars := c.generateLogicalBackupPodEnvVars() + envVars = append(envVars, secretEnvVarsList...) logicalBackupContainer := generateContainer( - "logical-backup", + logicalBackupContainerName, &c.OpConfig.LogicalBackup.LogicalBackupDockerImage, resourceRequirements, envVars, @@ -1924,40 +2397,39 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { nil, ) - labels := map[string]string{ - c.OpConfig.ClusterNameLabel: c.Name, - "application": "spilo-logical-backup", + logicalBackupJobLabel := map[string]string{ + "application": "spilo-logical-backup", } - podAffinityTerm := v1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - TopologyKey: "kubernetes.io/hostname", - } - podAffinity := v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{{ - Weight: 1, - PodAffinityTerm: podAffinityTerm, - }, - }, - }} + + labels := labels.Merge(c.labelsSet(true), logicalBackupJobLabel) + + nodeAffinity := c.nodeAffinity(c.OpConfig.NodeReadinessLabel, nil) + podAffinity := podAffinity( + labels, + "kubernetes.io/hostname", + nodeAffinity, + true, + false, + ) annotations := c.generatePodAnnotations(&c.Spec) + tolerationsSpec := tolerations(&spec.Tolerations, c.OpConfig.PodToleration) + + topologySpreadConstraintsSpec := generateTopologySpreadConstraints(labels, spec.TopologySpreadConstraints) + // re-use the method that generates DB pod templates if podTemplate, err = c.generatePodTemplate( c.Namespace, labels, - annotations, + c.annotationsSet(annotations), logicalBackupContainer, []v1.Container{}, []v1.Container{}, - &[]v1.Toleration{}, - nil, - nil, - nil, - nodeAffinity(c.OpConfig.NodeReadinessLabel, nil), + util.False(), + topologySpreadConstraintsSpec, + &tolerationsSpec, + c.nodeAffinity(c.OpConfig.NodeReadinessLabel, nil), nil, int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, @@ -1966,6 +2438,7 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { util.False(), false, "", + false, c.OpConfig.AdditionalSecretMount, c.OpConfig.AdditionalSecretMountPath, []acidv1.AdditionalVolume{}); err != nil { @@ -1973,18 +2446,28 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { } // overwrite specific params of logical backups pods - podTemplate.Spec.Affinity = &podAffinity + podTemplate.Spec.Affinity = podAffinity podTemplate.Spec.RestartPolicy = "Never" // affects containers within a pod // configure a batch job jobSpec := batchv1.JobSpec{ - Template: *podTemplate, + Template: *podTemplate, + TTLSecondsAfterFinished: c.OpConfig.LogicalBackup.LogicalBackupTTLSecondsAfterFinished, + } + + if jobSpec.TTLSecondsAfterFinished == nil { + defaultTTL := int32(86400) + jobSpec.TTLSecondsAfterFinished = &defaultTTL } // configure a cron job - jobTemplateSpec := batchv1beta1.JobTemplateSpec{ + jobTemplateSpec := batchv1.JobTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: c.annotationsSet(annotations), + }, Spec: jobSpec, } @@ -1993,17 +2476,32 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { schedule = c.OpConfig.LogicalBackupSchedule } - cronJob := &batchv1beta1.CronJob{ + successfulJobsHistoryLimit := c.OpConfig.LogicalBackup.LogicalBackupSuccessfulJobsHistoryLimit + if successfulJobsHistoryLimit == nil { + defaultLimit := int32(3) + successfulJobsHistoryLimit = &defaultLimit + } + + failedJobsHistoryLimit := c.OpConfig.LogicalBackup.LogicalBackupFailedJobsHistoryLimit + if failedJobsHistoryLimit == nil { + defaultLimit := int32(3) + failedJobsHistoryLimit = &defaultLimit + } + + cronJob := &batchv1.CronJob{ ObjectMeta: metav1.ObjectMeta{ - Name: c.getLogicalBackupJobName(), - Namespace: c.Namespace, - Labels: c.labelsSet(true), - Annotations: c.annotationsSet(nil), + Name: c.getLogicalBackupJobName(), + Namespace: c.Namespace, + Labels: labels, + Annotations: c.annotationsSet(annotations), + OwnerReferences: c.ownerReferences(), }, - Spec: batchv1beta1.CronJobSpec{ - Schedule: schedule, - JobTemplate: jobTemplateSpec, - ConcurrencyPolicy: batchv1beta1.ForbidConcurrent, + Spec: batchv1.CronJobSpec{ + Schedule: schedule, + JobTemplate: jobTemplateSpec, + ConcurrencyPolicy: batchv1.ForbidConcurrent, + SuccessfulJobsHistoryLimit: successfulJobsHistoryLimit, + FailedJobsHistoryLimit: failedJobsHistoryLimit, }, } @@ -2012,6 +2510,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { + backupProvider := c.OpConfig.LogicalBackup.LogicalBackupProvider + envVars := []v1.EnvVar{ { Name: "SCOPE", @@ -2030,35 +2530,6 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { }, }, }, - // Bucket env vars - { - Name: "LOGICAL_BACKUP_PROVIDER", - Value: c.OpConfig.LogicalBackup.LogicalBackupProvider, - }, - { - Name: "LOGICAL_BACKUP_S3_BUCKET", - Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket, - }, - { - Name: "LOGICAL_BACKUP_S3_REGION", - Value: c.OpConfig.LogicalBackup.LogicalBackupS3Region, - }, - { - Name: "LOGICAL_BACKUP_S3_ENDPOINT", - Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint, - }, - { - Name: "LOGICAL_BACKUP_S3_SSE", - Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE, - }, - { - Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", - Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), - }, - { - Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS", - Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials, - }, // Postgres env vars { Name: "PG_VERSION", @@ -2066,7 +2537,7 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { }, { Name: "PGPORT", - Value: "5432", + Value: fmt.Sprintf("%d", pgPort), }, { Name: "PGUSER", @@ -2091,24 +2562,88 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { }, }, }, + // Bucket env vars + { + Name: "LOGICAL_BACKUP_PROVIDER", + Value: backupProvider, + }, + { + Name: "LOGICAL_BACKUP_S3_BUCKET", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket, + }, + { + Name: "LOGICAL_BACKUP_S3_BUCKET_PREFIX", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3BucketPrefix, + }, + { + Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", + Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), + }, } - if c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID != "" { - envVars = append(envVars, v1.EnvVar{Name: "AWS_ACCESS_KEY_ID", Value: c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID}) + switch backupProvider { + case "s3": + envVars = appendEnvVars(envVars, []v1.EnvVar{ + { + Name: "LOGICAL_BACKUP_S3_REGION", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3Region, + }, + { + Name: "LOGICAL_BACKUP_S3_ENDPOINT", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint, + }, + { + Name: "LOGICAL_BACKUP_S3_SSE", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE, + }, + { + Name: "LOGICAL_BACKUP_S3_RETENTION_TIME", + Value: c.getLogicalBackupRetentionTime(), + }}...) + + if c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID != "" { + envVars = append(envVars, v1.EnvVar{Name: "AWS_ACCESS_KEY_ID", Value: c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID}) + } + + if c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey != "" { + envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey}) + } + + case "gcs": + if c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials != "" { + envVars = append(envVars, v1.EnvVar{Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials}) + } + + case "az": + envVars = appendEnvVars(envVars, []v1.EnvVar{ + { + Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME", + Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountName, + }, + { + Name: "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER", + Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageContainer, + }}...) + + if c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountKey != "" { + envVars = append(envVars, v1.EnvVar{Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountKey}) + } } - if c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey != "" { - envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey}) - } - - c.logger.Debugf("Generated logical backup env vars") - c.logger.Debugf("%v", envVars) return envVars } +func (c *Cluster) getLogicalBackupRetentionTime() (retentionTime string) { + if c.Spec.LogicalBackupRetention != "" { + return c.Spec.LogicalBackupRetention + } + + return c.OpConfig.LogicalBackup.LogicalBackupS3RetentionTime +} + // getLogicalBackupJobName returns the name; the job itself may not exists func (c *Cluster) getLogicalBackupJobName() (jobName string) { - return trimCronjobName(c.OpConfig.LogicalBackupJobPrefix + c.clusterName().Name) + return trimCronjobName(fmt.Sprintf("%s%s", c.OpConfig.LogicalBackupJobPrefix, c.clusterName().Name)) } // Return an array of ownerReferences to make an arbitraty object dependent on @@ -2118,22 +2653,26 @@ func (c *Cluster) getLogicalBackupJobName() (jobName string) { // survived, we can't delete an object because it will affect the functioning // cluster). func (c *Cluster) ownerReferences() []metav1.OwnerReference { - controller := true - - if c.Statefulset == nil { - c.logger.Warning("Cannot get owner reference, no statefulset") - return []metav1.OwnerReference{} + currentOwnerReferences := c.ObjectMeta.OwnerReferences + if c.OpConfig.EnableOwnerReferences == nil || !*c.OpConfig.EnableOwnerReferences { + return currentOwnerReferences } - return []metav1.OwnerReference{ - { - UID: c.Statefulset.ObjectMeta.UID, - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: c.Statefulset.ObjectMeta.Name, - Controller: &controller, - }, + for _, ownerRef := range currentOwnerReferences { + if ownerRef.UID == c.Postgresql.ObjectMeta.UID { + return currentOwnerReferences + } } + + controllerReference := metav1.OwnerReference{ + UID: c.Postgresql.ObjectMeta.UID, + APIVersion: acidv1.SchemeGroupVersion.Identifier(), + Kind: acidv1.PostgresCRDResourceKind, + Name: c.Postgresql.ObjectMeta.Name, + Controller: util.True(), + } + + return append(currentOwnerReferences, controllerReference) } func ensurePath(file string, defaultDir string, defaultFile string) string { @@ -2145,3 +2684,15 @@ func ensurePath(file string, defaultDir string, defaultFile string) string { } return file } + +func (c *Cluster) validateContainers(containers []v1.Container) error { + for i, container := range containers { + if container.Name == "" { + return fmt.Errorf("container[%d]: name is required", i) + } + if container.Image == "" { + return fmt.Errorf("container '%v': image is required", container.Name) + } + } + return nil +} diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 5acd4a159..8cbecb2fa 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -5,8 +5,8 @@ import ( "fmt" "reflect" "sort" - "testing" + "time" "github.com/stretchr/testify/assert" @@ -20,13 +20,16 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - policyv1beta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/fake" v1core "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" ) func newFakeK8sTestClient() (k8sutil.KubernetesClient, *fake.Clientset) { @@ -45,11 +48,7 @@ type ExpectedValue struct { envIndex int envVarConstant string envVarValue string -} - -func toIntStr(val int) *intstr.IntOrString { - b := intstr.FromInt(val) - return &b + envVarValueRef *v1.EnvVarSource } func TestGenerateSpiloJSONConfiguration(t *testing.T) { @@ -64,397 +63,1106 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - testName := "TestGenerateSpiloConfig" tests := []struct { subtest string pgParam *acidv1.PostgresqlParam patroni *acidv1.Patroni - role string - opConfig config.Config + opConfig *config.Config result string }{ { - subtest: "Patroni default configuration", - pgParam: &acidv1.PostgresqlParam{PgVersion: "9.6"}, - patroni: &acidv1.Patroni{}, - role: "zalandos", - opConfig: config.Config{}, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/9.6/bin"},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{}}}`, + subtest: "Patroni default configuration", + pgParam: &acidv1.PostgresqlParam{PgVersion: "18"}, + patroni: &acidv1.Patroni{}, + opConfig: &config.Config{ + Auth: config.Auth{ + PamRoleName: "zalandos", + }, + }, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/18/bin"},"bootstrap":{"initdb":[{"auth-host":"scram-sha-256"},{"auth-local":"trust"}],"dcs":{}}}`, }, { subtest: "Patroni configured", - pgParam: &acidv1.PostgresqlParam{PgVersion: "11"}, + pgParam: &acidv1.PostgresqlParam{PgVersion: "18"}, patroni: &acidv1.Patroni{ InitDB: map[string]string{ "encoding": "UTF8", "locale": "en_US.UTF-8", "data-checksums": "true", }, - PgHba: []string{"hostssl all all 0.0.0.0/0 md5", "host all all 0.0.0.0/0 md5"}, + PgHba: []string{"hostssl all all 0.0.0.0/0 scram-sha-256", "host all all 0.0.0.0/0 scram-sha-256"}, TTL: 30, LoopWait: 10, RetryTimeout: 10, MaximumLagOnFailover: 33554432, SynchronousMode: true, SynchronousModeStrict: true, + SynchronousNodeCount: 1, Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, + FailsafeMode: util.True(), }, - role: "zalandos", - opConfig: config.Config{}, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, + opConfig: &config.Config{}, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/18/bin","pg_hba":["hostssl all all 0.0.0.0/0 scram-sha-256","host all all 0.0.0.0/0 scram-sha-256"]},"bootstrap":{"initdb":[{"auth-host":"scram-sha-256"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"synchronous_node_count":1,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}},"failsafe_mode":true}}}`, + }, + { + subtest: "Patroni failsafe_mode configured globally", + pgParam: &acidv1.PostgresqlParam{PgVersion: "18"}, + patroni: &acidv1.Patroni{}, + opConfig: &config.Config{ + EnablePatroniFailsafeMode: util.True(), + }, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/18/bin"},"bootstrap":{"initdb":[{"auth-host":"scram-sha-256"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`, + }, + { + subtest: "Patroni failsafe_mode configured globally, disabled for cluster", + pgParam: &acidv1.PostgresqlParam{PgVersion: "18"}, + patroni: &acidv1.Patroni{ + FailsafeMode: util.False(), + }, + opConfig: &config.Config{ + EnablePatroniFailsafeMode: util.True(), + }, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/18/bin"},"bootstrap":{"initdb":[{"auth-host":"scram-sha-256"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":false}}}`, + }, + { + subtest: "Patroni failsafe_mode disabled globally, configured for cluster", + pgParam: &acidv1.PostgresqlParam{PgVersion: "18"}, + patroni: &acidv1.Patroni{ + FailsafeMode: util.True(), + }, + opConfig: &config.Config{ + EnablePatroniFailsafeMode: util.False(), + }, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/18/bin"},"bootstrap":{"initdb":[{"auth-host":"scram-sha-256"},{"auth-local":"trust"}],"dcs":{"failsafe_mode":true}}}`, }, } for _, tt := range tests { - cluster.OpConfig = tt.opConfig - result, err := generateSpiloJSONConfiguration(tt.pgParam, tt.patroni, tt.role, false, logger) + cluster.OpConfig = *tt.opConfig + result, err := generateSpiloJSONConfiguration(tt.pgParam, tt.patroni, tt.opConfig, logger) if err != nil { t.Errorf("Unexpected error: %v", err) } if tt.result != result { - t.Errorf("%s %s: Spilo Config is %v, expected %v for role %#v and param %#v", - testName, tt.subtest, result, tt.result, tt.role, tt.pgParam) + t.Errorf("%s %s: Spilo Config is %v, expected %v and param %#v", + t.Name(), tt.subtest, result, tt.result, tt.pgParam) } } } -func TestGenerateSpiloPodEnvVars(t *testing.T) { - var cluster = New( - Config{ - OpConfig: config.Config{ - WALGSBucket: "wale-gs-bucket", - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - - expectedValuesGSBucket := []ExpectedValue{ - ExpectedValue{ - envIndex: 15, - envVarConstant: "WAL_GS_BUCKET", - envVarValue: "wale-gs-bucket", - }, - ExpectedValue{ - envIndex: 16, - envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", - envVarValue: "/SomeUUID", - }, - ExpectedValue{ - envIndex: 17, - envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", - envVarValue: "", - }, - } - - expectedValuesGCPCreds := []ExpectedValue{ - ExpectedValue{ - envIndex: 15, - envVarConstant: "WAL_GS_BUCKET", - envVarValue: "wale-gs-bucket", - }, - ExpectedValue{ - envIndex: 16, - envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", - envVarValue: "/SomeUUID", - }, - ExpectedValue{ - envIndex: 17, - envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", - envVarValue: "", - }, - ExpectedValue{ - envIndex: 18, - envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS", - envVarValue: "some_path_to_credentials", - }, - } - - testName := "TestGenerateSpiloPodEnvVars" +func TestExtractPgVersionFromBinPath(t *testing.T) { tests := []struct { - subTest string - opConfig config.Config - uid types.UID - spiloConfig string - cloneDescription *acidv1.CloneDescription - standbyDescription *acidv1.StandbyDescription - customEnvList []v1.EnvVar - expectedValues []ExpectedValue + subTest string + binPath string + template string + expected string }{ { - subTest: "Will set WAL_GS_BUCKET env", - opConfig: config.Config{ - WALGSBucket: "wale-gs-bucket", - }, - uid: "SomeUUID", - spiloConfig: "someConfig", - cloneDescription: &acidv1.CloneDescription{}, - standbyDescription: &acidv1.StandbyDescription{}, - customEnvList: []v1.EnvVar{}, - expectedValues: expectedValuesGSBucket, + subTest: "test current bin path with decimal against hard coded template", + binPath: "/usr/lib/postgresql/9.6/bin", + template: pgBinariesLocationTemplate, + expected: "9.6", }, { - subTest: "Will set GOOGLE_APPLICATION_CREDENTIALS env", - opConfig: config.Config{ - WALGSBucket: "wale-gs-bucket", - GCPCredentials: "some_path_to_credentials", - }, - uid: "SomeUUID", - spiloConfig: "someConfig", - cloneDescription: &acidv1.CloneDescription{}, - standbyDescription: &acidv1.StandbyDescription{}, - customEnvList: []v1.EnvVar{}, - expectedValues: expectedValuesGCPCreds, + subTest: "test current bin path against hard coded template", + binPath: "/usr/lib/postgresql/18/bin", + template: pgBinariesLocationTemplate, + expected: "18", + }, + { + subTest: "test alternative bin path against a matching template", + binPath: "/usr/pgsql-18/bin", + template: "/usr/pgsql-%v/bin", + expected: "18", }, } for _, tt := range tests { - cluster.OpConfig = tt.opConfig + pgVersion, err := extractPgVersionFromBinPath(tt.binPath, tt.template) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if pgVersion != tt.expected { + t.Errorf("%s %s: Expected version %s, have %s instead", + t.Name(), tt.subTest, tt.expected, pgVersion) + } + } +} - actualEnvs := cluster.generateSpiloPodEnvVars(tt.uid, tt.spiloConfig, tt.cloneDescription, tt.standbyDescription, tt.customEnvList) +const ( + testPodEnvironmentConfigMapName = "pod_env_cm" + testPodEnvironmentSecretName = "pod_env_sc" + testCronjobEnvironmentSecretName = "pod_env_sc" + testPodEnvironmentObjectNotExists = "idonotexist" + testPodEnvironmentSecretNameAPIError = "pod_env_sc_apierror" + testResourceCheckInterval = 3 + testResourceCheckTimeout = 10 +) + +type mockSecret struct { + v1core.SecretInterface +} + +type mockConfigMap struct { + v1core.ConfigMapInterface +} + +func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { + if name == testPodEnvironmentSecretNameAPIError { + return nil, fmt.Errorf("Secret PodEnvironmentSecret API error") + } + if name != testPodEnvironmentSecretName { + return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "core", Resource: "secret"}, name) + } + secret := &v1.Secret{} + secret.Name = testPodEnvironmentSecretName + secret.Data = map[string][]byte{ + "clone_aws_access_key_id": []byte("0123456789abcdef0123456789abcdef"), + "custom_variable": []byte("secret-test"), + "standby_google_application_credentials": []byte("0123456789abcdef0123456789abcdef"), + } + return secret, nil +} + +func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { + if name != testPodEnvironmentConfigMapName { + return nil, fmt.Errorf("NotFound") + } + configmap := &v1.ConfigMap{} + configmap.Name = testPodEnvironmentConfigMapName + configmap.Data = map[string]string{ + // hard-coded clone env variable, can set when not specified in manifest + "clone_aws_endpoint": "s3.eu-west-1.amazonaws.com", + // custom variable, can be overridden by c.Spec.Env + "custom_variable": "configmap-test", + // hard-coded env variable, can not be overridden + "kubernetes_scope_label": "pgaas", + // hard-coded env variable, can be overridden + "wal_s3_bucket": "global-s3-bucket-configmap", + } + return configmap, nil +} + +type MockSecretGetter struct { +} + +type MockConfigMapsGetter struct { +} + +func (c *MockSecretGetter) Secrets(namespace string) v1core.SecretInterface { + return &mockSecret{} +} + +func (c *MockConfigMapsGetter) ConfigMaps(namespace string) v1core.ConfigMapInterface { + return &mockConfigMap{} +} + +func newMockKubernetesClient() k8sutil.KubernetesClient { + return k8sutil.KubernetesClient{ + SecretsGetter: &MockSecretGetter{}, + ConfigMapsGetter: &MockConfigMapsGetter{}, + } +} +func newMockCluster(opConfig config.Config) *Cluster { + cluster := &Cluster{ + Config: Config{OpConfig: opConfig}, + KubeClient: newMockKubernetesClient(), + logger: logger, + } + return cluster +} + +func TestPodEnvironmentConfigMapVariables(t *testing.T) { + tests := []struct { + subTest string + opConfig config.Config + envVars []v1.EnvVar + err error + }{ + { + subTest: "no PodEnvironmentConfigMap", + envVars: []v1.EnvVar{}, + }, + { + subTest: "missing PodEnvironmentConfigMap", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentObjectNotExists, + }, + }, + }, + err: fmt.Errorf("could not read PodEnvironmentConfigMap: NotFound"), + }, + { + subTest: "Pod environment vars configured by PodEnvironmentConfigMap", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + }, + envVars: []v1.EnvVar{ + { + Name: "clone_aws_endpoint", + Value: "s3.eu-west-1.amazonaws.com", + }, + { + Name: "custom_variable", + Value: "configmap-test", + }, + { + Name: "kubernetes_scope_label", + Value: "pgaas", + }, + { + Name: "wal_s3_bucket", + Value: "global-s3-bucket-configmap", + }, + }, + }, + } + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + vars, err := c.getPodEnvironmentConfigMapVariables() + if !reflect.DeepEqual(vars, tt.envVars) { + t.Errorf("%s %s: expected `%v` but got `%v`", + t.Name(), tt.subTest, tt.envVars, vars) + } + if tt.err != nil { + if err.Error() != tt.err.Error() { + t.Errorf("%s %s: expected error `%v` but got `%v`", + t.Name(), tt.subTest, tt.err, err) + } + } else { + if err != nil { + t.Errorf("%s %s: expected no error but got error: `%v`", + t.Name(), tt.subTest, err) + } + } + } +} + +// Test if the keys of an existing secret are properly referenced +func TestPodEnvironmentSecretVariables(t *testing.T) { + maxRetries := int(testResourceCheckTimeout / testResourceCheckInterval) + tests := []struct { + subTest string + opConfig config.Config + envVars []v1.EnvVar + err error + }{ + { + subTest: "No PodEnvironmentSecret configured", + envVars: []v1.EnvVar{}, + }, + { + subTest: "Secret referenced by PodEnvironmentSecret does not exist", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentObjectNotExists, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: still failing after %d retries: secret.core %q not found", maxRetries, testPodEnvironmentObjectNotExists), + }, + { + subTest: "API error during PodEnvironmentSecret retrieval", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretNameAPIError, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret API error"), + }, + { + subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + envVars: []v1.EnvVar{ + { + Name: "clone_aws_access_key_id", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "clone_aws_access_key_id", + }, + }, + }, + { + Name: "custom_variable", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "custom_variable", + }, + }, + }, + { + Name: "standby_google_application_credentials", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "standby_google_application_credentials", + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + vars, err := c.getPodEnvironmentSecretVariables() + sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) + if !reflect.DeepEqual(vars, tt.envVars) { + t.Errorf("%s %s: expected `%v` but got `%v`", + t.Name(), tt.subTest, tt.envVars, vars) + } + if tt.err != nil { + if err.Error() != tt.err.Error() { + t.Errorf("%s %s: expected error `%v` but got `%v`", + t.Name(), tt.subTest, tt.err, err) + } + } else { + if err != nil { + t.Errorf("%s %s: expected no error but got error: `%v`", + t.Name(), tt.subTest, err) + } + } + } + +} + +// Test if the keys of an existing secret are properly referenced +func TestCronjobEnvironmentSecretVariables(t *testing.T) { + testName := "TestCronjobEnvironmentSecretVariables" + tests := []struct { + subTest string + opConfig config.Config + envVars []v1.EnvVar + err error + }{ + { + subTest: "No CronjobEnvironmentSecret configured", + envVars: []v1.EnvVar{}, + }, + { + subTest: "Secret referenced by CronjobEnvironmentSecret does not exist", + opConfig: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupCronjobEnvironmentSecret: "idonotexist", + }, + }, + err: fmt.Errorf("could not read Secret CronjobEnvironmentSecretName: secret.core \"idonotexist\" not found"), + }, + { + subTest: "Cronjob environment vars reference all keys from secret configured by CronjobEnvironmentSecret", + opConfig: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupCronjobEnvironmentSecret: testCronjobEnvironmentSecretName, + }, + }, + envVars: []v1.EnvVar{ + { + Name: "clone_aws_access_key_id", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "clone_aws_access_key_id", + }, + }, + }, + { + Name: "custom_variable", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "custom_variable", + }, + }, + }, + { + Name: "standby_google_application_credentials", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "standby_google_application_credentials", + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + vars, err := c.getCronjobEnvironmentSecretVariables() + sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) + if !reflect.DeepEqual(vars, tt.envVars) { + t.Errorf("%s %s: expected `%v` but got `%v`", + testName, tt.subTest, tt.envVars, vars) + } + if tt.err != nil { + if err.Error() != tt.err.Error() { + t.Errorf("%s %s: expected error `%v` but got `%v`", + testName, tt.subTest, tt.err, err) + } + } else { + if err != nil { + t.Errorf("%s %s: expected no error but got error: `%v`", + testName, tt.subTest, err) + } + } + } + +} + +func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { + required := map[string]bool{ + "PGHOST": false, + "PGPORT": false, + "PGUSER": false, + "PGSCHEMA": false, + "PGPASSWORD": false, + "CONNECTION_POOLER_MODE": false, + "CONNECTION_POOLER_PORT": false, + } + + container := getPostgresContainer(&podSpec.Spec) + envs := container.Env + for _, env := range envs { + required[env.Name] = true + } + + for env, value := range required { + if !value { + return fmt.Errorf("Environment variable %s is not present", env) + } + } + + return nil +} + +func TestGenerateSpiloPodEnvVars(t *testing.T) { + var dummyUUID = "efd12e58-5786-11e8-b5a7-06148230260c" + + expectedClusterNameLabel := []ExpectedValue{ + { + envIndex: 5, + envVarConstant: "KUBERNETES_SCOPE_LABEL", + envVarValue: "cluster-name", + }, + } + expectedSpiloWalPathCompat := []ExpectedValue{ + { + envIndex: 12, + envVarConstant: "ENABLE_WAL_PATH_COMPAT", + envVarValue: "true", + }, + } + expectedValuesS3Bucket := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "WAL_S3_BUCKET", + envVarValue: "global-s3-bucket", + }, + { + envIndex: 16, + envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: fmt.Sprintf("/%s", dummyUUID), + }, + { + envIndex: 17, + envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", + envVarValue: "", + }, + } + expectedValuesGCPCreds := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "WAL_GS_BUCKET", + envVarValue: "global-gs-bucket", + }, + { + envIndex: 16, + envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: fmt.Sprintf("/%s", dummyUUID), + }, + { + envIndex: 17, + envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", + envVarValue: "", + }, + { + envIndex: 18, + envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS", + envVarValue: "some-path-to-credentials", + }, + } + expectedS3BucketConfigMap := []ExpectedValue{ + { + envIndex: 17, + envVarConstant: "wal_s3_bucket", + envVarValue: "global-s3-bucket-configmap", + }, + } + expectedCustomS3BucketSpec := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "WAL_S3_BUCKET", + envVarValue: "custom-s3-bucket", + }, + } + expectedCustomVariableSecret := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "custom_variable", + envVarValueRef: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "custom_variable", + }, + }, + }, + } + expectedCustomVariableConfigMap := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "custom_variable", + envVarValue: "configmap-test", + }, + } + expectedCustomVariableSpec := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "CUSTOM_VARIABLE", + envVarValue: "spec-env-test", + }, + } + expectedCloneEnvSpec := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "CLONE_WALE_S3_PREFIX", + envVarValue: "s3://another-bucket", + }, + { + envIndex: 19, + envVarConstant: "CLONE_WAL_BUCKET_SCOPE_PREFIX", + envVarValue: "", + }, + { + envIndex: 20, + envVarConstant: "CLONE_AWS_ENDPOINT", + envVarValue: "s3.eu-central-1.amazonaws.com", + }, + } + expectedCloneEnvSpecEnv := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "CLONE_WAL_BUCKET_SCOPE_PREFIX", + envVarValue: "test-cluster", + }, + { + envIndex: 17, + envVarConstant: "CLONE_WALE_S3_PREFIX", + envVarValue: "s3://another-bucket", + }, + { + envIndex: 21, + envVarConstant: "CLONE_AWS_ENDPOINT", + envVarValue: "s3.eu-central-1.amazonaws.com", + }, + } + expectedCloneEnvConfigMap := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "CLONE_WAL_S3_BUCKET", + envVarValue: "global-s3-bucket", + }, + { + envIndex: 17, + envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: fmt.Sprintf("/%s", dummyUUID), + }, + { + envIndex: 21, + envVarConstant: "clone_aws_endpoint", + envVarValue: "s3.eu-west-1.amazonaws.com", + }, + } + expectedCloneEnvSecret := []ExpectedValue{ + { + envIndex: 21, + envVarConstant: "clone_aws_access_key_id", + envVarValueRef: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "clone_aws_access_key_id", + }, + }, + }, + } + expectedStandbyEnvSecret := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "STANDBY_WALE_GS_PREFIX", + envVarValue: "gs://some/path/", + }, + { + envIndex: 20, + envVarConstant: "standby_google_application_credentials", + envVarValueRef: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "standby_google_application_credentials", + }, + }, + }, + } + + tests := []struct { + subTest string + opConfig config.Config + cloneDescription *acidv1.CloneDescription + standbyDescription *acidv1.StandbyDescription + expectedValues []ExpectedValue + pgsql acidv1.Postgresql + }{ + { + subTest: "will set ENABLE_WAL_PATH_COMPAT env", + opConfig: config.Config{ + EnableSpiloWalPathCompat: true, + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedSpiloWalPathCompat, + }, + { + subTest: "will set WAL_S3_BUCKET env", + opConfig: config.Config{ + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedValuesS3Bucket, + }, + { + subTest: "will set GOOGLE_APPLICATION_CREDENTIALS env", + opConfig: config.Config{ + WALGSBucket: "global-gs-bucket", + GCPCredentials: "some-path-to-credentials", + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedValuesGCPCreds, + }, + { + subTest: "will not override global config KUBERNETES_SCOPE_LABEL parameter", + opConfig: config.Config{ + Resources: config.Resources{ + ClusterNameLabel: "cluster-name", + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, // contains kubernetes_scope_label, too + }, + }, + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedClusterNameLabel, + pgsql: acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + Env: []v1.EnvVar{ + { + Name: "KUBERNETES_SCOPE_LABEL", + Value: "my-scope-label", + }, + }, + }, + }, + }, + { + subTest: "will override global WAL_S3_BUCKET parameter from pod environment config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedS3BucketConfigMap, + }, + { + subTest: "will override global WAL_S3_BUCKET parameter from manifest `env` section", + opConfig: config.Config{ + WALGSBucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCustomS3BucketSpec, + pgsql: acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + Env: []v1.EnvVar{ + { + Name: "WAL_S3_BUCKET", + Value: "custom-s3-bucket", + }, + }, + }, + }, + }, + { + subTest: "will set CUSTOM_VARIABLE from pod environment secret and not config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCustomVariableSecret, + }, + { + subTest: "will set CUSTOM_VARIABLE from pod environment config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCustomVariableConfigMap, + }, + { + subTest: "will override CUSTOM_VARIABLE of pod environment secret/configmap from manifest `env` section", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCustomVariableSpec, + pgsql: acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + Env: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "spec-env-test", + }, + }, + }, + }, + }, + { + subTest: "will set CLONE_ parameters from spec and not global config or pod environment config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: dummyUUID, + S3WalPath: "s3://another-bucket", + S3Endpoint: "s3.eu-central-1.amazonaws.com", + }, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCloneEnvSpec, + }, + { + subTest: "will set CLONE_ parameters from manifest `env` section, followed by other options", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: dummyUUID, + S3WalPath: "s3://another-bucket", + S3Endpoint: "s3.eu-central-1.amazonaws.com", + }, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCloneEnvSpecEnv, + pgsql: acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + Env: []v1.EnvVar{ + { + Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", + Value: "test-cluster", + }, + }, + }, + }, + }, + { + subTest: "will set CLONE_AWS_ENDPOINT parameter from pod environment config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: dummyUUID, + }, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCloneEnvConfigMap, + }, + { + subTest: "will set CLONE_AWS_ACCESS_KEY_ID parameter from pod environment secret", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: dummyUUID, + }, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCloneEnvSecret, + }, + { + subTest: "will set STANDBY_GOOGLE_APPLICATION_CREDENTIALS parameter from pod environment secret", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{ + GSWalPath: "gs://some/path/", + }, + expectedValues: expectedStandbyEnvSecret, + }, + } + + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + pgsql := tt.pgsql + pgsql.Spec.Clone = tt.cloneDescription + pgsql.Spec.StandbyCluster = tt.standbyDescription + c.Postgresql = pgsql + + actualEnvs, err := c.generateSpiloPodEnvVars(&pgsql.Spec, types.UID(dummyUUID), exampleSpiloConfig) + assert.NoError(t, err) for _, ev := range tt.expectedValues { env := actualEnvs[ev.envIndex] if env.Name != ev.envVarConstant { - t.Errorf("%s %s: Expected env name %s, have %s instead", - testName, tt.subTest, ev.envVarConstant, env.Name) + t.Errorf("%s %s: expected env name %s, have %s instead", + t.Name(), tt.subTest, ev.envVarConstant, env.Name) + } + + if ev.envVarValueRef != nil { + if !reflect.DeepEqual(env.ValueFrom, ev.envVarValueRef) { + t.Errorf("%s %s: expected env value reference %#v, have %#v instead", + t.Name(), tt.subTest, ev.envVarValueRef, env.ValueFrom) + } + continue } if env.Value != ev.envVarValue { - t.Errorf("%s %s: Expected env value %s, have %s instead", - testName, tt.subTest, ev.envVarValue, env.Value) + t.Errorf("%s %s: expected env value %s, have %s instead", + t.Name(), tt.subTest, ev.envVarValue, env.Value) } } } } -func TestCreateLoadBalancerLogic(t *testing.T) { - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - - testName := "TestCreateLoadBalancerLogic" +func TestGetNumberOfInstances(t *testing.T) { tests := []struct { - subtest string - role PostgresRole - spec *acidv1.PostgresSpec - opConfig config.Config - result bool + subTest string + config config.Config + annotationKey string + annotationValue string + desired int32 + provided int32 }{ { - subtest: "new format, load balancer is enabled for replica", - role: Replica, - spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.True()}, - opConfig: config.Config{}, - result: true, - }, - { - subtest: "new format, load balancer is disabled for replica", - role: Replica, - spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.False()}, - opConfig: config.Config{}, - result: false, - }, - { - subtest: "new format, load balancer isn't specified for replica", - role: Replica, - spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil}, - opConfig: config.Config{EnableReplicaLoadBalancer: true}, - result: true, - }, - { - subtest: "new format, load balancer isn't specified for replica", - role: Replica, - spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil}, - opConfig: config.Config{EnableReplicaLoadBalancer: false}, - result: false, - }, - } - for _, tt := range tests { - cluster.OpConfig = tt.opConfig - result := cluster.shouldCreateLoadBalancerForService(tt.role, tt.spec) - if tt.result != result { - t.Errorf("%s %s: Load balancer is %t, expect %t for role %#v and spec %#v", - testName, tt.subtest, result, tt.result, tt.role, tt.spec) - } - } -} - -func TestGeneratePodDisruptionBudget(t *testing.T) { - tests := []struct { - c *Cluster - out policyv1beta1.PodDisruptionBudget - }{ - // With multiple instances. - { - New( - Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, - k8sutil.KubernetesClient{}, - acidv1.Postgresql{ - ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, - Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger, - eventRecorder), - policyv1beta1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres-myapp-database-pdb", - Namespace: "myapp", - Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, - }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ - MinAvailable: toIntStr(1), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, - }, + subTest: "no constraints", + config: config.Config{ + Resources: config.Resources{ + MinInstances: -1, + MaxInstances: -1, + IgnoreInstanceLimitsAnnotationKey: "", }, }, + annotationKey: "", + annotationValue: "", + desired: 2, + provided: 2, }, - // With zero instances. { - New( - Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, - k8sutil.KubernetesClient{}, - acidv1.Postgresql{ - ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, - Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}}, - logger, - eventRecorder), - policyv1beta1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres-myapp-database-pdb", - Namespace: "myapp", - Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, - }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ - MinAvailable: toIntStr(0), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, - }, + subTest: "minInstances defined", + config: config.Config{ + Resources: config.Resources{ + MinInstances: 2, + MaxInstances: -1, + IgnoreInstanceLimitsAnnotationKey: "", }, }, + annotationKey: "", + annotationValue: "", + desired: 1, + provided: 2, }, - // With PodDisruptionBudget disabled. { - New( - Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.False()}}, - k8sutil.KubernetesClient{}, - acidv1.Postgresql{ - ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, - Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger, - eventRecorder), - policyv1beta1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres-myapp-database-pdb", - Namespace: "myapp", - Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, - }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ - MinAvailable: toIntStr(0), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, - }, + subTest: "maxInstances defined", + config: config.Config{ + Resources: config.Resources{ + MinInstances: -1, + MaxInstances: 5, + IgnoreInstanceLimitsAnnotationKey: "", }, }, + annotationKey: "", + annotationValue: "", + desired: 10, + provided: 5, }, - // With non-default PDBNameFormat and PodDisruptionBudget explicitly enabled. { - New( - Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-databass-budget", EnablePodDisruptionBudget: util.True()}}, - k8sutil.KubernetesClient{}, - acidv1.Postgresql{ - ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, - Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger, - eventRecorder), - policyv1beta1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres-myapp-database-databass-budget", - Namespace: "myapp", - Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, - }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ - MinAvailable: toIntStr(1), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, - }, + subTest: "ignore minInstances", + config: config.Config{ + Resources: config.Resources{ + MinInstances: 2, + MaxInstances: -1, + IgnoreInstanceLimitsAnnotationKey: "ignore-instance-limits", }, }, + annotationKey: "ignore-instance-limits", + annotationValue: "true", + desired: 1, + provided: 1, + }, + { + subTest: "want to ignore minInstances but wrong key", + config: config.Config{ + Resources: config.Resources{ + MinInstances: 2, + MaxInstances: -1, + IgnoreInstanceLimitsAnnotationKey: "ignore-instance-limits", + }, + }, + annotationKey: "ignoring-instance-limits", + annotationValue: "true", + desired: 1, + provided: 2, + }, + { + subTest: "want to ignore minInstances but wrong value", + config: config.Config{ + Resources: config.Resources{ + MinInstances: 2, + MaxInstances: -1, + IgnoreInstanceLimitsAnnotationKey: "ignore-instance-limits", + }, + }, + annotationKey: "ignore-instance-limits", + annotationValue: "active", + desired: 1, + provided: 2, + }, + { + subTest: "annotation set but no constraints to ignore", + config: config.Config{ + Resources: config.Resources{ + MinInstances: -1, + MaxInstances: -1, + IgnoreInstanceLimitsAnnotationKey: "ignore-instance-limits", + }, + }, + annotationKey: "ignore-instance-limits", + annotationValue: "true", + desired: 1, + provided: 1, }, } for _, tt := range tests { - result := tt.c.generatePodDisruptionBudget() - if !reflect.DeepEqual(*result, tt.out) { - t.Errorf("Expected PodDisruptionBudget: %#v, got %#v", tt.out, *result) + var cluster = New( + Config{ + OpConfig: tt.config, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + cluster.Spec.NumberOfInstances = tt.desired + if tt.annotationKey != "" { + cluster.ObjectMeta.Annotations = make(map[string]string) + cluster.ObjectMeta.Annotations[tt.annotationKey] = tt.annotationValue } - } -} + numInstances := cluster.getNumberOfInstances(&cluster.Spec) -func TestShmVolume(t *testing.T) { - testName := "TestShmVolume" - tests := []struct { - subTest string - podSpec *v1.PodSpec - shmPos int - }{ - { - subTest: "empty PodSpec", - podSpec: &v1.PodSpec{ - Volumes: []v1.Volume{}, - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{}, - }, - }, - }, - shmPos: 0, - }, - { - subTest: "non empty PodSpec", - podSpec: &v1.PodSpec{ - Volumes: []v1.Volume{{}}, - Containers: []v1.Container{ - { - Name: "postgres", - VolumeMounts: []v1.VolumeMount{ - {}, - }, - }, - }, - }, - shmPos: 1, - }, - } - for _, tt := range tests { - addShmVolume(tt.podSpec) - postgresContainer := getPostgresContainer(tt.podSpec) - - volumeName := tt.podSpec.Volumes[tt.shmPos].Name - volumeMountName := postgresContainer.VolumeMounts[tt.shmPos].Name - - if volumeName != constants.ShmVolumeName { - t.Errorf("%s %s: Expected volume %s was not created, have %s instead", - testName, tt.subTest, constants.ShmVolumeName, volumeName) - } - if volumeMountName != constants.ShmVolumeName { - t.Errorf("%s %s: Expected mount %s was not created, have %s instead", - testName, tt.subTest, constants.ShmVolumeName, volumeMountName) + if numInstances != tt.provided { + t.Errorf("%s %s: Expected to get %d instances, have %d instead", + t.Name(), tt.subTest, tt.provided, numInstances) } } } func TestCloneEnv(t *testing.T) { - testName := "TestCloneEnv" tests := []struct { subTest string cloneOpts *acidv1.CloneDescription @@ -521,365 +1229,223 @@ func TestCloneEnv(t *testing.T) { if env.Name != tt.env.Name { t.Errorf("%s %s: Expected env name %s, have %s instead", - testName, tt.subTest, tt.env.Name, env.Name) + t.Name(), tt.subTest, tt.env.Name, env.Name) } if env.Value != tt.env.Value { t.Errorf("%s %s: Expected env value %s, have %s instead", - testName, tt.subTest, tt.env.Value, env.Value) + t.Name(), tt.subTest, tt.env.Value, env.Value) } } } -func TestExtractPgVersionFromBinPath(t *testing.T) { - testName := "TestExtractPgVersionFromBinPath" +func TestAppendEnvVar(t *testing.T) { tests := []struct { - subTest string - binPath string - template string - expected string + subTest string + envs []v1.EnvVar + envsToAppend []v1.EnvVar + expectedSize int }{ { - subTest: "test current bin path with decimal against hard coded template", - binPath: "/usr/lib/postgresql/9.6/bin", - template: pgBinariesLocationTemplate, - expected: "9.6", + subTest: "append two variables - one with same key that should get rejected", + envs: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "test", + }, + }, + envsToAppend: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "new-test", + }, + { + Name: "ANOTHER_CUSTOM_VARIABLE", + Value: "another-test", + }, + }, + expectedSize: 2, }, { - subTest: "test current bin path against hard coded template", - binPath: "/usr/lib/postgresql/12/bin", - template: pgBinariesLocationTemplate, - expected: "12", + subTest: "append empty slice", + envs: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "test", + }, + }, + envsToAppend: []v1.EnvVar{}, + expectedSize: 1, }, { - subTest: "test alternative bin path against a matching template", - binPath: "/usr/pgsql-12/bin", - template: "/usr/pgsql-%v/bin", - expected: "12", + subTest: "append nil", + envs: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "test", + }, + }, + envsToAppend: nil, + expectedSize: 1, }, } for _, tt := range tests { - pgVersion, err := extractPgVersionFromBinPath(tt.binPath, tt.template) - if err != nil { - t.Errorf("Unexpected error: %v", err) + finalEnvs := appendEnvVars(tt.envs, tt.envsToAppend...) + + if len(finalEnvs) != tt.expectedSize { + t.Errorf("%s %s: expected %d env variables, got %d", + t.Name(), tt.subTest, tt.expectedSize, len(finalEnvs)) } - if pgVersion != tt.expected { - t.Errorf("%s %s: Expected version %s, have %s instead", - testName, tt.subTest, tt.expected, pgVersion) + + for _, env := range tt.envs { + for _, finalEnv := range finalEnvs { + if env.Name == finalEnv.Name { + if env.Value != finalEnv.Value { + t.Errorf("%s %s: expected env value %s of variable %s, got %s instead", + t.Name(), tt.subTest, env.Value, env.Name, finalEnv.Value) + } + } + } } } } -func TestSecretVolume(t *testing.T) { - testName := "TestSecretVolume" +func TestStandbyEnv(t *testing.T) { tests := []struct { - subTest string - podSpec *v1.PodSpec - secretPos int + subTest string + standbyOpts *acidv1.StandbyDescription + env v1.EnvVar + envPos int + envLen int }{ { - subTest: "empty PodSpec", - podSpec: &v1.PodSpec{ - Volumes: []v1.Volume{}, - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{}, - }, - }, + subTest: "from custom s3 path", + standbyOpts: &acidv1.StandbyDescription{ + S3WalPath: "s3://some/path/", }, - secretPos: 0, + env: v1.EnvVar{ + Name: "STANDBY_WALE_S3_PREFIX", + Value: "s3://some/path/", + }, + envPos: 0, + envLen: 3, }, { - subTest: "non empty PodSpec", - podSpec: &v1.PodSpec{ - Volumes: []v1.Volume{{}}, - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "data", - ReadOnly: false, - MountPath: "/data", - }, - }, - }, - }, + subTest: "ignore gs path if s3 is set", + standbyOpts: &acidv1.StandbyDescription{ + S3WalPath: "s3://some/path/", + GSWalPath: "gs://some/path/", }, - secretPos: 1, - }, - } - for _, tt := range tests { - additionalSecretMount := "aws-iam-s3-role" - additionalSecretMountPath := "/meta/credentials" - postgresContainer := getPostgresContainer(tt.podSpec) - - numMounts := len(postgresContainer.VolumeMounts) - - addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath) - - volumeName := tt.podSpec.Volumes[tt.secretPos].Name - - if volumeName != additionalSecretMount { - t.Errorf("%s %s: Expected volume %s was not created, have %s instead", - testName, tt.subTest, additionalSecretMount, volumeName) - } - - for i := range tt.podSpec.Containers { - volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name - - if volumeMountName != additionalSecretMount { - t.Errorf("%s %s: Expected mount %s was not created, have %s instead", - testName, tt.subTest, additionalSecretMount, volumeMountName) - } - } - - postgresContainer = getPostgresContainer(tt.podSpec) - numMountsCheck := len(postgresContainer.VolumeMounts) - - if numMountsCheck != numMounts+1 { - t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", - numMountsCheck, numMounts+1) - } - } -} - -const ( - testPodEnvironmentConfigMapName = "pod_env_cm" - testPodEnvironmentSecretName = "pod_env_sc" -) - -type mockSecret struct { - v1core.SecretInterface -} - -type mockConfigMap struct { - v1core.ConfigMapInterface -} - -func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { - if name != testPodEnvironmentSecretName { - return nil, fmt.Errorf("Secret PodEnvironmentSecret not found") - } - secret := &v1.Secret{} - secret.Name = testPodEnvironmentSecretName - secret.Data = map[string][]byte{ - "minio_access_key": []byte("alpha"), - "minio_secret_key": []byte("beta"), - } - return secret, nil -} - -func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { - if name != testPodEnvironmentConfigMapName { - return nil, fmt.Errorf("NotFound") - } - configmap := &v1.ConfigMap{} - configmap.Name = testPodEnvironmentConfigMapName - configmap.Data = map[string]string{ - "foo1": "bar1", - "foo2": "bar2", - } - return configmap, nil -} - -type MockSecretGetter struct { -} - -type MockConfigMapsGetter struct { -} - -func (c *MockSecretGetter) Secrets(namespace string) v1core.SecretInterface { - return &mockSecret{} -} - -func (c *MockConfigMapsGetter) ConfigMaps(namespace string) v1core.ConfigMapInterface { - return &mockConfigMap{} -} - -func newMockKubernetesClient() k8sutil.KubernetesClient { - return k8sutil.KubernetesClient{ - SecretsGetter: &MockSecretGetter{}, - ConfigMapsGetter: &MockConfigMapsGetter{}, - } -} -func newMockCluster(opConfig config.Config) *Cluster { - cluster := &Cluster{ - Config: Config{OpConfig: opConfig}, - KubeClient: newMockKubernetesClient(), - } - return cluster -} - -func TestPodEnvironmentConfigMapVariables(t *testing.T) { - testName := "TestPodEnvironmentConfigMapVariables" - tests := []struct { - subTest string - opConfig config.Config - envVars []v1.EnvVar - err error - }{ - { - subTest: "no PodEnvironmentConfigMap", - envVars: []v1.EnvVar{}, + env: v1.EnvVar{ + Name: "STANDBY_METHOD", + Value: "STANDBY_WITH_WALE", + }, + envPos: 1, + envLen: 3, }, { - subTest: "missing PodEnvironmentConfigMap", - opConfig: config.Config{ - Resources: config.Resources{ - PodEnvironmentConfigMap: spec.NamespacedName{ - Name: "idonotexist", - }, - }, + subTest: "from remote primary", + standbyOpts: &acidv1.StandbyDescription{ + StandbyHost: "remote-primary", }, - err: fmt.Errorf("could not read PodEnvironmentConfigMap: NotFound"), + env: v1.EnvVar{ + Name: "STANDBY_HOST", + Value: "remote-primary", + }, + envPos: 0, + envLen: 1, }, { - subTest: "simple PodEnvironmentConfigMap", - opConfig: config.Config{ - Resources: config.Resources{ - PodEnvironmentConfigMap: spec.NamespacedName{ - Name: testPodEnvironmentConfigMapName, - }, - }, + subTest: "from remote primary with port", + standbyOpts: &acidv1.StandbyDescription{ + StandbyHost: "remote-primary", + StandbyPort: "9876", }, - envVars: []v1.EnvVar{ - { - Name: "foo1", - Value: "bar1", - }, - { - Name: "foo2", - Value: "bar2", - }, + env: v1.EnvVar{ + Name: "STANDBY_PORT", + Value: "9876", }, + envPos: 1, + envLen: 2, + }, + { + subTest: "from remote primary with S3 WAL path", + standbyOpts: &acidv1.StandbyDescription{ + S3WalPath: "s3://some/path/", + StandbyHost: "remote-primary", + }, + env: v1.EnvVar{ + Name: "STANDBY_HOST", + Value: "remote-primary", + }, + envPos: 0, + envLen: 4, + }, + { + subTest: "verify S3 WAL env with standby host", + standbyOpts: &acidv1.StandbyDescription{ + S3WalPath: "s3://some/path/", + StandbyHost: "remote-primary", + }, + env: v1.EnvVar{ + Name: "STANDBY_WALE_S3_PREFIX", + Value: "s3://some/path/", + }, + envPos: 1, + envLen: 4, + }, + { + subTest: "from remote primary with GCS WAL path", + standbyOpts: &acidv1.StandbyDescription{ + GSWalPath: "gs://some/path/", + StandbyHost: "remote-primary", + }, + env: v1.EnvVar{ + Name: "STANDBY_HOST", + Value: "remote-primary", + }, + envPos: 0, + envLen: 4, + }, + { + subTest: "from remote primary with slot name", + standbyOpts: &acidv1.StandbyDescription{ + StandbyHost: "remote-primary", + StandbyPrimarySlotName: "my_slot", + }, + env: v1.EnvVar{ + Name: "STANDBY_PRIMARY_SLOT_NAME", + Value: "my_slot", + }, + envPos: 1, + envLen: 2, }, } - for _, tt := range tests { - c := newMockCluster(tt.opConfig) - vars, err := c.getPodEnvironmentConfigMapVariables() - sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) - if !reflect.DeepEqual(vars, tt.envVars) { - t.Errorf("%s %s: expected `%v` but got `%v`", - testName, tt.subTest, tt.envVars, vars) - } - if tt.err != nil { - if err.Error() != tt.err.Error() { - t.Errorf("%s %s: expected error `%v` but got `%v`", - testName, tt.subTest, tt.err, err) - } - } else { - if err != nil { - t.Errorf("%s %s: expected no error but got error: `%v`", - testName, tt.subTest, err) - } - } - } -} -// Test if the keys of an existing secret are properly referenced -func TestPodEnvironmentSecretVariables(t *testing.T) { - testName := "TestPodEnvironmentSecretVariables" - tests := []struct { - subTest string - opConfig config.Config - envVars []v1.EnvVar - err error - }{ - { - subTest: "No PodEnvironmentSecret configured", - envVars: []v1.EnvVar{}, - }, - { - subTest: "Secret referenced by PodEnvironmentSecret does not exist", - opConfig: config.Config{ - Resources: config.Resources{ - PodEnvironmentSecret: "idonotexist", - }, - }, - err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret not found"), - }, - { - subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret", - opConfig: config.Config{ - Resources: config.Resources{ - PodEnvironmentSecret: testPodEnvironmentSecretName, - }, - }, - envVars: []v1.EnvVar{ - { - Name: "minio_access_key", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: testPodEnvironmentSecretName, - }, - Key: "minio_access_key", - }, - }, - }, - { - Name: "minio_secret_key", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: testPodEnvironmentSecretName, - }, - Key: "minio_secret_key", - }, - }, - }, - }, - }, - } + var cluster = New( + Config{}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) for _, tt := range tests { - c := newMockCluster(tt.opConfig) - vars, err := c.getPodEnvironmentSecretVariables() - sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) - if !reflect.DeepEqual(vars, tt.envVars) { - t.Errorf("%s %s: expected `%v` but got `%v`", - testName, tt.subTest, tt.envVars, vars) + envs := cluster.generateStandbyEnvironment(tt.standbyOpts) + + env := envs[tt.envPos] + + if env.Name != tt.env.Name { + t.Errorf("%s %s: Expected env name %s, have %s instead", + t.Name(), tt.subTest, tt.env.Name, env.Name) } - if tt.err != nil { - if err.Error() != tt.err.Error() { - t.Errorf("%s %s: expected error `%v` but got `%v`", - testName, tt.subTest, tt.err, err) - } - } else { - if err != nil { - t.Errorf("%s %s: expected no error but got error: `%v`", - testName, tt.subTest, err) - } + + if env.Value != tt.env.Value { + t.Errorf("%s %s: Expected env value %s, have %s instead", + t.Name(), tt.subTest, tt.env.Value, env.Value) + } + + if len(envs) != tt.envLen { + t.Errorf("%s %s: Expected number of env variables %d, have %d instead", + t.Name(), tt.subTest, tt.envLen, len(envs)) } } - -} - -func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { - required := map[string]bool{ - "PGHOST": false, - "PGPORT": false, - "PGUSER": false, - "PGSCHEMA": false, - "PGPASSWORD": false, - "CONNECTION_POOLER_MODE": false, - "CONNECTION_POOLER_PORT": false, - } - - container := getPostgresContainer(&podSpec.Spec) - envs := container.Env - for _, env := range envs { - required[env.Name] = true - } - - for env, value := range required { - if !value { - return fmt.Errorf("Environment variable %s is not present", env) - } - } - - return nil } func TestNodeAffinity(t *testing.T) { @@ -893,9 +1459,9 @@ func TestNodeAffinity(t *testing.T) { makeSpec := func(nodeAffinity *v1.NodeAffinity) acidv1.PostgresSpec { return acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ - ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, - ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, }, Volume: acidv1.Volume{ Size: "1G", @@ -924,9 +1490,9 @@ func TestNodeAffinity(t *testing.T) { nodeAff := &v1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ NodeSelectorTerms: []v1.NodeSelectorTerm{ - v1.NodeSelectorTerm{ + { MatchExpressions: []v1.NodeSelectorRequirement{ - v1.NodeSelectorRequirement{ + { Key: "test-label", Operator: v1.NodeSelectorOpIn, Values: []string{ @@ -948,30 +1514,177 @@ func TestNodeAffinity(t *testing.T) { assert.Equal(t, s.Spec.Template.Spec.Affinity.NodeAffinity, nodeAff, "cluster template has correct node affinity") } +func TestPodAffinity(t *testing.T) { + clusterName := "acid-test-cluster" + namespace := "default" + + tests := []struct { + subTest string + preferred bool + anti bool + }{ + { + subTest: "generate affinity RequiredDuringSchedulingIgnoredDuringExecution", + preferred: false, + anti: false, + }, + { + subTest: "generate affinity PreferredDuringSchedulingIgnoredDuringExecution", + preferred: true, + anti: false, + }, + { + subTest: "generate anitAffinity RequiredDuringSchedulingIgnoredDuringExecution", + preferred: false, + anti: true, + }, + { + subTest: "generate anitAffinity PreferredDuringSchedulingIgnoredDuringExecution", + preferred: true, + anti: true, + }, + } + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + } + + for _, tt := range tests { + cluster := New( + Config{ + OpConfig: config.Config{ + EnablePodAntiAffinity: tt.anti, + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + PodAntiAffinityPreferredDuringScheduling: tt.preferred, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, k8sutil.KubernetesClient{}, pg, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + + s, err := cluster.generateStatefulSet(&pg.Spec) + if err != nil { + assert.NoError(t, err) + } + + if !tt.anti { + assert.Nil(t, s.Spec.Template.Spec.Affinity, "pod affinity should not be set") + } else { + if tt.preferred { + assert.NotNil(t, s.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, "pod anti-affinity should use preferredDuringScheduling") + assert.Nil(t, s.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, "pod anti-affinity should not use requiredDuringScheduling") + } else { + assert.Nil(t, s.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, "pod anti-affinity should not use preferredDuringScheduling") + assert.NotNil(t, s.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, "pod anti-affinity should use requiredDuringScheduling") + } + } + } +} + func testDeploymentOwnerReference(cluster *Cluster, deployment *appsv1.Deployment) error { + if len(deployment.ObjectMeta.OwnerReferences) == 0 { + return nil + } owner := deployment.ObjectMeta.OwnerReferences[0] - if owner.Name != cluster.Statefulset.ObjectMeta.Name { - return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", - owner.Name, cluster.Statefulset.ObjectMeta.Name) + if owner.Name != cluster.Postgresql.ObjectMeta.Name { + return fmt.Errorf("Owner reference is incorrect, got %s, expected %s", + owner.Name, cluster.Postgresql.ObjectMeta.Name) } return nil } func testServiceOwnerReference(cluster *Cluster, service *v1.Service, role PostgresRole) error { + if len(service.ObjectMeta.OwnerReferences) == 0 { + return nil + } owner := service.ObjectMeta.OwnerReferences[0] - if owner.Name != cluster.Statefulset.ObjectMeta.Name { - return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", - owner.Name, cluster.Statefulset.ObjectMeta.Name) + if owner.Name != cluster.Postgresql.ObjectMeta.Name { + return fmt.Errorf("Owner reference is incorrect, got %s, expected %s", + owner.Name, cluster.Postgresql.ObjectMeta.Name) } return nil } -func TestTLS(t *testing.T) { +func TestSharePgSocketWithSidecars(t *testing.T) { + tests := []struct { + subTest string + podSpec *v1.PodSpec + runVolPos int + }{ + { + subTest: "empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + runVolPos: 0, + }, + { + subTest: "non empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + Name: "postgres", + VolumeMounts: []v1.VolumeMount{ + {}, + }, + }, + }, + }, + runVolPos: 1, + }, + } + for _, tt := range tests { + addVarRunVolume(tt.podSpec) + postgresContainer := getPostgresContainer(tt.podSpec) + volumeName := tt.podSpec.Volumes[tt.runVolPos].Name + volumeMountName := postgresContainer.VolumeMounts[tt.runVolPos].Name + + if volumeName != constants.RunVolumeName { + t.Errorf("%s %s: Expected volume %s was not created, have %s instead", + t.Name(), tt.subTest, constants.RunVolumeName, volumeName) + } + if volumeMountName != constants.RunVolumeName { + t.Errorf("%s %s: Expected mount %s was not created, have %s instead", + t.Name(), tt.subTest, constants.RunVolumeName, volumeMountName) + } + } +} + +func TestTLS(t *testing.T) { client, _ := newFakeK8sTestClient() clusterName := "acid-test-cluster" namespace := "default" @@ -989,9 +1702,9 @@ func TestTLS(t *testing.T) { }, Spec: acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ - ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, - ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, }, Volume: acidv1.Volume{ Size: "1G", @@ -999,7 +1712,7 @@ func TestTLS(t *testing.T) { TLS: &acidv1.TLSDescription{ SecretName: tlsSecretName, CAFile: "ca.crt"}, AdditionalVolumes: []acidv1.AdditionalVolume{ - acidv1.AdditionalVolume{ + { Name: tlsSecretName, MountPath: mountPath, VolumeSource: v1.VolumeSource{ @@ -1059,9 +1772,131 @@ func TestTLS(t *testing.T) { assert.Contains(t, postgresContainer.Env, v1.EnvVar{Name: "SSL_CA_FILE", Value: "/tls/ca.crt"}) } -func TestAdditionalVolume(t *testing.T) { - testName := "TestAdditionalVolume" +func TestShmVolume(t *testing.T) { + tests := []struct { + subTest string + podSpec *v1.PodSpec + shmPos int + }{ + { + subTest: "empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + shmPos: 0, + }, + { + subTest: "non empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + Name: "postgres", + VolumeMounts: []v1.VolumeMount{ + {}, + }, + }, + }, + }, + shmPos: 1, + }, + } + for _, tt := range tests { + addShmVolume(tt.podSpec) + postgresContainer := getPostgresContainer(tt.podSpec) + volumeName := tt.podSpec.Volumes[tt.shmPos].Name + volumeMountName := postgresContainer.VolumeMounts[tt.shmPos].Name + + if volumeName != constants.ShmVolumeName { + t.Errorf("%s %s: Expected volume %s was not created, have %s instead", + t.Name(), tt.subTest, constants.ShmVolumeName, volumeName) + } + if volumeMountName != constants.ShmVolumeName { + t.Errorf("%s %s: Expected mount %s was not created, have %s instead", + t.Name(), tt.subTest, constants.ShmVolumeName, volumeMountName) + } + } +} + +func TestSecretVolume(t *testing.T) { + tests := []struct { + subTest string + podSpec *v1.PodSpec + secretPos int + }{ + { + subTest: "empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + secretPos: 0, + }, + { + subTest: "non empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + }, + }, + secretPos: 1, + }, + } + for _, tt := range tests { + additionalSecretMount := "aws-iam-s3-role" + additionalSecretMountPath := "/meta/credentials" + postgresContainer := getPostgresContainer(tt.podSpec) + + numMounts := len(postgresContainer.VolumeMounts) + + addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath) + + volumeName := tt.podSpec.Volumes[tt.secretPos].Name + + if volumeName != additionalSecretMount { + t.Errorf("%s %s: Expected volume %s was not created, have %s instead", + t.Name(), tt.subTest, additionalSecretMount, volumeName) + } + + for i := range tt.podSpec.Containers { + volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name + + if volumeMountName != additionalSecretMount { + t.Errorf("%s %s: Expected mount %s was not created, have %s instead", + t.Name(), tt.subTest, additionalSecretMount, volumeMountName) + } + } + + postgresContainer = getPostgresContainer(tt.podSpec) + numMountsCheck := len(postgresContainer.VolumeMounts) + + if numMountsCheck != numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts+1) + } + } +} + +func TestAdditionalVolume(t *testing.T) { client, _ := newFakeK8sTestClient() clusterName := "acid-test-cluster" namespace := "default" @@ -1099,6 +1934,25 @@ func TestAdditionalVolume(t *testing.T) { EmptyDir: &v1.EmptyDirVolumeSource{}, }, }, + { + Name: "test5", + MountPath: "/test5", + SubPath: "subpath", + TargetContainers: nil, // should mount only to postgres + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "test6", + MountPath: "/test6", + SubPath: "$(POD_NAME)", + IsSubPathExpr: util.True(), + TargetContainers: nil, // should mount only to postgres + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, } pg := acidv1.Postgresql{ @@ -1108,17 +1962,20 @@ func TestAdditionalVolume(t *testing.T) { }, Spec: acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ - ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, - ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, }, Volume: acidv1.Volume{ - Size: "1G", + Size: "1G", + SubPath: "$(POD_NAME)", + IsSubPathExpr: util.True(), }, AdditionalVolumes: additionalVolumes, Sidecars: []acidv1.Sidecar{ { - Name: sidecarName, + Name: sidecarName, + DockerImage: "test-image", }, }, }, @@ -1145,19 +2002,25 @@ func TestAdditionalVolume(t *testing.T) { assert.NoError(t, err) tests := []struct { - subTest string - container string - expectedMounts []string + subTest string + container string + expectedMounts []string + expectedSubPaths []string + expectedSubPathExprs []string }{ { - subTest: "checking volume mounts of postgres container", - container: constants.PostgresContainerName, - expectedMounts: []string{"pgdata", "test1", "test3", "test4"}, + subTest: "checking volume mounts of postgres container", + container: constants.PostgresContainerName, + expectedMounts: []string{"pgdata", "test1", "test3", "test4", "test5", "test6"}, + expectedSubPaths: []string{"", "", "", "", "subpath", ""}, + expectedSubPathExprs: []string{"$(POD_NAME)", "", "", "", "", "$(POD_NAME)"}, }, { - subTest: "checking volume mounts of sidecar container", - container: "sidecar", - expectedMounts: []string{"pgdata", "test1", "test2"}, + subTest: "checking volume mounts of sidecar container", + container: "sidecar", + expectedMounts: []string{"pgdata", "test1", "test2"}, + expectedSubPaths: []string{"", "", ""}, + expectedSubPathExprs: []string{"$(POD_NAME)", "", ""}, }, } @@ -1167,14 +2030,131 @@ func TestAdditionalVolume(t *testing.T) { continue } mounts := []string{} + subPaths := []string{} + subPathExprs := []string{} + for _, volumeMounts := range container.VolumeMounts { mounts = append(mounts, volumeMounts.Name) + subPaths = append(subPaths, volumeMounts.SubPath) + subPathExprs = append(subPathExprs, volumeMounts.SubPathExpr) } if !util.IsEqualIgnoreOrder(mounts, tt.expectedMounts) { - t.Errorf("%s %s: different volume mounts: got %v, epxected %v", - testName, tt.subTest, mounts, tt.expectedMounts) + t.Errorf("%s %s: different volume mounts: got %v, expected %v", + t.Name(), tt.subTest, mounts, tt.expectedMounts) } + + if !util.IsEqualIgnoreOrder(subPaths, tt.expectedSubPaths) { + t.Errorf("%s %s: different volume subPaths: got %v, expected %v", + t.Name(), tt.subTest, subPaths, tt.expectedSubPaths) + } + + if !util.IsEqualIgnoreOrder(subPathExprs, tt.expectedSubPathExprs) { + t.Errorf("%s %s: different volume subPathExprs: got %v, expected %v", + t.Name(), tt.subTest, subPathExprs, tt.expectedSubPathExprs) + } + } + } +} + +func TestVolumeSelector(t *testing.T) { + makeSpec := func(volume acidv1.Volume) acidv1.PostgresSpec { + return acidv1.PostgresSpec{ + TeamID: "myapp", + NumberOfInstances: 0, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + Volume: volume, + } + } + + tests := []struct { + subTest string + volume acidv1.Volume + wantSelector *metav1.LabelSelector + }{ + { + subTest: "PVC template has no selector", + volume: acidv1.Volume{ + Size: "1G", + }, + wantSelector: nil, + }, + { + subTest: "PVC template has simple label selector", + volume: acidv1.Volume{ + Size: "1G", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "unittest"}, + }, + }, + wantSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "unittest"}, + }, + }, + { + subTest: "PVC template has full selector", + volume: acidv1.Volume{ + Size: "1G", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "unittest"}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "flavour", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"banana", "chocolate"}, + }, + }, + }, + }, + wantSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "unittest"}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "flavour", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"banana", "chocolate"}, + }, + }, + }, + }, + } + + cluster := New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + for _, tt := range tests { + pgSpec := makeSpec(tt.volume) + sts, err := cluster.generateStatefulSet(&pgSpec) + if err != nil { + t.Fatalf("%s %s: no statefulset created %v", t.Name(), tt.subTest, err) + } + + volIdx := len(sts.Spec.VolumeClaimTemplates) + for i, ct := range sts.Spec.VolumeClaimTemplates { + if ct.ObjectMeta.Name == constants.DataVolumeName { + volIdx = i + break + } + } + if volIdx == len(sts.Spec.VolumeClaimTemplates) { + t.Errorf("%s %s: no datavolume found in sts", t.Name(), tt.subTest) + } + + selector := sts.Spec.VolumeClaimTemplates[volIdx].Spec.Selector + if !reflect.DeepEqual(selector, tt.wantSelector) { + t.Errorf("%s %s: expected: %#v but got: %#v", t.Name(), tt.subTest, tt.wantSelector, selector) } } } @@ -1207,28 +2187,36 @@ func TestSidecars(t *testing.T) { } spec = acidv1.PostgresSpec{ + PostgresqlParam: acidv1.PostgresqlParam{ + PgVersion: "18", + Parameters: map[string]string{ + "max_connections": "100", + }, + }, TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ - ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, - ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, }, Volume: acidv1.Volume{ Size: "1G", }, Sidecars: []acidv1.Sidecar{ - acidv1.Sidecar{ - Name: "cluster-specific-sidecar", + { + Name: "cluster-specific-sidecar", + DockerImage: "test-image", }, - acidv1.Sidecar{ - Name: "cluster-specific-sidecar-with-resources", - Resources: acidv1.Resources{ - ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"}, - ResourceLimits: acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"}, + { + Name: "cluster-specific-sidecar-with-resources", + DockerImage: "test-image", + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("210m"), Memory: k8sutil.StringToPointer("0.8Gi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("510m"), Memory: k8sutil.StringToPointer("1.4Gi")}, }, }, - acidv1.Sidecar{ + { Name: "replace-sidecar", - DockerImage: "overwrite-image", + DockerImage: "override-image", }, }, } @@ -1244,19 +2232,22 @@ func TestSidecars(t *testing.T) { }, Resources: config.Resources{ DefaultCPURequest: "200m", + MaxCPURequest: "300m", DefaultCPULimit: "500m", DefaultMemoryRequest: "0.7Gi", + MaxMemoryRequest: "1.0Gi", DefaultMemoryLimit: "1.3Gi", }, SidecarImages: map[string]string{ "deprecated-global-sidecar": "image:123", }, SidecarContainers: []v1.Container{ - v1.Container{ - Name: "global-sidecar", + { + Name: "global-sidecar", + Image: "test-image", }, // will be replaced by a cluster specific sidecar with the same name - v1.Container{ + { Name: "replace-sidecar", Image: "replaced-image", }, @@ -1311,7 +2302,7 @@ func TestSidecars(t *testing.T) { }, } mounts := []v1.VolumeMount{ - v1.VolumeMount{ + { Name: "pgdata", MountPath: "/home/postgres/pgdata", }, @@ -1323,6 +2314,7 @@ func TestSidecars(t *testing.T) { // cluster specific sidecar assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ Name: "cluster-specific-sidecar", + Image: "test-image", Env: env, Resources: generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"), ImagePullPolicy: v1.PullIfNotPresent, @@ -1349,6 +2341,7 @@ func TestSidecars(t *testing.T) { // global sidecar assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ Name: "global-sidecar", + Image: "test-image", Env: env, VolumeMounts: mounts, }) @@ -1356,7 +2349,7 @@ func TestSidecars(t *testing.T) { // replaced sidecar assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ Name: "replace-sidecar", - Image: "overwrite-image", + Image: "override-image", Resources: generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"), ImagePullPolicy: v1.PullIfNotPresent, Env: env, @@ -1377,33 +2370,483 @@ func TestSidecars(t *testing.T) { } +func TestContainerValidation(t *testing.T) { + testCases := []struct { + name string + spec acidv1.PostgresSpec + clusterConfig Config + expectedError string + }{ + { + name: "init container without image", + spec: acidv1.PostgresSpec{ + PostgresqlParam: acidv1.PostgresqlParam{ + PgVersion: "18", + }, + TeamID: "myapp", + NumberOfInstances: 1, + Volume: acidv1.Volume{ + Size: "1G", + }, + InitContainers: []v1.Container{ + { + Name: "invalid-initcontainer", + }, + }, + }, + clusterConfig: Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, + expectedError: "image is required", + }, + { + name: "sidecar without name", + spec: acidv1.PostgresSpec{ + PostgresqlParam: acidv1.PostgresqlParam{ + PgVersion: "18", + }, + TeamID: "myapp", + NumberOfInstances: 1, + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + clusterConfig: Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + SidecarContainers: []v1.Container{ + { + Image: "test-image", + }, + }, + }, + }, + expectedError: "name is required", + }, + { + name: "sidecar without image", + spec: acidv1.PostgresSpec{ + PostgresqlParam: acidv1.PostgresqlParam{ + PgVersion: "18", + }, + TeamID: "myapp", + NumberOfInstances: 1, + Volume: acidv1.Volume{ + Size: "1G", + }, + Sidecars: []acidv1.Sidecar{ + { + Name: "invalid-sidecar", + }, + }, + }, + clusterConfig: Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, + expectedError: "image is required", + }, + { + name: "valid containers pass validation", + spec: acidv1.PostgresSpec{ + PostgresqlParam: acidv1.PostgresqlParam{ + PgVersion: "18", + }, + TeamID: "myapp", + NumberOfInstances: 1, + Volume: acidv1.Volume{ + Size: "1G", + }, + Sidecars: []acidv1.Sidecar{ + { + Name: "valid-sidecar", + DockerImage: "busybox:latest", + }, + }, + InitContainers: []v1.Container{ + { + Name: "valid-initcontainer", + Image: "alpine:latest", + }, + }, + }, + clusterConfig: Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, + expectedError: "", + }, + { + name: "multiple invalid sidecars", + spec: acidv1.PostgresSpec{ + Sidecars: []acidv1.Sidecar{ + { + Name: "sidecar1", + }, + { + Name: "sidecar2", + }, + }, + }, + expectedError: "image is required", + }, + { + name: "empty container name and image", + spec: acidv1.PostgresSpec{ + InitContainers: []v1.Container{ + { + Name: "", + Image: "", + }, + }, + }, + expectedError: "name is required", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cluster := New(tc.clusterConfig, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + _, err := cluster.generateStatefulSet(&tc.spec) + + if tc.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tc.expectedError) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestGeneratePodDisruptionBudget(t *testing.T) { + testName := "Test PodDisruptionBudget spec generation" + + hasName := func(pdbName string) func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { + return func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { + if pdbName != podDisruptionBudget.ObjectMeta.Name { + return fmt.Errorf("PodDisruptionBudget name is incorrect, got %s, expected %s", + podDisruptionBudget.ObjectMeta.Name, pdbName) + } + return nil + } + } + + hasMinAvailable := func(expectedMinAvailable int) func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { + return func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { + actual := podDisruptionBudget.Spec.MinAvailable.IntVal + if actual != int32(expectedMinAvailable) { + return fmt.Errorf("PodDisruptionBudget MinAvailable is incorrect, got %d, expected %d", + actual, expectedMinAvailable) + } + return nil + } + } + + testLabelsAndSelectors := func(isPrimary bool) func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { + return func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { + masterLabelSelectorDisabled := cluster.OpConfig.PDBMasterLabelSelector != nil && !*cluster.OpConfig.PDBMasterLabelSelector + if podDisruptionBudget.ObjectMeta.Namespace != "myapp" { + return fmt.Errorf("Object Namespace incorrect.") + } + expectedLabels := map[string]string{"team": "myapp", "cluster-name": "myapp-database"} + if !reflect.DeepEqual(podDisruptionBudget.Labels, expectedLabels) { + return fmt.Errorf("Labels incorrect, got %#v, expected %#v", podDisruptionBudget.Labels, expectedLabels) + } + if !masterLabelSelectorDisabled { + if isPrimary { + expectedLabels := &metav1.LabelSelector{ + MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}} + if !reflect.DeepEqual(podDisruptionBudget.Spec.Selector, expectedLabels) { + return fmt.Errorf("MatchLabels incorrect, got %#v, expected %#v", podDisruptionBudget.Spec.Selector, expectedLabels) + } + } else { + expectedLabels := &metav1.LabelSelector{ + MatchLabels: map[string]string{"cluster-name": "myapp-database", "critical-operation": "true"}} + if !reflect.DeepEqual(podDisruptionBudget.Spec.Selector, expectedLabels) { + return fmt.Errorf("MatchLabels incorrect, got %#v, expected %#v", podDisruptionBudget.Spec.Selector, expectedLabels) + } + } + } + + return nil + } + } + + testPodDisruptionBudgetOwnerReference := func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error { + if len(podDisruptionBudget.ObjectMeta.OwnerReferences) == 0 { + return nil + } + owner := podDisruptionBudget.ObjectMeta.OwnerReferences[0] + + if owner.Name != cluster.Postgresql.ObjectMeta.Name { + return fmt.Errorf("Owner reference is incorrect, got %s, expected %s", + owner.Name, cluster.Postgresql.ObjectMeta.Name) + } + + return nil + } + + tests := []struct { + scenario string + spec *Cluster + check []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error + }{ + { + scenario: "With multiple instances", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-pdb"), + hasMinAvailable(1), + testLabelsAndSelectors(true), + }, + }, + { + scenario: "With zero instances", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-pdb"), + hasMinAvailable(0), + testLabelsAndSelectors(true), + }, + }, + { + scenario: "With PodDisruptionBudget disabled", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.False()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-pdb"), + hasMinAvailable(0), + testLabelsAndSelectors(true), + }, + }, + { + scenario: "With non-default PDBNameFormat and PodDisruptionBudget explicitly enabled", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-databass-budget", EnablePodDisruptionBudget: util.True()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-databass-budget"), + hasMinAvailable(1), + testLabelsAndSelectors(true), + }, + }, + { + scenario: "With PDBMasterLabelSelector disabled", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.True(), PDBMasterLabelSelector: util.False()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-pdb"), + hasMinAvailable(1), + testLabelsAndSelectors(true), + }, + }, + { + scenario: "With OwnerReference enabled", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role", EnableOwnerReferences: util.True()}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.True()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-pdb"), + hasMinAvailable(1), + testLabelsAndSelectors(true), + }, + }, + } + + for _, tt := range tests { + result := tt.spec.generatePrimaryPodDisruptionBudget() + for _, check := range tt.check { + err := check(tt.spec, result) + if err != nil { + t.Errorf("%s [%s]: PodDisruptionBudget spec is incorrect, %+v", + testName, tt.scenario, err) + } + } + } + + testCriticalOp := []struct { + scenario string + spec *Cluster + check []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error + }{ + { + scenario: "With multiple instances", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-critical-op-pdb"), + hasMinAvailable(3), + testLabelsAndSelectors(false), + }, + }, + { + scenario: "With zero instances", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-critical-op-pdb"), + hasMinAvailable(0), + testLabelsAndSelectors(false), + }, + }, + { + scenario: "With PodDisruptionBudget disabled", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.False()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-critical-op-pdb"), + hasMinAvailable(0), + testLabelsAndSelectors(false), + }, + }, + { + scenario: "With OwnerReference enabled", + spec: New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role", EnableOwnerReferences: util.True()}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.True()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + check: []func(cluster *Cluster, podDisruptionBudget *policyv1.PodDisruptionBudget) error{ + testPodDisruptionBudgetOwnerReference, + hasName("postgres-myapp-database-critical-op-pdb"), + hasMinAvailable(3), + testLabelsAndSelectors(false), + }, + }, + } + + for _, tt := range testCriticalOp { + result := tt.spec.generateCriticalOpPodDisruptionBudget() + for _, check := range tt.check { + err := check(tt.spec, result) + if err != nil { + t.Errorf("%s [%s]: PodDisruptionBudget spec is incorrect, %+v", + testName, tt.scenario, err) + } + } + } +} + func TestGenerateService(t *testing.T) { var spec acidv1.PostgresSpec var cluster *Cluster var enableLB bool = true spec = acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ - ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, - ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, }, Volume: acidv1.Volume{ Size: "1G", }, Sidecars: []acidv1.Sidecar{ - acidv1.Sidecar{ + { Name: "cluster-specific-sidecar", }, - acidv1.Sidecar{ - Name: "cluster-specific-sidecar-with-resources", - Resources: acidv1.Resources{ - ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"}, - ResourceLimits: acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"}, + { + Name: "cluster-specific-sidecar-with-resources", + DockerImage: "test-image", + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("210m"), Memory: k8sutil.StringToPointer("0.8Gi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("510m"), Memory: k8sutil.StringToPointer("1.4Gi")}, }, }, - acidv1.Sidecar{ + { Name: "replace-sidecar", - DockerImage: "overwrite-image", + DockerImage: "override-image", }, }, EnableMasterLoadBalancer: &enableLB, @@ -1420,19 +2863,21 @@ func TestGenerateService(t *testing.T) { }, Resources: config.Resources{ DefaultCPURequest: "200m", + MaxCPURequest: "300m", DefaultCPULimit: "500m", DefaultMemoryRequest: "0.7Gi", + MaxMemoryRequest: "1.0Gi", DefaultMemoryLimit: "1.3Gi", }, SidecarImages: map[string]string{ "deprecated-global-sidecar": "image:123", }, SidecarContainers: []v1.Container{ - v1.Container{ + { Name: "global-sidecar", }, // will be replaced by a cluster specific sidecar with the same name - v1.Container{ + { Name: "replace-sidecar", Image: "replaced-image", }, @@ -1457,9 +2902,1548 @@ func TestGenerateService(t *testing.T) { } -func TestGenerateCapabilities(t *testing.T) { +func TestCreateLoadBalancerLogic(t *testing.T) { + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - testName := "TestGenerateCapabilities" + tests := []struct { + subtest string + role PostgresRole + spec *acidv1.PostgresSpec + opConfig config.Config + result bool + }{ + { + subtest: "new format, load balancer is enabled for replica", + role: Replica, + spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.True()}, + opConfig: config.Config{}, + result: true, + }, + { + subtest: "new format, load balancer is disabled for replica", + role: Replica, + spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.False()}, + opConfig: config.Config{}, + result: false, + }, + { + subtest: "new format, load balancer isn't specified for replica", + role: Replica, + spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil}, + opConfig: config.Config{EnableReplicaLoadBalancer: true}, + result: true, + }, + { + subtest: "new format, load balancer isn't specified for replica", + role: Replica, + spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil}, + opConfig: config.Config{EnableReplicaLoadBalancer: false}, + result: false, + }, + } + for _, tt := range tests { + cluster.OpConfig = tt.opConfig + result := cluster.shouldCreateLoadBalancerForService(tt.role, tt.spec) + if tt.result != result { + t.Errorf("%s %s: Load balancer is %t, expect %t for role %#v and spec %#v", + t.Name(), tt.subtest, result, tt.result, tt.role, tt.spec) + } + } +} + +func newLBFakeClient() (k8sutil.KubernetesClient, *fake.Clientset) { + clientSet := fake.NewSimpleClientset() + + return k8sutil.KubernetesClient{ + DeploymentsGetter: clientSet.AppsV1(), + PodsGetter: clientSet.CoreV1(), + ServicesGetter: clientSet.CoreV1(), + SecretsGetter: clientSet.CoreV1(), + }, clientSet +} + +func getServices(serviceType v1.ServiceType, sourceRanges []string, extTrafficPolicy, clusterName string, nodePort int32) []v1.ServiceSpec { + return []v1.ServiceSpec{ + { + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyType(extTrafficPolicy), + LoadBalancerSourceRanges: sourceRanges, + Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}, NodePort: nodePort}}, + Type: serviceType, + }, + { + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyType(extTrafficPolicy), + LoadBalancerSourceRanges: sourceRanges, + Ports: []v1.ServicePort{{Name: clusterName + "-pooler", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}, NodePort: nodePort}}, + Selector: map[string]string{"connection-pooler": clusterName + "-pooler"}, + Type: serviceType, + }, + { + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyType(extTrafficPolicy), + LoadBalancerSourceRanges: sourceRanges, + Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}, NodePort: nodePort}}, + Selector: map[string]string{"spilo-role": "replica", "application": "spilo", "cluster-name": clusterName}, + Type: serviceType, + }, + { + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyType(extTrafficPolicy), + LoadBalancerSourceRanges: sourceRanges, + Ports: []v1.ServicePort{{Name: clusterName + "-pooler-repl", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}, NodePort: nodePort}}, + Selector: map[string]string{"connection-pooler": clusterName + "-pooler-repl"}, + Type: serviceType, + }, + } +} + +func TestEnableLoadBalancers(t *testing.T) { + client, _ := newLBFakeClient() + clusterName := "acid-test-cluster" + namespace := "default" + clusterNameLabel := "cluster-name" + roleLabel := "spilo-role" + roles := []PostgresRole{Master, Replica} + sourceRanges := []string{"192.186.1.2/22"} + extTrafficPolicy := "Cluster" + + tests := []struct { + subTest string + config config.Config + pgSpec acidv1.Postgresql + expectedServices []v1.ServiceSpec + }{ + { + subTest: "LBs enabled in config, disabled in manifest", + config: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(1), + }, + EnableMasterLoadBalancer: true, + EnableMasterPoolerLoadBalancer: true, + EnableReplicaLoadBalancer: true, + EnableReplicaPoolerLoadBalancer: true, + ExternalTrafficPolicy: extTrafficPolicy, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + PodRoleLabel: roleLabel, + }, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + AllowedSourceRanges: sourceRanges, + EnableConnectionPooler: util.True(), + EnableReplicaConnectionPooler: util.True(), + EnableMasterLoadBalancer: util.False(), + EnableMasterPoolerLoadBalancer: util.False(), + EnableReplicaLoadBalancer: util.False(), + EnableReplicaPoolerLoadBalancer: util.False(), + NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedServices: getServices(v1.ServiceTypeClusterIP, nil, "", clusterName, 0), + }, + { + subTest: "LBs enabled in manifest, disabled in config", + config: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(1), + }, + EnableMasterLoadBalancer: false, + EnableMasterPoolerLoadBalancer: false, + EnableReplicaLoadBalancer: false, + EnableReplicaPoolerLoadBalancer: false, + ExternalTrafficPolicy: extTrafficPolicy, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + PodRoleLabel: roleLabel, + }, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + AllowedSourceRanges: sourceRanges, + EnableConnectionPooler: util.True(), + EnableReplicaConnectionPooler: util.True(), + EnableMasterLoadBalancer: util.True(), + EnableMasterPoolerLoadBalancer: util.True(), + EnableReplicaLoadBalancer: util.True(), + EnableReplicaPoolerLoadBalancer: util.True(), + NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedServices: getServices(v1.ServiceTypeLoadBalancer, sourceRanges, extTrafficPolicy, clusterName, 0), + }, + } + + for _, tt := range tests { + var cluster = New( + Config{ + OpConfig: tt.config, + }, client, tt.pgSpec, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{} + generatedServices := make([]v1.ServiceSpec, 0) + for _, role := range roles { + cluster.syncService(role) + cluster.ConnectionPooler[role] = &ConnectionPoolerObjects{ + Name: cluster.connectionPoolerName(role), + ClusterName: cluster.Name, + Namespace: cluster.Namespace, + Role: role, + } + cluster.syncConnectionPoolerWorker(&tt.pgSpec, &tt.pgSpec, role) + generatedServices = append(generatedServices, cluster.Services[role].Spec) + generatedServices = append(generatedServices, cluster.ConnectionPooler[role].Service.Spec) + } + if !reflect.DeepEqual(tt.expectedServices, generatedServices) { + t.Errorf("%s %s: expected %#v but got %#v", t.Name(), tt.subTest, tt.expectedServices, generatedServices) + } + } +} + +func TestEnableNodePorts(t *testing.T) { + clusterName := "acid-test-cluster" + namespace := "default" + clusterNameLabel := "cluster-name" + roleLabel := "spilo-role" + roles := []PostgresRole{Master, Replica} + extTrafficPolicy := "Cluster" + port := int32(1337) + + tests := []struct { + subTest string + config config.Config + pgSpec acidv1.Postgresql + expectedServices []v1.ServiceSpec + }{ + { + subTest: "NodePorts enabled in config, disabled in manifest", + config: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(1), + }, + EnableMasterNodePort: true, + EnableMasterPoolerNodePort: true, + EnableReplicaNodePort: true, + EnableReplicaPoolerNodePort: true, + ExternalTrafficPolicy: extTrafficPolicy, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + PodRoleLabel: roleLabel, + }, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: util.True(), + EnableReplicaConnectionPooler: util.True(), + EnableMasterNodePort: util.False(), + EnableMasterPoolerNodePort: util.False(), + EnableReplicaNodePort: util.False(), + EnableReplicaPoolerNodePort: util.False(), + NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedServices: getServices(v1.ServiceTypeClusterIP, nil, "", clusterName, 0), + }, + { + subTest: "NodePorts configured in manifest, disabled in config", + config: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(1), + }, + EnableMasterNodePort: false, + EnableMasterPoolerNodePort: false, + EnableReplicaNodePort: false, + EnableReplicaPoolerNodePort: false, + ExternalTrafficPolicy: extTrafficPolicy, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + PodRoleLabel: roleLabel, + }, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: util.True(), + EnableReplicaConnectionPooler: util.True(), + EnableMasterNodePort: util.True(), + EnableMasterPoolerNodePort: util.True(), + EnableReplicaNodePort: util.True(), + EnableReplicaPoolerNodePort: util.True(), + NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedServices: getServices(v1.ServiceTypeNodePort, nil, extTrafficPolicy, clusterName, 0), + }, + { + subTest: "NodePorts configured in manifest, disabled in config, custom port specified", + config: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(1), + }, + EnableMasterNodePort: false, + EnableMasterPoolerNodePort: false, + EnableReplicaNodePort: false, + EnableReplicaPoolerNodePort: false, + ExternalTrafficPolicy: extTrafficPolicy, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + PodRoleLabel: roleLabel, + }, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: util.True(), + EnableReplicaConnectionPooler: util.True(), + EnableMasterNodePort: util.True(), + MasterNodePort: &port, + EnableMasterPoolerNodePort: util.True(), + MasterPoolerNodePort: &port, + EnableReplicaNodePort: util.True(), + ReplicaNodePort: &port, + EnableReplicaPoolerNodePort: util.True(), + ReplicaPoolerNodePort: &port, + NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedServices: getServices(v1.ServiceTypeNodePort, nil, extTrafficPolicy, clusterName, port), + }, + } + + for _, tt := range tests { + client, _ := newLBFakeClient() + + var cluster = New( + Config{ + OpConfig: tt.config, + }, client, tt.pgSpec, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{} + generatedServices := make([]v1.ServiceSpec, 0) + for _, role := range roles { + cluster.syncService(role) + cluster.ConnectionPooler[role] = &ConnectionPoolerObjects{ + Name: cluster.connectionPoolerName(role), + ClusterName: cluster.Name, + Namespace: cluster.Namespace, + Role: role, + } + cluster.syncConnectionPoolerWorker(&tt.pgSpec, &tt.pgSpec, role) + generatedServices = append(generatedServices, cluster.Services[role].Spec) + generatedServices = append(generatedServices, cluster.ConnectionPooler[role].Service.Spec) + } + if !reflect.DeepEqual(tt.expectedServices, generatedServices) { + t.Errorf("%s %s: expected %#v but got %#v", t.Name(), tt.subTest, tt.expectedServices, generatedServices) + } + } +} + +func TestGenerateResourceRequirements(t *testing.T) { + client, _ := newFakeK8sTestClient() + clusterName := "acid-test-cluster" + namespace := "default" + clusterNameLabel := "cluster-name" + sidecarName := "postgres-exporter" + dockerImage := "test-image" + + // enforceMinResourceLimits will be called 2 times emitting 4 events (2x cpu, 2x memory raise) + // enforceMaxResourceRequests will be called 4 times emitting 6 events (2x cpu, 4x memory cap) + // hence event bufferSize of 10 is required + newEventRecorder := record.NewFakeRecorder(10) + + configResources := config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + DefaultCPURequest: "100m", + DefaultCPULimit: "1", + MaxCPURequest: "500m", + MinCPULimit: "250m", + DefaultMemoryRequest: "100Mi", + DefaultMemoryLimit: "500Mi", + MaxMemoryRequest: "1Gi", + MinMemoryLimit: "250Mi", + PodRoleLabel: "spilo-role", + } + + configWithEnabledIgnoreResourcesLimits := configResources + configWithEnabledIgnoreResourcesLimits.IgnoreResourcesLimitsAnnotationKey = "zalando.org/ignore-resources-limits" + + tests := []struct { + subTest string + config config.Config + pgSpec acidv1.Postgresql + expectedResources acidv1.Resources + }{ + { + subTest: "test generation of default resources when empty in manifest", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("500Mi")}, + }, + }, + { + subTest: "test generation of default resources for sidecar", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Sidecars: []acidv1.Sidecar{ + { + Name: sidecarName, + DockerImage: dockerImage, + }, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("500Mi")}, + }, + }, + { + subTest: "test generation of resources when only requests are defined in manifest", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("50m"), Memory: k8sutil.StringToPointer("50Mi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("50m"), Memory: k8sutil.StringToPointer("50Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("500Mi")}, + }, + }, + { + subTest: "test generation of resources when only memory is defined in manifest", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("1Gi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("1Gi")}, + }, + }, + { + subTest: "test generation of resources when default is not defined", + config: config.Config{ + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + DefaultCPURequest: "100m", + DefaultMemoryRequest: "100Mi", + PodRoleLabel: "spilo-role", + }, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + }, + }, + { + subTest: "test generation of resources when min limits are all set to zero", + config: config.Config{ + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + DefaultCPURequest: "0", + DefaultCPULimit: "0", + MaxCPURequest: "0", + MinCPULimit: "0", + DefaultMemoryRequest: "0", + DefaultMemoryLimit: "0", + MaxMemoryRequest: "0", + MinMemoryLimit: "0", + PodRoleLabel: "spilo-role", + }, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("5m"), Memory: k8sutil.StringToPointer("5Mi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("5m"), Memory: k8sutil.StringToPointer("5Mi")}, + }, + }, + { + subTest: "test matchLimitsWithRequestsIfSmaller", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("750Mi")}, + ResourceLimits: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("300Mi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("750Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("750Mi")}, + }, + }, + { + subTest: "defaults are not defined but minimum limit is", + config: config.Config{ + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + MinMemoryLimit: "250Mi", + PodRoleLabel: "spilo-role", + }, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("500Mi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("500Mi")}, + ResourceLimits: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("500Mi")}, + }, + }, + { + subTest: "test SetMemoryRequestToLimit flag", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: true, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("200Mi")}, + ResourceLimits: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("300Mi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("300Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("300Mi")}, + }, + }, + { + subTest: "test SetMemoryRequestToLimit flag for sidecar container, too", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: true, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Sidecars: []acidv1.Sidecar{ + { + Name: sidecarName, + DockerImage: dockerImage, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("10m"), Memory: k8sutil.StringToPointer("10Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + }, + }, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("10m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + }, + }, + { + subTest: "test generating resources from manifest", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("10m"), Memory: k8sutil.StringToPointer("250Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("400m"), Memory: k8sutil.StringToPointer("800Mi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("10m"), Memory: k8sutil.StringToPointer("250Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("400m"), Memory: k8sutil.StringToPointer("800Mi")}, + }, + }, + { + subTest: "test enforcing min cpu and memory limit", + config: config.Config{ + Resources: configWithEnabledIgnoreResourcesLimits, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + Annotations: map[string]string{"zalando.org/ignore-resources-limits": "false"}, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("200m"), Memory: k8sutil.StringToPointer("200Mi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("250m"), Memory: k8sutil.StringToPointer("250Mi")}, + }, + }, + { + subTest: "ingnore min cpu and memory limit threshold", + config: config.Config{ + Resources: configWithEnabledIgnoreResourcesLimits, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + Annotations: map[string]string{"zalando.org/ignore-resources-limits": "true"}, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("200m"), Memory: k8sutil.StringToPointer("200Mi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("200m"), Memory: k8sutil.StringToPointer("200Mi")}, + }, + }, + { + subTest: "test min cpu and memory limit are not enforced on sidecar", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Sidecars: []acidv1.Sidecar{ + { + Name: sidecarName, + DockerImage: dockerImage, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("10m"), Memory: k8sutil.StringToPointer("10Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + }, + }, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("10m"), Memory: k8sutil.StringToPointer("10Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + }, + }, + { + subTest: "test enforcing max cpu and memory requests", + config: config.Config{ + Resources: configWithEnabledIgnoreResourcesLimits, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + Annotations: map[string]string{"zalando.org/ignore-resources-limits": "yes"}, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("2Gi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("2"), Memory: k8sutil.StringToPointer("4Gi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("500m"), Memory: k8sutil.StringToPointer("1Gi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("2"), Memory: k8sutil.StringToPointer("4Gi")}, + }, + }, + { + subTest: "ignore max cpu and memory requests limit", + config: config.Config{ + Resources: configWithEnabledIgnoreResourcesLimits, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + Annotations: map[string]string{"zalando.org/ignore-resources-limits": "true"}, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("2Gi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("2"), Memory: k8sutil.StringToPointer("4Gi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("2Gi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("2"), Memory: k8sutil.StringToPointer("4Gi")}, + }, + }, + { + subTest: "test SetMemoryRequestToLimit flag but raise only until max memory request", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: true, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("500Mi")}, + ResourceLimits: acidv1.ResourceDescription{Memory: k8sutil.StringToPointer("2Gi")}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("1Gi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("2Gi")}, + }, + }, + { + subTest: "test HugePages are not set on container when not requested in manifest", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{}, + ResourceLimits: acidv1.ResourceDescription{}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{ + CPU: k8sutil.StringToPointer("100m"), + Memory: k8sutil.StringToPointer("100Mi"), + }, + ResourceLimits: acidv1.ResourceDescription{ + CPU: k8sutil.StringToPointer("1"), + Memory: k8sutil.StringToPointer("500Mi"), + }, + }, + }, + { + subTest: "test HugePages are passed through to the postgres container", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{ + HugePages2Mi: k8sutil.StringToPointer("128Mi"), + HugePages1Gi: k8sutil.StringToPointer("1Gi"), + }, + ResourceLimits: acidv1.ResourceDescription{ + HugePages2Mi: k8sutil.StringToPointer("256Mi"), + HugePages1Gi: k8sutil.StringToPointer("2Gi"), + }, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{ + CPU: k8sutil.StringToPointer("100m"), + Memory: k8sutil.StringToPointer("100Mi"), + HugePages2Mi: k8sutil.StringToPointer("128Mi"), + HugePages1Gi: k8sutil.StringToPointer("1Gi"), + }, + ResourceLimits: acidv1.ResourceDescription{ + CPU: k8sutil.StringToPointer("1"), + Memory: k8sutil.StringToPointer("500Mi"), + HugePages2Mi: k8sutil.StringToPointer("256Mi"), + HugePages1Gi: k8sutil.StringToPointer("2Gi"), + }, + }, + }, + { + subTest: "test HugePages are passed through on sidecars", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Sidecars: []acidv1.Sidecar{ + { + Name: "test-sidecar", + DockerImage: "test-image", + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{ + HugePages2Mi: k8sutil.StringToPointer("128Mi"), + HugePages1Gi: k8sutil.StringToPointer("1Gi"), + }, + ResourceLimits: acidv1.ResourceDescription{ + HugePages2Mi: k8sutil.StringToPointer("256Mi"), + HugePages1Gi: k8sutil.StringToPointer("2Gi"), + }, + }, + }, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{ + CPU: k8sutil.StringToPointer("100m"), + Memory: k8sutil.StringToPointer("100Mi"), + HugePages2Mi: k8sutil.StringToPointer("128Mi"), + HugePages1Gi: k8sutil.StringToPointer("1Gi"), + }, + ResourceLimits: acidv1.ResourceDescription{ + CPU: k8sutil.StringToPointer("1"), + Memory: k8sutil.StringToPointer("500Mi"), + HugePages2Mi: k8sutil.StringToPointer("256Mi"), + HugePages1Gi: k8sutil.StringToPointer("2Gi"), + }, + }, + }, + } + + for _, tt := range tests { + var cluster = New( + Config{ + OpConfig: tt.config, + }, client, tt.pgSpec, logger, newEventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + _, err := cluster.createStatefulSet() + if k8sutil.ResourceAlreadyExists(err) { + err = cluster.syncStatefulSet() + } + assert.NoError(t, err) + + containers := cluster.Statefulset.Spec.Template.Spec.Containers + clusterResources, err := parseResourceRequirements(containers[0].Resources) + if len(containers) > 1 { + clusterResources, err = parseResourceRequirements(containers[1].Resources) + } + assert.NoError(t, err) + if !reflect.DeepEqual(tt.expectedResources, clusterResources) { + t.Errorf("%s - %s: expected %#v but got %#v", t.Name(), tt.subTest, tt.expectedResources, clusterResources) + } + } +} + +func TestGenerateLogicalBackupJob(t *testing.T) { + clusterName := "acid-test-cluster" + teamId := "test" + configResources := config.Resources{ + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "100m", + DefaultCPULimit: "1", + DefaultMemoryRequest: "100Mi", + DefaultMemoryLimit: "500Mi", + } + + tests := []struct { + subTest string + config config.Config + specSchedule string + expectedSchedule string + expectedJobName string + expectedResources acidv1.Resources + expectedAnnotation map[string]string + expectedLabel map[string]string + }{ + { + subTest: "test generation of logical backup pod resources when not configured", + config: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupJobPrefix: "logical-backup-", + LogicalBackupSchedule: "30 00 * * *", + }, + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + specSchedule: "", + expectedSchedule: "30 00 * * *", + expectedJobName: "logical-backup-acid-test-cluster", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("500Mi")}, + }, + expectedLabel: map[string]string{"application": "spilo-logical-backup", configResources.ClusterNameLabel: clusterName, "team": teamId}, + expectedAnnotation: nil, + }, + { + subTest: "test generation of logical backup pod resources when configured", + config: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupCPURequest: "10m", + LogicalBackupCPULimit: "300m", + LogicalBackupMemoryRequest: "50Mi", + LogicalBackupMemoryLimit: "300Mi", + LogicalBackupJobPrefix: "lb-", + LogicalBackupSchedule: "30 00 * * *", + }, + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + specSchedule: "30 00 * * 7", + expectedSchedule: "30 00 * * 7", + expectedJobName: "lb-acid-test-cluster", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("10m"), Memory: k8sutil.StringToPointer("50Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("300m"), Memory: k8sutil.StringToPointer("300Mi")}, + }, + expectedLabel: map[string]string{"application": "spilo-logical-backup", configResources.ClusterNameLabel: clusterName, "team": teamId}, + expectedAnnotation: nil, + }, + { + subTest: "test generation of logical backup pod resources when partly configured", + config: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupCPURequest: "50m", + LogicalBackupCPULimit: "250m", + LogicalBackupJobPrefix: "", + LogicalBackupSchedule: "30 00 * * *", + }, + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + specSchedule: "", + expectedSchedule: "30 00 * * *", + expectedJobName: "acid-test-cluster", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("50m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("250m"), Memory: k8sutil.StringToPointer("500Mi")}, + }, + expectedLabel: map[string]string{"application": "spilo-logical-backup", configResources.ClusterNameLabel: clusterName, "team": teamId}, + expectedAnnotation: nil, + }, + { + subTest: "test generation of logical backup pod resources with SetMemoryRequestToLimit enabled", + config: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupMemoryRequest: "80Mi", + LogicalBackupMemoryLimit: "200Mi", + LogicalBackupJobPrefix: "test-long-prefix-so-name-must-be-trimmed-", + LogicalBackupSchedule: "30 00 * * *", + }, + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: true, + }, + specSchedule: "", + expectedSchedule: "30 00 * * *", + expectedJobName: "test-long-prefix-so-name-must-be-trimmed-acid-test-c", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("200Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("200Mi")}, + }, + expectedLabel: map[string]string{"application": "spilo-logical-backup", configResources.ClusterNameLabel: clusterName, "team": teamId}, + expectedAnnotation: nil, + }, + { + subTest: "test generation of pod annotations when cluster InheritedLabel is set", + config: config.Config{ + Resources: config.Resources{ + ClusterNameLabel: "cluster-name", + InheritedLabels: []string{"labelKey"}, + DefaultCPURequest: "100m", + DefaultCPULimit: "1", + DefaultMemoryRequest: "100Mi", + DefaultMemoryLimit: "500Mi", + }, + }, + specSchedule: "", + expectedJobName: "acid-test-cluster", + expectedSchedule: "", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("500Mi")}, + }, + expectedLabel: map[string]string{"application": "spilo-logical-backup", "labelKey": "labelValue", "cluster-name": clusterName, "team": teamId}, + expectedAnnotation: nil, + }, + { + subTest: "test generation of pod annotations when cluster InheritedAnnotations is set", + config: config.Config{ + Resources: config.Resources{ + ClusterNameLabel: "cluster-name", + InheritedAnnotations: []string{"annotationKey"}, + DefaultCPURequest: "100m", + DefaultCPULimit: "1", + DefaultMemoryRequest: "100Mi", + DefaultMemoryLimit: "500Mi", + }, + }, + specSchedule: "", + expectedJobName: "acid-test-cluster", + expectedSchedule: "", + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("100m"), Memory: k8sutil.StringToPointer("100Mi")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("500Mi")}, + }, + expectedLabel: map[string]string{"application": "spilo-logical-backup", configResources.ClusterNameLabel: clusterName, "team": teamId}, + expectedAnnotation: map[string]string{"annotationKey": "annotationValue"}, + }, + } + + for _, tt := range tests { + var cluster = New( + Config{ + OpConfig: tt.config, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger, eventRecorder) + cluster.ObjectMeta.Name = clusterName + cluster.Spec.TeamID = teamId + if cluster.ObjectMeta.Labels == nil { + cluster.ObjectMeta.Labels = make(map[string]string) + } + if cluster.ObjectMeta.Annotations == nil { + cluster.ObjectMeta.Annotations = make(map[string]string) + } + cluster.ObjectMeta.Labels["labelKey"] = "labelValue" + cluster.ObjectMeta.Annotations["annotationKey"] = "annotationValue" + cluster.Spec.LogicalBackupSchedule = tt.specSchedule + cronJob, err := cluster.generateLogicalBackupJob() + assert.NoError(t, err) + + if !reflect.DeepEqual(cronJob.ObjectMeta.OwnerReferences, cluster.ownerReferences()) { + t.Errorf("%s - %s: expected owner references %#v, got %#v", t.Name(), tt.subTest, cluster.ownerReferences(), cronJob.ObjectMeta.OwnerReferences) + } + + if cronJob.Spec.Schedule != tt.expectedSchedule { + t.Errorf("%s - %s: expected schedule %s, got %s", t.Name(), tt.subTest, tt.expectedSchedule, cronJob.Spec.Schedule) + } + + if cronJob.Name != tt.expectedJobName { + t.Errorf("%s - %s: expected job name %s, got %s", t.Name(), tt.subTest, tt.expectedJobName, cronJob.Name) + } + + if !reflect.DeepEqual(cronJob.Labels, tt.expectedLabel) { + t.Errorf("%s - %s: expected labels %s, got %s", t.Name(), tt.subTest, tt.expectedLabel, cronJob.Labels) + } + + if !reflect.DeepEqual(cronJob.Annotations, tt.expectedAnnotation) { + t.Errorf("%s - %s: expected annotations %s, got %s", t.Name(), tt.subTest, tt.expectedAnnotation, cronJob.Annotations) + } + + containers := cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers + clusterResources, err := parseResourceRequirements(containers[0].Resources) + assert.NoError(t, err) + if !reflect.DeepEqual(tt.expectedResources, clusterResources) { + t.Errorf("%s - %s: expected resources %#v, got %#v", t.Name(), tt.subTest, tt.expectedResources, clusterResources) + } + + expectedSuccessfulJobsHistoryLimit := int32(3) + if cluster.OpConfig.LogicalBackup.LogicalBackupSuccessfulJobsHistoryLimit != nil { + expectedSuccessfulJobsHistoryLimit = *cluster.OpConfig.LogicalBackup.LogicalBackupSuccessfulJobsHistoryLimit + } + if *cronJob.Spec.SuccessfulJobsHistoryLimit != expectedSuccessfulJobsHistoryLimit { + t.Errorf("%s - %s: expected successfulJobsHistoryLimit %d, got %d", t.Name(), tt.subTest, expectedSuccessfulJobsHistoryLimit, *cronJob.Spec.SuccessfulJobsHistoryLimit) + } + + expectedFailedJobsHistoryLimit := int32(3) + if cluster.OpConfig.LogicalBackup.LogicalBackupFailedJobsHistoryLimit != nil { + expectedFailedJobsHistoryLimit = *cluster.OpConfig.LogicalBackup.LogicalBackupFailedJobsHistoryLimit + } + if *cronJob.Spec.FailedJobsHistoryLimit != expectedFailedJobsHistoryLimit { + t.Errorf("%s - %s: expected failedJobsHistoryLimit %d, got %d", t.Name(), tt.subTest, expectedFailedJobsHistoryLimit, *cronJob.Spec.FailedJobsHistoryLimit) + } + + expectedTTL := int32(86400) + if cluster.OpConfig.LogicalBackup.LogicalBackupTTLSecondsAfterFinished != nil { + expectedTTL = *cluster.OpConfig.LogicalBackup.LogicalBackupTTLSecondsAfterFinished + } + if cronJob.Spec.JobTemplate.Spec.TTLSecondsAfterFinished == nil { + t.Errorf("%s - %s: expected TTLSecondsAfterFinished to be set", t.Name(), tt.subTest) + } else if *cronJob.Spec.JobTemplate.Spec.TTLSecondsAfterFinished != expectedTTL { + t.Errorf("%s - %s: expected TTLSecondsAfterFinished %d, got %d", t.Name(), tt.subTest, expectedTTL, *cronJob.Spec.JobTemplate.Spec.TTLSecondsAfterFinished) + } + } +} + +func TestGenerateLogicalBackupPodEnvVars(t *testing.T) { + var ( + dummyUUID = "efd12e58-5786-11e8-b5a7-06148230260c" + dummyBucket = "dummy-backup-location" + ) + + expectedLogicalBackupS3Bucket := []ExpectedValue{ + { + envIndex: 9, + envVarConstant: "LOGICAL_BACKUP_PROVIDER", + envVarValue: "s3", + }, + { + envIndex: 10, + envVarConstant: "LOGICAL_BACKUP_S3_BUCKET", + envVarValue: dummyBucket, + }, + { + envIndex: 11, + envVarConstant: "LOGICAL_BACKUP_S3_BUCKET_PREFIX", + envVarValue: "spilo", + }, + { + envIndex: 12, + envVarConstant: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", + envVarValue: "/" + dummyUUID, + }, + { + envIndex: 13, + envVarConstant: "LOGICAL_BACKUP_S3_REGION", + envVarValue: "eu-central-1", + }, + { + envIndex: 14, + envVarConstant: "LOGICAL_BACKUP_S3_ENDPOINT", + envVarValue: "", + }, + { + envIndex: 15, + envVarConstant: "LOGICAL_BACKUP_S3_SSE", + envVarValue: "", + }, + { + envIndex: 16, + envVarConstant: "LOGICAL_BACKUP_S3_RETENTION_TIME", + envVarValue: "1 month", + }, + } + + expectedLogicalBackupGCPCreds := []ExpectedValue{ + { + envIndex: 9, + envVarConstant: "LOGICAL_BACKUP_PROVIDER", + envVarValue: "gcs", + }, + { + envIndex: 13, + envVarConstant: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS", + envVarValue: "some-path-to-credentials", + }, + } + + expectedLogicalBackupAzureStorage := []ExpectedValue{ + { + envIndex: 9, + envVarConstant: "LOGICAL_BACKUP_PROVIDER", + envVarValue: "az", + }, + { + envIndex: 13, + envVarConstant: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME", + envVarValue: "some-azure-storage-account-name", + }, + { + envIndex: 14, + envVarConstant: "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER", + envVarValue: "some-azure-storage-container", + }, + { + envIndex: 15, + envVarConstant: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY", + envVarValue: "some-azure-storage-account-key", + }, + } + + expectedLogicalBackupRetentionTime := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "LOGICAL_BACKUP_S3_RETENTION_TIME", + envVarValue: "3 months", + }, + } + + tests := []struct { + subTest string + opConfig config.Config + expectedValues []ExpectedValue + pgsql acidv1.Postgresql + }{ + { + subTest: "logical backup with provider: s3", + opConfig: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupProvider: "s3", + LogicalBackupS3Bucket: dummyBucket, + LogicalBackupS3BucketPrefix: "spilo", + LogicalBackupS3Region: "eu-central-1", + LogicalBackupS3RetentionTime: "1 month", + }, + }, + expectedValues: expectedLogicalBackupS3Bucket, + }, + { + subTest: "logical backup with provider: gcs", + opConfig: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupProvider: "gcs", + LogicalBackupS3Bucket: dummyBucket, + LogicalBackupGoogleApplicationCredentials: "some-path-to-credentials", + }, + }, + expectedValues: expectedLogicalBackupGCPCreds, + }, + { + subTest: "logical backup with provider: az", + opConfig: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupProvider: "az", + LogicalBackupS3Bucket: dummyBucket, + LogicalBackupAzureStorageAccountName: "some-azure-storage-account-name", + LogicalBackupAzureStorageContainer: "some-azure-storage-container", + LogicalBackupAzureStorageAccountKey: "some-azure-storage-account-key", + }, + }, + expectedValues: expectedLogicalBackupAzureStorage, + }, + { + subTest: "will override retention time parameter", + opConfig: config.Config{ + LogicalBackup: config.LogicalBackup{ + LogicalBackupProvider: "s3", + LogicalBackupS3RetentionTime: "1 month", + }, + }, + expectedValues: expectedLogicalBackupRetentionTime, + pgsql: acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + LogicalBackupRetention: "3 months", + }, + }, + }, + } + + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + pgsql := tt.pgsql + c.Postgresql = pgsql + c.UID = types.UID(dummyUUID) + + actualEnvs := c.generateLogicalBackupPodEnvVars() + + for _, ev := range tt.expectedValues { + env := actualEnvs[ev.envIndex] + + if env.Name != ev.envVarConstant { + t.Errorf("%s %s: expected env name %s, have %s instead", + t.Name(), tt.subTest, ev.envVarConstant, env.Name) + } + + if ev.envVarValueRef != nil { + if !reflect.DeepEqual(env.ValueFrom, ev.envVarValueRef) { + t.Errorf("%s %s: expected env value reference %#v, have %#v instead", + t.Name(), tt.subTest, ev.envVarValueRef, env.ValueFrom) + } + continue + } + + if env.Value != ev.envVarValue { + t.Errorf("%s %s: expected env value %s, have %s instead", + t.Name(), tt.subTest, ev.envVarValue, env.Value) + } + } + } +} + +func TestGenerateCapabilities(t *testing.T) { tests := []struct { subTest string configured []string @@ -1499,7 +4483,60 @@ func TestGenerateCapabilities(t *testing.T) { caps := generateCapabilities(tt.configured) if !reflect.DeepEqual(caps, tt.capabilities) { t.Errorf("%s %s: expected `%v` but got `%v`", - testName, tt.subTest, tt.capabilities, caps) + t.Name(), tt.subTest, tt.capabilities, caps) } } } + +func TestTopologySpreadConstraints(t *testing.T) { + clusterName := "acid-test-cluster" + namespace := "default" + labelSelector := &metav1.LabelSelector{ + MatchLabels: cluster.labelsSet(true), + } + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + ResourceLimits: acidv1.ResourceDescription{CPU: k8sutil.StringToPointer("1"), Memory: k8sutil.StringToPointer("10")}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "topology.kubernetes.io/zone", + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: labelSelector, + }, + }, + }, + } + + cluster := New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + cluster.Name = clusterName + cluster.Namespace = namespace + cluster.labelsSet(true) + + s, err := cluster.generateStatefulSet(&pg.Spec) + assert.NoError(t, err) + assert.Contains(t, s.Spec.Template.Spec.TopologySpreadConstraints, v1.TopologySpreadConstraint{ + MaxSkew: int32(1), + TopologyKey: "topology.kubernetes.io/zone", + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: labelSelector, + }, + ) +} diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index c997a675a..6995c50b5 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -1,39 +1,43 @@ package cluster import ( + "context" + "encoding/json" "fmt" + "strconv" + "strings" + "github.com/Masterminds/semver/v3" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) -// VersionMap Map of version numbers -var VersionMap = map[string]int{ - "9.5": 90500, - "9.6": 90600, - "10": 100000, - "11": 110000, - "12": 120000, - "13": 130000, -} +const ( + majorVersionUpgradeSuccessAnnotation = "last-major-upgrade-success" + majorVersionUpgradeFailureAnnotation = "last-major-upgrade-failure" +) // IsBiggerPostgresVersion Compare two Postgres version numbers func IsBiggerPostgresVersion(old string, new string) bool { - oldN := VersionMap[old] - newN := VersionMap[new] + oldN, _ := strconv.Atoi(old) + newN, _ := strconv.Atoi(new) return newN > oldN } // GetDesiredMajorVersionAsInt Convert string to comparable integer of PG version func (c *Cluster) GetDesiredMajorVersionAsInt() int { - return VersionMap[c.GetDesiredMajorVersion()] + version, _ := strconv.Atoi(c.GetDesiredMajorVersion()) + return version * 10000 } // GetDesiredMajorVersion returns major version to use, incl. potential auto upgrade func (c *Cluster) GetDesiredMajorVersion() string { if c.Config.OpConfig.MajorVersionUpgradeMode == "full" { - // current is 9.5, minimal is 11 allowing 11 to 13 clusters, everything below is upgraded + // e.g. current is 14, minimal is 14 allowing 14 to 18 clusters, everything below is upgraded if IsBiggerPostgresVersion(c.Spec.PgVersion, c.Config.OpConfig.MinimalMajorVersion) { c.logger.Infof("overwriting configured major version %s to %s", c.Spec.PgVersion, c.Config.OpConfig.TargetMajorVersion) return c.Config.OpConfig.TargetMajorVersion @@ -43,15 +47,92 @@ func (c *Cluster) GetDesiredMajorVersion() string { return c.Spec.PgVersion } +func (c *Cluster) isUpgradeAllowedForTeam(owningTeam string) bool { + allowedTeams := c.OpConfig.MajorVersionUpgradeTeamAllowList + + if len(allowedTeams) == 0 { + return false + } + + return util.SliceContains(allowedTeams, owningTeam) +} + +func (c *Cluster) annotatePostgresResource(isSuccess bool) error { + annotations := make(map[string]string) + currentTime := metav1.Now().Format("2006-01-02T15:04:05Z") + if isSuccess { + annotations[majorVersionUpgradeSuccessAnnotation] = currentTime + } else { + annotations[majorVersionUpgradeFailureAnnotation] = currentTime + } + patchData, err := metaAnnotationsPatch(annotations) + if err != nil { + c.logger.Errorf("could not form patch for %s postgresql resource: %v", c.Name, err) + return err + } + _, err = c.KubeClient.Postgresqls(c.Namespace).Patch(context.Background(), c.Name, types.MergePatchType, patchData, metav1.PatchOptions{}) + if err != nil { + c.logger.Errorf("failed to patch annotations to postgresql resource: %v", err) + return err + } + return nil +} + +func (c *Cluster) removeFailuresAnnotation() error { + annotationToRemove := []map[string]string{ + { + "op": "remove", + "path": fmt.Sprintf("/metadata/annotations/%s", majorVersionUpgradeFailureAnnotation), + }, + } + removePatch, err := json.Marshal(annotationToRemove) + if err != nil { + c.logger.Errorf("could not form removal patch for %s postgresql resource: %v", c.Name, err) + return err + } + _, err = c.KubeClient.Postgresqls(c.Namespace).Patch(context.Background(), c.Name, types.JSONPatchType, removePatch, metav1.PatchOptions{}) + if err != nil { + c.logger.Errorf("failed to remove annotations from postgresql resource: %v", err) + return err + } + return nil +} + +func (c *Cluster) criticalOperationLabel(pods []v1.Pod, value *string) error { + metadataReq := map[string]map[string]map[string]*string{"metadata": {"labels": {"critical-operation": value}}} + + patchReq, err := json.Marshal(metadataReq) + if err != nil { + return fmt.Errorf("could not marshal ObjectMeta: %v", err) + } + for _, pod := range pods { + _, err = c.KubeClient.Pods(c.Namespace).Patch(context.TODO(), pod.Name, types.StrategicMergePatchType, patchReq, metav1.PatchOptions{}) + if err != nil { + return err + } + } + return nil +} + +/* +Execute upgrade when mode is set to manual or full or when the owning team is allowed for upgrade (and mode is "off"). + +Manual upgrade means, it is triggered by the user via manifest version change +Full upgrade means, operator also determines the minimal version used accross all clusters and upgrades violators. +*/ func (c *Cluster) majorVersionUpgrade() error { - if c.OpConfig.MajorVersionUpgradeMode == "off" { + if c.OpConfig.MajorVersionUpgradeMode == "off" && !c.isUpgradeAllowedForTeam(c.Spec.TeamID) { return nil } desiredVersion := c.GetDesiredMajorVersionAsInt() if c.currentMajorVersion >= desiredVersion { + if _, exists := c.ObjectMeta.Annotations[majorVersionUpgradeFailureAnnotation]; exists { // if failure annotation exists, remove it + c.removeFailuresAnnotation() + c.logger.Infof("removing failure annotation as the cluster is already up to date") + } c.logger.Infof("cluster version up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion) return nil } @@ -62,40 +143,142 @@ func (c *Cluster) majorVersionUpgrade() error { } allRunning := true + isStandbyCluster := false var masterPod *v1.Pod - for _, pod := range pods { + for i, pod := range pods { ps, _ := c.patroni.GetMemberData(&pod) + if ps.Role == "standby_leader" { + isStandbyCluster = true + c.currentMajorVersion = ps.ServerVersion + break + } + if ps.State != "running" { allRunning = false c.logger.Infof("identified non running pod, potentially skipping major version upgrade") } - if ps.Role == "master" { - masterPod = &pod + if ps.Role == "master" || ps.Role == "primary" { + masterPod = &pods[i] c.currentMajorVersion = ps.ServerVersion } } + if masterPod == nil { + c.logger.Infof("no master in the cluster, skipping major version upgrade") + return nil + } + + // Recheck version with newest data from Patroni + if c.currentMajorVersion >= desiredVersion { + if _, exists := c.ObjectMeta.Annotations[majorVersionUpgradeFailureAnnotation]; exists { // if failure annotation exists, remove it + c.removeFailuresAnnotation() + c.logger.Infof("removing failure annotation as the cluster is already up to date") + } + c.logger.Infof("recheck cluster version is already up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion) + return nil + } else if isStandbyCluster { + c.logger.Warnf("skipping major version upgrade for %s/%s standby cluster. Re-deploy standby cluster with the required Postgres version specified", c.Namespace, c.Name) + return nil + } + + if _, exists := c.ObjectMeta.Annotations[majorVersionUpgradeFailureAnnotation]; exists { + c.logger.Infof("last major upgrade failed, skipping upgrade") + return nil + } + + if !c.isInMaintenanceWindow(c.Spec.MaintenanceWindows) { + c.logger.Infof("skipping major version upgrade, not in maintenance window") + return nil + } + + members, err := c.patroni.GetClusterMembers(masterPod) + if err != nil { + c.logger.Error("could not get cluster members data from Patroni API, skipping major version upgrade") + return err + } + patroniData, err := c.patroni.GetMemberData(masterPod) + if err != nil { + c.logger.Error("could not get members data from Patroni API, skipping major version upgrade") + return err + } + patroniVer, err := semver.NewVersion(patroniData.Patroni.Version) + if err != nil { + c.logger.Error("error parsing Patroni version") + patroniVer, _ = semver.NewVersion("3.0.4") + } + verConstraint, _ := semver.NewConstraint(">= 3.0.4") + checkStreaming, _ := verConstraint.Validate(patroniVer) + + for _, member := range members { + if PostgresRole(member.Role) == Leader { + continue + } + if checkStreaming && member.State != "streaming" { + c.logger.Infof("skipping major version upgrade, replica %s is not streaming from primary", member.Name) + return nil + } + if member.Lag > 16*1024*1024 { + c.logger.Infof("skipping major version upgrade, replication lag on member %s is too high", member.Name) + return nil + } + } + + isUpgradeSuccess := true numberOfPods := len(pods) - if allRunning && masterPod != nil { + if allRunning { c.logger.Infof("healthy cluster ready to upgrade, current: %d desired: %d", c.currentMajorVersion, desiredVersion) if c.currentMajorVersion < desiredVersion { - podName := &spec.NamespacedName{Namespace: masterPod.Namespace, Name: masterPod.Name} - c.logger.Infof("triggering major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Starting major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) - upgradeCommand := fmt.Sprintf("/usr/bin/python3 /scripts/inplace_upgrade.py %d 2>&1 | tee last_upgrade.log", numberOfPods) - - result, err := c.ExecCommand(podName, "/bin/su", "postgres", "-c", upgradeCommand) - if err != nil { - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, err) - return err + defer func() error { + if err = c.criticalOperationLabel(pods, nil); err != nil { + return fmt.Errorf("failed to remove critical-operation label: %s", err) + } + return nil + }() + val := "true" + if err = c.criticalOperationLabel(pods, &val); err != nil { + return fmt.Errorf("failed to assign critical-operation label: %s", err) } - c.logger.Infof("upgrade action triggered and command completed: %s", result[:50]) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Upgrade from %d to %d finished", c.currentMajorVersion, desiredVersion) + podName := &spec.NamespacedName{Namespace: masterPod.Namespace, Name: masterPod.Name} + c.logger.Infof("triggering major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "starting major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) + upgradeCommand := fmt.Sprintf("set -o pipefail && /usr/bin/python3 /scripts/inplace_upgrade.py %d 2>&1 | tee last_upgrade.log", numberOfPods) + + c.logger.Debug("checking if the spilo image runs with root or non-root (check for user id=0)") + resultIdCheck, errIdCheck := c.ExecCommand(podName, "/bin/bash", "-c", "/usr/bin/id -u") + if errIdCheck != nil { + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "checking user id to run upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, errIdCheck) + } + + resultIdCheck = strings.TrimSuffix(resultIdCheck, "\n") + var result, scriptErrMsg string + if resultIdCheck != "0" { + c.logger.Infof("user id was identified as: %s, hence default user is non-root already", resultIdCheck) + result, err = c.ExecCommand(podName, "/bin/bash", "-c", upgradeCommand) + scriptErrMsg, _ = c.ExecCommand(podName, "/bin/bash", "-c", "tail -n 1 last_upgrade.log") + } else { + c.logger.Infof("user id was identified as: %s, using su to reach the postgres user", resultIdCheck) + result, err = c.ExecCommand(podName, "/bin/su", "postgres", "-c", upgradeCommand) + scriptErrMsg, _ = c.ExecCommand(podName, "/bin/bash", "-c", "tail -n 1 last_upgrade.log") + } + if err != nil { + isUpgradeSuccess = false + c.annotatePostgresResource(isUpgradeSuccess) + c.logger.Errorf("upgrade action triggered but command failed: %v", err) + if strings.TrimSpace(scriptErrMsg) == "" { + scriptErrMsg = err.Error() + } + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, scriptErrMsg) + return fmt.Errorf("%s", scriptErrMsg) + } + + c.annotatePostgresResource(isUpgradeSuccess) + c.logger.Infof("upgrade action triggered and command completed: %s", result[:100]) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "upgrade from %d to %d finished", c.currentMajorVersion, desiredVersion) } } diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index c8dfe2a13..6658ba414 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -3,7 +3,8 @@ package cluster import ( "context" "fmt" - "math/rand" + "slices" + "sort" "strconv" "time" @@ -12,6 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/patroni" @@ -56,7 +58,7 @@ func (c *Cluster) markRollingUpdateFlagForPod(pod *v1.Pod, msg string) error { return nil } - c.logger.Debugf("mark rolling update annotation for %s: reason %s", pod.Name, msg) + c.logger.Infof("mark rolling update annotation for %s: reason %s", pod.Name, msg) flag := make(map[string]string) flag[rollingUpdatePodAnnotationKey] = strconv.FormatBool(true) @@ -107,7 +109,7 @@ func (c *Cluster) getRollingUpdateFlagFromPod(pod *v1.Pod) (flag bool) { } func (c *Cluster) deletePods() error { - c.logger.Debugln("deleting pods") + c.logger.Debug("deleting pods") pods, err := c.listPods() if err != nil { return err @@ -124,9 +126,9 @@ func (c *Cluster) deletePods() error { } } if len(pods) > 0 { - c.logger.Debugln("pods have been deleted") + c.logger.Debug("pods have been deleted") } else { - c.logger.Debugln("no pods to delete") + c.logger.Debug("no pods to delete") } return nil @@ -149,12 +151,13 @@ func (c *Cluster) unregisterPodSubscriber(podName spec.NamespacedName) { c.podSubscribersMu.Lock() defer c.podSubscribersMu.Unlock() - if _, ok := c.podSubscribers[podName]; !ok { + ch, ok := c.podSubscribers[podName] + if !ok { panic("subscriber for pod '" + podName.String() + "' is not found") } - close(c.podSubscribers[podName]) delete(c.podSubscribers, podName) + close(ch) } func (c *Cluster) registerPodSubscriber(podName spec.NamespacedName) chan PodEvent { @@ -209,57 +212,24 @@ func (c *Cluster) movePodFromEndOfLifeNode(pod *v1.Pod) (*v1.Pod, error) { return newPod, nil } -func (c *Cluster) masterCandidate(oldNodeName string) (*v1.Pod, error) { - - // Wait until at least one replica pod will come up - if err := c.waitForAnyReplicaLabelReady(); err != nil { - c.logger.Warningf("could not find at least one ready replica: %v", err) - } - - replicas, err := c.getRolePods(Replica) - if err != nil { - return nil, fmt.Errorf("could not get replica pods: %v", err) - } - - if len(replicas) == 0 { - c.logger.Warningf("no available master candidates, migration will cause longer downtime of Postgres cluster") - return nil, nil - } - - for i, pod := range replicas { - // look for replicas running on live nodes. Ignore errors when querying the nodes. - if pod.Spec.NodeName != oldNodeName { - eol, err := c.podIsEndOfLife(&pod) - if err == nil && !eol { - return &replicas[i], nil - } - } - } - c.logger.Warningf("no available master candidates on live nodes") - return &replicas[rand.Intn(len(replicas))], nil -} - // MigrateMasterPod migrates master pod via failover to a replica func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { var ( - masterCandidatePod *v1.Pod - err error - eol bool + err error + eol bool ) oldMaster, err := c.KubeClient.Pods(podName.Namespace).Get(context.TODO(), podName.Name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("could not get pod: %v", err) + return fmt.Errorf("could not get master pod: %v", err) } c.logger.Infof("starting process to migrate master pod %q", podName) - if eol, err = c.podIsEndOfLife(oldMaster); err != nil { return fmt.Errorf("could not get node %q: %v", oldMaster.Spec.NodeName, err) } if !eol { - c.logger.Debugf("no action needed: master pod is already on a live node") + c.logger.Debug("no action needed: master pod is already on a live node") return nil } @@ -278,11 +248,17 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { } c.Statefulset = sset } - // We may not have a cached statefulset if the initial cluster sync has aborted, revert to the spec in that case. + // we may not have a cached statefulset if the initial cluster sync has aborted, revert to the spec in that case + masterCandidateName := podName + masterCandidatePod := oldMaster if *c.Statefulset.Spec.Replicas > 1 { - if masterCandidatePod, err = c.masterCandidate(oldMaster.Spec.NodeName); err != nil { + if masterCandidateName, err = c.getSwitchoverCandidate(oldMaster); err != nil { return fmt.Errorf("could not find suitable replica pod as candidate for failover: %v", err) } + masterCandidatePod, err = c.KubeClient.Pods(masterCandidateName.Namespace).Get(context.TODO(), masterCandidateName.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("could not get master candidate pod: %v", err) + } } else { c.logger.Warningf("migrating single pod cluster %q, this will cause downtime of the Postgres cluster until pod is back", c.clusterName()) } @@ -299,13 +275,28 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { return nil } - if masterCandidatePod, err = c.movePodFromEndOfLifeNode(masterCandidatePod); err != nil { + if _, err = c.movePodFromEndOfLifeNode(masterCandidatePod); err != nil { return fmt.Errorf("could not move pod: %v", err) } - masterCandidateName := util.NameFromMeta(masterCandidatePod.ObjectMeta) - if err := c.Switchover(oldMaster, masterCandidateName); err != nil { - return fmt.Errorf("could not failover to pod %q: %v", masterCandidateName, err) + scheduleSwitchover := false + if !c.isInMaintenanceWindow(c.Spec.MaintenanceWindows) { + c.logger.Infof("postponing switchover, not in maintenance window") + scheduleSwitchover = true + } + err = retryutil.Retry(1*time.Minute, 5*time.Minute, + func() (bool, error) { + err := c.Switchover(oldMaster, masterCandidateName, scheduleSwitchover) + if err != nil { + c.logger.Errorf("could not switchover to pod %q: %v", masterCandidateName, err) + return false, nil + } + return true, nil + }, + ) + + if err != nil { + return fmt.Errorf("could not migrate master pod: %v", err) } return nil @@ -337,10 +328,89 @@ func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName st return nil } +func (c *Cluster) getPatroniConfig(pod *v1.Pod) (acidv1.Patroni, map[string]string, error) { + var ( + patroniConfig acidv1.Patroni + pgParameters map[string]string + ) + podName := util.NameFromMeta(pod.ObjectMeta) + err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, + func() (bool, error) { + var err error + patroniConfig, pgParameters, err = c.patroni.GetConfig(pod) + + if err != nil { + return false, err + } + return true, nil + }, + ) + + if err != nil { + return acidv1.Patroni{}, nil, fmt.Errorf("could not get Postgres config from pod %s: %v", podName, err) + } + + return patroniConfig, pgParameters, nil +} + +func (c *Cluster) getPatroniMemberData(pod *v1.Pod) (patroni.MemberData, error) { + var memberData patroni.MemberData + err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, + func() (bool, error) { + var err error + memberData, err = c.patroni.GetMemberData(pod) + + if err != nil { + return false, err + } + return true, nil + }, + ) + if err != nil { + return patroni.MemberData{}, fmt.Errorf("could not get member data: %v", err) + } + if memberData.State == "creating replica" { + return patroni.MemberData{}, fmt.Errorf("replica currently being initialized") + } + + return memberData, nil +} + +// podIsNotRunning returns true if a pod is known to be in a non-running state, +// e.g. stuck in CreateContainerConfigError, CrashLoopBackOff, ImagePullBackOff, etc. +// Pods with no status information are not considered non-running, as they may +// simply not have reported status yet. +func podIsNotRunning(pod *v1.Pod) bool { + if pod.Status.Phase == "" { + // No status reported yet — don't treat as non-running + return false + } + if pod.Status.Phase != v1.PodRunning { + return true + } + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil || cs.State.Terminated != nil { + return true + } + } + return false +} + +// allPodsRunning returns true only if every pod in the list is in a healthy running state. +func (c *Cluster) allPodsRunning(pods []v1.Pod) bool { + for i := range pods { + if podIsNotRunning(&pods[i]) { + return false + } + } + return true +} + func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { + stopCh := make(chan struct{}) ch := c.registerPodSubscriber(podName) defer c.unregisterPodSubscriber(podName) - stopChan := make(chan struct{}) + defer close(stopCh) err := retryutil.Retry(1*time.Second, 5*time.Second, func() (bool, error) { @@ -360,7 +430,7 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { if err := c.waitForPodDeletion(ch); err != nil { return nil, err } - pod, err := c.waitForPodLabel(ch, stopChan, nil) + pod, err := c.waitForPodLabel(ch, stopCh, nil) if err != nil { return nil, err } @@ -368,54 +438,10 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { return pod, nil } -func (c *Cluster) isSafeToRecreatePods(pods []v1.Pod) bool { - - /* - Operator should not re-create pods if there is at least one replica being bootstrapped - because Patroni might use other replicas to take basebackup from (see Patroni's "clonefrom" tag). - - XXX operator cannot forbid replica re-init, so we might still fail if re-init is started - after this check succeeds but before a pod is re-created - */ - for _, pod := range pods { - c.logger.Debugf("name=%s phase=%s ip=%s", pod.Name, pod.Status.Phase, pod.Status.PodIP) - } - - for _, pod := range pods { - - var data patroni.MemberData - - err := retryutil.Retry(1*time.Second, 5*time.Second, - func() (bool, error) { - var err error - data, err = c.patroni.GetMemberData(&pod) - - if err != nil { - return false, err - } - return true, nil - }, - ) - - if err != nil { - c.logger.Errorf("failed to get Patroni state for pod: %s", err) - return false - } else if data.State == "creating replica" { - c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name) - return false - } - } - return true -} - func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.NamespacedName) error { c.setProcessName("starting to recreate pods") c.logger.Infof("there are %d pods in the cluster to recreate", len(pods)) - if !c.isSafeToRecreatePods(pods) { - return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initialized") - } - var ( masterPod, newMasterPod *v1.Pod ) @@ -429,16 +455,17 @@ func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.Namesp continue } - podName := util.NameFromMeta(pod.ObjectMeta) + podName := util.NameFromMeta(pods[i].ObjectMeta) newPod, err := c.recreatePod(podName) if err != nil { return fmt.Errorf("could not recreate replica pod %q: %v", util.NameFromMeta(pod.ObjectMeta), err) } newRole := PostgresRole(newPod.Labels[c.OpConfig.PodRoleLabel]) - if newRole == Replica { + switch newRole { + case Replica: replicas = append(replicas, util.NameFromMeta(pod.ObjectMeta)) - } else if newRole == Master { + case Master: newMasterPod = newPod } } @@ -447,12 +474,21 @@ func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.Namesp // switchover if // 1. we have not observed a new master pod when re-creating former replicas // 2. we know possible switchover targets even when no replicas were recreated - if newMasterPod == nil && len(replicas) > 0 { - if err := c.Switchover(masterPod, masterCandidate(replicas)); err != nil { - c.logger.Warningf("could not perform switch over: %v", err) + // 3. the master pod is actually running (can't switchover a dead master) + if newMasterPod == nil && len(replicas) > 0 && !podIsNotRunning(masterPod) { + masterCandidate, err := c.getSwitchoverCandidate(masterPod) + if err != nil { + // do not recreate master now so it will keep the update flag and switchover will be retried on next sync + return fmt.Errorf("skipping switchover: %v", err) + } + if err := c.Switchover(masterPod, masterCandidate, false); err != nil { + return fmt.Errorf("could not perform switch over: %v", err) } } else if newMasterPod == nil && len(replicas) == 0 { c.logger.Warningf("cannot perform switch over before re-creating the pod: no replicas") + } else if podIsNotRunning(masterPod) { + c.logger.Warningf("master pod %q is not running, skipping switchover and recreating directly", + util.NameFromMeta(masterPod.ObjectMeta)) } c.logger.Infof("recreating old master pod %q", util.NameFromMeta(masterPod.ObjectMeta)) @@ -464,6 +500,67 @@ func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.Namesp return nil } +func (c *Cluster) getSwitchoverCandidate(master *v1.Pod) (spec.NamespacedName, error) { + + var members []patroni.ClusterMember + candidates := make([]patroni.ClusterMember, 0) + syncCandidates := make([]patroni.ClusterMember, 0) + + err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, + func() (bool, error) { + var err error + members, err = c.patroni.GetClusterMembers(master) + if err != nil { + return false, err + } + + // look for SyncStandby candidates (which also implies pod is in running state) + for _, member := range members { + if PostgresRole(member.Role) == SyncStandby { + syncCandidates = append(syncCandidates, member) + } + if PostgresRole(member.Role) != Leader && PostgresRole(member.Role) != StandbyLeader && slices.Contains([]string{"running", "streaming", "in archive recovery"}, member.State) { + candidates = append(candidates, member) + } + } + + // if synchronous mode is enabled and no SyncStandy was found + // return false for retry - cannot failover with no sync candidate + if c.Spec.Patroni.SynchronousMode && len(syncCandidates) == 0 { + c.logger.Warnf("no sync standby found - retrying fetching cluster members") + return false, nil + } + + // retry also in asynchronous mode when no replica candidate was found + if !c.Spec.Patroni.SynchronousMode && len(candidates) == 0 { + c.logger.Warnf("no replica candidate found - retrying fetching cluster members") + return false, nil + } + + return true, nil + }, + ) + if err != nil { + return spec.NamespacedName{}, fmt.Errorf("failed to get Patroni cluster members: %s", err) + } + + // pick candidate with lowest lag + if len(syncCandidates) > 0 { + sort.Slice(syncCandidates, func(i, j int) bool { + return syncCandidates[i].Lag < syncCandidates[j].Lag + }) + return spec.NamespacedName{Namespace: master.Namespace, Name: syncCandidates[0].Name}, nil + } + if len(candidates) > 0 { + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].Lag < candidates[j].Lag + }) + return spec.NamespacedName{Namespace: master.Namespace, Name: candidates[0].Name}, nil + } + + return spec.NamespacedName{}, fmt.Errorf("no switchover candidate found") +} + func (c *Cluster) podIsEndOfLife(pod *v1.Pod) (bool, error) { node, err := c.KubeClient.Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{}) if err != nil { diff --git a/pkg/cluster/pod_test.go b/pkg/cluster/pod_test.go new file mode 100644 index 000000000..6ab3f9207 --- /dev/null +++ b/pkg/cluster/pod_test.go @@ -0,0 +1,414 @@ +package cluster + +import ( + "bytes" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/zalando/postgres-operator/mocks" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "github.com/zalando/postgres-operator/pkg/util/patroni" + v1 "k8s.io/api/core/v1" +) + +func TestGetSwitchoverCandidate(t *testing.T) { + testName := "test getting right switchover candidate" + namespace := "default" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + var cluster = New( + Config{ + OpConfig: config.Config{ + PatroniAPICheckInterval: time.Duration(1), + PatroniAPICheckTimeout: time.Duration(5), + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + // simulate different member scenarios + tests := []struct { + subtest string + clusterJson string + syncModeEnabled bool + expectedCandidate spec.NamespacedName + expectedError error + }{ + { + subtest: "choose sync_standby over replica", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "streaming", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, {"name": "acid-test-cluster-2", "role": "replica", "state": "streaming", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 0}]}`, + syncModeEnabled: true, + expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-1"}, + expectedError: nil, + }, + { + subtest: "no running sync_standby available", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "replica", "state": "streaming", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}]}`, + syncModeEnabled: true, + expectedCandidate: spec.NamespacedName{}, + expectedError: fmt.Errorf("failed to get Patroni cluster members: unexpected end of JSON input"), + }, + { + subtest: "choose replica with lowest lag", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "replica", "state": "streaming", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 5}, {"name": "acid-test-cluster-2", "role": "replica", "state": "streaming", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 2}]}`, + syncModeEnabled: false, + expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-2"}, + expectedError: nil, + }, + { + subtest: "choose first replica when lag is equal everywhere", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "replica", "state": "streaming", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 5}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 5}]}`, + syncModeEnabled: false, + expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-1"}, + expectedError: nil, + }, + { + subtest: "no running replica available", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 2}, {"name": "acid-test-cluster-1", "role": "replica", "state": "starting", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 2}]}`, + syncModeEnabled: false, + expectedCandidate: spec.NamespacedName{}, + expectedError: fmt.Errorf("failed to get Patroni cluster members: unexpected end of JSON input"), + }, + { + subtest: "replicas with different status", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "replica", "state": "streaming", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 5}, {"name": "acid-test-cluster-2", "role": "replica", "state": "in archive recovery", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 2}]}`, + syncModeEnabled: false, + expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-2"}, + expectedError: nil, + }, + } + + for _, tt := range tests { + // mocking cluster members + r := io.NopCloser(bytes.NewReader([]byte(tt.clusterJson))) + + response := http.Response{ + StatusCode: 200, + Body: r, + } + + mockClient := mocks.NewMockHTTPClient(ctrl) + mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil).AnyTimes() + + p := patroni.New(patroniLogger, mockClient) + cluster.patroni = p + mockMasterPod := newMockPod("192.168.100.1") + mockMasterPod.Namespace = namespace + cluster.Spec.Patroni.SynchronousMode = tt.syncModeEnabled + + candidate, err := cluster.getSwitchoverCandidate(mockMasterPod) + if err != nil && err.Error() != tt.expectedError.Error() { + t.Errorf("%s - %s: unexpected error, %v", testName, tt.subtest, err) + } + + if candidate != tt.expectedCandidate { + t.Errorf("%s - %s: unexpect switchover candidate, got %s, expected %s", testName, tt.subtest, candidate, tt.expectedCandidate) + } + } +} + +func TestPodIsNotRunning(t *testing.T) { + tests := []struct { + subtest string + pod v1.Pod + expected bool + }{ + { + subtest: "pod with no status reported yet", + pod: v1.Pod{ + Status: v1.PodStatus{}, + }, + expected: false, + }, + { + subtest: "pod running with all containers ready", + pod: v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + }, + expected: false, + }, + { + subtest: "pod in pending phase", + pod: v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodPending, + }, + }, + expected: true, + }, + { + subtest: "pod running but container in CreateContainerConfigError", + pod: v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: "CreateContainerConfigError", + Message: `secret "some-secret" not found`, + }, + }, + }, + }, + }, + }, + expected: true, + }, + { + subtest: "pod running but container in CrashLoopBackOff", + pod: v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: "CrashLoopBackOff", + }, + }, + }, + }, + }, + }, + expected: true, + }, + { + subtest: "pod running but container terminated", + pod: v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + ExitCode: 137, + }, + }, + }, + }, + }, + }, + expected: true, + }, + { + subtest: "pod running with mixed container states - one healthy one broken", + pod: v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: "CreateContainerConfigError", + }, + }, + }, + }, + }, + }, + expected: true, + }, + { + subtest: "pod in failed phase", + pod: v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodFailed, + }, + }, + expected: true, + }, + { + subtest: "pod running with multiple healthy containers", + pod: v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + { + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + }, + expected: false, + }, + { + subtest: "pod running with ImagePullBackOff", + pod: v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: "ImagePullBackOff", + }, + }, + }, + }, + }, + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.subtest, func(t *testing.T) { + result := podIsNotRunning(&tt.pod) + if result != tt.expected { + t.Errorf("podIsNotRunning() = %v, expected %v", result, tt.expected) + } + }) + } +} + +func TestAllPodsRunning(t *testing.T) { + client, _ := newFakeK8sSyncClient() + + var cluster = New( + Config{ + OpConfig: config.Config{ + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + PodRoleLabel: "spilo-role", + }, + }, + }, client, acidv1.Postgresql{}, logger, eventRecorder) + + tests := []struct { + subtest string + pods []v1.Pod + expected bool + }{ + { + subtest: "all pods running", + pods: []v1.Pod{ + { + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + {State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}}, + }, + }, + }, + { + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + {State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}}, + }, + }, + }, + }, + expected: true, + }, + { + subtest: "one pod not running", + pods: []v1.Pod{ + { + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + {State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}}, + }, + }, + }, + { + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: "CreateContainerConfigError", + }, + }, + }, + }, + }, + }, + }, + expected: false, + }, + { + subtest: "all pods not running", + pods: []v1.Pod{ + { + Status: v1.PodStatus{ + Phase: v1.PodPending, + }, + }, + { + Status: v1.PodStatus{ + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: "CrashLoopBackOff", + }, + }, + }, + }, + }, + }, + }, + expected: false, + }, + { + subtest: "empty pod list", + pods: []v1.Pod{}, + expected: true, + }, + { + subtest: "pods with no status reported yet", + pods: []v1.Pod{ + { + Status: v1.PodStatus{}, + }, + { + Status: v1.PodStatus{}, + }, + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.subtest, func(t *testing.T) { + result := cluster.allPodsRunning(tt.pods) + if result != tt.expected { + t.Errorf("allPodsRunning() = %v, expected %v", result, tt.expected) + } + }) + } +} diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 48b17f532..1fdad3e5e 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -7,9 +7,9 @@ import ( "strings" appsv1 "k8s.io/api/apps/v1" - batchv1beta1 "k8s.io/api/batch/v1beta1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -23,24 +23,47 @@ const ( ) func (c *Cluster) listResources() error { - if c.PodDisruptionBudget != nil { - c.logger.Infof("found pod disruption budget: %q (uid: %q)", util.NameFromMeta(c.PodDisruptionBudget.ObjectMeta), c.PodDisruptionBudget.UID) + if c.PrimaryPodDisruptionBudget != nil { + c.logger.Infof("found primary pod disruption budget: %q (uid: %q)", util.NameFromMeta(c.PrimaryPodDisruptionBudget.ObjectMeta), c.PrimaryPodDisruptionBudget.UID) + } + + if c.CriticalOpPodDisruptionBudget != nil { + c.logger.Infof("found pod disruption budget for critical operations: %q (uid: %q)", util.NameFromMeta(c.CriticalOpPodDisruptionBudget.ObjectMeta), c.CriticalOpPodDisruptionBudget.UID) + } if c.Statefulset != nil { c.logger.Infof("found statefulset: %q (uid: %q)", util.NameFromMeta(c.Statefulset.ObjectMeta), c.Statefulset.UID) } - for _, obj := range c.Secrets { - c.logger.Infof("found secret: %q (uid: %q)", util.NameFromMeta(obj.ObjectMeta), obj.UID) + for appId, stream := range c.Streams { + c.logger.Infof("found stream: %q with application id %q (uid: %q)", util.NameFromMeta(stream.ObjectMeta), appId, stream.UID) + } + + if c.LogicalBackupJob != nil { + c.logger.Infof("found logical backup job: %q (uid: %q)", util.NameFromMeta(c.LogicalBackupJob.ObjectMeta), c.LogicalBackupJob.UID) + } + + for uid, secret := range c.Secrets { + c.logger.Infof("found secret: %q (uid: %q) namespace: %s", util.NameFromMeta(secret.ObjectMeta), uid, secret.ObjectMeta.Namespace) + } + + for role, service := range c.Services { + c.logger.Infof("found %s service: %q (uid: %q)", role, util.NameFromMeta(service.ObjectMeta), service.UID) } for role, endpoint := range c.Endpoints { c.logger.Infof("found %s endpoint: %q (uid: %q)", role, util.NameFromMeta(endpoint.ObjectMeta), endpoint.UID) } - for role, service := range c.Services { - c.logger.Infof("found %s service: %q (uid: %q)", role, util.NameFromMeta(service.ObjectMeta), service.UID) + if c.patroniKubernetesUseConfigMaps() { + for suffix, configmap := range c.PatroniConfigMaps { + c.logger.Infof("found %s Patroni config map: %q (uid: %q)", suffix, util.NameFromMeta(configmap.ObjectMeta), configmap.UID) + } + } else { + for suffix, endpoint := range c.PatroniEndpoints { + c.logger.Infof("found %s Patroni endpoint: %q (uid: %q)", suffix, util.NameFromMeta(endpoint.ObjectMeta), endpoint.UID) + } } pods, err := c.listPods() @@ -52,13 +75,17 @@ func (c *Cluster) listResources() error { c.logger.Infof("found pod: %q (uid: %q)", util.NameFromMeta(obj.ObjectMeta), obj.UID) } - pvcs, err := c.listPersistentVolumeClaims() - if err != nil { - return fmt.Errorf("could not get the list of PVCs: %v", err) + for uid, pvc := range c.VolumeClaims { + c.logger.Infof("found persistent volume claim: %q (uid: %q)", util.NameFromMeta(pvc.ObjectMeta), uid) } - for _, obj := range pvcs { - c.logger.Infof("found PVC: %q (uid: %q)", util.NameFromMeta(obj.ObjectMeta), obj.UID) + for role, poolerObjs := range c.ConnectionPooler { + if poolerObjs.Deployment != nil { + c.logger.Infof("found %s pooler deployment: %q (uid: %q) ", role, util.NameFromMeta(poolerObjs.Deployment.ObjectMeta), poolerObjs.Deployment.UID) + } + if poolerObjs.Service != nil { + c.logger.Infof("found %s pooler service: %q (uid: %q) ", role, util.NameFromMeta(poolerObjs.Service.ObjectMeta), poolerObjs.Service.UID) + } } return nil @@ -67,12 +94,12 @@ func (c *Cluster) listResources() error { func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { c.setProcessName("creating statefulset") // check if it's allowed that spec contains initContainers - if c.Spec.InitContainers != nil && len(c.Spec.InitContainers) > 0 && + if len(c.Spec.InitContainers) > 0 && c.OpConfig.EnableInitContainers != nil && !(*c.OpConfig.EnableInitContainers) { return nil, fmt.Errorf("initContainers specified but disabled in configuration") } // check if it's allowed that spec contains sidecars - if c.Spec.Sidecars != nil && len(c.Spec.Sidecars) > 0 && + if len(c.Spec.Sidecars) > 0 && c.OpConfig.EnableSidecars != nil && !(*c.OpConfig.EnableSidecars) { return nil, fmt.Errorf("sidecar containers specified but disabled in configuration") } @@ -140,8 +167,8 @@ func (c *Cluster) preScaleDown(newStatefulSet *appsv1.StatefulSet) error { return fmt.Errorf("pod %q does not belong to cluster", podName) } - if err := c.patroni.Switchover(&masterPod[0], masterCandidatePod.Name); err != nil { - return fmt.Errorf("could not failover: %v", err) + if err := c.patroni.Switchover(&masterPod[0], masterCandidatePod.Name, ""); err != nil { + return fmt.Errorf("could not switchover: %v", err) } return nil @@ -160,7 +187,7 @@ func (c *Cluster) updateStatefulSet(newStatefulSet *appsv1.StatefulSet) error { c.logger.Warningf("could not scale down: %v", err) } } - c.logger.Debugf("updating statefulset") + c.logger.Debug("updating statefulset") patchData, err := specPatch(newStatefulSet.Spec) if err != nil { @@ -191,7 +218,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { } statefulSetName := util.NameFromMeta(c.Statefulset.ObjectMeta) - c.logger.Debugf("replacing statefulset") + c.logger.Debug("replacing statefulset") // Delete the current statefulset without deleting the pods deletePropagationPolicy := metav1.DeletePropagationOrphan @@ -205,7 +232,7 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { // make sure we clear the stored statefulset status if the subsequent create fails. c.Statefulset = nil // wait until the statefulset is truly deleted - c.logger.Debugf("waiting for the statefulset to be deleted") + c.logger.Debug("waiting for the statefulset to be deleted") err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, func() (bool, error) { @@ -239,15 +266,19 @@ func (c *Cluster) replaceStatefulSet(newStatefulSet *appsv1.StatefulSet) error { func (c *Cluster) deleteStatefulSet() error { c.setProcessName("deleting statefulset") - c.logger.Debugln("deleting statefulset") + c.logger.Debug("deleting statefulset") if c.Statefulset == nil { - return fmt.Errorf("there is no statefulset in the cluster") + c.logger.Debug("there is no statefulset in the cluster") + return nil } err := c.KubeClient.StatefulSets(c.Statefulset.Namespace).Delete(context.TODO(), c.Statefulset.Name, c.deleteOptions) - if err != nil { + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("statefulset %q has already been deleted", util.NameFromMeta(c.Statefulset.ObjectMeta)) + } else if err != nil { return err } + c.logger.Infof("statefulset %q has been deleted", util.NameFromMeta(c.Statefulset.ObjectMeta)) c.Statefulset = nil @@ -255,8 +286,12 @@ func (c *Cluster) deleteStatefulSet() error { return fmt.Errorf("could not delete pods: %v", err) } - if err := c.deletePersistentVolumeClaims(); err != nil { - return fmt.Errorf("could not delete PersistentVolumeClaims: %v", err) + if c.OpConfig.EnablePersistentVolumeClaimDeletion != nil && *c.OpConfig.EnablePersistentVolumeClaimDeletion { + if err := c.deletePersistentVolumeClaims(); err != nil { + return fmt.Errorf("could not delete persistent volume claims: %v", err) + } + } else { + c.logger.Info("not deleting persistent volume claims because disabled in configuration") } return nil @@ -275,82 +310,67 @@ func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) { return service, nil } -func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error { - var ( - svc *v1.Service - err error - ) +func (c *Cluster) updateService(role PostgresRole, oldService *v1.Service, newService *v1.Service) (*v1.Service, error) { + var err error + svc := oldService - c.setProcessName("updating %v service", role) + serviceName := util.NameFromMeta(oldService.ObjectMeta) + match, reason := c.compareServices(oldService, newService) + if !match { + c.logServiceChanges(role, oldService, newService, false, reason) + c.setProcessName("updating %v service", role) - if c.Services[role] == nil { - return fmt.Errorf("there is no service in the cluster") - } - - serviceName := util.NameFromMeta(c.Services[role].ObjectMeta) - - // update the service annotation in order to propagate ELB notation. - if len(newService.ObjectMeta.Annotations) > 0 { - if annotationsPatchData, err := metaAnnotationsPatch(newService.ObjectMeta.Annotations); err == nil { - _, err = c.KubeClient.Services(serviceName.Namespace).Patch( - context.TODO(), - serviceName.Name, - types.MergePatchType, - []byte(annotationsPatchData), - metav1.PatchOptions{}, - "") - - if err != nil { - return fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err) + // now, patch the service spec, but when disabling LoadBalancers do update instead + // patch does not work because of LoadBalancerSourceRanges field (even if set to nil) + oldServiceType := oldService.Spec.Type + newServiceType := newService.Spec.Type + if newServiceType != oldServiceType && oldServiceType == v1.ServiceTypeLoadBalancer { + // Kubernetes rejects updates that change type away from LoadBalancer while + // loadBalancerSourceRanges is still set; clear it before updating + newService.Spec.LoadBalancerSourceRanges = nil + newService.ResourceVersion = oldService.ResourceVersion + if newServiceType == v1.ServiceTypeClusterIP { + newService.Spec.ClusterIP = oldService.Spec.ClusterIP } - } else { - return fmt.Errorf("could not form patch for the service metadata: %v", err) } - } - - // now, patch the service spec, but when disabling LoadBalancers do update instead - // patch does not work because of LoadBalancerSourceRanges field (even if set to nil) - oldServiceType := c.Services[role].Spec.Type - newServiceType := newService.Spec.Type - if newServiceType == "ClusterIP" && newServiceType != oldServiceType { - newService.ResourceVersion = c.Services[role].ResourceVersion - newService.Spec.ClusterIP = c.Services[role].Spec.ClusterIP svc, err = c.KubeClient.Services(serviceName.Namespace).Update(context.TODO(), newService, metav1.UpdateOptions{}) if err != nil { - return fmt.Errorf("could not update service %q: %v", serviceName, err) - } - } else { - patchData, err := specPatch(newService.Spec) - if err != nil { - return fmt.Errorf("could not form patch for the service %q: %v", serviceName, err) - } - - svc, err = c.KubeClient.Services(serviceName.Namespace).Patch( - context.TODO(), serviceName.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "") - if err != nil { - return fmt.Errorf("could not patch service %q: %v", serviceName, err) + return nil, fmt.Errorf("could not update service %q: %v", serviceName, err) } } - c.Services[role] = svc - return nil + if changed, _ := c.compareAnnotations(oldService.Annotations, newService.Annotations, nil); changed { + patchData, err := metaAnnotationsPatch(newService.Annotations) + if err != nil { + return nil, fmt.Errorf("could not form patch for service %q annotations: %v", oldService.Name, err) + } + svc, err = c.KubeClient.Services(serviceName.Namespace).Patch(context.TODO(), newService.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return nil, fmt.Errorf("could not patch annotations for service %q: %v", oldService.Name, err) + } + } + + return svc, nil } func (c *Cluster) deleteService(role PostgresRole) error { - c.logger.Debugf("deleting service %s", role) + c.setProcessName("deleting service") + c.logger.Debugf("deleting %s service", role) - service, ok := c.Services[role] - if !ok { + if c.Services[role] == nil { c.logger.Debugf("No service for %s role was found, nothing to delete", role) return nil } - if err := c.KubeClient.Services(service.Namespace).Delete(context.TODO(), service.Name, c.deleteOptions); err != nil { - return err + if err := c.KubeClient.Services(c.Services[role].Namespace).Delete(context.TODO(), c.Services[role].Name, c.deleteOptions); err != nil { + if !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("could not delete %s service: %v", role, err) + } + c.logger.Debugf("%s service has already been deleted", role) } - c.logger.Infof("%s service %q has been deleted", role, util.NameFromMeta(service.ObjectMeta)) - c.Services[role] = nil + c.logger.Infof("%s service %q has been deleted", role, util.NameFromMeta(c.Services[role].ObjectMeta)) + delete(c.Services, role) return nil } @@ -407,55 +427,128 @@ func (c *Cluster) generateEndpointSubsets(role PostgresRole) []v1.EndpointSubset return result } -func (c *Cluster) createPodDisruptionBudget() (*policybeta1.PodDisruptionBudget, error) { - podDisruptionBudgetSpec := c.generatePodDisruptionBudget() +func (c *Cluster) createPrimaryPodDisruptionBudget() error { + c.logger.Debug("creating primary pod disruption budget") + if c.PrimaryPodDisruptionBudget != nil { + c.logger.Warning("primary pod disruption budget already exists in the cluster") + return nil + } + + podDisruptionBudgetSpec := c.generatePrimaryPodDisruptionBudget() podDisruptionBudget, err := c.KubeClient. PodDisruptionBudgets(podDisruptionBudgetSpec.Namespace). Create(context.TODO(), podDisruptionBudgetSpec, metav1.CreateOptions{}) if err != nil { - return nil, err + return err } - c.PodDisruptionBudget = podDisruptionBudget + c.logger.Infof("primary pod disruption budget %q has been successfully created", util.NameFromMeta(podDisruptionBudget.ObjectMeta)) + c.PrimaryPodDisruptionBudget = podDisruptionBudget - return podDisruptionBudget, nil + return nil } -func (c *Cluster) updatePodDisruptionBudget(pdb *policybeta1.PodDisruptionBudget) error { - if c.PodDisruptionBudget == nil { - return fmt.Errorf("there is no pod disruption budget in the cluster") +func (c *Cluster) createCriticalOpPodDisruptionBudget() error { + c.logger.Debug("creating pod disruption budget for critical operations") + if c.CriticalOpPodDisruptionBudget != nil { + c.logger.Warning("pod disruption budget for critical operations already exists in the cluster") + return nil } - if err := c.deletePodDisruptionBudget(); err != nil { - return fmt.Errorf("could not delete pod disruption budget: %v", err) + podDisruptionBudgetSpec := c.generateCriticalOpPodDisruptionBudget() + podDisruptionBudget, err := c.KubeClient. + PodDisruptionBudgets(podDisruptionBudgetSpec.Namespace). + Create(context.TODO(), podDisruptionBudgetSpec, metav1.CreateOptions{}) + + if err != nil { + return err + } + c.logger.Infof("pod disruption budget for critical operations %q has been successfully created", util.NameFromMeta(podDisruptionBudget.ObjectMeta)) + c.CriticalOpPodDisruptionBudget = podDisruptionBudget + + return nil +} + +func (c *Cluster) createPodDisruptionBudgets() error { + errors := make([]string, 0) + + err := c.createPrimaryPodDisruptionBudget() + if err != nil { + errors = append(errors, fmt.Sprintf("could not create primary pod disruption budget: %v", err)) + } + + err = c.createCriticalOpPodDisruptionBudget() + if err != nil { + errors = append(errors, fmt.Sprintf("could not create pod disruption budget for critical operations: %v", err)) + } + + if len(errors) > 0 { + return fmt.Errorf("%v", strings.Join(errors, `', '`)) + } + return nil +} + +func (c *Cluster) updatePrimaryPodDisruptionBudget(pdb *policyv1.PodDisruptionBudget) error { + c.logger.Debug("updating primary pod disruption budget") + if c.PrimaryPodDisruptionBudget == nil { + return fmt.Errorf("there is no primary pod disruption budget in the cluster") + } + + if err := c.deletePrimaryPodDisruptionBudget(); err != nil { + return fmt.Errorf("could not delete primary pod disruption budget: %v", err) } newPdb, err := c.KubeClient. PodDisruptionBudgets(pdb.Namespace). Create(context.TODO(), pdb, metav1.CreateOptions{}) if err != nil { - return fmt.Errorf("could not create pod disruption budget: %v", err) + return fmt.Errorf("could not create primary pod disruption budget: %v", err) } - c.PodDisruptionBudget = newPdb + c.PrimaryPodDisruptionBudget = newPdb return nil } -func (c *Cluster) deletePodDisruptionBudget() error { - c.logger.Debug("deleting pod disruption budget") - if c.PodDisruptionBudget == nil { - return fmt.Errorf("there is no pod disruption budget in the cluster") +func (c *Cluster) updateCriticalOpPodDisruptionBudget(pdb *policyv1.PodDisruptionBudget) error { + c.logger.Debug("updating pod disruption budget for critical operations") + if c.CriticalOpPodDisruptionBudget == nil { + return fmt.Errorf("there is no pod disruption budget for critical operations in the cluster") } - pdbName := util.NameFromMeta(c.PodDisruptionBudget.ObjectMeta) - err := c.KubeClient. - PodDisruptionBudgets(c.PodDisruptionBudget.Namespace). - Delete(context.TODO(), c.PodDisruptionBudget.Name, c.deleteOptions) - if err != nil { - return fmt.Errorf("could not delete pod disruption budget: %v", err) + if err := c.deleteCriticalOpPodDisruptionBudget(); err != nil { + return fmt.Errorf("could not delete pod disruption budget for critical operations: %v", err) } - c.logger.Infof("pod disruption budget %q has been deleted", util.NameFromMeta(c.PodDisruptionBudget.ObjectMeta)) - c.PodDisruptionBudget = nil + + newPdb, err := c.KubeClient. + PodDisruptionBudgets(pdb.Namespace). + Create(context.TODO(), pdb, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("could not create pod disruption budget for critical operations: %v", err) + } + c.CriticalOpPodDisruptionBudget = newPdb + + return nil +} + +func (c *Cluster) deletePrimaryPodDisruptionBudget() error { + c.logger.Debug("deleting primary pod disruption budget") + if c.PrimaryPodDisruptionBudget == nil { + c.logger.Debug("there is no primary pod disruption budget in the cluster") + return nil + } + + pdbName := util.NameFromMeta(c.PrimaryPodDisruptionBudget.ObjectMeta) + err := c.KubeClient. + PodDisruptionBudgets(c.PrimaryPodDisruptionBudget.Namespace). + Delete(context.TODO(), c.PrimaryPodDisruptionBudget.Name, c.deleteOptions) + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("PodDisruptionBudget %q has already been deleted", util.NameFromMeta(c.PrimaryPodDisruptionBudget.ObjectMeta)) + } else if err != nil { + return fmt.Errorf("could not delete primary pod disruption budget: %v", err) + } + + c.logger.Infof("pod disruption budget %q has been deleted", util.NameFromMeta(c.PrimaryPodDisruptionBudget.ObjectMeta)) + c.PrimaryPodDisruptionBudget = nil err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, func() (bool, error) { @@ -469,60 +562,190 @@ func (c *Cluster) deletePodDisruptionBudget() error { return false, err2 }) if err != nil { - return fmt.Errorf("could not delete pod disruption budget: %v", err) + return fmt.Errorf("could not delete primary pod disruption budget: %v", err) } return nil } +func (c *Cluster) deleteCriticalOpPodDisruptionBudget() error { + c.logger.Debug("deleting pod disruption budget for critical operations") + if c.CriticalOpPodDisruptionBudget == nil { + c.logger.Debug("there is no pod disruption budget for critical operations in the cluster") + return nil + } + + pdbName := util.NameFromMeta(c.CriticalOpPodDisruptionBudget.ObjectMeta) + err := c.KubeClient. + PodDisruptionBudgets(c.CriticalOpPodDisruptionBudget.Namespace). + Delete(context.TODO(), c.CriticalOpPodDisruptionBudget.Name, c.deleteOptions) + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("PodDisruptionBudget %q has already been deleted", util.NameFromMeta(c.CriticalOpPodDisruptionBudget.ObjectMeta)) + } else if err != nil { + return fmt.Errorf("could not delete pod disruption budget for critical operations: %v", err) + } + + c.logger.Infof("pod disruption budget %q has been deleted", util.NameFromMeta(c.CriticalOpPodDisruptionBudget.ObjectMeta)) + c.CriticalOpPodDisruptionBudget = nil + + err = retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, + func() (bool, error) { + _, err2 := c.KubeClient.PodDisruptionBudgets(pdbName.Namespace).Get(context.TODO(), pdbName.Name, metav1.GetOptions{}) + if err2 == nil { + return false, nil + } + if k8sutil.ResourceNotFound(err2) { + return true, nil + } + return false, err2 + }) + if err != nil { + return fmt.Errorf("could not delete pod disruption budget for critical operations: %v", err) + } + + return nil +} + +func (c *Cluster) deletePodDisruptionBudgets() error { + errors := make([]string, 0) + + if err := c.deletePrimaryPodDisruptionBudget(); err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + + if err := c.deleteCriticalOpPodDisruptionBudget(); err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + + if len(errors) > 0 { + return fmt.Errorf("%v", strings.Join(errors, `', '`)) + } + return nil +} + func (c *Cluster) deleteEndpoint(role PostgresRole) error { c.setProcessName("deleting endpoint") - c.logger.Debugln("deleting endpoint") + c.logger.Debugf("deleting %s endpoint", role) if c.Endpoints[role] == nil { - return fmt.Errorf("there is no %s endpoint in the cluster", role) + c.logger.Debugf("there is no %s endpoint in the cluster", role) + return nil } - if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete( - context.TODO(), c.Endpoints[role].Name, c.deleteOptions); err != nil { - return fmt.Errorf("could not delete endpoint: %v", err) + if err := c.KubeClient.Endpoints(c.Endpoints[role].Namespace).Delete(context.TODO(), c.Endpoints[role].Name, c.deleteOptions); err != nil { + if !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("could not delete %s endpoint: %v", role, err) + } + c.logger.Debugf("%s endpoint has already been deleted", role) } - c.logger.Infof("endpoint %q has been deleted", util.NameFromMeta(c.Endpoints[role].ObjectMeta)) + c.logger.Infof("%s endpoint %q has been deleted", role, util.NameFromMeta(c.Endpoints[role].ObjectMeta)) + delete(c.Endpoints, role) - c.Endpoints[role] = nil + return nil +} + +func (c *Cluster) deletePatroniResources() error { + c.setProcessName("deleting Patroni resources") + errors := make([]string, 0) + + if err := c.deleteService(Patroni); err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + + for _, suffix := range patroniObjectSuffixes { + if c.patroniKubernetesUseConfigMaps() { + if err := c.deletePatroniConfigMap(suffix); err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + } else { + if err := c.deletePatroniEndpoint(suffix); err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + } + } + + if len(errors) > 0 { + return fmt.Errorf("%v", strings.Join(errors, `', '`)) + } + + return nil +} + +func (c *Cluster) deletePatroniConfigMap(suffix string) error { + c.setProcessName("deleting Patroni config map") + c.logger.Debugf("deleting %s Patroni config map", suffix) + cm := c.PatroniConfigMaps[suffix] + if cm == nil { + c.logger.Debugf("there is no %s Patroni config map in the cluster", suffix) + return nil + } + + if err := c.KubeClient.ConfigMaps(cm.Namespace).Delete(context.TODO(), cm.Name, c.deleteOptions); err != nil { + if !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("could not delete %s Patroni config map %q: %v", suffix, cm.Name, err) + } + c.logger.Debugf("%s Patroni config map has already been deleted", suffix) + } + + c.logger.Infof("%s Patroni config map %q has been deleted", suffix, util.NameFromMeta(cm.ObjectMeta)) + delete(c.PatroniConfigMaps, suffix) + + return nil +} + +func (c *Cluster) deletePatroniEndpoint(suffix string) error { + c.setProcessName("deleting Patroni endpoint") + c.logger.Debugf("deleting %s Patroni endpoint", suffix) + ep := c.PatroniEndpoints[suffix] + if ep == nil { + c.logger.Debugf("there is no %s Patroni endpoint in the cluster", suffix) + return nil + } + + if err := c.KubeClient.Endpoints(ep.Namespace).Delete(context.TODO(), ep.Name, c.deleteOptions); err != nil { + if !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("could not delete %s Patroni endpoint %q: %v", suffix, ep.Name, err) + } + c.logger.Debugf("%s Patroni endpoint has already been deleted", suffix) + } + + c.logger.Infof("%s Patroni endpoint %q has been deleted", suffix, util.NameFromMeta(ep.ObjectMeta)) + delete(c.PatroniEndpoints, suffix) return nil } func (c *Cluster) deleteSecrets() error { c.setProcessName("deleting secrets") - var errors []string - errorCount := 0 - for uid, secret := range c.Secrets { - err := c.deleteSecret(uid, *secret) + errors := make([]string, 0) + + for uid := range c.Secrets { + err := c.deleteSecret(uid) if err != nil { errors = append(errors, fmt.Sprintf("%v", err)) - errorCount++ } } - if errorCount > 0 { - return fmt.Errorf("could not delete all secrets: %v", errors) + if len(errors) > 0 { + return fmt.Errorf("could not delete all secrets: %v", strings.Join(errors, `', '`)) } return nil } -func (c *Cluster) deleteSecret(uid types.UID, secret v1.Secret) error { +func (c *Cluster) deleteSecret(uid types.UID) error { c.setProcessName("deleting secret") + secret := c.Secrets[uid] secretName := util.NameFromMeta(secret.ObjectMeta) c.logger.Debugf("deleting secret %q", secretName) err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) - if err != nil { + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("secret %q has already been deleted", secretName) + } else if err != nil { return fmt.Errorf("could not delete secret %q: %v", secretName, err) } c.logger.Infof("secret %q has been deleted", secretName) - c.Secrets[uid] = nil + delete(c.Secrets, uid) return nil } @@ -540,17 +763,17 @@ func (c *Cluster) createLogicalBackupJob() (err error) { if err != nil { return fmt.Errorf("could not generate k8s cron job spec: %v", err) } - c.logger.Debugf("Generated cronJobSpec: %v", logicalBackupJobSpec) - _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(context.TODO(), logicalBackupJobSpec, metav1.CreateOptions{}) + cronJob, err := c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Create(context.TODO(), logicalBackupJobSpec, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create k8s cron job: %v", err) } + c.LogicalBackupJob = cronJob return nil } -func (c *Cluster) patchLogicalBackupJob(newJob *batchv1beta1.CronJob) error { +func (c *Cluster) patchLogicalBackupJob(newJob *batchv1.CronJob) error { c.setProcessName("patching logical backup job") patchData, err := specPatch(newJob.Spec) @@ -559,7 +782,7 @@ func (c *Cluster) patchLogicalBackupJob(newJob *batchv1beta1.CronJob) error { } // update the backup job spec - _, err = c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Patch( + cronJob, err := c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Patch( context.TODO(), c.getLogicalBackupJobName(), types.MergePatchType, @@ -569,15 +792,26 @@ func (c *Cluster) patchLogicalBackupJob(newJob *batchv1beta1.CronJob) error { if err != nil { return fmt.Errorf("could not patch logical backup job: %v", err) } + c.LogicalBackupJob = cronJob return nil } func (c *Cluster) deleteLogicalBackupJob() error { - + if c.LogicalBackupJob == nil { + return nil + } c.logger.Info("removing the logical backup job") - return c.KubeClient.CronJobsGetter.CronJobs(c.Namespace).Delete(context.TODO(), c.getLogicalBackupJobName(), c.deleteOptions) + err := c.KubeClient.CronJobsGetter.CronJobs(c.LogicalBackupJob.Namespace).Delete(context.TODO(), c.getLogicalBackupJobName(), c.deleteOptions) + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("logical backup cron job %q has already been deleted", c.getLogicalBackupJobName()) + } else if err != nil { + return err + } + c.LogicalBackupJob = nil + + return nil } // GetServiceMaster returns cluster's kubernetes master Service @@ -595,7 +829,7 @@ func (c *Cluster) GetEndpointMaster() *v1.Endpoints { return c.Endpoints[Master] } -// GetEndpointReplica returns cluster's kubernetes master Endpoint +// GetEndpointReplica returns cluster's kubernetes replica Endpoint func (c *Cluster) GetEndpointReplica() *v1.Endpoints { return c.Endpoints[Replica] } @@ -605,7 +839,12 @@ func (c *Cluster) GetStatefulSet() *appsv1.StatefulSet { return c.Statefulset } -// GetPodDisruptionBudget returns cluster's kubernetes PodDisruptionBudget -func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget { - return c.PodDisruptionBudget +// GetPrimaryPodDisruptionBudget returns cluster's primary kubernetes PodDisruptionBudget +func (c *Cluster) GetPrimaryPodDisruptionBudget() *policyv1.PodDisruptionBudget { + return c.PrimaryPodDisruptionBudget +} + +// GetCriticalOpPodDisruptionBudget returns cluster's kubernetes PodDisruptionBudget for critical operations +func (c *Cluster) GetCriticalOpPodDisruptionBudget() *policyv1.PodDisruptionBudget { + return c.CriticalOpPodDisruptionBudget } diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go new file mode 100644 index 000000000..bf9be3fb4 --- /dev/null +++ b/pkg/cluster/streams.go @@ -0,0 +1,622 @@ +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "sort" + "strings" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + "github.com/zalando/postgres-operator/pkg/util" + "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func (c *Cluster) createStreams(appId string) (*zalandov1.FabricEventStream, error) { + c.setProcessName("creating streams") + + fes := c.generateFabricEventStream(appId) + streamCRD, err := c.KubeClient.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + + return streamCRD, nil +} + +func (c *Cluster) updateStreams(newEventStreams *zalandov1.FabricEventStream) (patchedStream *zalandov1.FabricEventStream, err error) { + c.setProcessName("updating event streams") + + patch, err := json.Marshal(newEventStreams) + if err != nil { + return nil, fmt.Errorf("could not marshal new event stream CRD %q: %v", newEventStreams.Name, err) + } + if patchedStream, err = c.KubeClient.FabricEventStreams(newEventStreams.Namespace).Patch( + context.TODO(), newEventStreams.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return nil, err + } + + return patchedStream, nil +} + +func (c *Cluster) deleteStream(appId string) error { + c.setProcessName("deleting event stream") + c.logger.Debugf("deleting event stream with applicationId %s", appId) + + err := c.KubeClient.FabricEventStreams(c.Streams[appId].Namespace).Delete(context.TODO(), c.Streams[appId].Name, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("could not delete event stream %q with applicationId %s: %v", c.Streams[appId].Name, appId, err) + } + c.logger.Infof("event stream %q with applicationId %s has been successfully deleted", c.Streams[appId].Name, appId) + delete(c.Streams, appId) + + return nil +} + +func (c *Cluster) deleteStreams() error { + // check if stream CRD is installed before trying a delete + _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamCRDName, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + return nil + } + c.setProcessName("deleting event streams") + errors := make([]string, 0) + + for appId := range c.Streams { + err := c.deleteStream(appId) + if err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + } + + if len(errors) > 0 { + return fmt.Errorf("could not delete all event stream custom resources: %v", strings.Join(errors, `', '`)) + } + + return nil +} + +func getDistinctApplicationIds(streams []acidv1.Stream) []string { + appIds := make([]string, 0) + for _, stream := range streams { + if !util.SliceContains(appIds, stream.ApplicationId) { + appIds = append(appIds, stream.ApplicationId) + } + } + + return appIds +} + +func (c *Cluster) syncPublication(dbName string, databaseSlotsList map[string]zalandov1.Slot, slotsToSync *map[string]map[string]string) error { + createPublications := make(map[string]string) + alterPublications := make(map[string]string) + deletePublications := []string{} + + defer func() { + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + + // check for existing publications + if err := c.initDbConnWithName(dbName); err != nil { + return fmt.Errorf("could not init database connection: %v", err) + } + + currentPublications, err := c.getPublications() + if err != nil { + return fmt.Errorf("could not get current publications: %v", err) + } + + for slotName, slotAndPublication := range databaseSlotsList { + newTables := slotAndPublication.Publication + tableNames := make([]string, len(newTables)) + i := 0 + for t := range newTables { + tableName, schemaName := getTableSchema(t) + tableNames[i] = fmt.Sprintf("%s.%s", schemaName, tableName) + i++ + } + sort.Strings(tableNames) + tableList := strings.Join(tableNames, ", ") + + currentTables, exists := currentPublications[slotName] + // if newTables is empty it means that it's definition was removed from streams section + // but when slot is defined in manifest we should sync publications, too + // by reusing current tables we make sure it is not + if len(newTables) == 0 { + tableList = currentTables + } + if !exists { + createPublications[slotName] = tableList + } else if currentTables != tableList { + alterPublications[slotName] = tableList + } else { + (*slotsToSync)[slotName] = slotAndPublication.Slot + } + } + + // check if there is any deletion + for slotName := range currentPublications { + if _, exists := databaseSlotsList[slotName]; !exists { + deletePublications = append(deletePublications, slotName) + } + } + + if len(createPublications)+len(alterPublications)+len(deletePublications) == 0 { + return nil + } + + errors := make([]string, 0) + for publicationName, tables := range createPublications { + if err = c.executeCreatePublication(publicationName, tables); err != nil { + errors = append(errors, fmt.Sprintf("creation of publication %q failed: %v", publicationName, err)) + continue + } + (*slotsToSync)[publicationName] = databaseSlotsList[publicationName].Slot + } + for publicationName, tables := range alterPublications { + if err = c.executeAlterPublication(publicationName, tables); err != nil { + errors = append(errors, fmt.Sprintf("update of publication %q failed: %v", publicationName, err)) + continue + } + (*slotsToSync)[publicationName] = databaseSlotsList[publicationName].Slot + } + for _, publicationName := range deletePublications { + if err = c.executeDropPublication(publicationName); err != nil { + errors = append(errors, fmt.Sprintf("deletion of publication %q failed: %v", publicationName, err)) + continue + } + (*slotsToSync)[publicationName] = nil + } + + if len(errors) > 0 { + return fmt.Errorf("%v", strings.Join(errors, `', '`)) + } + + return nil +} + +func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEventStream { + eventStreams := make([]zalandov1.EventStream, 0) + resourceAnnotations := map[string]string{} + var err, err2 error + + for _, stream := range c.Spec.Streams { + if stream.ApplicationId != appId { + continue + } + + err = setResourceAnnotation(&resourceAnnotations, stream.CPU, constants.EventStreamCpuAnnotationKey) + err2 = setResourceAnnotation(&resourceAnnotations, stream.Memory, constants.EventStreamMemoryAnnotationKey) + if err != nil || err2 != nil { + c.logger.Warningf("could not set resource annotation for event stream: %v", err) + } + + for tableName, table := range stream.Tables { + streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn) + streamFlow := getEventStreamFlow(table.PayloadColumn) + streamSink := getEventStreamSink(stream, table.EventType) + streamRecovery := getEventStreamRecovery(stream, table.RecoveryEventType, table.EventType, table.IgnoreRecovery) + + eventStreams = append(eventStreams, zalandov1.EventStream{ + EventStreamFlow: streamFlow, + EventStreamRecovery: streamRecovery, + EventStreamSink: streamSink, + EventStreamSource: streamSource}) + } + } + + return &zalandov1.FabricEventStream{ + TypeMeta: metav1.TypeMeta{ + APIVersion: constants.EventStreamCRDApiVersion, + Kind: constants.EventStreamCRDKind, + }, + ObjectMeta: metav1.ObjectMeta{ + // max length for cluster name is 58 so we can only add 5 more characters / numbers + Name: fmt.Sprintf("%s-%s", c.Name, strings.ToLower(util.RandomPassword(5))), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: c.AnnotationsToPropagate(c.annotationsSet(resourceAnnotations)), + OwnerReferences: c.ownerReferences(), + }, + Spec: zalandov1.FabricEventStreamSpec{ + ApplicationId: appId, + EventStreams: eventStreams, + }, + } +} + +func setResourceAnnotation(annotations *map[string]string, resource *string, key string) error { + var ( + isSmaller bool + err error + ) + if resource != nil { + currentValue, exists := (*annotations)[key] + if exists { + isSmaller, err = util.IsSmallerQuantity(currentValue, *resource) + if err != nil { + return fmt.Errorf("could not compare resource in %q annotation: %v", key, err) + } + } + if isSmaller || !exists { + (*annotations)[key] = *resource + } + } + + return nil +} + +func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName string, idColumn *string) zalandov1.EventStreamSource { + table, schema := getTableSchema(tableName) + streamFilter := stream.Filter[tableName] + return zalandov1.EventStreamSource{ + Type: constants.EventStreamSourcePGType, + Schema: schema, + EventStreamTable: getOutboxTable(table, idColumn), + Filter: streamFilter, + Connection: c.getStreamConnection( + stream.Database, + constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix, + stream.ApplicationId), + } +} + +func getEventStreamFlow(payloadColumn *string) zalandov1.EventStreamFlow { + return zalandov1.EventStreamFlow{ + Type: constants.EventStreamFlowPgGenericType, + PayloadColumn: payloadColumn, + } +} + +func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1.EventStreamSink { + return zalandov1.EventStreamSink{ + Type: constants.EventStreamSinkNakadiType, + EventType: eventType, + MaxBatchSize: stream.BatchSize, + } +} + +func getEventStreamRecovery(stream acidv1.Stream, recoveryEventType, eventType string, ignoreRecovery *bool) zalandov1.EventStreamRecovery { + if (stream.EnableRecovery != nil && !*stream.EnableRecovery) || + (stream.EnableRecovery == nil && recoveryEventType == "") { + return zalandov1.EventStreamRecovery{ + Type: constants.EventStreamRecoveryNoneType, + } + } + + if ignoreRecovery != nil && *ignoreRecovery { + return zalandov1.EventStreamRecovery{ + Type: constants.EventStreamRecoveryIgnoreType, + } + } + + if stream.EnableRecovery != nil && *stream.EnableRecovery && recoveryEventType == "" { + recoveryEventType = fmt.Sprintf("%s-%s", eventType, constants.EventStreamRecoverySuffix) + } + + return zalandov1.EventStreamRecovery{ + Type: constants.EventStreamRecoveryDLQType, + Sink: &zalandov1.EventStreamSink{ + Type: constants.EventStreamSinkNakadiType, + EventType: recoveryEventType, + MaxBatchSize: stream.BatchSize, + }, + } +} + +func getTableSchema(fullTableName string) (tableName, schemaName string) { + schemaName = "public" + tableName = fullTableName + if strings.Contains(fullTableName, ".") { + schemaName = strings.Split(fullTableName, ".")[0] + tableName = strings.Split(fullTableName, ".")[1] + } + + return tableName, schemaName +} + +func getOutboxTable(tableName string, idColumn *string) zalandov1.EventStreamTable { + return zalandov1.EventStreamTable{ + Name: tableName, + IDColumn: idColumn, + } +} + +func getSlotName(dbName, appId string) string { + return fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)) +} + +func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1.Connection { + return zalandov1.Connection{ + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), + SlotName: getSlotName(database, appId), + PluginType: constants.EventStreamSourcePluginType, + DBAuth: zalandov1.DBAuth{ + Type: constants.EventStreamSourceAuthType, + Name: c.credentialSecretNameForCluster(user, c.Name), + UserKey: "username", + PasswordKey: "password", + }, + } +} + +func (c *Cluster) syncStreams() error { + c.setProcessName("syncing streams") + + _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamCRDName, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + c.logger.Debug("event stream CRD not installed, skipping") + return nil + } + + // create map with every database and empty slot defintion + // we need it to detect removal of streams from databases + if err := c.initDbConn(); err != nil { + return fmt.Errorf("could not init database connection") + } + defer func() { + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + listDatabases, err := c.getDatabases() + if err != nil { + return fmt.Errorf("could not get list of databases: %v", err) + } + databaseSlots := make(map[string]map[string]zalandov1.Slot) + for dbName := range listDatabases { + if dbName != "template0" && dbName != "template1" { + databaseSlots[dbName] = map[string]zalandov1.Slot{} + } + } + + // need to take explicitly defined slots into account whey syncing Patroni config + slotsToSync := make(map[string]map[string]string) + requiredPatroniConfig := c.Spec.Patroni + if len(requiredPatroniConfig.Slots) > 0 { + for slotName, slotConfig := range requiredPatroniConfig.Slots { + slotsToSync[slotName] = slotConfig + if _, exists := databaseSlots[slotConfig["database"]]; exists { + databaseSlots[slotConfig["database"]][slotName] = zalandov1.Slot{ + Slot: slotConfig, + Publication: make(map[string]acidv1.StreamTable), + } + } + } + } + + // get list of required slots and publications, group by database + for _, stream := range c.Spec.Streams { + if _, exists := databaseSlots[stream.Database]; !exists { + c.logger.Warningf("database %q does not exist in the cluster", stream.Database) + continue + } + slot := map[string]string{ + "database": stream.Database, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + } + slotName := getSlotName(stream.Database, stream.ApplicationId) + slotAndPublication, exists := databaseSlots[stream.Database][slotName] + if !exists { + databaseSlots[stream.Database][slotName] = zalandov1.Slot{ + Slot: slot, + Publication: stream.Tables, + } + } else { + streamTables := slotAndPublication.Publication + for tableName, table := range stream.Tables { + if _, exists := streamTables[tableName]; !exists { + streamTables[tableName] = table + } + } + slotAndPublication.Publication = streamTables + databaseSlots[stream.Database][slotName] = slotAndPublication + } + } + + // sync publication in a database + c.logger.Debug("syncing database publications") + for dbName, databaseSlotsList := range databaseSlots { + err := c.syncPublication(dbName, databaseSlotsList, &slotsToSync) + if err != nil { + c.logger.Warningf("could not sync all publications in database %q: %v", dbName, err) + continue + } + } + + c.logger.Debug("syncing logical replication slots") + pods, err := c.listPods() + if err != nil { + return fmt.Errorf("could not get list of pods to sync logical replication slots via Patroni API: %v", err) + } + + // sync logical replication slots in Patroni config + requiredPatroniConfig.Slots = slotsToSync + configPatched, _, _, err := c.syncPatroniConfig(pods, requiredPatroniConfig, nil) + if err != nil { + c.logger.Warningf("Patroni config updated? %v - errors during config sync: %v", configPatched, err) + } + + // finally sync stream CRDs + // get distinct application IDs from streams section + // there will be a separate event stream resource for each ID + appIds := getDistinctApplicationIds(c.Spec.Streams) + for _, appId := range appIds { + if hasSlotsInSync(appId, databaseSlots, slotsToSync) { + if err = c.syncStream(appId); err != nil { + c.logger.Warningf("could not sync event streams with applicationId %s: %v", appId, err) + } + } else { + c.logger.Warningf("database replication slots %#v for streams with applicationId %s not in sync, skipping event stream sync", slotsToSync, appId) + } + } + + // check if there is any deletion + if err = c.cleanupRemovedStreams(appIds); err != nil { + return fmt.Errorf("%v", err) + } + + return nil +} + +func hasSlotsInSync(appId string, databaseSlots map[string]map[string]zalandov1.Slot, slotsToSync map[string]map[string]string) bool { + allSlotsInSync := true + for dbName, slots := range databaseSlots { + for slotName := range slots { + if slotName == getSlotName(dbName, appId) { + if slot, exists := slotsToSync[slotName]; !exists || slot == nil { + allSlotsInSync = false + continue + } + } + } + } + + return allSlotsInSync +} + +func (c *Cluster) syncStream(appId string) error { + var ( + streams *zalandov1.FabricEventStreamList + err error + ) + c.setProcessName("syncing stream with applicationId %s", appId) + c.logger.Debugf("syncing stream with applicationId %s", appId) + + listOptions := metav1.ListOptions{ + LabelSelector: c.labelsSet(false).String(), + } + streams, err = c.KubeClient.FabricEventStreams(c.Namespace).List(context.TODO(), listOptions) + if err != nil { + return fmt.Errorf("could not list of FabricEventStreams for applicationId %s: %v", appId, err) + } + + streamExists := false + for _, stream := range streams.Items { + if stream.Spec.ApplicationId != appId { + continue + } + streamExists = true + c.Streams[appId] = &stream + desiredStreams := c.generateFabricEventStream(appId) + if !reflect.DeepEqual(stream.ObjectMeta.OwnerReferences, desiredStreams.ObjectMeta.OwnerReferences) { + c.logger.Infof("owner references of event streams with applicationId %s do not match the current ones", appId) + stream.ObjectMeta.OwnerReferences = desiredStreams.ObjectMeta.OwnerReferences + c.setProcessName("updating event streams with applicationId %s", appId) + updatedStream, err := c.KubeClient.FabricEventStreams(stream.Namespace).Update(context.TODO(), &stream, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update event streams with applicationId %s: %v", appId, err) + } + c.Streams[appId] = updatedStream + } + if match, reason := c.compareStreams(&stream, desiredStreams); !match { + c.logger.Infof("updating event streams with applicationId %s: %s", appId, reason) + // make sure to keep the old name with randomly generated suffix + desiredStreams.ObjectMeta.Name = stream.ObjectMeta.Name + updatedStream, err := c.updateStreams(desiredStreams) + if err != nil { + return fmt.Errorf("failed updating event streams %s with applicationId %s: %v", stream.Name, appId, err) + } + c.Streams[appId] = updatedStream + c.logger.Infof("event streams %q with applicationId %s have been successfully updated", updatedStream.Name, appId) + } + break + } + + if !streamExists { + c.logger.Infof("event streams with applicationId %s do not exist, create it", appId) + createdStream, err := c.createStreams(appId) + if err != nil { + return fmt.Errorf("failed creating event streams with applicationId %s: %v", appId, err) + } + c.logger.Infof("event streams %q have been successfully created", createdStream.Name) + c.Streams[appId] = createdStream + } + + return nil +} + +func (c *Cluster) compareStreams(curEventStreams, newEventStreams *zalandov1.FabricEventStream) (match bool, reason string) { + reasons := make([]string, 0) + desiredAnnotations := make(map[string]string) + match = true + + // stream operator can add extra annotations so incl. current annotations in desired annotations + for curKey, curValue := range curEventStreams.Annotations { + if _, exists := desiredAnnotations[curKey]; !exists { + desiredAnnotations[curKey] = curValue + } + } + // add/or override annotations if cpu and memory values were changed + for newKey, newValue := range newEventStreams.Annotations { + desiredAnnotations[newKey] = newValue + } + if changed, reason := c.compareAnnotations(curEventStreams.ObjectMeta.Annotations, desiredAnnotations, nil); changed { + match = false + reasons = append(reasons, fmt.Sprintf("new streams annotations do not match: %s", reason)) + } + + if !reflect.DeepEqual(curEventStreams.ObjectMeta.Labels, newEventStreams.ObjectMeta.Labels) { + match = false + reasons = append(reasons, "new streams labels do not match the current ones") + } + + if changed, reason := sameEventStreams(curEventStreams.Spec.EventStreams, newEventStreams.Spec.EventStreams); !changed { + match = false + reasons = append(reasons, fmt.Sprintf("new streams EventStreams array does not match : %s", reason)) + } + + return match, strings.Join(reasons, ", ") +} + +func sameEventStreams(curEventStreams, newEventStreams []zalandov1.EventStream) (match bool, reason string) { + if len(newEventStreams) != len(curEventStreams) { + return false, "number of defined streams is different" + } + + for _, newStream := range newEventStreams { + match = false + reason = "event stream specs differ" + for _, curStream := range curEventStreams { + if reflect.DeepEqual(newStream.EventStreamSource, curStream.EventStreamSource) && + reflect.DeepEqual(newStream.EventStreamFlow, curStream.EventStreamFlow) && + reflect.DeepEqual(newStream.EventStreamSink, curStream.EventStreamSink) && + reflect.DeepEqual(newStream.EventStreamRecovery, curStream.EventStreamRecovery) { + match = true + break + } + } + if !match { + return false, reason + } + } + + return true, "" +} + +func (c *Cluster) cleanupRemovedStreams(appIds []string) error { + errors := make([]string, 0) + for appId := range c.Streams { + if !util.SliceContains(appIds, appId) { + c.logger.Infof("event streams with applicationId %s do not exist in the manifest, delete it", appId) + err := c.deleteStream(appId) + if err != nil { + errors = append(errors, fmt.Sprintf("failed deleting event streams with applicationId %s: %v", appId, err)) + } + } + } + + if len(errors) > 0 { + return fmt.Errorf("could not delete all removed event streams: %v", strings.Join(errors, `', '`)) + } + + return nil +} diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go new file mode 100644 index 000000000..934f2bfd4 --- /dev/null +++ b/pkg/cluster/streams_test.go @@ -0,0 +1,891 @@ +package cluster + +import ( + "fmt" + "reflect" + "strings" + + "context" + "testing" + + "github.com/stretchr/testify/assert" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + fakezalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" + "github.com/zalando/postgres-operator/pkg/util" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var ( + clusterName string = "acid-stream-cluster" + namespace string = "default" + appId string = "test-app" + dbName string = "foo" + fesUser string = fmt.Sprintf("%s%s", constants.EventStreamSourceSlotPrefix, constants.UserRoleNameSuffix) + slotName string = fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)) + + zalandoClientSet = fakezalandov1.NewSimpleClientset() + + client = k8sutil.KubernetesClient{ + FabricEventStreamsGetter: zalandoClientSet.ZalandoV1(), + PostgresqlsGetter: zalandoClientSet.AcidV1(), + PodsGetter: clientSet.CoreV1(), + StatefulSetsGetter: clientSet.AppsV1(), + } + + pg = acidv1.Postgresql{ + TypeMeta: metav1.TypeMeta{ + Kind: "Postgresql", + APIVersion: "acid.zalan.do/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Databases: map[string]string{ + dbName: fmt.Sprintf("%s%s", dbName, constants.UserRoleNameSuffix), + }, + Streams: []acidv1.Stream{ + { + ApplicationId: appId, + Database: "foo", + Tables: map[string]acidv1.StreamTable{ + "data.bar": { + EventType: "stream-type-a", + IdColumn: k8sutil.StringToPointer("b_id"), + PayloadColumn: k8sutil.StringToPointer("b_payload"), + }, + "data.foobar": { + EventType: "stream-type-b", + RecoveryEventType: "stream-type-b-dlq", + }, + "data.foofoobar": { + EventType: "stream-type-c", + IgnoreRecovery: util.True(), + }, + }, + EnableRecovery: util.True(), + Filter: map[string]*string{ + "data.bar": k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"), + }, + BatchSize: k8sutil.UInt32ToPointer(uint32(100)), + CPU: k8sutil.StringToPointer("250m"), + }, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + fes = &zalandov1.FabricEventStream{ + TypeMeta: metav1.TypeMeta{ + APIVersion: constants.EventStreamCRDApiVersion, + Kind: constants.EventStreamCRDKind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-12345", clusterName), + Namespace: namespace, + Annotations: map[string]string{ + constants.EventStreamCpuAnnotationKey: "250m", + }, + Labels: map[string]string{ + "application": "spilo", + "cluster-name": clusterName, + "team": "acid", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "acid-test-cluster", + Controller: util.True(), + }, + }, + }, + Spec: zalandov1.FabricEventStreamSpec{ + ApplicationId: appId, + EventStreams: []zalandov1.EventStream{ + { + EventStreamFlow: zalandov1.EventStreamFlow{ + PayloadColumn: k8sutil.StringToPointer("b_payload"), + Type: constants.EventStreamFlowPgGenericType, + }, + EventStreamRecovery: zalandov1.EventStreamRecovery{ + Type: constants.EventStreamRecoveryDLQType, + Sink: &zalandov1.EventStreamSink{ + EventType: fmt.Sprintf("%s-%s", "stream-type-a", constants.EventStreamRecoverySuffix), + MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)), + Type: constants.EventStreamSinkNakadiType, + }, + }, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-a", + MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)), + Type: constants.EventStreamSinkNakadiType, + }, + EventStreamSource: zalandov1.EventStreamSource{ + Filter: k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"), + Connection: zalandov1.Connection{ + DBAuth: zalandov1.DBAuth{ + Name: fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName), + PasswordKey: "password", + Type: constants.EventStreamSourceAuthType, + UserKey: "username", + }, + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), + SlotName: slotName, + PluginType: constants.EventStreamSourcePluginType, + }, + Schema: "data", + EventStreamTable: zalandov1.EventStreamTable{ + IDColumn: k8sutil.StringToPointer("b_id"), + Name: "bar", + }, + Type: constants.EventStreamSourcePGType, + }, + }, + { + EventStreamFlow: zalandov1.EventStreamFlow{ + Type: constants.EventStreamFlowPgGenericType, + }, + EventStreamRecovery: zalandov1.EventStreamRecovery{ + Type: constants.EventStreamRecoveryDLQType, + Sink: &zalandov1.EventStreamSink{ + EventType: "stream-type-b-dlq", + MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)), + Type: constants.EventStreamSinkNakadiType, + }, + }, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-b", + MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)), + Type: constants.EventStreamSinkNakadiType, + }, + EventStreamSource: zalandov1.EventStreamSource{ + Connection: zalandov1.Connection{ + DBAuth: zalandov1.DBAuth{ + Name: fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName), + PasswordKey: "password", + Type: constants.EventStreamSourceAuthType, + UserKey: "username", + }, + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), + SlotName: slotName, + PluginType: constants.EventStreamSourcePluginType, + }, + Schema: "data", + EventStreamTable: zalandov1.EventStreamTable{ + Name: "foobar", + }, + Type: constants.EventStreamSourcePGType, + }, + }, + { + EventStreamFlow: zalandov1.EventStreamFlow{ + Type: constants.EventStreamFlowPgGenericType, + }, + EventStreamRecovery: zalandov1.EventStreamRecovery{ + Type: constants.EventStreamRecoveryIgnoreType, + }, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-c", + MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)), + Type: constants.EventStreamSinkNakadiType, + }, + EventStreamSource: zalandov1.EventStreamSource{ + Connection: zalandov1.Connection{ + DBAuth: zalandov1.DBAuth{ + Name: fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName), + PasswordKey: "password", + Type: constants.EventStreamSourceAuthType, + UserKey: "username", + }, + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), + SlotName: slotName, + PluginType: constants.EventStreamSourcePluginType, + }, + Schema: "data", + EventStreamTable: zalandov1.EventStreamTable{ + Name: "foofoobar", + }, + Type: constants.EventStreamSourcePGType, + }, + }, + }, + }, + } + + cluster = New( + Config{ + OpConfig: config.Config{ + Auth: config.Auth{ + SecretNameTemplate: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}", + }, + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) +) + +func TestGatherApplicationIds(t *testing.T) { + testAppIds := []string{appId} + appIds := getDistinctApplicationIds(pg.Spec.Streams) + + if !util.IsEqualIgnoreOrder(testAppIds, appIds) { + t.Errorf("list of applicationIds does not match, expected %#v, got %#v", testAppIds, appIds) + } +} + +func TestHasSlotsInSync(t *testing.T) { + cluster.Name = clusterName + cluster.Namespace = namespace + + appId2 := fmt.Sprintf("%s-2", appId) + dbNotExists := "dbnotexists" + slotNotExists := fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbNotExists, strings.Replace(appId, "-", "_", -1)) + slotNotExistsAppId2 := fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbNotExists, strings.Replace(appId2, "-", "_", -1)) + + tests := []struct { + subTest string + applicationId string + expectedSlots map[string]map[string]zalandov1.Slot + actualSlots map[string]map[string]string + slotsInSync bool + }{ + { + subTest: fmt.Sprintf("slots in sync for applicationId %s", appId), + applicationId: appId, + expectedSlots: map[string]map[string]zalandov1.Slot{ + dbName: { + slotName: zalandov1.Slot{ + Slot: map[string]string{ + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + Publication: map[string]acidv1.StreamTable{ + "test1": { + EventType: "stream-type-a", + }, + }, + }, + }, + }, + actualSlots: map[string]map[string]string{ + slotName: { + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + }, + slotsInSync: true, + }, { + subTest: fmt.Sprintf("slots empty for applicationId %s after create or update of publication failed", appId), + applicationId: appId, + expectedSlots: map[string]map[string]zalandov1.Slot{ + dbNotExists: { + slotNotExists: zalandov1.Slot{ + Slot: map[string]string{ + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + Publication: map[string]acidv1.StreamTable{ + "test1": { + EventType: "stream-type-a", + }, + }, + }, + }, + }, + actualSlots: map[string]map[string]string{}, + slotsInSync: false, + }, { + subTest: fmt.Sprintf("slot with empty definition for applicationId %s after publication git deleted", appId), + applicationId: appId, + expectedSlots: map[string]map[string]zalandov1.Slot{ + dbNotExists: { + slotNotExists: zalandov1.Slot{ + Slot: map[string]string{ + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + Publication: map[string]acidv1.StreamTable{ + "test1": { + EventType: "stream-type-a", + }, + }, + }, + }, + }, + actualSlots: map[string]map[string]string{ + slotName: nil, + }, + slotsInSync: false, + }, { + subTest: fmt.Sprintf("one slot not in sync for applicationId %s because database does not exist", appId), + applicationId: appId, + expectedSlots: map[string]map[string]zalandov1.Slot{ + dbName: { + slotName: zalandov1.Slot{ + Slot: map[string]string{ + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + Publication: map[string]acidv1.StreamTable{ + "test1": { + EventType: "stream-type-a", + }, + }, + }, + }, + dbNotExists: { + slotNotExists: zalandov1.Slot{ + Slot: map[string]string{ + "databases": "dbnotexists", + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + Publication: map[string]acidv1.StreamTable{ + "test2": { + EventType: "stream-type-b", + }, + }, + }, + }, + }, + actualSlots: map[string]map[string]string{ + slotName: { + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + }, + slotsInSync: false, + }, { + subTest: fmt.Sprintf("slots in sync for applicationId %s, but not for %s - checking %s should return true", appId, appId2, appId), + applicationId: appId, + expectedSlots: map[string]map[string]zalandov1.Slot{ + dbName: { + slotName: zalandov1.Slot{ + Slot: map[string]string{ + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + Publication: map[string]acidv1.StreamTable{ + "test1": { + EventType: "stream-type-a", + }, + }, + }, + }, + dbNotExists: { + slotNotExistsAppId2: zalandov1.Slot{ + Slot: map[string]string{ + "databases": "dbnotexists", + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + Publication: map[string]acidv1.StreamTable{ + "test2": { + EventType: "stream-type-b", + }, + }, + }, + }, + }, + actualSlots: map[string]map[string]string{ + slotName: { + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + }, + slotsInSync: true, + }, { + subTest: fmt.Sprintf("slots in sync for applicationId %s, but not for %s - checking %s should return false", appId, appId2, appId2), + applicationId: appId2, + expectedSlots: map[string]map[string]zalandov1.Slot{ + dbName: { + slotName: zalandov1.Slot{ + Slot: map[string]string{ + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + Publication: map[string]acidv1.StreamTable{ + "test1": { + EventType: "stream-type-a", + }, + }, + }, + }, + dbNotExists: { + slotNotExistsAppId2: zalandov1.Slot{ + Slot: map[string]string{ + "databases": "dbnotexists", + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + Publication: map[string]acidv1.StreamTable{ + "test2": { + EventType: "stream-type-b", + }, + }, + }, + }, + }, + actualSlots: map[string]map[string]string{ + slotName: { + "databases": dbName, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + }, + }, + slotsInSync: false, + }, + } + + for _, tt := range tests { + result := hasSlotsInSync(tt.applicationId, tt.expectedSlots, tt.actualSlots) + if result != tt.slotsInSync { + t.Errorf("%s: unexpected result for slot test of applicationId: %v, expected slots %#v, actual slots %#v", tt.subTest, tt.applicationId, tt.expectedSlots, tt.actualSlots) + } + } +} + +func TestGenerateFabricEventStream(t *testing.T) { + cluster.Name = clusterName + cluster.Namespace = namespace + + // create the streams + err := cluster.syncStream(appId) + assert.NoError(t, err) + + // compare generated stream with expected stream + result := cluster.generateFabricEventStream(appId) + if match, _ := cluster.compareStreams(result, fes); !match { + t.Errorf("malformed FabricEventStream, expected %#v, got %#v", fes, result) + } + + listOptions := metav1.ListOptions{ + LabelSelector: cluster.labelsSet(false).String(), + } + streams, err := cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + assert.Equalf(t, 1, len(streams.Items), "unexpected number of streams found: got %d, but expected only one", len(streams.Items)) + + // compare stream returned from API with expected stream + if match, _ := cluster.compareStreams(&streams.Items[0], fes); !match { + t.Errorf("malformed FabricEventStream returned from API, expected %#v, got %#v", fes, streams.Items[0]) + } + + // sync streams once again + err = cluster.syncStream(appId) + assert.NoError(t, err) + + streams, err = cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + assert.Equalf(t, 1, len(streams.Items), "unexpected number of streams found: got %d, but expected only one", len(streams.Items)) + + // compare stream resturned from API with generated stream + if match, _ := cluster.compareStreams(&streams.Items[0], result); !match { + t.Errorf("returned FabricEventStream differs from generated one, expected %#v, got %#v", result, streams.Items[0]) + } +} + +func newFabricEventStream(streams []zalandov1.EventStream, annotations map[string]string) *zalandov1.FabricEventStream { + return &zalandov1.FabricEventStream{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-12345", clusterName), + Annotations: annotations, + }, + Spec: zalandov1.FabricEventStreamSpec{ + ApplicationId: appId, + EventStreams: streams, + }, + } +} + +func TestSyncStreams(t *testing.T) { + newClusterName := fmt.Sprintf("%s-2", pg.Name) + pg.Name = newClusterName + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + _, err := cluster.KubeClient.Postgresqls(namespace).Create( + context.TODO(), &pg, metav1.CreateOptions{}) + assert.NoError(t, err) + + // create the stream + err = cluster.syncStream(appId) + assert.NoError(t, err) + + // sync the stream again + err = cluster.syncStream(appId) + assert.NoError(t, err) + + // check that only one stream remains after sync + listOptions := metav1.ListOptions{ + LabelSelector: cluster.labelsSet(false).String(), + } + streams, err := cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + assert.Equalf(t, 1, len(streams.Items), "unexpected number of streams found: got %d, but expected only 1", len(streams.Items)) +} + +func TestSameStreams(t *testing.T) { + testName := "TestSameStreams" + annotationsA := map[string]string{constants.EventStreamMemoryAnnotationKey: "500Mi"} + annotationsB := map[string]string{constants.EventStreamMemoryAnnotationKey: "1Gi"} + + stream1 := zalandov1.EventStream{ + EventStreamFlow: zalandov1.EventStreamFlow{}, + EventStreamRecovery: zalandov1.EventStreamRecovery{}, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-a", + }, + EventStreamSource: zalandov1.EventStreamSource{ + EventStreamTable: zalandov1.EventStreamTable{ + Name: "foo", + }, + }, + } + + stream2 := zalandov1.EventStream{ + EventStreamFlow: zalandov1.EventStreamFlow{}, + EventStreamRecovery: zalandov1.EventStreamRecovery{}, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-b", + }, + EventStreamSource: zalandov1.EventStreamSource{ + EventStreamTable: zalandov1.EventStreamTable{ + Name: "bar", + }, + }, + } + + stream3 := zalandov1.EventStream{ + EventStreamFlow: zalandov1.EventStreamFlow{}, + EventStreamRecovery: zalandov1.EventStreamRecovery{ + Type: constants.EventStreamRecoveryNoneType, + }, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-b", + }, + EventStreamSource: zalandov1.EventStreamSource{ + EventStreamTable: zalandov1.EventStreamTable{ + Name: "bar", + }, + }, + } + + tests := []struct { + subTest string + streamsA *zalandov1.FabricEventStream + streamsB *zalandov1.FabricEventStream + match bool + reason string + }{ + { + subTest: "identical streams", + streamsA: newFabricEventStream([]zalandov1.EventStream{stream1, stream2}, annotationsA), + streamsB: newFabricEventStream([]zalandov1.EventStream{stream1, stream2}, annotationsA), + match: true, + reason: "", + }, + { + subTest: "same streams different order", + streamsA: newFabricEventStream([]zalandov1.EventStream{stream1, stream2}, nil), + streamsB: newFabricEventStream([]zalandov1.EventStream{stream2, stream1}, nil), + match: true, + reason: "", + }, + { + subTest: "same streams different order", + streamsA: newFabricEventStream([]zalandov1.EventStream{stream1}, nil), + streamsB: newFabricEventStream([]zalandov1.EventStream{stream1, stream2}, nil), + match: false, + reason: "new streams EventStreams array does not match : number of defined streams is different", + }, + { + subTest: "different number of streams", + streamsA: newFabricEventStream([]zalandov1.EventStream{stream1}, nil), + streamsB: newFabricEventStream([]zalandov1.EventStream{stream1, stream2}, nil), + match: false, + reason: "new streams EventStreams array does not match : number of defined streams is different", + }, + { + subTest: "event stream specs differ", + streamsA: newFabricEventStream([]zalandov1.EventStream{stream1, stream2}, nil), + streamsB: fes, + match: false, + reason: "new streams annotations do not match: Added \"fes.zalando.org/FES_CPU\" with value \"250m\"., new streams labels do not match the current ones, new streams EventStreams array does not match : number of defined streams is different", + }, + { + subTest: "event stream recovery specs differ", + streamsA: newFabricEventStream([]zalandov1.EventStream{stream2}, nil), + streamsB: newFabricEventStream([]zalandov1.EventStream{stream3}, nil), + match: false, + reason: "new streams EventStreams array does not match : event stream specs differ", + }, + { + subTest: "event stream with new annotations", + streamsA: newFabricEventStream([]zalandov1.EventStream{stream2}, nil), + streamsB: newFabricEventStream([]zalandov1.EventStream{stream2}, annotationsA), + match: false, + reason: "new streams annotations do not match: Added \"fes.zalando.org/FES_MEMORY\" with value \"500Mi\".", + }, + { + subTest: "event stream annotations differ", + streamsA: newFabricEventStream([]zalandov1.EventStream{stream3}, annotationsA), + streamsB: newFabricEventStream([]zalandov1.EventStream{stream3}, annotationsB), + match: false, + reason: "new streams annotations do not match: \"fes.zalando.org/FES_MEMORY\" changed from \"500Mi\" to \"1Gi\".", + }, + } + + for _, tt := range tests { + streamsMatch, matchReason := cluster.compareStreams(tt.streamsA, tt.streamsB) + if streamsMatch != tt.match || matchReason != tt.reason { + t.Errorf("%s %s: unexpected match result when comparing streams: got %s, expected %s", + testName, tt.subTest, matchReason, tt.reason) + } + } +} + +func TestUpdateStreams(t *testing.T) { + pg.Name = fmt.Sprintf("%s-3", pg.Name) + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + EnableOwnerReferences: util.True(), + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + _, err := cluster.KubeClient.Postgresqls(namespace).Create( + context.TODO(), &pg, metav1.CreateOptions{}) + assert.NoError(t, err) + + // create stream with different owner reference + fes.ObjectMeta.Name = fmt.Sprintf("%s-12345", pg.Name) + fes.ObjectMeta.Labels["cluster-name"] = pg.Name + createdStream, err := cluster.KubeClient.FabricEventStreams(namespace).Create( + context.TODO(), fes, metav1.CreateOptions{}) + assert.NoError(t, err) + assert.Equal(t, createdStream.Spec.ApplicationId, appId) + + // sync the stream which should update the owner reference + err = cluster.syncStream(appId) + assert.NoError(t, err) + + // check that only one stream exists after sync + listOptions := metav1.ListOptions{ + LabelSelector: cluster.labelsSet(true).String(), + } + streams, err := cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + assert.Equalf(t, 1, len(streams.Items), "unexpected number of streams found: got %d, but expected only 1", len(streams.Items)) + + // compare owner references + if !reflect.DeepEqual(streams.Items[0].OwnerReferences, cluster.ownerReferences()) { + t.Errorf("unexpected owner references, expected %#v, got %#v", cluster.ownerReferences(), streams.Items[0].OwnerReferences) + } + + // change specs of streams and patch CRD + for i, stream := range pg.Spec.Streams { + if stream.ApplicationId == appId { + streamTable := stream.Tables["data.bar"] + streamTable.EventType = "stream-type-c" + stream.Tables["data.bar"] = streamTable + stream.BatchSize = k8sutil.UInt32ToPointer(uint32(250)) + pg.Spec.Streams[i] = stream + } + } + + // compare stream returned from API with expected stream + streams = patchPostgresqlStreams(t, cluster, &pg.Spec, listOptions) + result := cluster.generateFabricEventStream(appId) + if match, _ := cluster.compareStreams(&streams.Items[0], result); !match { + t.Errorf("Malformed FabricEventStream after updating manifest, expected %#v, got %#v", streams.Items[0], result) + } + + // disable recovery + for idx, stream := range pg.Spec.Streams { + if stream.ApplicationId == appId { + stream.EnableRecovery = util.False() + pg.Spec.Streams[idx] = stream + } + } + + streams = patchPostgresqlStreams(t, cluster, &pg.Spec, listOptions) + result = cluster.generateFabricEventStream(appId) + if match, _ := cluster.compareStreams(&streams.Items[0], result); !match { + t.Errorf("Malformed FabricEventStream after disabling event recovery, expected %#v, got %#v", streams.Items[0], result) + } +} + +func patchPostgresqlStreams(t *testing.T, cluster *Cluster, pgSpec *acidv1.PostgresSpec, listOptions metav1.ListOptions) (streams *zalandov1.FabricEventStreamList) { + patchData, err := specPatch(pgSpec) + assert.NoError(t, err) + + pgPatched, err := cluster.KubeClient.Postgresqls(namespace).Patch( + context.TODO(), cluster.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "spec") + assert.NoError(t, err) + + cluster.Postgresql.Spec = pgPatched.Spec + err = cluster.syncStream(appId) + assert.NoError(t, err) + + streams, err = cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + + return streams +} + +func TestDeleteStreams(t *testing.T) { + pg.Name = fmt.Sprintf("%s-4", pg.Name) + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + _, err := cluster.KubeClient.Postgresqls(namespace).Create( + context.TODO(), &pg, metav1.CreateOptions{}) + assert.NoError(t, err) + + // create the stream + err = cluster.syncStream(appId) + assert.NoError(t, err) + + // change specs of streams and patch CRD + for i, stream := range pg.Spec.Streams { + if stream.ApplicationId == appId { + streamTable := stream.Tables["data.bar"] + streamTable.EventType = "stream-type-c" + stream.Tables["data.bar"] = streamTable + stream.BatchSize = k8sutil.UInt32ToPointer(uint32(250)) + pg.Spec.Streams[i] = stream + } + } + + // compare stream returned from API with expected stream + listOptions := metav1.ListOptions{ + LabelSelector: cluster.labelsSet(false).String(), + } + streams := patchPostgresqlStreams(t, cluster, &pg.Spec, listOptions) + result := cluster.generateFabricEventStream(appId) + if match, _ := cluster.compareStreams(&streams.Items[0], result); !match { + t.Errorf("Malformed FabricEventStream after updating manifest, expected %#v, got %#v", streams.Items[0], result) + } + + // change teamId and check that stream is updated + pg.Spec.TeamID = "new-team" + streams = patchPostgresqlStreams(t, cluster, &pg.Spec, listOptions) + result = cluster.generateFabricEventStream(appId) + if match, _ := cluster.compareStreams(&streams.Items[0], result); !match { + t.Errorf("Malformed FabricEventStream after updating teamId, expected %#v, got %#v", streams.Items[0].ObjectMeta.Labels, result.ObjectMeta.Labels) + } + + // disable recovery + for idx, stream := range pg.Spec.Streams { + if stream.ApplicationId == appId { + stream.EnableRecovery = util.False() + pg.Spec.Streams[idx] = stream + } + } + + streams = patchPostgresqlStreams(t, cluster, &pg.Spec, listOptions) + result = cluster.generateFabricEventStream(appId) + if match, _ := cluster.compareStreams(&streams.Items[0], result); !match { + t.Errorf("Malformed FabricEventStream after disabling event recovery, expected %#v, got %#v", streams.Items[0], result) + } + + // remove streams from manifest + pg.Spec.Streams = nil + pgUpdated, err := cluster.KubeClient.Postgresqls(namespace).Update( + context.TODO(), &pg, metav1.UpdateOptions{}) + assert.NoError(t, err) + + appIds := getDistinctApplicationIds(pgUpdated.Spec.Streams) + cluster.cleanupRemovedStreams(appIds) + + // check that streams have been deleted + streams, err = cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + assert.Equalf(t, 0, len(streams.Items), "unexpected number of streams found: got %d, but expected none", len(streams.Items)) + + // create stream to test deleteStreams code + fes.ObjectMeta.Name = fmt.Sprintf("%s-12345", pg.Name) + fes.ObjectMeta.Labels["cluster-name"] = pg.Name + _, err = cluster.KubeClient.FabricEventStreams(namespace).Create( + context.TODO(), fes, metav1.CreateOptions{}) + assert.NoError(t, err) + + // sync it once to cluster struct + err = cluster.syncStream(appId) + assert.NoError(t, err) + + // we need a mock client because deleteStreams checks for CRD existance + mockClient := k8sutil.NewMockKubernetesClient() + cluster.KubeClient.CustomResourceDefinitionsGetter = mockClient.CustomResourceDefinitionsGetter + cluster.deleteStreams() + + // check that streams have been deleted + streams, err = cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions) + assert.NoError(t, err) + assert.Equalf(t, 0, len(streams.Items), "unexpected number of streams found: got %d, but expected none", len(streams.Items)) +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 94e930290..e15b5fedc 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -2,21 +2,37 @@ package cluster import ( "context" + "encoding/json" "fmt" + "maps" + "reflect" "regexp" + "slices" + "strconv" "strings" + "time" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" - batchv1beta1 "k8s.io/api/batch/v1beta1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) +var requirePrimaryRestartWhenDecreased = []string{ + "max_connections", + "max_prepared_transactions", + "max_locks_per_transaction", + "max_worker_processes", + "max_wal_senders", +} + // Sync syncs the cluster, making sure the actual Kubernetes objects correspond to what is defined in the manifest. // Unlike the update, sync does not error out if some objects do not exist and takes care of creating them. func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { @@ -25,17 +41,36 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { defer c.mu.Unlock() oldSpec := c.Postgresql + + if !c.isInMaintenanceWindow(newSpec.Spec.MaintenanceWindows) { + // do not apply any major version related changes yet + newSpec.Spec.PostgresqlParam.PgVersion = oldSpec.Spec.PostgresqlParam.PgVersion + } + c.setSpec(newSpec) defer func() { if err != nil { c.logger.Warningf("error while syncing cluster state: %v", err) - c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusSyncFailed) + newSpec.Status.PostgresClusterStatus = acidv1.ClusterStatusSyncFailed } else if !c.Status.Running() { - c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusRunning) + newSpec.Status.PostgresClusterStatus = acidv1.ClusterStatusRunning + } + + if !equality.Semantic.DeepEqual(oldSpec.Status, newSpec.Status) { + pgUpdatedStatus, err := c.KubeClient.SetPostgresCRDStatus(c.clusterName(), newSpec) + if err != nil { + c.logger.Warningf("could not set cluster status: %v", err) + return + } + c.setSpec(pgUpdatedStatus) } }() + if err = c.syncFinalizer(); err != nil { + c.logger.Debugf("could not sync finalizers: %v", err) + } + if err = c.initUsers(); err != nil { err = fmt.Errorf("could not init users: %v", err) return err @@ -52,24 +87,22 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } + if err = c.syncPatroniResources(); err != nil { + c.logger.Errorf("could not sync Patroni resources: %v", err) + } + // sync volume may already transition volumes to gp3, if iops/throughput or type is specified if err = c.syncVolumes(); err != nil { return err } - if c.OpConfig.EnableEBSGp3Migration { + if c.OpConfig.EnableEBSGp3Migration && len(c.EBSVolumes) > 0 { err = c.executeEBSMigration() if nil != err { return err } } - if err = c.enforceMinResourceLimits(&c.Spec); err != nil { - err = fmt.Errorf("could not enforce minimum resource limits: %v", err) - return err - } - - c.logger.Debugf("syncing statefulsets") if err = c.syncStatefulSet(); err != nil { if !k8sutil.ResourceAlreadyExists(err) { err = fmt.Errorf("could not sync statefulsets: %v", err) @@ -77,9 +110,16 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } + // add or remove standby_cluster section from Patroni config depending on changes in standby section + if !reflect.DeepEqual(oldSpec.Spec.StandbyCluster, newSpec.Spec.StandbyCluster) { + if err := c.syncStandbyClusterConfiguration(); err != nil { + return fmt.Errorf("could not sync StandbyCluster configuration: %v", err) + } + } + c.logger.Debug("syncing pod disruption budgets") - if err = c.syncPodDisruptionBudget(false); err != nil { - err = fmt.Errorf("could not sync pod disruption budget: %v", err) + if err = c.syncPodDisruptionBudgets(false); err != nil { + err = fmt.Errorf("could not sync pod disruption budgets: %v", err) return err } @@ -95,20 +135,17 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { // create database objects unless we are running without pods or disabled that feature explicitly if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&newSpec.Spec) <= 0 || c.Spec.StandbyCluster != nil) { - c.logger.Debugf("syncing roles") + c.logger.Debug("syncing roles") if err = c.syncRoles(); err != nil { - err = fmt.Errorf("could not sync roles: %v", err) - return err + c.logger.Errorf("could not sync roles: %v", err) } - c.logger.Debugf("syncing databases") + c.logger.Debug("syncing databases") if err = c.syncDatabases(); err != nil { - err = fmt.Errorf("could not sync databases: %v", err) - return err + c.logger.Errorf("could not sync databases: %v", err) } - c.logger.Debugf("syncing prepared databases with schemas") + c.logger.Debug("syncing prepared databases with schemas") if err = c.syncPreparedDatabases(); err != nil { - err = fmt.Errorf("could not sync prepared database: %v", err) - return err + c.logger.Errorf("could not sync prepared database: %v", err) } } @@ -117,6 +154,17 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return fmt.Errorf("could not sync connection pooler: %v", err) } + // sync if manifest stream count is different from stream CR count + // it can be that they are always different due to grouping of manifest streams + // but we would catch missed removals on update + if len(c.Spec.Streams) != len(c.Streams) { + c.logger.Debug("syncing streams") + if err = c.syncStreams(); err != nil { + err = fmt.Errorf("could not sync streams: %v", err) + return err + } + } + // Major version upgrade must only run after success of all earlier operations, must remain last item in sync if err := c.majorVersionUpgrade(); err != nil { c.logger.Errorf("major version upgrade failed: %v", err) @@ -125,6 +173,181 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } +func (c *Cluster) syncFinalizer() error { + var err error + if c.OpConfig.EnableFinalizers != nil && *c.OpConfig.EnableFinalizers { + err = c.addFinalizer() + } else { + err = c.removeFinalizer() + } + if err != nil { + return fmt.Errorf("could not sync finalizer: %v", err) + } + + return nil +} + +func (c *Cluster) syncPatroniResources() error { + errors := make([]string, 0) + + if err := c.syncPatroniService(); err != nil { + errors = append(errors, fmt.Sprintf("could not sync %s service: %v", Patroni, err)) + } + + for _, suffix := range patroniObjectSuffixes { + if c.patroniKubernetesUseConfigMaps() { + if err := c.syncPatroniConfigMap(suffix); err != nil { + errors = append(errors, fmt.Sprintf("could not sync %s Patroni config map: %v", suffix, err)) + } + } else { + if err := c.syncPatroniEndpoint(suffix); err != nil { + errors = append(errors, fmt.Sprintf("could not sync %s Patroni endpoint: %v", suffix, err)) + } + } + } + + if len(errors) > 0 { + return fmt.Errorf("%v", strings.Join(errors, `', '`)) + } + + return nil +} + +func (c *Cluster) syncPatroniConfigMap(suffix string) error { + var ( + cm *v1.ConfigMap + err error + ) + configMapName := fmt.Sprintf("%s-%s", c.Name, suffix) + c.logger.Debugf("syncing %s config map", configMapName) + c.setProcessName("syncing %s config map", configMapName) + + if cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}); err == nil { + c.PatroniConfigMaps[suffix] = cm + desiredOwnerRefs := c.ownerReferences() + if !reflect.DeepEqual(cm.ObjectMeta.OwnerReferences, desiredOwnerRefs) { + c.logger.Infof("new %s config map's owner references do not match the current ones", configMapName) + cm.ObjectMeta.OwnerReferences = desiredOwnerRefs + c.setProcessName("updating %s config map", configMapName) + cm, err = c.KubeClient.ConfigMaps(c.Namespace).Update(context.TODO(), cm, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update %s config map: %v", configMapName, err) + } + c.PatroniConfigMaps[suffix] = cm + } + annotations := make(map[string]string) + maps.Copy(annotations, cm.Annotations) + // Patroni can add extra annotations so incl. current annotations in desired annotations + desiredAnnotations := c.annotationsSet(cm.Annotations) + if changed, _ := c.compareAnnotations(annotations, desiredAnnotations, nil); changed { + patchData, err := metaAnnotationsPatch(desiredAnnotations) + if err != nil { + return fmt.Errorf("could not form patch for %s config map: %v", configMapName, err) + } + cm, err = c.KubeClient.ConfigMaps(c.Namespace).Patch(context.TODO(), configMapName, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("could not patch annotations of %s config map: %v", configMapName, err) + } + c.PatroniConfigMaps[suffix] = cm + } + } else if !k8sutil.ResourceNotFound(err) { + // if config map does not exist yet, Patroni should create it + return fmt.Errorf("could not get %s config map: %v", configMapName, err) + } + + return nil +} + +func (c *Cluster) syncPatroniEndpoint(suffix string) error { + var ( + ep *v1.Endpoints + err error + ) + endpointName := fmt.Sprintf("%s-%s", c.Name, suffix) + c.logger.Debugf("syncing %s endpoint", endpointName) + c.setProcessName("syncing %s endpoint", endpointName) + + if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), endpointName, metav1.GetOptions{}); err == nil { + c.PatroniEndpoints[suffix] = ep + desiredOwnerRefs := c.ownerReferences() + if !reflect.DeepEqual(ep.ObjectMeta.OwnerReferences, desiredOwnerRefs) { + c.logger.Infof("new %s endpoints's owner references do not match the current ones", endpointName) + ep.ObjectMeta.OwnerReferences = desiredOwnerRefs + c.setProcessName("updating %s endpoint", endpointName) + ep, err = c.KubeClient.Endpoints(c.Namespace).Update(context.TODO(), ep, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update %s endpoint: %v", endpointName, err) + } + c.PatroniEndpoints[suffix] = ep + } + annotations := make(map[string]string) + maps.Copy(annotations, ep.Annotations) + // Patroni can add extra annotations so incl. current annotations in desired annotations + desiredAnnotations := c.annotationsSet(ep.Annotations) + if changed, _ := c.compareAnnotations(annotations, desiredAnnotations, nil); changed { + patchData, err := metaAnnotationsPatch(desiredAnnotations) + if err != nil { + return fmt.Errorf("could not form patch for %s endpoint: %v", endpointName, err) + } + ep, err = c.KubeClient.Endpoints(c.Namespace).Patch(context.TODO(), endpointName, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("could not patch annotations of %s endpoint: %v", endpointName, err) + } + c.PatroniEndpoints[suffix] = ep + } + } else if !k8sutil.ResourceNotFound(err) { + // if endpoint does not exist yet, Patroni should create it + return fmt.Errorf("could not get %s endpoint: %v", endpointName, err) + } + + return nil +} + +func (c *Cluster) syncPatroniService() error { + var ( + svc *v1.Service + err error + ) + serviceName := fmt.Sprintf("%s-%s", c.Name, Patroni) + c.logger.Debugf("syncing %s service", serviceName) + c.setProcessName("syncing %s service", serviceName) + + if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), serviceName, metav1.GetOptions{}); err == nil { + c.Services[Patroni] = svc + desiredOwnerRefs := c.ownerReferences() + if !reflect.DeepEqual(svc.ObjectMeta.OwnerReferences, desiredOwnerRefs) { + c.logger.Infof("new %s service's owner references do not match the current ones", serviceName) + svc.ObjectMeta.OwnerReferences = desiredOwnerRefs + c.setProcessName("updating %v service", serviceName) + svc, err = c.KubeClient.Services(c.Namespace).Update(context.TODO(), svc, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update %s service: %v", serviceName, err) + } + c.Services[Patroni] = svc + } + annotations := make(map[string]string) + maps.Copy(annotations, svc.Annotations) + // Patroni can add extra annotations so incl. current annotations in desired annotations + desiredAnnotations := c.annotationsSet(svc.Annotations) + if changed, _ := c.compareAnnotations(annotations, desiredAnnotations, nil); changed { + patchData, err := metaAnnotationsPatch(desiredAnnotations) + if err != nil { + return fmt.Errorf("could not form patch for %s service: %v", serviceName, err) + } + svc, err = c.KubeClient.Services(c.Namespace).Patch(context.TODO(), serviceName, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("could not patch annotations of %s service: %v", serviceName, err) + } + c.Services[Patroni] = svc + } + } else if !k8sutil.ResourceNotFound(err) { + // if config service does not exist yet, Patroni should create it + return fmt.Errorf("could not get %s service: %v", serviceName, err) + } + + return nil +} + func (c *Cluster) syncServices() error { for _, role := range []PostgresRole{Master, Replica} { c.logger.Debugf("syncing %s service", role) @@ -152,20 +375,17 @@ func (c *Cluster) syncService(role PostgresRole) error { if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil { c.Services[role] = svc desiredSvc := c.generateService(role, &c.Spec) - if match, reason := k8sutil.SameService(svc, desiredSvc); !match { - c.logServiceChanges(role, svc, desiredSvc, false, reason) - if err = c.updateService(role, desiredSvc); err != nil { - return fmt.Errorf("could not update %s service to match desired state: %v", role, err) - } - c.logger.Infof("%s service %q is in the desired state now", role, util.NameFromMeta(desiredSvc.ObjectMeta)) + updatedSvc, err := c.updateService(role, svc, desiredSvc) + if err != nil { + return fmt.Errorf("could not update %s service to match desired state: %v", role, err) } + c.Services[role] = updatedSvc return nil } if !k8sutil.ResourceNotFound(err) { return fmt.Errorf("could not get %s service: %v", role, err) } // no existing service, create new one - c.Services[role] = nil c.logger.Infof("could not find the cluster's %s service", role) if svc, err = c.createService(role); err == nil { @@ -190,8 +410,28 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error { ) c.setProcessName("syncing %s endpoint", role) - if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err == nil { - // TODO: No syncing of endpoints here, is this covered completely by updateService? + if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil { + desiredEp := c.generateEndpoint(role, ep.Subsets) + // if owner references differ we update which would also change annotations + if !reflect.DeepEqual(ep.ObjectMeta.OwnerReferences, desiredEp.ObjectMeta.OwnerReferences) { + c.logger.Infof("new %s endpoints's owner references do not match the current ones", role) + c.setProcessName("updating %v endpoint", role) + ep, err = c.KubeClient.Endpoints(c.Namespace).Update(context.TODO(), desiredEp, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update %s endpoint: %v", role, err) + } + } else { + if changed, _ := c.compareAnnotations(ep.Annotations, desiredEp.Annotations, nil); changed { + patchData, err := metaAnnotationsPatch(desiredEp.Annotations) + if err != nil { + return fmt.Errorf("could not form patch for %s endpoint: %v", role, err) + } + ep, err = c.KubeClient.Endpoints(c.Namespace).Patch(context.TODO(), c.serviceName(role), types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("could not patch annotations of %s endpoint: %v", role, err) + } + } + } c.Endpoints[role] = ep return nil } @@ -199,7 +439,6 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error { return fmt.Errorf("could not get %s endpoint: %v", role, err) } // no existing endpoint, create new one - c.Endpoints[role] = nil c.logger.Infof("could not find the cluster's %s endpoint", role) if ep, err = c.createEndpoint(role); err == nil { @@ -209,7 +448,7 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error { return fmt.Errorf("could not create missing %s endpoint: %v", role, err) } c.logger.Infof("%s endpoint %q already exists", role, util.NameFromMeta(ep.ObjectMeta)) - if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.endpointName(role), metav1.GetOptions{}); err != nil { + if ep, err = c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing %s endpoint: %v", role, err) } } @@ -217,21 +456,22 @@ func (c *Cluster) syncEndpoint(role PostgresRole) error { return nil } -func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { +func (c *Cluster) syncPrimaryPodDisruptionBudget(isUpdate bool) error { var ( - pdb *policybeta1.PodDisruptionBudget + pdb *policyv1.PodDisruptionBudget err error ) - if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err == nil { - c.PodDisruptionBudget = pdb - newPDB := c.generatePodDisruptionBudget() - if match, reason := k8sutil.SamePDB(pdb, newPDB); !match { + if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.PrimaryPodDisruptionBudgetName(), metav1.GetOptions{}); err == nil { + c.PrimaryPodDisruptionBudget = pdb + newPDB := c.generatePrimaryPodDisruptionBudget() + match, reason := c.comparePodDisruptionBudget(pdb, newPDB) + if !match { c.logPDBChanges(pdb, newPDB, isUpdate, reason) - if err = c.updatePodDisruptionBudget(newPDB); err != nil { + if err = c.updatePrimaryPodDisruptionBudget(newPDB); err != nil { return err } } else { - c.PodDisruptionBudget = pdb + c.PrimaryPodDisruptionBudget = pdb } return nil @@ -240,43 +480,101 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { return fmt.Errorf("could not get pod disruption budget: %v", err) } // no existing pod disruption budget, create new one - c.PodDisruptionBudget = nil - c.logger.Infof("could not find the cluster's pod disruption budget") + c.logger.Infof("could not find the primary pod disruption budget") - if pdb, err = c.createPodDisruptionBudget(); err != nil { + if err = c.createPrimaryPodDisruptionBudget(); err != nil { if !k8sutil.ResourceAlreadyExists(err) { - return fmt.Errorf("could not create pod disruption budget: %v", err) + return fmt.Errorf("could not create primary pod disruption budget: %v", err) } c.logger.Infof("pod disruption budget %q already exists", util.NameFromMeta(pdb.ObjectMeta)) - if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.podDisruptionBudgetName(), metav1.GetOptions{}); err != nil { + if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.PrimaryPodDisruptionBudgetName(), metav1.GetOptions{}); err != nil { return fmt.Errorf("could not fetch existing %q pod disruption budget", util.NameFromMeta(pdb.ObjectMeta)) } } - c.logger.Infof("created missing pod disruption budget %q", util.NameFromMeta(pdb.ObjectMeta)) - c.PodDisruptionBudget = pdb + return nil +} + +func (c *Cluster) syncCriticalOpPodDisruptionBudget(isUpdate bool) error { + var ( + pdb *policyv1.PodDisruptionBudget + err error + ) + if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.criticalOpPodDisruptionBudgetName(), metav1.GetOptions{}); err == nil { + c.CriticalOpPodDisruptionBudget = pdb + newPDB := c.generateCriticalOpPodDisruptionBudget() + match, reason := c.comparePodDisruptionBudget(pdb, newPDB) + if !match { + c.logPDBChanges(pdb, newPDB, isUpdate, reason) + if err = c.updateCriticalOpPodDisruptionBudget(newPDB); err != nil { + return err + } + } else { + c.CriticalOpPodDisruptionBudget = pdb + } + return nil + + } + if !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("could not get pod disruption budget: %v", err) + } + // no existing pod disruption budget, create new one + c.logger.Infof("could not find pod disruption budget for critical operations") + + if err = c.createCriticalOpPodDisruptionBudget(); err != nil { + if !k8sutil.ResourceAlreadyExists(err) { + return fmt.Errorf("could not create pod disruption budget for critical operations: %v", err) + } + c.logger.Infof("pod disruption budget %q already exists", util.NameFromMeta(pdb.ObjectMeta)) + if pdb, err = c.KubeClient.PodDisruptionBudgets(c.Namespace).Get(context.TODO(), c.criticalOpPodDisruptionBudgetName(), metav1.GetOptions{}); err != nil { + return fmt.Errorf("could not fetch existing %q pod disruption budget", util.NameFromMeta(pdb.ObjectMeta)) + } + } return nil } -func (c *Cluster) syncStatefulSet() error { +func (c *Cluster) syncPodDisruptionBudgets(isUpdate bool) error { + errors := make([]string, 0) + if err := c.syncPrimaryPodDisruptionBudget(isUpdate); err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + + if err := c.syncCriticalOpPodDisruptionBudget(isUpdate); err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + + if len(errors) > 0 { + return fmt.Errorf("%v", strings.Join(errors, `', '`)) + } + return nil +} + +func (c *Cluster) syncStatefulSet() error { + var ( + restartWait uint32 + configPatched bool + restartPrimaryFirst bool + ) podsToRecreate := make([]v1.Pod, 0) + isSafeToRecreatePods := true + postponeReasons := make([]string, 0) switchoverCandidates := make([]spec.NamespacedName, 0) pods, err := c.listPods() if err != nil { - c.logger.Infof("could not list pods of the statefulset: %v", err) + c.logger.Warnf("could not list pods of the statefulset: %v", err) } // NB: Be careful to consider the codepath that acts on podsRollingUpdateRequired before returning early. sset, err := c.KubeClient.StatefulSets(c.Namespace).Get(context.TODO(), c.statefulSetName(), metav1.GetOptions{}) + if err != nil && !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("error during reading of statefulset: %v", err) + } + if err != nil { - if !k8sutil.ResourceNotFound(err) { - return fmt.Errorf("error during reading of statefulset: %v", err) - } // statefulset does not exist, try to re-create it - c.Statefulset = nil c.logger.Infof("cluster's statefulset does not exist") sset, err = c.createStatefulSet() @@ -299,6 +597,11 @@ func (c *Cluster) syncStatefulSet() error { c.logger.Infof("created missing statefulset %q", util.NameFromMeta(sset.ObjectMeta)) } else { + desiredSts, err := c.generateStatefulSet(&c.Spec) + if err != nil { + return fmt.Errorf("could not generate statefulset: %v", err) + } + c.logger.Debug("syncing statefulsets") // check if there are still pods with a rolling update flag for _, pod := range pods { if c.getRollingUpdateFlagFromPod(&pod) { @@ -313,18 +616,36 @@ func (c *Cluster) syncStatefulSet() error { } if len(podsToRecreate) > 0 { - c.logger.Debugf("%d / %d pod(s) still need to be rotated", len(podsToRecreate), len(pods)) + c.logger.Infof("%d / %d pod(s) still need to be rotated", len(podsToRecreate), len(pods)) } // statefulset is already there, make sure we use its definition in order to compare with the spec. c.Statefulset = sset - desiredSts, err := c.generateStatefulSet(&c.Spec) - if err != nil { - return fmt.Errorf("could not generate statefulset: %v", err) - } - cmp := c.compareStatefulSetWith(desiredSts) + if !cmp.rollingUpdate { + updatedPodAnnotations := map[string]*string{} + for _, anno := range cmp.deletedPodAnnotations { + updatedPodAnnotations[anno] = nil + } + for anno, val := range desiredSts.Spec.Template.Annotations { + updatedPodAnnotations[anno] = &val + } + metadataReq := map[string]map[string]map[string]*string{"metadata": {"annotations": updatedPodAnnotations}} + patch, err := json.Marshal(metadataReq) + if err != nil { + return fmt.Errorf("could not form patch for pod annotations: %v", err) + } + + for _, pod := range pods { + if changed, _ := c.compareAnnotations(pod.Annotations, desiredSts.Spec.Template.Annotations, nil); changed { + _, err = c.KubeClient.Pods(c.Namespace).Patch(context.TODO(), pod.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("could not patch annotations for pod %q: %v", pod.Name, err) + } + } + } + } if !cmp.match { if cmp.rollingUpdate { podsToRecreate = make([]v1.Pod, 0) @@ -376,23 +697,172 @@ func (c *Cluster) syncStatefulSet() error { } } - // Apply special PostgreSQL parameters that can only be set via the Patroni API. + // apply PostgreSQL parameters that can only be set via the Patroni API. // it is important to do it after the statefulset pods are there, but before the rolling update // since those parameters require PostgreSQL restart. - if err := c.checkAndSetGlobalPostgreSQLConfiguration(); err != nil { - return fmt.Errorf("could not set cluster-wide PostgreSQL configuration options: %v", err) + pods, err = c.listPods() + if err != nil { + c.logger.Warnf("could not get list of pods to apply PostgreSQL parameters only to be set via Patroni API: %v", err) + } + + requiredPgParameters := make(map[string]string) + for k, v := range c.Spec.Parameters { + requiredPgParameters[k] = v + } + // if streams are defined wal_level must be switched to logical + if len(c.Spec.Streams) > 0 { + requiredPgParameters["wal_level"] = "logical" + } + + // sync Patroni config + c.logger.Debug("syncing Patroni config") + if configPatched, restartPrimaryFirst, restartWait, err = c.syncPatroniConfig(pods, c.Spec.Patroni, requiredPgParameters); err != nil { + c.logger.Warningf("Patroni config updated? %v - errors during config sync: %v", configPatched, err) + postponeReasons = append(postponeReasons, "errors during Patroni config sync") + // Only mark unsafe if all pods are running. If some pods are not running, + // Patroni API errors are expected and should not block pod recreation, + // which is the only way to fix non-running pods. + if c.allPodsRunning(pods) { + isSafeToRecreatePods = false + } else { + c.logger.Warningf("ignoring Patroni config sync errors because some pods are not running") + } + } + + // restart Postgres where it is still pending + if err = c.restartInstances(pods, restartWait, restartPrimaryFirst); err != nil { + c.logger.Errorf("errors while restarting Postgres in pods via Patroni API: %v", err) + postponeReasons = append(postponeReasons, "errors while restarting Postgres via Patroni API") + // Same logic: don't let unreachable non-running pods block recreation. + if c.allPodsRunning(pods) { + isSafeToRecreatePods = false + } else { + c.logger.Warningf("ignoring Patroni restart errors because some pods are not running") + } } // if we get here we also need to re-create the pods (either leftovers from the old // statefulset or those that got their configuration from the outdated statefulset) if len(podsToRecreate) > 0 { - c.logger.Debugln("performing rolling update") - c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update") - if err := c.recreatePods(podsToRecreate, switchoverCandidates); err != nil { - return fmt.Errorf("could not recreate pods: %v", err) + if isSafeToRecreatePods { + c.logger.Info("performing rolling update") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update") + if err := c.recreatePods(podsToRecreate, switchoverCandidates); err != nil { + return fmt.Errorf("could not recreate pods: %v", err) + } + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated") + } else { + c.logger.Warningf("postpone pod recreation until next sync - reason: %s", strings.Join(postponeReasons, `', '`)) } - c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated") } + + return nil +} + +func (c *Cluster) syncPatroniConfig(pods []v1.Pod, requiredPatroniConfig acidv1.Patroni, requiredPgParameters map[string]string) (bool, bool, uint32, error) { + var ( + effectivePatroniConfig acidv1.Patroni + effectivePgParameters map[string]string + loopWait uint32 + configPatched bool + restartPrimaryFirst bool + err error + ) + + errors := make([]string, 0) + + // get Postgres config, compare with manifest and update via Patroni PATCH endpoint if it differs + for i, pod := range pods { + podName := util.NameFromMeta(pods[i].ObjectMeta) + effectivePatroniConfig, effectivePgParameters, err = c.getPatroniConfig(&pod) + if err != nil { + errors = append(errors, fmt.Sprintf("could not get Postgres config from pod %s: %v", podName, err)) + continue + } + loopWait = effectivePatroniConfig.LoopWait + + // empty config probably means cluster is not fully initialized yet, e.g. restoring from backup + if reflect.DeepEqual(effectivePatroniConfig, acidv1.Patroni{}) || len(effectivePgParameters) == 0 { + errors = append(errors, fmt.Sprintf("empty Patroni config on pod %s - skipping config patch", podName)) + } else { + configPatched, restartPrimaryFirst, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, effectivePatroniConfig, requiredPatroniConfig, effectivePgParameters, requiredPgParameters) + if err != nil { + errors = append(errors, fmt.Sprintf("could not set PostgreSQL configuration options for pod %s: %v", podName, err)) + continue + } + + // it could take up to LoopWait to apply the config + if configPatched { + time.Sleep(time.Duration(loopWait)*time.Second + time.Second*2) + // Patroni's config endpoint is just a "proxy" to DCS. + // It is enough to patch it only once and it doesn't matter which pod is used + break + } + } + } + + if len(errors) > 0 { + err = fmt.Errorf("%v", strings.Join(errors, `', '`)) + } + + return configPatched, restartPrimaryFirst, loopWait, err +} + +func (c *Cluster) restartInstances(pods []v1.Pod, restartWait uint32, restartPrimaryFirst bool) (err error) { + errors := make([]string, 0) + remainingPods := make([]*v1.Pod, 0) + + skipRole := Master + if restartPrimaryFirst { + skipRole = Replica + } + + for i, pod := range pods { + role := PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel]) + if role == skipRole { + remainingPods = append(remainingPods, &pods[i]) + continue + } + if err = c.restartInstance(&pod, restartWait); err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + } + + // in most cases only the master should be left to restart + if len(remainingPods) > 0 { + for _, remainingPod := range remainingPods { + if err = c.restartInstance(remainingPod, restartWait); err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) + } + } + } + + if len(errors) > 0 { + return fmt.Errorf("%v", strings.Join(errors, `', '`)) + } + + return nil +} + +func (c *Cluster) restartInstance(pod *v1.Pod, restartWait uint32) error { + // if the config update requires a restart, call Patroni restart + podName := util.NameFromMeta(pod.ObjectMeta) + role := PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel]) + memberData, err := c.getPatroniMemberData(pod) + if err != nil { + return fmt.Errorf("could not restart Postgres in %s pod %s: %v", role, podName, err) + } + + // do restart only when it is pending + if memberData.PendingRestart { + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("restarting Postgres server within %s pod %s", role, podName)) + if err := c.patroni.Restart(pod); err != nil { + return err + } + time.Sleep(time.Duration(restartWait) * time.Second) + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("Postgres server restart done for %s pod %s", role, podName)) + } + return nil } @@ -429,25 +899,170 @@ func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[stri } // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters -// (like max_connections) has changed and if necessary sets it via the Patroni API -func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration() error { +// (like max_connections) have changed and if necessary sets it via the Patroni API +func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectivePatroniConfig, desiredPatroniConfig acidv1.Patroni, effectivePgParameters, desiredPgParameters map[string]string) (bool, bool, error) { + configToSet := make(map[string]interface{}) + parametersToSet := make(map[string]string) + restartPrimary := make([]bool, 0) + configPatched := false + requiresMasterRestart := false + + // compare effective and desired Patroni config options + if desiredPatroniConfig.LoopWait > 0 && desiredPatroniConfig.LoopWait != effectivePatroniConfig.LoopWait { + configToSet["loop_wait"] = desiredPatroniConfig.LoopWait + } + if desiredPatroniConfig.MaximumLagOnFailover > 0 && desiredPatroniConfig.MaximumLagOnFailover != effectivePatroniConfig.MaximumLagOnFailover { + configToSet["maximum_lag_on_failover"] = desiredPatroniConfig.MaximumLagOnFailover + } + if desiredPatroniConfig.PgHba != nil && !reflect.DeepEqual(desiredPatroniConfig.PgHba, effectivePatroniConfig.PgHba) { + configToSet["pg_hba"] = desiredPatroniConfig.PgHba + } + if desiredPatroniConfig.RetryTimeout > 0 && desiredPatroniConfig.RetryTimeout != effectivePatroniConfig.RetryTimeout { + configToSet["retry_timeout"] = desiredPatroniConfig.RetryTimeout + } + if desiredPatroniConfig.SynchronousMode != effectivePatroniConfig.SynchronousMode { + configToSet["synchronous_mode"] = desiredPatroniConfig.SynchronousMode + } + if desiredPatroniConfig.SynchronousModeStrict != effectivePatroniConfig.SynchronousModeStrict { + configToSet["synchronous_mode_strict"] = desiredPatroniConfig.SynchronousModeStrict + } + if desiredPatroniConfig.SynchronousNodeCount != effectivePatroniConfig.SynchronousNodeCount { + configToSet["synchronous_node_count"] = desiredPatroniConfig.SynchronousNodeCount + } + if desiredPatroniConfig.TTL > 0 && desiredPatroniConfig.TTL != effectivePatroniConfig.TTL { + configToSet["ttl"] = desiredPatroniConfig.TTL + } + + var desiredFailsafe *bool + if desiredPatroniConfig.FailsafeMode != nil { + desiredFailsafe = desiredPatroniConfig.FailsafeMode + } else if c.OpConfig.EnablePatroniFailsafeMode != nil { + desiredFailsafe = c.OpConfig.EnablePatroniFailsafeMode + } + + effectiveFailsafe := effectivePatroniConfig.FailsafeMode + + if desiredFailsafe != nil { + if effectiveFailsafe == nil || *desiredFailsafe != *effectiveFailsafe { + configToSet["failsafe_mode"] = *desiredFailsafe + } + } + + slotsToSet := make(map[string]interface{}) + // check if there is any slot deletion + for slotName, effectiveSlot := range c.replicationSlots { + if desiredSlot, exists := desiredPatroniConfig.Slots[slotName]; exists { + if reflect.DeepEqual(effectiveSlot, desiredSlot) { + continue + } + } + slotsToSet[slotName] = nil + delete(c.replicationSlots, slotName) + } + // check if specified slots exist in config and if they differ + for slotName, desiredSlot := range desiredPatroniConfig.Slots { + // only add slots specified in manifest to c.replicationSlots + for manifestSlotName := range c.Spec.Patroni.Slots { + if manifestSlotName == slotName { + c.replicationSlots[slotName] = desiredSlot + } + } + if effectiveSlot, exists := effectivePatroniConfig.Slots[slotName]; exists { + if reflect.DeepEqual(desiredSlot, effectiveSlot) { + continue + } + } + slotsToSet[slotName] = desiredSlot + } + if len(slotsToSet) > 0 { + configToSet["slots"] = slotsToSet + } + + // compare effective and desired parameters under postgresql section in Patroni config + for desiredOption, desiredValue := range desiredPgParameters { + effectiveValue := effectivePgParameters[desiredOption] + if isBootstrapOnlyParameter(desiredOption) && (effectiveValue != desiredValue) { + parametersToSet[desiredOption] = desiredValue + if slices.Contains(requirePrimaryRestartWhenDecreased, desiredOption) { + effectiveValueNum, errConv := strconv.Atoi(effectiveValue) + desiredValueNum, errConv2 := strconv.Atoi(desiredValue) + if errConv != nil || errConv2 != nil { + continue + } + if effectiveValueNum > desiredValueNum { + restartPrimary = append(restartPrimary, true) + continue + } + } + restartPrimary = append(restartPrimary, false) + } + } + + // check if there exist only config updates that require a restart of the primary + if len(restartPrimary) > 0 && !slices.Contains(restartPrimary, false) && len(configToSet) == 0 { + requiresMasterRestart = true + } + + if len(parametersToSet) > 0 { + configToSet["postgresql"] = map[string]interface{}{constants.PatroniPGParametersParameterName: parametersToSet} + } + + if len(configToSet) == 0 { + return configPatched, requiresMasterRestart, nil + } + + configToSetJson, err := json.Marshal(configToSet) + if err != nil { + c.logger.Debugf("could not convert config patch to JSON: %v", err) + } + + // try all pods until the first one that is successful, as it doesn't matter which pod + // carries the request to change configuration through + podName := util.NameFromMeta(pod.ObjectMeta) + c.logger.Debugf("patching Postgres config via Patroni API on pod %s with following options: %s", + podName, configToSetJson) + if err = c.patroni.SetConfig(pod, configToSet); err != nil { + return configPatched, requiresMasterRestart, fmt.Errorf("could not patch postgres parameters within pod %s: %v", podName, err) + } + configPatched = true + + return configPatched, requiresMasterRestart, nil +} + +// syncStandbyClusterConfiguration checks whether standby cluster +// parameters have changed and if necessary sets it via the Patroni API +func (c *Cluster) syncStandbyClusterConfiguration() error { var ( err error pods []v1.Pod ) - // we need to extract those options from the cluster manifest. - optionsToSet := make(map[string]string) - pgOptions := c.Spec.Parameters + standbyOptionsToSet := make(map[string]interface{}) + if c.Spec.StandbyCluster != nil { + c.logger.Infof("turning %q into a standby cluster", c.Name) + standbyOptionsToSet["create_replica_methods"] = []string{"bootstrap_standby_with_wale", "basebackup_fast_xlog"} + standbyOptionsToSet["restore_command"] = "envdir \"/run/etc/wal-e.d/env-standby\" /scripts/restore_command.sh \"%f\" \"%p\"" - for k, v := range pgOptions { - if isBootstrapOnlyParameter(k) { - optionsToSet[k] = v + if c.Spec.StandbyCluster.StandbyHost != "" { + standbyOptionsToSet["host"] = c.Spec.StandbyCluster.StandbyHost + } else { + standbyOptionsToSet["host"] = nil } - } - if len(optionsToSet) == 0 { - return nil + if c.Spec.StandbyCluster.StandbyPort != "" { + standbyOptionsToSet["port"] = c.Spec.StandbyCluster.StandbyPort + } else { + standbyOptionsToSet["port"] = nil + } + + if c.Spec.StandbyCluster.StandbyPrimarySlotName != "" { + standbyOptionsToSet["primary_slot_name"] = c.Spec.StandbyCluster.StandbyPrimarySlotName + } else { + standbyOptionsToSet["primary_slot_name"] = nil + } + } else { + c.logger.Infof("promoting standby cluster and detach from source") + standbyOptionsToSet = nil } if pods, err = c.listPods(); err != nil { @@ -460,79 +1075,348 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration() error { // carries the request to change configuration through for _, pod := range pods { podName := util.NameFromMeta(pod.ObjectMeta) - c.logger.Debugf("calling Patroni API on a pod %s to set the following Postgres options: %v", - podName, optionsToSet) - if err = c.patroni.SetPostgresParameters(&pod, optionsToSet); err == nil { + c.logger.Infof("patching Postgres config via Patroni API on pod %s with following options: %s", + podName, standbyOptionsToSet) + if err = c.patroni.SetStandbyClusterParameters(&pod, standbyOptionsToSet); err == nil { return nil } - c.logger.Warningf("could not patch postgres parameters with a pod %s: %v", podName, err) + c.logger.Warningf("could not patch postgres parameters within pod %s: %v", podName, err) } return fmt.Errorf("could not reach Patroni API to set Postgres options: failed on every pod (%d total)", len(pods)) } func (c *Cluster) syncSecrets() error { - var ( - err error - secret *v1.Secret - ) - c.logger.Info("syncing secrets") + c.logger.Debug("syncing secrets") c.setProcessName("syncing secrets") - secrets := c.generateUserSecrets() + errors := make([]string, 0) + generatedSecrets := c.generateUserSecrets() + retentionUsers := make([]string, 0) + currentTime := time.Now() - for secretUsername, secretSpec := range secrets { - if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil { - c.Secrets[secret.UID] = secret - c.logger.Debugf("created new secret %s, uid: %s", util.NameFromMeta(secret.ObjectMeta), secret.UID) + for secretUsername, generatedSecret := range generatedSecrets { + pgUserDegraded := false + createdSecret, err := c.KubeClient.Secrets(generatedSecret.Namespace).Create(context.TODO(), generatedSecret, metav1.CreateOptions{}) + if err == nil { + c.Secrets[createdSecret.UID] = createdSecret + c.logger.Infof("created new secret %s, namespace: %s, uid: %s", util.NameFromMeta(createdSecret.ObjectMeta), generatedSecret.Namespace, createdSecret.UID) continue } if k8sutil.ResourceAlreadyExists(err) { - var userMap map[string]spec.PgUser - if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(context.TODO(), secretSpec.Name, metav1.GetOptions{}); err != nil { - return fmt.Errorf("could not get current secret: %v", err) - } - if secretUsername != string(secret.Data["username"]) { - c.logger.Errorf("secret %s does not contain the role %s", secretSpec.Name, secretUsername) + updatedSecret, err := c.updateSecret(secretUsername, generatedSecret, &retentionUsers, currentTime) + if err == nil { + c.Secrets[updatedSecret.UID] = updatedSecret continue } - c.Secrets[secret.UID] = secret - c.logger.Debugf("secret %s already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta)) - if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name { - secretUsername = constants.SuperuserKeyName - userMap = c.systemUsers - } else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name { - secretUsername = constants.ReplicationUserKeyName - userMap = c.systemUsers - } else { - userMap = c.pgUsers - } - pwdUser := userMap[secretUsername] - // if this secret belongs to the infrastructure role and the password has changed - replace it in the secret - if pwdUser.Password != string(secret.Data["password"]) && - pwdUser.Origin == spec.RoleOriginInfrastructure { - - c.logger.Debugf("updating the secret %s from the infrastructure roles", secretSpec.Name) - if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secretSpec, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err) - } - } else { - // for non-infrastructure role - update the role with the password from the secret - pwdUser.Password = string(secret.Data["password"]) - userMap[secretUsername] = pwdUser - } + errors = append(errors, fmt.Sprintf("syncing secret %s failed: %v", util.NameFromMeta(generatedSecret.ObjectMeta), err)) + pgUserDegraded = true } else { - return fmt.Errorf("could not create secret for user %s: %v", secretUsername, err) + errors = append(errors, fmt.Sprintf("could not create secret for user %s: in namespace %s: %v", secretUsername, generatedSecret.Namespace, err)) + pgUserDegraded = true + } + c.updatePgUser(secretUsername, pgUserDegraded) + } + + // remove rotation users that exceed the retention interval + if len(retentionUsers) > 0 { + if err := c.cleanupRotatedUsers(retentionUsers); err != nil { + errors = append(errors, fmt.Sprintf("error removing users exceeding configured retention interval: %v", err)) } } + if len(errors) > 0 { + return fmt.Errorf("%v", strings.Join(errors, `', '`)) + } + return nil } +func (c *Cluster) getNextRotationDate(currentDate time.Time) (time.Time, string) { + nextRotationDate := currentDate.AddDate(0, 0, int(c.OpConfig.PasswordRotationInterval)) + return nextRotationDate, nextRotationDate.Format(time.RFC3339) +} + +func (c *Cluster) updateSecret( + secretUsername string, + generatedSecret *v1.Secret, + retentionUsers *[]string, + currentTime time.Time) (*v1.Secret, error) { + var ( + secret *v1.Secret + err error + updateSecret bool + updateSecretMsg string + ) + + // get the secret first + if secret, err = c.KubeClient.Secrets(generatedSecret.Namespace).Get(context.TODO(), generatedSecret.Name, metav1.GetOptions{}); err != nil { + return generatedSecret, fmt.Errorf("could not get current secret: %v", err) + } + c.Secrets[secret.UID] = secret + + // fetch user map to update later + var userMap map[string]spec.PgUser + var userKey string + switch secretUsername { + case c.systemUsers[constants.SuperuserKeyName].Name: + userKey = constants.SuperuserKeyName + userMap = c.systemUsers + case c.systemUsers[constants.ReplicationUserKeyName].Name: + userKey = constants.ReplicationUserKeyName + userMap = c.systemUsers + default: + userKey = secretUsername + userMap = c.pgUsers + } + + // use system user when pooler is enabled and pooler user is specfied in manifest + if _, exists := c.systemUsers[constants.ConnectionPoolerUserKeyName]; exists { + if secretUsername == c.systemUsers[constants.ConnectionPoolerUserKeyName].Name { + userKey = constants.ConnectionPoolerUserKeyName + userMap = c.systemUsers + } + } + // use system user when streams are defined and fes_user is specfied in manifest + if _, exists := c.systemUsers[constants.EventStreamUserKeyName]; exists { + if secretUsername == c.systemUsers[constants.EventStreamUserKeyName].Name { + userKey = constants.EventStreamUserKeyName + userMap = c.systemUsers + } + } + + pwdUser := userMap[userKey] + secretName := util.NameFromMeta(secret.ObjectMeta) + + // do not perform any rotation of reset for standby clusters + if !isStandbyCluster(&c.Spec) { + updateSecretMsg, err = c.checkForPasswordRotation(secret, secretUsername, pwdUser, retentionUsers, currentTime) + if err != nil { + return nil, fmt.Errorf("error while checking for password rotation: %v", err) + } + if updateSecretMsg != "" { + updateSecret = true + } + } + + // if this secret belongs to the infrastructure role and the password has changed - replace it in the secret + if pwdUser.Password != string(secret.Data["password"]) && pwdUser.Origin == spec.RoleOriginInfrastructure { + secret = generatedSecret + updateSecret = true + updateSecretMsg = fmt.Sprintf("updating the secret %s from the infrastructure roles", secretName) + } else { + // for non-infrastructure role - update the role with username and password from secret + pwdUser.Name = string(secret.Data["username"]) + pwdUser.Password = string(secret.Data["password"]) + // update membership if we deal with a rotation user + if secretUsername != pwdUser.Name { + pwdUser.Rotated = true + pwdUser.MemberOf = []string{secretUsername} + } + userMap[userKey] = pwdUser + } + + if !reflect.DeepEqual(secret.ObjectMeta.OwnerReferences, generatedSecret.ObjectMeta.OwnerReferences) { + updateSecret = true + updateSecretMsg = fmt.Sprintf("secret %s owner references do not match the current ones", secretName) + secret.ObjectMeta.OwnerReferences = generatedSecret.ObjectMeta.OwnerReferences + } + + if updateSecret { + c.logger.Infof("%s", updateSecretMsg) + if secret, err = c.KubeClient.Secrets(secret.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil { + return nil, fmt.Errorf("could not update secret: %v", err) + } + } + + if changed, _ := c.compareAnnotations(secret.Annotations, generatedSecret.Annotations, nil); changed { + patchData, err := metaAnnotationsPatch(generatedSecret.Annotations) + if err != nil { + return nil, fmt.Errorf("could not form patch for secret annotations: %v", err) + } + secret, err = c.KubeClient.Secrets(secret.Namespace).Patch(context.TODO(), secret.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return nil, fmt.Errorf("could not patch annotations for secret: %v", err) + } + } + + return secret, nil +} + +func (c *Cluster) checkForPasswordRotation( + secret *v1.Secret, + secretUsername string, + pwdUser spec.PgUser, + retentionUsers *[]string, + currentTime time.Time) (string, error) { + + var ( + passwordRotationMsg string + err error + ) + + // if password rotation is enabled update password and username if rotation interval has been passed + // rotation can be enabled globally or via the manifest (excluding the Postgres superuser) + rotationEnabledInManifest := secretUsername != constants.SuperuserKeyName && + (slices.Contains(c.Spec.UsersWithSecretRotation, secretUsername) || + slices.Contains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername)) + + // globally enabled rotation is only allowed for manifest and bootstrapped roles + allowedRoleTypes := []spec.RoleOrigin{spec.RoleOriginManifest, spec.RoleOriginBootstrap} + rotationAllowed := !pwdUser.IsDbOwner && slices.Contains(allowedRoleTypes, pwdUser.Origin) + + // users can ignore any kind of rotation + isIgnoringRotation := slices.Contains(c.Spec.UsersIgnoringSecretRotation, secretUsername) + + if ((c.OpConfig.EnablePasswordRotation && rotationAllowed) || rotationEnabledInManifest) && !isIgnoringRotation { + passwordRotationMsg, err = c.rotatePasswordInSecret(secret, secretUsername, pwdUser.Origin, currentTime, retentionUsers) + if err != nil { + c.logger.Warnf("password rotation failed for user %s: %v", secretUsername, err) + } + } else { + // username might not match if password rotation has been disabled again + usernameFromSecret := string(secret.Data["username"]) + if secretUsername != usernameFromSecret { + // handle edge case when manifest user conflicts with a user from prepared databases + if strings.Replace(usernameFromSecret, "-", "_", -1) == strings.Replace(secretUsername, "-", "_", -1) { + return "", fmt.Errorf("could not update secret because of user name mismatch: expected: %s, got: %s", secretUsername, usernameFromSecret) + } + *retentionUsers = append(*retentionUsers, secretUsername) + secret.Data["username"] = []byte(secretUsername) + secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength)) + secret.Data["nextRotation"] = []byte{} + passwordRotationMsg = fmt.Sprintf("secret does not contain the role %s - updating username and resetting password", secretUsername) + } + } + + return passwordRotationMsg, nil +} + +func (c *Cluster) rotatePasswordInSecret( + secret *v1.Secret, + secretUsername string, + roleOrigin spec.RoleOrigin, + currentTime time.Time, + retentionUsers *[]string) (string, error) { + var ( + err error + nextRotationDate time.Time + nextRotationDateStr string + expectedUsername string + rotationModeChanged bool + updateSecretMsg string + ) + + secretName := util.NameFromMeta(secret.ObjectMeta) + + // initialize password rotation setting first rotation date + nextRotationDateStr = string(secret.Data["nextRotation"]) + if nextRotationDate, err = time.ParseInLocation(time.RFC3339, nextRotationDateStr, currentTime.UTC().Location()); err != nil { + nextRotationDate, nextRotationDateStr = c.getNextRotationDate(currentTime) + secret.Data["nextRotation"] = []byte(nextRotationDateStr) + updateSecretMsg = fmt.Sprintf("rotation date not found in secret %s. Setting it to %s", secretName, nextRotationDateStr) + } + + // check if next rotation can happen sooner + // if rotation interval has been decreased + currentRotationDate, nextRotationDateStr := c.getNextRotationDate(currentTime) + if nextRotationDate.After(currentRotationDate) { + nextRotationDate = currentRotationDate + } + + // set username and check if it differs from current value in secret + currentUsername := string(secret.Data["username"]) + if !slices.Contains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) { + expectedUsername = fmt.Sprintf("%s%s", secretUsername, currentTime.Format(constants.RotationUserDateFormat)) + } else { + expectedUsername = secretUsername + } + + // when changing to in-place rotation update secret immediatly + // if currentUsername is longer we know it has a date suffix + // the other way around we can wait until the next rotation date + if len(currentUsername) > len(expectedUsername) { + rotationModeChanged = true + c.logger.Infof("updating secret %s after switching to in-place rotation mode for username: %s", secretName, string(secret.Data["username"])) + } + + // update password and next rotation date if configured interval has passed + if currentTime.After(nextRotationDate) || rotationModeChanged { + // create rotation user if role is not listed for in-place password update + if !slices.Contains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) { + secret.Data["username"] = []byte(expectedUsername) + c.logger.Infof("updating username in secret %s and creating rotation user %s in the database", secretName, expectedUsername) + // whenever there is a rotation, check if old rotation users can be deleted + *retentionUsers = append(*retentionUsers, secretUsername) + } else { + // when passwords of system users are rotated in-place, pods have to be replaced + if roleOrigin == spec.RoleOriginSystem { + pods, err := c.listPods() + if err != nil { + return "", fmt.Errorf("could not list pods of the statefulset: %v", err) + } + for _, pod := range pods { + if err = c.markRollingUpdateFlagForPod(&pod, + fmt.Sprintf("replace pod due to password rotation of system user %s", secretUsername)); err != nil { + c.logger.Warnf("marking pod for rolling update due to password rotation failed: %v", err) + } + } + } + + // when password of connection pooler is rotated in-place, pooler pods have to be replaced + if roleOrigin == spec.RoleOriginConnectionPooler { + listOptions := metav1.ListOptions{ + LabelSelector: c.poolerLabelsSet(true).String(), + } + poolerPods, err := c.listPoolerPods(listOptions) + if err != nil { + return "", fmt.Errorf("could not list pods of the pooler deployment: %v", err) + } + for _, poolerPod := range poolerPods { + if err = c.markRollingUpdateFlagForPod(&poolerPod, + fmt.Sprintf("replace pooler pod due to password rotation of pooler user %s", secretUsername)); err != nil { + c.logger.Warnf("marking pooler pod for rolling update due to password rotation failed: %v", err) + } + } + } + + // when password of stream user is rotated in-place, it should trigger rolling update in FES deployment + if roleOrigin == spec.RoleOriginStream { + c.logger.Warnf("password in secret of stream user %s changed", constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix) + } + + secret.Data["username"] = []byte(secretUsername) + } + secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength)) + secret.Data["nextRotation"] = []byte(nextRotationDateStr) + updateSecretMsg = fmt.Sprintf("updating secret %s due to password rotation - next rotation date: %s", secretName, nextRotationDateStr) + } + + return updateSecretMsg, nil +} + +func (c *Cluster) updatePgUser(secretUsername string, degraded bool) { + for key, pgUser := range c.pgUsers { + if pgUser.Name == secretUsername { + pgUser.Degraded = degraded + c.pgUsers[key] = pgUser + return + } + } + for key, pgUser := range c.systemUsers { + if pgUser.Name == secretUsername { + pgUser.Degraded = degraded + c.systemUsers[key] = pgUser + return + } + } +} + func (c *Cluster) syncRoles() (err error) { c.setProcessName("syncing roles") var ( dbUsers spec.PgUserMap + newUsers spec.PgUserMap userNames []string ) @@ -553,14 +1437,22 @@ func (c *Cluster) syncRoles() (err error) { // mapping between original role name and with deletion suffix deletedUsers := map[string]string{} + newUsers = make(map[string]spec.PgUser) // create list of database roles to query for _, u := range c.pgUsers { - userNames = append(userNames, u.Name) + pgRole := u.Name + userNames = append(userNames, pgRole) + + // when a rotation happened add group role to query its rolconfig + if u.Rotated { + userNames = append(userNames, u.MemberOf[0]) + } + // add team member role name with rename suffix in case we need to rename it back if u.Origin == spec.RoleOriginTeamsAPI && c.OpConfig.EnableTeamMemberDeprecation { - deletedUsers[u.Name+c.OpConfig.RoleDeletionSuffix] = u.Name - userNames = append(userNames, u.Name+c.OpConfig.RoleDeletionSuffix) + deletedUsers[pgRole+c.OpConfig.RoleDeletionSuffix] = pgRole + userNames = append(userNames, pgRole+c.OpConfig.RoleDeletionSuffix) } } @@ -572,15 +1464,10 @@ func (c *Cluster) syncRoles() (err error) { } } - // add pooler user to list of pgUsers, too - // to check if the pooler user exists or has to be created - if needMasterConnectionPooler(&c.Spec) || needReplicaConnectionPooler(&c.Spec) { - connectionPoolerUser := c.systemUsers[constants.ConnectionPoolerUserKeyName] - userNames = append(userNames, connectionPoolerUser.Name) - - if _, exists := c.pgUsers[connectionPoolerUser.Name]; !exists { - c.pgUsers[connectionPoolerUser.Name] = connectionPoolerUser - } + // search also for system users + for _, systemUser := range c.systemUsers { + userNames = append(userNames, systemUser.Name) + newUsers[systemUser.Name] = systemUser } dbUsers, err = c.readPgUsersFromDatabase(userNames) @@ -588,17 +1475,37 @@ func (c *Cluster) syncRoles() (err error) { return fmt.Errorf("error getting users from the database: %v", err) } - // update pgUsers where a deleted role was found - // so that they are skipped in ProduceSyncRequests +DBUSERS: for _, dbUser := range dbUsers { - if originalUser, exists := deletedUsers[dbUser.Name]; exists { - recreatedUser := c.pgUsers[originalUser] + // copy rolconfig to rotation users + for pgUserName, pgUser := range c.pgUsers { + if pgUser.Rotated && pgUser.MemberOf[0] == dbUser.Name { + pgUser.Parameters = dbUser.Parameters + c.pgUsers[pgUserName] = pgUser + // remove group role from dbUsers to not count as deleted role + delete(dbUsers, dbUser.Name) + continue DBUSERS + } + } + + // update pgUsers where a deleted role was found + // so that they are skipped in ProduceSyncRequests + originalUsername, foundDeletedUser := deletedUsers[dbUser.Name] + // check if original user does not exist in dbUsers + _, originalUserAlreadyExists := dbUsers[originalUsername] + if foundDeletedUser && !originalUserAlreadyExists { + recreatedUser := c.pgUsers[originalUsername] recreatedUser.Deleted = true - c.pgUsers[originalUser] = recreatedUser + c.pgUsers[originalUsername] = recreatedUser } } - pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(dbUsers, c.pgUsers) + // last but not least copy pgUsers to newUsers to send to ProduceSyncRequests + for _, pgUser := range c.pgUsers { + newUsers[pgUser.Name] = pgUser + } + + pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(dbUsers, newUsers) if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil { return fmt.Errorf("error executing sync statements: %v", err) } @@ -608,7 +1515,7 @@ func (c *Cluster) syncRoles() (err error) { func (c *Cluster) syncDatabases() error { c.setProcessName("syncing databases") - + errors := make([]string, 0) createDatabases := make(map[string]string) alterOwnerDatabases := make(map[string]string) preparedDatabases := make([]string, 0) @@ -634,7 +1541,7 @@ func (c *Cluster) syncDatabases() error { for preparedDatabaseName := range c.Spec.PreparedDatabases { _, exists := currentDatabases[preparedDatabaseName] if !exists { - createDatabases[preparedDatabaseName] = preparedDatabaseName + constants.OwnerRoleNameSuffix + createDatabases[preparedDatabaseName] = fmt.Sprintf("%s%s", preparedDatabaseName, constants.OwnerRoleNameSuffix) preparedDatabases = append(preparedDatabases, preparedDatabaseName) } } @@ -654,33 +1561,53 @@ func (c *Cluster) syncDatabases() error { for databaseName, owner := range createDatabases { if err = c.executeCreateDatabase(databaseName, owner); err != nil { - return err + errors = append(errors, err.Error()) } } for databaseName, owner := range alterOwnerDatabases { if err = c.executeAlterDatabaseOwner(databaseName, owner); err != nil { - return err + errors = append(errors, err.Error()) + } + } + + if len(createDatabases) > 0 { + // trigger creation of pooler objects in new database in syncConnectionPooler + if c.ConnectionPooler != nil { + for _, role := range [2]PostgresRole{Master, Replica} { + c.ConnectionPooler[role].LookupFunction = false + } } } // set default privileges for prepared database for _, preparedDatabase := range preparedDatabases { if err := c.initDbConnWithName(preparedDatabase); err != nil { - return fmt.Errorf("could not init database connection to %s", preparedDatabase) + errors = append(errors, fmt.Sprintf("could not init database connection to %s", preparedDatabase)) + continue } - if err = c.execAlterGlobalDefaultPrivileges(preparedDatabase+constants.OwnerRoleNameSuffix, preparedDatabase); err != nil { - return err + + for _, owner := range c.getOwnerRoles(preparedDatabase, c.Spec.PreparedDatabases[preparedDatabase].DefaultUsers) { + if err = c.execAlterGlobalDefaultPrivileges(owner, preparedDatabase); err != nil { + errors = append(errors, err.Error()) + } } } + if len(errors) > 0 { + return fmt.Errorf("error(s) while syncing databases: %v", strings.Join(errors, `', '`)) + } + return nil } func (c *Cluster) syncPreparedDatabases() error { c.setProcessName("syncing prepared databases") + errors := make([]string, 0) + for preparedDbName, preparedDB := range c.Spec.PreparedDatabases { if err := c.initDbConnWithName(preparedDbName); err != nil { - return fmt.Errorf("could not init connection to database %s: %v", preparedDbName, err) + errors = append(errors, fmt.Sprintf("could not init connection to database %s: %v", preparedDbName, err)) + continue } c.logger.Debugf("syncing prepared database %q", preparedDbName) @@ -690,12 +1617,13 @@ func (c *Cluster) syncPreparedDatabases() error { preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} } if err := c.syncPreparedSchemas(preparedDbName, preparedSchemas); err != nil { - return err + errors = append(errors, err.Error()) + continue } // install extensions if err := c.syncExtensions(preparedDB.Extensions); err != nil { - return err + errors = append(errors, err.Error()) } if err := c.closeDbConn(); err != nil { @@ -703,11 +1631,16 @@ func (c *Cluster) syncPreparedDatabases() error { } } + if len(errors) > 0 { + return fmt.Errorf("error(s) while syncing prepared databases: %v", strings.Join(errors, `', '`)) + } + return nil } func (c *Cluster) syncPreparedSchemas(databaseName string, preparedSchemas map[string]acidv1.PreparedSchema) error { c.setProcessName("syncing prepared schemas") + errors := make([]string, 0) currentSchemas, err := c.getSchemas() if err != nil { @@ -723,24 +1656,28 @@ func (c *Cluster) syncPreparedSchemas(databaseName string, preparedSchemas map[s if createPreparedSchemas, equal := util.SubstractStringSlices(schemas, currentSchemas); !equal { for _, schemaName := range createPreparedSchemas { owner := constants.OwnerRoleNameSuffix - dbOwner := databaseName + owner + dbOwner := fmt.Sprintf("%s%s", databaseName, owner) if preparedSchemas[schemaName].DefaultRoles == nil || *preparedSchemas[schemaName].DefaultRoles { - owner = databaseName + "_" + schemaName + owner + owner = fmt.Sprintf("%s_%s%s", databaseName, schemaName, owner) } else { owner = dbOwner } if err = c.executeCreateDatabaseSchema(databaseName, schemaName, dbOwner, owner); err != nil { - return err + errors = append(errors, err.Error()) } } } + if len(errors) > 0 { + return fmt.Errorf("error(s) while syncing schemas of prepared databases: %v", strings.Join(errors, `', '`)) + } + return nil } func (c *Cluster) syncExtensions(extensions map[string]string) error { c.setProcessName("syncing database extensions") - + errors := make([]string, 0) createExtensions := make(map[string]string) alterExtensions := make(map[string]string) @@ -760,22 +1697,26 @@ func (c *Cluster) syncExtensions(extensions map[string]string) error { for extName, schema := range createExtensions { if err = c.executeCreateExtension(extName, schema); err != nil { - return err + errors = append(errors, err.Error()) } } for extName, schema := range alterExtensions { if err = c.executeAlterExtension(extName, schema); err != nil { - return err + errors = append(errors, err.Error()) } } + if len(errors) > 0 { + return fmt.Errorf("error(s) while syncing database extensions: %v", strings.Join(errors, `', '`)) + } + return nil } func (c *Cluster) syncLogicalBackupJob() error { var ( - job *batchv1beta1.CronJob - desiredJob *batchv1beta1.CronJob + job *batchv1.CronJob + desiredJob *batchv1.CronJob err error ) c.setProcessName("syncing the logical backup job") @@ -789,18 +1730,66 @@ func (c *Cluster) syncLogicalBackupJob() error { if err != nil { return fmt.Errorf("could not generate the desired logical backup job state: %v", err) } - if match, reason := k8sutil.SameLogicalBackupJob(job, desiredJob); !match { + if !reflect.DeepEqual(job.ObjectMeta.OwnerReferences, desiredJob.ObjectMeta.OwnerReferences) { + c.logger.Info("new logical backup job's owner references do not match the current ones") + job, err = c.KubeClient.CronJobs(job.Namespace).Update(context.TODO(), desiredJob, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update owner references for logical backup job %q: %v", job.Name, err) + } + c.logger.Infof("logical backup job %s updated", c.getLogicalBackupJobName()) + } + if cmp := c.compareLogicalBackupJob(job, desiredJob); !cmp.match { c.logger.Infof("logical job %s is not in the desired state and needs to be updated", c.getLogicalBackupJobName(), ) - if reason != "" { - c.logger.Infof("reason: %s", reason) + if len(cmp.reasons) != 0 { + for _, reason := range cmp.reasons { + c.logger.Infof("reason: %s", reason) + } + } + if len(cmp.deletedPodAnnotations) != 0 { + templateMetadataReq := map[string]map[string]map[string]map[string]map[string]map[string]map[string]*string{ + "spec": {"jobTemplate": {"spec": {"template": {"metadata": {"annotations": {}}}}}}} + for _, anno := range cmp.deletedPodAnnotations { + templateMetadataReq["spec"]["jobTemplate"]["spec"]["template"]["metadata"]["annotations"][anno] = nil + } + patch, err := json.Marshal(templateMetadataReq) + if err != nil { + return fmt.Errorf("could not marshal ObjectMeta for logical backup job %q pod template: %v", jobName, err) + } + + job, err = c.KubeClient.CronJobs(c.Namespace).Patch(context.TODO(), jobName, types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "") + if err != nil { + c.logger.Errorf("failed to remove annotations from the logical backup job %q pod template: %v", jobName, err) + return err + } } if err = c.patchLogicalBackupJob(desiredJob); err != nil { return fmt.Errorf("could not update logical backup job to match desired state: %v", err) } c.logger.Info("the logical backup job is synced") } + if !reflect.DeepEqual(job.Labels, desiredJob.Labels) { + patchData, err := metaLabelsPatch(desiredJob.Labels) + if err != nil { + return fmt.Errorf("could not form patch for the logical backup job %q labels: %v", jobName, err) + } + _, err = c.KubeClient.CronJobs(c.Namespace).Patch(context.TODO(), jobName, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("could not patch labels of the logical backup job %q: %v", jobName, err) + } + } + if changed, _ := c.compareAnnotations(job.Annotations, desiredJob.Annotations, nil); changed { + patchData, err := metaAnnotationsPatch(desiredJob.Annotations) + if err != nil { + return fmt.Errorf("could not form patch for the logical backup job %q: %v", jobName, err) + } + _, err = c.KubeClient.CronJobs(c.Namespace).Patch(context.TODO(), jobName, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("could not patch annotations of the logical backup job %q: %v", jobName, err) + } + } + c.LogicalBackupJob = desiredJob return nil } if !k8sutil.ResourceNotFound(err) { diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index e6f23914b..f7d46d427 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -1,26 +1,50 @@ package cluster import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "slices" "testing" "time" - "context" - + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + k8stesting "k8s.io/client-go/testing" + "github.com/golang/mock/gomock" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/zalando/postgres-operator/mocks" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" + "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "github.com/zalando/postgres-operator/pkg/util/patroni" "k8s.io/client-go/kubernetes/fake" ) -func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) { - acidClientSet := fakeacidv1.NewSimpleClientset() - clientSet := fake.NewSimpleClientset() +var patroniLogger = logrus.New().WithField("test", "patroni") +var acidClientSet = fakeacidv1.NewSimpleClientset() +var clientSet = fake.NewSimpleClientset() +func newMockPod(ip string) *v1.Pod { + return &v1.Pod{ + Status: v1.PodStatus{ + PodIP: ip, + }, + } +} + +func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) { return k8sutil.KubernetesClient{ PodsGetter: clientSet.CoreV1(), PostgresqlsGetter: acidClientSet.AcidV1(), @@ -28,6 +52,22 @@ func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) { }, clientSet } +func newFakeK8sSyncSecretsClient() (k8sutil.KubernetesClient, *fake.Clientset) { + // add a reactor that checks namespace existence before creating secrets + clientSet.PrependReactor("create", "secrets", func(action k8stesting.Action) (bool, runtime.Object, error) { + createAction := action.(k8stesting.CreateAction) + secret := createAction.GetObject().(*v1.Secret) + if secret.Namespace != "default" { + return true, nil, errors.New("namespace does not exist") + } + return false, nil, nil + }) + + return k8sutil.KubernetesClient{ + SecretsGetter: clientSet.CoreV1(), + }, clientSet +} + func TestSyncStatefulSetsAnnotations(t *testing.T) { testName := "test syncing statefulsets annotations" client, _ := newFakeK8sSyncClient() @@ -113,3 +153,903 @@ func TestSyncStatefulSetsAnnotations(t *testing.T) { t.Errorf("%s: inherited annotation not found in desired statefulset: %#v", testName, desiredSts.Annotations) } } + +func TestPodAnnotationsSync(t *testing.T) { + clusterName := "acid-test-cluster-2" + namespace := "default" + podAnnotation := "no-scale-down" + podAnnotations := map[string]string{podAnnotation: "true"} + customPodAnnotation := "foo" + customPodAnnotations := map[string]string{customPodAnnotation: "true"} + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockHTTPClient(ctrl) + client, _ := newFakeK8sAnnotationsClient() + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Volume: acidv1.Volume{ + Size: "1Gi", + }, + EnableConnectionPooler: boolToPointer(true), + EnableLogicalBackup: true, + EnableReplicaConnectionPooler: boolToPointer(true), + PodAnnotations: podAnnotations, + NumberOfInstances: 2, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + PatroniAPICheckInterval: time.Duration(1), + PatroniAPICheckTimeout: time.Duration(5), + PodManagementPolicy: "ordered_ready", + CustomPodAnnotations: customPodAnnotations, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(1), + }, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + MaxInstances: -1, + PodRoleLabel: "spilo-role", + ResourceCheckInterval: time.Duration(3), + ResourceCheckTimeout: time.Duration(10), + }, + }, + }, client, pg, logger, eventRecorder) + + configJson := `{"postgresql": {"parameters": {"log_min_duration_statement": 200, "max_connections": 50}}}, "ttl": 20}` + response := http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader([]byte(configJson))), + } + + mockClient.EXPECT().Do(gomock.Any()).Return(&response, nil).AnyTimes() + cluster.patroni = patroni.New(patroniLogger, mockClient) + cluster.Name = clusterName + cluster.Namespace = namespace + clusterOptions := clusterLabelsOptions(cluster) + + // create a statefulset + _, err := cluster.createStatefulSet() + assert.NoError(t, err) + // create a pods + podsList := createPods(cluster) + for _, pod := range podsList { + _, err = cluster.KubeClient.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) + assert.NoError(t, err) + } + // create connection pooler + _, err = cluster.createConnectionPooler(mockInstallLookupFunction) + assert.NoError(t, err) + + // create cron job + err = cluster.createLogicalBackupJob() + assert.NoError(t, err) + + annotateResources(cluster) + err = cluster.Sync(&cluster.Postgresql) + assert.NoError(t, err) + + // 1. PodAnnotations set + stsList, err := cluster.KubeClient.StatefulSets(namespace).List(context.TODO(), clusterOptions) + assert.NoError(t, err) + for _, sts := range stsList.Items { + for _, annotation := range []string{podAnnotation, customPodAnnotation} { + assert.Contains(t, sts.Spec.Template.Annotations, annotation) + } + } + + for _, role := range []PostgresRole{Master, Replica} { + deploy, err := cluster.KubeClient.Deployments(namespace).Get(context.TODO(), cluster.connectionPoolerName(role), metav1.GetOptions{}) + assert.NoError(t, err) + for _, annotation := range []string{podAnnotation, customPodAnnotation} { + assert.Contains(t, deploy.Spec.Template.Annotations, annotation, + fmt.Sprintf("pooler deployment pod template %s should contain annotation %s, found %#v", + deploy.Name, annotation, deploy.Spec.Template.Annotations)) + } + } + + podList, err := cluster.KubeClient.Pods(namespace).List(context.TODO(), clusterOptions) + assert.NoError(t, err) + for _, pod := range podList.Items { + for _, annotation := range []string{podAnnotation, customPodAnnotation} { + assert.Contains(t, pod.Annotations, annotation, + fmt.Sprintf("pod %s should contain annotation %s, found %#v", pod.Name, annotation, pod.Annotations)) + } + } + + cronJobList, err := cluster.KubeClient.CronJobs(namespace).List(context.TODO(), clusterOptions) + assert.NoError(t, err) + for _, cronJob := range cronJobList.Items { + for _, annotation := range []string{podAnnotation, customPodAnnotation} { + assert.Contains(t, cronJob.Spec.JobTemplate.Spec.Template.Annotations, annotation, + fmt.Sprintf("logical backup cron job's pod template should contain annotation %s, found %#v", + annotation, cronJob.Spec.JobTemplate.Spec.Template.Annotations)) + } + } + + // 2 PodAnnotations removed + newSpec := cluster.Postgresql.DeepCopy() + newSpec.Spec.PodAnnotations = nil + cluster.OpConfig.CustomPodAnnotations = nil + err = cluster.Sync(newSpec) + assert.NoError(t, err) + + stsList, err = cluster.KubeClient.StatefulSets(namespace).List(context.TODO(), clusterOptions) + assert.NoError(t, err) + for _, sts := range stsList.Items { + for _, annotation := range []string{podAnnotation, customPodAnnotation} { + assert.NotContains(t, sts.Spec.Template.Annotations, annotation) + } + } + + for _, role := range []PostgresRole{Master, Replica} { + deploy, err := cluster.KubeClient.Deployments(namespace).Get(context.TODO(), cluster.connectionPoolerName(role), metav1.GetOptions{}) + assert.NoError(t, err) + for _, annotation := range []string{podAnnotation, customPodAnnotation} { + assert.NotContains(t, deploy.Spec.Template.Annotations, annotation, + fmt.Sprintf("pooler deployment pod template %s should not contain annotation %s, found %#v", + deploy.Name, annotation, deploy.Spec.Template.Annotations)) + } + } + + podList, err = cluster.KubeClient.Pods(namespace).List(context.TODO(), clusterOptions) + assert.NoError(t, err) + for _, pod := range podList.Items { + for _, annotation := range []string{podAnnotation, customPodAnnotation} { + assert.NotContains(t, pod.Annotations, annotation, + fmt.Sprintf("pod %s should not contain annotation %s, found %#v", pod.Name, annotation, pod.Annotations)) + } + } + + cronJobList, err = cluster.KubeClient.CronJobs(namespace).List(context.TODO(), clusterOptions) + assert.NoError(t, err) + for _, cronJob := range cronJobList.Items { + for _, annotation := range []string{podAnnotation, customPodAnnotation} { + assert.NotContains(t, cronJob.Spec.JobTemplate.Spec.Template.Annotations, annotation, + fmt.Sprintf("logical backup cron job's pod template should not contain annotation %s, found %#v", + annotation, cronJob.Spec.JobTemplate.Spec.Template.Annotations)) + } + } +} + +func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) { + testName := "test config comparison" + client, _ := newFakeK8sSyncClient() + clusterName := "acid-test-cluster" + namespace := "default" + testSlots := map[string]map[string]string{ + "slot1": { + "type": "logical", + "plugin": "wal2json", + "database": "foo", + }, + } + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + defaultPgParameters := map[string]string{ + "log_min_duration_statement": "200", + "max_connections": "50", + } + defaultPatroniParameters := acidv1.Patroni{ + TTL: 20, + } + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Patroni: defaultPatroniParameters, + PostgresqlParam: acidv1.PostgresqlParam{ + Parameters: defaultPgParameters, + }, + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + ResourceCheckInterval: time.Duration(3), + ResourceCheckTimeout: time.Duration(10), + }, + }, + }, client, pg, logger, eventRecorder) + + // mocking a config after setConfig is called + configJson := `{"postgresql": {"parameters": {"log_min_duration_statement": 200, "max_connections": 50}}}, "ttl": 20}` + r := io.NopCloser(bytes.NewReader([]byte(configJson))) + + response := http.Response{ + StatusCode: 200, + Body: r, + } + + mockClient := mocks.NewMockHTTPClient(ctrl) + mockClient.EXPECT().Do(gomock.Any()).Return(&response, nil).AnyTimes() + + p := patroni.New(patroniLogger, mockClient) + cluster.patroni = p + mockPod := newMockPod("192.168.100.1") + + // simulate existing config that differs from cluster.Spec + tests := []struct { + subtest string + patroni acidv1.Patroni + desiredSlots map[string]map[string]string + removedSlots map[string]map[string]string + pgParams map[string]string + shouldBePatched bool + restartPrimary bool + }{ + { + subtest: "Patroni and Postgresql.Parameters do not differ", + patroni: acidv1.Patroni{ + TTL: 20, + }, + pgParams: map[string]string{ + "log_min_duration_statement": "200", + "max_connections": "50", + }, + shouldBePatched: false, + restartPrimary: false, + }, + { + subtest: "Patroni and Postgresql.Parameters differ - restart replica first", + patroni: acidv1.Patroni{ + TTL: 30, // desired 20 + }, + pgParams: map[string]string{ + "log_min_duration_statement": "500", // desired 200 + "max_connections": "100", // desired 50 + }, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "multiple Postgresql.Parameters differ - restart replica first", + patroni: defaultPatroniParameters, + pgParams: map[string]string{ + "log_min_duration_statement": "500", // desired 200 + "max_connections": "100", // desired 50 + }, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "desired max_connections bigger - restart replica first", + patroni: defaultPatroniParameters, + pgParams: map[string]string{ + "log_min_duration_statement": "200", + "max_connections": "30", // desired 50 + }, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "desired max_connections smaller - restart master first", + patroni: defaultPatroniParameters, + pgParams: map[string]string{ + "log_min_duration_statement": "200", + "max_connections": "100", // desired 50 + }, + shouldBePatched: true, + restartPrimary: true, + }, + { + subtest: "slot does not exist but is desired", + patroni: acidv1.Patroni{ + TTL: 20, + }, + desiredSlots: testSlots, + pgParams: map[string]string{ + "log_min_duration_statement": "200", + "max_connections": "50", + }, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "slot exist, nothing specified in manifest", + patroni: acidv1.Patroni{ + TTL: 20, + Slots: map[string]map[string]string{ + "slot1": { + "type": "logical", + "plugin": "pgoutput", + "database": "foo", + }, + }, + }, + pgParams: map[string]string{ + "log_min_duration_statement": "200", + "max_connections": "50", + }, + shouldBePatched: false, + restartPrimary: false, + }, + { + subtest: "slot is removed from manifest", + patroni: acidv1.Patroni{ + TTL: 20, + Slots: map[string]map[string]string{ + "slot1": { + "type": "logical", + "plugin": "pgoutput", + "database": "foo", + }, + }, + }, + removedSlots: testSlots, + pgParams: map[string]string{ + "log_min_duration_statement": "200", + "max_connections": "50", + }, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "slot plugin differs", + patroni: acidv1.Patroni{ + TTL: 20, + Slots: map[string]map[string]string{ + "slot1": { + "type": "logical", + "plugin": "pgoutput", + "database": "foo", + }, + }, + }, + desiredSlots: testSlots, + pgParams: map[string]string{ + "log_min_duration_statement": "200", + "max_connections": "50", + }, + shouldBePatched: true, + restartPrimary: false, + }, + } + + for _, tt := range tests { + if len(tt.desiredSlots) > 0 { + cluster.Spec.Patroni.Slots = tt.desiredSlots + } + if len(tt.removedSlots) > 0 { + for slotName, removedSlot := range tt.removedSlots { + cluster.replicationSlots[slotName] = removedSlot + } + } + + configPatched, requirePrimaryRestart, err := cluster.checkAndSetGlobalPostgreSQLConfiguration(mockPod, tt.patroni, cluster.Spec.Patroni, tt.pgParams, cluster.Spec.Parameters) + assert.NoError(t, err) + if configPatched != tt.shouldBePatched { + t.Errorf("%s - %s: expected config update did not happen", testName, tt.subtest) + } + if requirePrimaryRestart != tt.restartPrimary { + t.Errorf("%s - %s: wrong master restart strategy, got restart %v, expected restart %v", testName, tt.subtest, requirePrimaryRestart, tt.restartPrimary) + } + + // reset slots for next tests + cluster.Spec.Patroni.Slots = nil + cluster.replicationSlots = make(map[string]interface{}) + } + + testsFailsafe := []struct { + subtest string + operatorVal *bool + effectiveVal *bool + desiredVal bool + shouldBePatched bool + restartPrimary bool + }{ + { + subtest: "Not set in operator config, not set for pg cluster. Set to true in the pg config.", + operatorVal: nil, + effectiveVal: nil, + desiredVal: true, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "Not set in operator config, disabled for pg cluster. Set to true in the pg config.", + operatorVal: nil, + effectiveVal: util.False(), + desiredVal: true, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "Not set in operator config, not set for pg cluster. Set to false in the pg config.", + operatorVal: nil, + effectiveVal: nil, + desiredVal: false, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "Not set in operator config, enabled for pg cluster. Set to false in the pg config.", + operatorVal: nil, + effectiveVal: util.True(), + desiredVal: false, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "Enabled in operator config, not set for pg cluster. Set to false in the pg config.", + operatorVal: util.True(), + effectiveVal: nil, + desiredVal: false, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "Enabled in operator config, disabled for pg cluster. Set to true in the pg config.", + operatorVal: util.True(), + effectiveVal: util.False(), + desiredVal: true, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "Disabled in operator config, not set for pg cluster. Set to true in the pg config.", + operatorVal: util.False(), + effectiveVal: nil, + desiredVal: true, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "Disabled in operator config, enabled for pg cluster. Set to false in the pg config.", + operatorVal: util.False(), + effectiveVal: util.True(), + desiredVal: false, + shouldBePatched: true, + restartPrimary: false, + }, + { + subtest: "Disabled in operator config, enabled for pg cluster. Set to true in the pg config.", + operatorVal: util.False(), + effectiveVal: util.True(), + desiredVal: true, + shouldBePatched: false, // should not require patching + restartPrimary: false, + }, + } + + for _, tt := range testsFailsafe { + patroniConf := defaultPatroniParameters + + if tt.operatorVal != nil { + cluster.OpConfig.EnablePatroniFailsafeMode = tt.operatorVal + } + if tt.effectiveVal != nil { + patroniConf.FailsafeMode = tt.effectiveVal + } + cluster.Spec.Patroni.FailsafeMode = &tt.desiredVal + + configPatched, requirePrimaryRestart, err := cluster.checkAndSetGlobalPostgreSQLConfiguration(mockPod, patroniConf, cluster.Spec.Patroni, defaultPgParameters, cluster.Spec.Parameters) + assert.NoError(t, err) + if configPatched != tt.shouldBePatched { + t.Errorf("%s - %s: expected update went wrong", testName, tt.subtest) + } + if requirePrimaryRestart != tt.restartPrimary { + t.Errorf("%s - %s: wrong master restart strategy, got restart %v, expected restart %v", testName, tt.subtest, requirePrimaryRestart, tt.restartPrimary) + } + } +} + +func TestSyncStandbyClusterConfiguration(t *testing.T) { + client, _ := newFakeK8sSyncClient() + clusterName := "acid-standby-cluster" + applicationLabel := "spilo" + namespace := "default" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + NumberOfInstances: int32(1), + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + PatroniAPICheckInterval: time.Duration(1), + PatroniAPICheckTimeout: time.Duration(5), + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": applicationLabel}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + MinInstances: int32(-1), + MaxInstances: int32(-1), + PodRoleLabel: "spilo-role", + ResourceCheckInterval: time.Duration(3), + ResourceCheckTimeout: time.Duration(10), + }, + }, + }, client, pg, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + + // mocking a config after getConfig is called + mockClient := mocks.NewMockHTTPClient(ctrl) + configJson := `{"ttl": 20}` + r := io.NopCloser(bytes.NewReader([]byte(configJson))) + response := http.Response{ + StatusCode: 200, + Body: r, + } + mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil).AnyTimes() + + // mocking a config after setConfig is called + standbyJson := `{"standby_cluster":{"create_replica_methods":["bootstrap_standby_with_wale","basebackup_fast_xlog"],"restore_command":"envdir \"/run/etc/wal-e.d/env-standby\" /scripts/restore_command.sh \"%f\" \"%p\""}}` + r = io.NopCloser(bytes.NewReader([]byte(standbyJson))) + response = http.Response{ + StatusCode: 200, + Body: r, + } + mockClient.EXPECT().Do(gomock.Any()).Return(&response, nil).AnyTimes() + p := patroni.New(patroniLogger, mockClient) + cluster.patroni = p + + mockPod := newMockPod("192.168.100.1") + mockPod.Name = fmt.Sprintf("%s-0", clusterName) + mockPod.Namespace = namespace + podLabels := map[string]string{ + "cluster-name": clusterName, + "application": applicationLabel, + "spilo-role": "master", + } + mockPod.Labels = podLabels + client.PodsGetter.Pods(namespace).Create(context.TODO(), mockPod, metav1.CreateOptions{}) + + // create a statefulset + sts, err := cluster.createStatefulSet() + assert.NoError(t, err) + + // check that pods do not have a STANDBY_* environment variable + assert.NotContains(t, sts.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) + + // add standby section + cluster.Spec.StandbyCluster = &acidv1.StandbyDescription{ + S3WalPath: "s3://custom/path/to/bucket/", + } + cluster.syncStatefulSet() + updatedSts := cluster.Statefulset + + // check that pods do not have a STANDBY_* environment variable + assert.Contains(t, updatedSts.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) + + // this should update the Patroni config + err = cluster.syncStandbyClusterConfiguration() + assert.NoError(t, err) + + configJson = `{"standby_cluster":{"create_replica_methods":["bootstrap_standby_with_wale","basebackup_fast_xlog"],"restore_command":"envdir \"/run/etc/wal-e.d/env-standby\" /scripts/restore_command.sh \"%f\" \"%p\""}, "ttl": 20}` + r = io.NopCloser(bytes.NewReader([]byte(configJson))) + response = http.Response{ + StatusCode: 200, + Body: r, + } + mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil).AnyTimes() + + pods, err := cluster.listPods() + assert.NoError(t, err) + + _, _, err = cluster.patroni.GetConfig(&pods[0]) + assert.NoError(t, err) + // ToDo extend GetConfig to return standy_cluster setting to compare + /* + defaultStandbyParameters := map[string]interface{}{ + "create_replica_methods": []string{"bootstrap_standby_with_wale", "basebackup_fast_xlog"}, + "restore_command": "envdir \"/run/etc/wal-e.d/env-standby\" /scripts/restore_command.sh \"%f\" \"%p\"", + } + assert.True(t, reflect.DeepEqual(defaultStandbyParameters, standbyCluster)) + */ + // remove standby section + cluster.Spec.StandbyCluster = &acidv1.StandbyDescription{} + cluster.syncStatefulSet() + updatedSts2 := cluster.Statefulset + + // check that pods do not have a STANDBY_* environment variable + assert.NotContains(t, updatedSts2.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) + + // this should update the Patroni config again + err = cluster.syncStandbyClusterConfiguration() + assert.NoError(t, err) + + // test with standby_host, standby_port and standby_primary_slot_name + cluster.Spec.StandbyCluster = &acidv1.StandbyDescription{ + StandbyHost: "remote-primary.example.com", + StandbyPort: "5433", + StandbyPrimarySlotName: "standby_slot", + } + cluster.syncStatefulSet() + updatedSts4 := cluster.Statefulset + + // check that pods have all three STANDBY_* environment variables + assert.Contains(t, updatedSts4.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "STANDBY_HOST", Value: "remote-primary.example.com"}) + assert.Contains(t, updatedSts4.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "STANDBY_PORT", Value: "5433"}) + assert.Contains(t, updatedSts4.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "STANDBY_PRIMARY_SLOT_NAME", Value: "standby_slot"}) + + // this should update the Patroni config with host, port and primary_slot_name + err = cluster.syncStandbyClusterConfiguration() + assert.NoError(t, err) + + // test property deletion: remove standby_primary_slot_name + cluster.Spec.StandbyCluster = &acidv1.StandbyDescription{ + StandbyHost: "remote-primary.example.com", + StandbyPort: "5433", + } + cluster.syncStatefulSet() + updatedSts5 := cluster.Statefulset + + // check that STANDBY_PRIMARY_SLOT_NAME is not present + assert.Contains(t, updatedSts5.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "STANDBY_HOST", Value: "remote-primary.example.com"}) + assert.Contains(t, updatedSts5.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "STANDBY_PORT", Value: "5433"}) + assert.NotContains(t, updatedSts5.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "STANDBY_PRIMARY_SLOT_NAME", Value: "standby_slot"}) + + // this should update the Patroni config and set primary_slot_name to nil + err = cluster.syncStandbyClusterConfiguration() + assert.NoError(t, err) +} + +func TestUpdateSecret(t *testing.T) { + testName := "test syncing secrets" + client, _ := newFakeK8sSyncSecretsClient() + + clusterName := "acid-test-cluster" + namespace := "default" + dbname := "app" + dbowner := "appowner" + appUser := "foo" + secretTemplate := config.StringTemplate("{username}.{cluster}.credentials") + retentionUsers := make([]string, 0) + + // define manifest users and enable rotation for dbowner + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Databases: map[string]string{dbname: dbowner}, + Users: map[string]acidv1.UserFlags{appUser: {}, "bar": {}, dbowner: {}, "not-exist.test_user": {}}, + UsersIgnoringSecretRotation: []string{"bar"}, + UsersWithInPlaceSecretRotation: []string{dbowner}, + Streams: []acidv1.Stream{ + { + ApplicationId: appId, + Database: dbname, + Tables: map[string]acidv1.StreamTable{ + "data.foo": { + EventType: "stream-type-b", + }, + }, + }, + }, + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + // new cluster with enabled password rotation + var cluster = New( + Config{ + OpConfig: config.Config{ + Auth: config.Auth{ + SuperUsername: "postgres", + ReplicationUsername: "standby", + SecretNameTemplate: secretTemplate, + EnablePasswordRotation: true, + PasswordRotationInterval: 1, + PasswordRotationUserRetention: 3, + }, + EnableCrossNamespaceSecret: true, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + }, + }, + }, client, pg, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + cluster.pgUsers = map[string]spec.PgUser{} + + // init all users + cluster.initUsers() + // create secrets + cluster.syncSecrets() + // initialize rotation with current time + cluster.syncSecrets() + + dayAfterTomorrow := time.Now().AddDate(0, 0, 2) + + allUsers := make(map[string]spec.PgUser) + for _, pgUser := range cluster.pgUsers { + if !pgUser.Degraded { + allUsers[pgUser.Name] = pgUser + } + } + for _, systemUser := range cluster.systemUsers { + allUsers[systemUser.Name] = systemUser + } + + for username, pgUser := range allUsers { + // first, get the secret + secretName := cluster.credentialSecretName(username) + secret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + assert.NoError(t, err) + secretPassword := string(secret.Data["password"]) + + // now update the secret setting a next rotation date (tomorrow + interval) + cluster.updateSecret(username, secret, &retentionUsers, dayAfterTomorrow) + updatedSecret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + assert.NoError(t, err) + + // check that passwords are different + rotatedPassword := string(updatedSecret.Data["password"]) + if secretPassword == rotatedPassword { + // passwords for system users should not have been rotated + if pgUser.Origin != spec.RoleOriginManifest { + continue + } + if slices.Contains(pg.Spec.UsersIgnoringSecretRotation, username) { + continue + } + t.Errorf("%s: password unchanged in updated secret for %s", testName, username) + } + + // check that next rotation date is tomorrow + interval, not date in secret + interval + nextRotation := string(updatedSecret.Data["nextRotation"]) + _, nextRotationDate := cluster.getNextRotationDate(dayAfterTomorrow) + if nextRotation != nextRotationDate { + t.Errorf("%s: updated secret of %s does not contain correct rotation date: expected %s, got %s", testName, username, nextRotationDate, nextRotation) + } + + // compare username, when it's dbowner they should be equal because of UsersWithInPlaceSecretRotation + secretUsername := string(updatedSecret.Data["username"]) + if pgUser.IsDbOwner { + if secretUsername != username { + t.Errorf("%s: username differs in updated secret: expected %s, got %s", testName, username, secretUsername) + } + } else { + rotatedUsername := username + dayAfterTomorrow.Format(constants.RotationUserDateFormat) + if secretUsername != rotatedUsername { + t.Errorf("%s: updated secret does not contain correct username: expected %s, got %s", testName, rotatedUsername, secretUsername) + } + // whenever there's a rotation the retentionUsers list is extended or updated + if len(retentionUsers) != 1 { + t.Errorf("%s: unexpected number of users to drop - expected only %s, found %d", testName, username, len(retentionUsers)) + } + } + } + + // switch rotation for foo to in-place + inPlaceRotationUsers := []string{dbowner, appUser} + cluster.Spec.UsersWithInPlaceSecretRotation = inPlaceRotationUsers + cluster.initUsers() + cluster.syncSecrets() + updatedSecret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), cluster.credentialSecretName(appUser), metav1.GetOptions{}) + assert.NoError(t, err) + + // username in secret should be switched to original user + currentUsername := string(updatedSecret.Data["username"]) + if currentUsername != appUser { + t.Errorf("%s: updated secret does not contain correct username: expected %s, got %s", testName, appUser, currentUsername) + } + + // switch rotation back to rotation user + inPlaceRotationUsers = []string{dbowner} + cluster.Spec.UsersWithInPlaceSecretRotation = inPlaceRotationUsers + cluster.initUsers() + cluster.syncSecrets() + updatedSecret, err = cluster.KubeClient.Secrets(namespace).Get(context.TODO(), cluster.credentialSecretName(appUser), metav1.GetOptions{}) + assert.NoError(t, err) + + // username in secret will only be switched after next rotation date is passed + currentUsername = string(updatedSecret.Data["username"]) + if currentUsername != appUser { + t.Errorf("%s: updated secret does not contain expected username: expected %s, got %s", testName, appUser, currentUsername) + } +} + +func TestUpdateSecretNameConflict(t *testing.T) { + client, _ := newFakeK8sSyncSecretsClient() + + clusterName := "acid-test-cluster" + namespace := "default" + secretTemplate := config.StringTemplate("{username}.{cluster}.credentials") + + // define manifest user that has the same name as a prepared database owner user except for dashes vs underscores + // because of this the operator cannot create both secrets because underscores are not allowed in k8s secret names + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + PreparedDatabases: map[string]acidv1.PreparedDatabase{"prepared": {DefaultUsers: true}}, + Users: map[string]acidv1.UserFlags{"prepared-owner-user": {}}, + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + Auth: config.Auth{ + SuperUsername: "postgres", + ReplicationUsername: "standby", + SecretNameTemplate: secretTemplate, + }, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + }, + }, + }, client, pg, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + cluster.pgUsers = map[string]spec.PgUser{} + + // init all users + cluster.initUsers() + // create secrets and fail because of user name mismatch + // prepared-owner-user from manifest vs prepared_owner_user from prepared database + err := cluster.syncSecrets() + assert.Error(t, err) + + // the order of secrets to sync is not deterministic, check only first part of the error message + expectedError := fmt.Sprintf("syncing secret %s failed: error while checking for password rotation: could not update secret because of user name mismatch", "default/prepared-owner-user.acid-test-cluster.credentials") + assert.Contains(t, err.Error(), expectedError) +} diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 8aa519817..17c4e705e 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -6,7 +6,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/types" ) @@ -14,11 +14,15 @@ import ( type PostgresRole string const ( - // Master role - Master PostgresRole = "master" - - // Replica role + // spilo roles + Master PostgresRole = "master" Replica PostgresRole = "replica" + Patroni PostgresRole = "config" + + // roles returned by Patroni cluster endpoint + Leader PostgresRole = "leader" + StandbyLeader PostgresRole = "standby_leader" + SyncStandby PostgresRole = "sync_standby" ) // PodEventType represents the type of a pod-related event @@ -54,14 +58,16 @@ type WorkerStatus struct { // ClusterStatus describes status of the cluster type ClusterStatus struct { - Team string - Cluster string - MasterService *v1.Service - ReplicaService *v1.Service - MasterEndpoint *v1.Endpoints - ReplicaEndpoint *v1.Endpoints - StatefulSet *appsv1.StatefulSet - PodDisruptionBudget *policybeta1.PodDisruptionBudget + Team string + Cluster string + Namespace string + MasterService *v1.Service + ReplicaService *v1.Service + MasterEndpoint *v1.Endpoints + ReplicaEndpoint *v1.Endpoints + StatefulSet *appsv1.StatefulSet + PrimaryPodDisruptionBudget *policyv1.PodDisruptionBudget + CriticalOpPodDisruptionBudget *policyv1.PodDisruptionBudget CurrentProcess Process Worker uint32 @@ -72,7 +78,7 @@ type ClusterStatus struct { type TemplateParams map[string]interface{} -type InstallFunction func(schema string, user string, role PostgresRole) error +type InstallFunction func(schema string, user string) error type SyncReason []string diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 4350f9d56..cbcccd16e 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -6,15 +6,16 @@ import ( "encoding/gob" "encoding/json" "fmt" - "math/rand" + "net/http" "reflect" + "regexp" "sort" "strings" "time" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -78,7 +79,14 @@ func (c *Cluster) isProtectedUsername(username string) bool { } func (c *Cluster) isSystemUsername(username string) bool { - return (username == c.OpConfig.SuperUsername || username == c.OpConfig.ReplicationUsername) + // is there a pooler system user defined + for _, systemUser := range c.systemUsers { + if username == systemUser.Name { + return true + } + } + + return false } func isValidFlag(flag string) bool { @@ -159,7 +167,15 @@ func metaAnnotationsPatch(annotations map[string]string) ([]byte, error) { }{&meta}) } -func (c *Cluster) logPDBChanges(old, new *policybeta1.PodDisruptionBudget, isUpdate bool, reason string) { +func metaLabelsPatch(labels map[string]string) ([]byte, error) { + var meta metav1.ObjectMeta + meta.Labels = labels + return json.Marshal(struct { + ObjMeta interface{} `json:"metadata"` + }{&meta}) +} + +func (c *Cluster) logPDBChanges(old, new *policyv1.PodDisruptionBudget, isUpdate bool, reason string) { if isUpdate { c.logger.Infof("pod disruption budget %q has been changed", util.NameFromMeta(old.ObjectMeta)) } else { @@ -169,6 +185,10 @@ func (c *Cluster) logPDBChanges(old, new *policybeta1.PodDisruptionBudget, isUpd } logNiceDiff(c.logger, old.Spec, new.Spec) + + if reason != "" { + c.logger.Infof("reason: %s", reason) + } } func logNiceDiff(log *logrus.Entry, old, new interface{}) { @@ -182,7 +202,7 @@ func logNiceDiff(log *logrus.Entry, old, new interface{}) { nice := nicediff.Diff(string(o), string(n), true) for _, s := range strings.Split(nice, "\n") { // " is not needed in the value to understand - log.Debugf(strings.ReplaceAll(s, "\"", "")) + log.Debug(strings.ReplaceAll(s, "\"", "")) } } @@ -198,7 +218,7 @@ func (c *Cluster) logStatefulSetChanges(old, new *appsv1.StatefulSet, isUpdate b logNiceDiff(c.logger, old.Spec, new.Spec) if !reflect.DeepEqual(old.Annotations, new.Annotations) { - c.logger.Debugf("metadata.annotation are different") + c.logger.Debug("metadata.annotation are different") logNiceDiff(c.logger, old.Annotations, new.Annotations) } @@ -244,7 +264,12 @@ func getPostgresContainer(podSpec *v1.PodSpec) (pgContainer v1.Container) { func (c *Cluster) getTeamMembers(teamID string) ([]string, error) { if teamID == "" { - return nil, fmt.Errorf("no teamId specified") + msg := "no teamId specified" + if c.OpConfig.EnableTeamIdClusternamePrefix { + return nil, fmt.Errorf("%s", msg) + } + c.logger.Warnf("%s", msg) + return nil, nil } members := []string{} @@ -264,7 +289,7 @@ func (c *Cluster) getTeamMembers(teamID string) ([]string, error) { } if !c.OpConfig.EnableTeamsAPI { - c.logger.Debugf("team API is disabled") + c.logger.Debug("team API is disabled") return members, nil } @@ -273,17 +298,21 @@ func (c *Cluster) getTeamMembers(teamID string) ([]string, error) { return nil, fmt.Errorf("could not get oauth token to authenticate to team service API: %v", err) } - teamInfo, err := c.teamsAPIClient.TeamInfo(teamID, token) - if err != nil { - return nil, fmt.Errorf("could not get team info for team %q: %v", teamID, err) - } + teamInfo, statusCode, err := c.teamsAPIClient.TeamInfo(teamID, token) - for _, member := range teamInfo.Members { - if !(util.SliceContains(members, member)) { - members = append(members, member) + if err != nil { + if statusCode == http.StatusNotFound { + c.logger.Warningf("could not get team info for team %q: %v", teamID, err) + } else { + return nil, fmt.Errorf("could not get team info for team %q: %v", teamID, err) + } + } else { + for _, member := range teamInfo.Members { + if !(util.SliceContains(members, member)) { + members = append(members, member) + } } } - return members, nil } @@ -312,7 +341,7 @@ func (c *Cluster) annotationsSet(annotations map[string]string) map[string]strin return nil } -func (c *Cluster) waitForPodLabel(podEvents chan PodEvent, stopChan chan struct{}, role *PostgresRole) (*v1.Pod, error) { +func (c *Cluster) waitForPodLabel(podEvents chan PodEvent, stopCh chan struct{}, role *PostgresRole) (*v1.Pod, error) { timeout := time.After(c.OpConfig.PodLabelWaitTimeout) for { select { @@ -328,7 +357,7 @@ func (c *Cluster) waitForPodLabel(podEvents chan PodEvent, stopChan chan struct{ } case <-timeout: return nil, fmt.Errorf("pod label wait timeout") - case <-stopChan: + case <-stopCh: return nil, fmt.Errorf("pod label wait cancelled") } } @@ -396,7 +425,7 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error { podsNumber = len(pods.Items) c.logger.Debugf("Waiting for %d pods to become ready", podsNumber) } else { - c.logger.Debugf("Waiting for any replica pod to become ready") + c.logger.Debug("Waiting for any replica pod to become ready") } err := retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, @@ -429,10 +458,6 @@ func (c *Cluster) _waitPodLabelsReady(anyReplica bool) error { return err } -func (c *Cluster) waitForAnyReplicaLabelReady() error { - return c._waitPodLabelsReady(true) -} - func (c *Cluster) waitForAllPodsLabelReady() error { return c._waitPodLabelsReady(false) } @@ -496,16 +521,55 @@ func (c *Cluster) roleLabelsSet(shouldAddExtraLabels bool, role PostgresRole) la return lbls } -func (c *Cluster) masterDNSName() string { +func (c *Cluster) dnsName(role PostgresRole) string { + var dnsString, oldDnsString string + + if role == Master { + dnsString = c.masterDNSName(c.Name) + } else { + dnsString = c.replicaDNSName(c.Name) + } + + // if cluster name starts with teamID we might need to provide backwards compatibility + clusterNameWithoutTeamPrefix, _ := acidv1.ExtractClusterName(c.Name, c.Spec.TeamID) + if clusterNameWithoutTeamPrefix != "" { + if role == Master { + oldDnsString = c.oldMasterDNSName(clusterNameWithoutTeamPrefix) + } else { + oldDnsString = c.oldReplicaDNSName(clusterNameWithoutTeamPrefix) + } + dnsString = fmt.Sprintf("%s,%s", dnsString, oldDnsString) + } + + return dnsString +} + +func (c *Cluster) masterDNSName(clusterName string) string { return strings.ToLower(c.OpConfig.MasterDNSNameFormat.Format( - "cluster", c.Spec.ClusterName, + "cluster", clusterName, + "namespace", c.Namespace, "team", c.teamName(), "hostedzone", c.OpConfig.DbHostedZone)) } -func (c *Cluster) replicaDNSName() string { +func (c *Cluster) replicaDNSName(clusterName string) string { return strings.ToLower(c.OpConfig.ReplicaDNSNameFormat.Format( - "cluster", c.Spec.ClusterName, + "cluster", clusterName, + "namespace", c.Namespace, + "team", c.teamName(), + "hostedzone", c.OpConfig.DbHostedZone)) +} + +func (c *Cluster) oldMasterDNSName(clusterName string) string { + return strings.ToLower(c.OpConfig.MasterLegacyDNSNameFormat.Format( + "cluster", clusterName, + "team", c.teamName(), + "hostedzone", c.OpConfig.DbHostedZone)) +} + +func (c *Cluster) oldReplicaDNSName(clusterName string) string { + return strings.ToLower(c.OpConfig.ReplicaLegacyDNSNameFormat.Format( + "cluster", clusterName, "team", c.teamName(), "hostedzone", c.OpConfig.DbHostedZone)) } @@ -525,10 +589,6 @@ func (c *Cluster) credentialSecretNameForCluster(username string, clusterName st "tprgroup", acidzalando.GroupName) } -func masterCandidate(replicas []spec.NamespacedName) spec.NamespacedName { - return replicas[rand.Intn(len(replicas))] -} - func cloneSpec(from *acidv1.Postgresql) (*acidv1.Postgresql, error) { var ( buf bytes.Buffer @@ -599,3 +659,61 @@ func trimCronjobName(name string) string { } return name } + +func parseResourceRequirements(resourcesRequirement v1.ResourceRequirements) (acidv1.Resources, error) { + var resources acidv1.Resources + resourcesJSON, err := json.Marshal(resourcesRequirement) + if err != nil { + return acidv1.Resources{}, fmt.Errorf("could not marshal K8s resources requirements") + } + if err = json.Unmarshal(resourcesJSON, &resources); err != nil { + return acidv1.Resources{}, fmt.Errorf("could not unmarshal K8s resources requirements into acidv1.Resources struct") + } + return resources, nil +} + +func isStandbyCluster(spec *acidv1.PostgresSpec) bool { + for _, env := range spec.Env { + hasStandbyEnv, _ := regexp.MatchString(`^STANDBY_WALE_(S3|GS|GSC|SWIFT)_PREFIX$`, env.Name) + if hasStandbyEnv && env.Value != "" { + return true + } + } + return spec.StandbyCluster != nil +} + +func (c *Cluster) isInMaintenanceWindow(specMaintenanceWindows []acidv1.MaintenanceWindow) bool { + ignoreMaintenanceWindows := c.OpConfig.EnableMaintenanceWindows != nil && !*c.OpConfig.EnableMaintenanceWindows + noWindowsDefined := len(specMaintenanceWindows) == 0 && len(c.OpConfig.MaintenanceWindows) == 0 + if noWindowsDefined || ignoreMaintenanceWindows { + return true + } + now := time.Now() + currentDay := now.Weekday() + currentTime := now.Format("15:04") + + maintenanceWindows := specMaintenanceWindows + if len(maintenanceWindows) == 0 { + maintenanceWindows = make([]acidv1.MaintenanceWindow, 0, len(c.OpConfig.MaintenanceWindows)) + for _, windowStr := range c.OpConfig.MaintenanceWindows { + var window acidv1.MaintenanceWindow + if err := window.UnmarshalJSON([]byte(windowStr)); err != nil { + c.logger.Errorf("could not parse default maintenance window %q: %v", windowStr, err) + continue + } + maintenanceWindows = append(maintenanceWindows, window) + } + } + + for _, window := range maintenanceWindows { + startTime := window.StartTime.Format("15:04") + endTime := window.EndTime.Format("15:04") + + if window.Everyday || window.Weekday == currentDay { + if currentTime >= startTime && currentTime <= endTime { + return true + } + } + } + return false +} diff --git a/pkg/cluster/util_test.go b/pkg/cluster/util_test.go index 20214a085..8413ca396 100644 --- a/pkg/cluster/util_test.go +++ b/pkg/cluster/util_test.go @@ -1,143 +1,619 @@ package cluster import ( + "bytes" "context" + "fmt" + "io" + "maps" + "net/http" + "reflect" "testing" + "time" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "github.com/zalando/postgres-operator/mocks" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "github.com/zalando/postgres-operator/pkg/util/patroni" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" k8sFake "k8s.io/client-go/kubernetes/fake" ) +var externalAnnotations = map[string]string{"existing": "annotation"} + +func mustParseTime(s string) metav1.Time { + v, err := time.Parse("15:04", s) + if err != nil { + panic(err) + } + + return metav1.Time{Time: v.UTC()} +} + func newFakeK8sAnnotationsClient() (k8sutil.KubernetesClient, *k8sFake.Clientset) { clientSet := k8sFake.NewSimpleClientset() acidClientSet := fakeacidv1.NewSimpleClientset() return k8sutil.KubernetesClient{ - PodDisruptionBudgetsGetter: clientSet.PolicyV1beta1(), - ServicesGetter: clientSet.CoreV1(), - StatefulSetsGetter: clientSet.AppsV1(), - PostgresqlsGetter: acidClientSet.AcidV1(), + PodDisruptionBudgetsGetter: clientSet.PolicyV1(), + SecretsGetter: clientSet.CoreV1(), + ServicesGetter: clientSet.CoreV1(), + StatefulSetsGetter: clientSet.AppsV1(), + PostgresqlsGetter: acidClientSet.AcidV1(), + PersistentVolumeClaimsGetter: clientSet.CoreV1(), + PersistentVolumesGetter: clientSet.CoreV1(), + EndpointsGetter: clientSet.CoreV1(), + ConfigMapsGetter: clientSet.CoreV1(), + PodsGetter: clientSet.CoreV1(), + DeploymentsGetter: clientSet.AppsV1(), + CronJobsGetter: clientSet.BatchV1(), }, clientSet } -func TestInheritedAnnotations(t *testing.T) { - testName := "test inheriting annotations from manifest" - client, _ := newFakeK8sAnnotationsClient() - clusterName := "acid-test-cluster" - namespace := "default" - annotationValue := "acid" - role := Master +func clusterLabelsOptions(cluster *Cluster) metav1.ListOptions { + clusterLabel := labels.Set(map[string]string{cluster.OpConfig.ClusterNameLabel: cluster.Name}) + return metav1.ListOptions{ + LabelSelector: clusterLabel.String(), + } +} +func checkResourcesInheritedAnnotations(cluster *Cluster, resultAnnotations map[string]string) error { + clusterOptions := clusterLabelsOptions(cluster) + // helper functions + containsAnnotations := func(expected map[string]string, actual map[string]string, objName string, objType string) error { + if !util.MapContains(actual, expected) { + return fmt.Errorf("%s %v expected annotations %#v to be contained in %#v", objType, objName, expected, actual) + } + return nil + } + + updateAnnotations := func(annotations map[string]string) map[string]string { + result := make(map[string]string, 0) + for anno := range annotations { + if _, ok := externalAnnotations[anno]; !ok { + result[anno] = annotations[anno] + } + } + return result + } + + checkSts := func(annotations map[string]string) error { + stsList, err := cluster.KubeClient.StatefulSets(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + stsAnnotations := updateAnnotations(annotations) + + for _, sts := range stsList.Items { + if err := containsAnnotations(stsAnnotations, sts.Annotations, sts.ObjectMeta.Name, "StatefulSet"); err != nil { + return err + } + // pod template + if err := containsAnnotations(stsAnnotations, sts.Spec.Template.Annotations, sts.ObjectMeta.Name, "StatefulSet pod template"); err != nil { + return err + } + // pvc template + if err := containsAnnotations(stsAnnotations, sts.Spec.VolumeClaimTemplates[0].Annotations, sts.ObjectMeta.Name, "StatefulSet pvc template"); err != nil { + return err + } + } + return nil + } + + checkPods := func(annotations map[string]string) error { + podList, err := cluster.KubeClient.Pods(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, pod := range podList.Items { + if err := containsAnnotations(annotations, pod.Annotations, pod.ObjectMeta.Name, "Pod"); err != nil { + return err + } + } + return nil + } + + checkSvc := func(annotations map[string]string) error { + svcList, err := cluster.KubeClient.Services(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, svc := range svcList.Items { + if err := containsAnnotations(annotations, svc.Annotations, svc.ObjectMeta.Name, "Service"); err != nil { + return err + } + } + return nil + } + + checkPdb := func(annotations map[string]string) error { + pdbList, err := cluster.KubeClient.PodDisruptionBudgets(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, pdb := range pdbList.Items { + if err := containsAnnotations(updateAnnotations(annotations), pdb.Annotations, pdb.ObjectMeta.Name, "Pod Disruption Budget"); err != nil { + return err + } + } + return nil + } + + checkPvc := func(annotations map[string]string) error { + pvcList, err := cluster.KubeClient.PersistentVolumeClaims(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, pvc := range pvcList.Items { + if err := containsAnnotations(annotations, pvc.Annotations, pvc.ObjectMeta.Name, "Volume claim"); err != nil { + return err + } + } + return nil + } + + checkPooler := func(annotations map[string]string) error { + for _, role := range []PostgresRole{Master, Replica} { + deploy, err := cluster.KubeClient.Deployments(namespace).Get(context.TODO(), cluster.connectionPoolerName(role), metav1.GetOptions{}) + if err != nil { + return err + } + if err := containsAnnotations(annotations, deploy.Annotations, deploy.Name, "Deployment"); err != nil { + return err + } + if err := containsAnnotations(updateAnnotations(annotations), deploy.Spec.Template.Annotations, deploy.Name, "Pooler pod template"); err != nil { + return err + } + } + return nil + } + + checkCronJob := func(annotations map[string]string) error { + cronJobList, err := cluster.KubeClient.CronJobs(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, cronJob := range cronJobList.Items { + if err := containsAnnotations(annotations, cronJob.Annotations, cronJob.ObjectMeta.Name, "Logical backup cron job"); err != nil { + return err + } + if err := containsAnnotations(updateAnnotations(annotations), cronJob.Spec.JobTemplate.Spec.Template.Annotations, cronJob.Name, "Logical backup cron job pod template"); err != nil { + return err + } + } + return nil + } + + checkSecrets := func(annotations map[string]string) error { + secretList, err := cluster.KubeClient.Secrets(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, secret := range secretList.Items { + if err := containsAnnotations(annotations, secret.Annotations, secret.Name, "Secret"); err != nil { + return err + } + } + return nil + } + + checkEndpoints := func(annotations map[string]string) error { + endpointsList, err := cluster.KubeClient.Endpoints(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, ep := range endpointsList.Items { + if err := containsAnnotations(annotations, ep.Annotations, ep.Name, "Endpoints"); err != nil { + return err + } + } + return nil + } + + checkConfigMaps := func(annotations map[string]string) error { + cmList, err := cluster.KubeClient.ConfigMaps(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, cm := range cmList.Items { + if err := containsAnnotations(annotations, cm.Annotations, cm.ObjectMeta.Name, "ConfigMap"); err != nil { + return err + } + } + return nil + } + + checkFuncs := []func(map[string]string) error{ + checkSts, checkPods, checkSvc, checkPdb, checkPooler, checkCronJob, checkPvc, checkSecrets, checkEndpoints, checkConfigMaps, + } + for _, f := range checkFuncs { + if err := f(resultAnnotations); err != nil { + return err + } + } + return nil +} + +func createPods(cluster *Cluster) []v1.Pod { + podsList := make([]v1.Pod, 0) + for i, role := range []PostgresRole{Master, Replica} { + podsList = append(podsList, v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", cluster.Name, i), + Namespace: namespace, + Labels: map[string]string{ + "application": "spilo", + "cluster-name": cluster.Name, + "spilo-role": string(role), + }, + }, + }) + podsList = append(podsList, v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-pooler-%s", cluster.Name, role), + Namespace: namespace, + Labels: cluster.connectionPoolerLabels(role, true).MatchLabels, + }, + }) + } + + return podsList +} + +func newInheritedAnnotationsCluster(client k8sutil.KubernetesClient) (*Cluster, error) { pg := acidv1.Postgresql{ ObjectMeta: metav1.ObjectMeta{ Name: clusterName, Annotations: map[string]string{ - "owned-by": annotationValue, + "owned-by": "acid", + "foo": "bar", // should not be inherited }, }, Spec: acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), EnableReplicaConnectionPooler: boolToPointer(true), + EnableLogicalBackup: true, Volume: acidv1.Volume{ Size: "1Gi", }, + NumberOfInstances: 2, }, } - var cluster = New( + // add postgresql cluster to fake client + _, err := client.PostgresqlsGetter.Postgresqls(namespace).Create(context.TODO(), &pg, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + + cluster := New( Config{ OpConfig: config.Config{ + PatroniAPICheckInterval: time.Duration(1), + PatroniAPICheckTimeout: time.Duration(5), + KubernetesUseConfigMaps: true, ConnectionPooler: config.ConnectionPooler{ ConnectionPoolerDefaultCPURequest: "100m", ConnectionPoolerDefaultCPULimit: "100m", ConnectionPoolerDefaultMemoryRequest: "100Mi", ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + NumberOfInstances: k8sutil.Int32ToPointer(1), }, + PDBNameFormat: "postgres-{cluster}-pdb", PodManagementPolicy: "ordered_ready", Resources: config.Resources{ - ClusterLabels: map[string]string{"application": "spilo"}, - ClusterNameLabel: "cluster-name", - DefaultCPURequest: "300m", - DefaultCPULimit: "300m", - DefaultMemoryRequest: "300Mi", - DefaultMemoryLimit: "300Mi", - InheritedAnnotations: []string{"owned-by"}, - PodRoleLabel: "spilo-role", + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + InheritedAnnotations: []string{"owned-by"}, + PodRoleLabel: "spilo-role", + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + MinInstances: -1, + MaxInstances: -1, }, }, }, client, pg, logger, eventRecorder) - cluster.Name = clusterName cluster.Namespace = namespace - - // test annotationsSet function - inheritedAnnotations := cluster.annotationsSet(nil) - - listOptions := metav1.ListOptions{ - LabelSelector: cluster.labelsSet(false).String(), + _, err = cluster.createStatefulSet() + if err != nil { + return nil, err + } + _, err = cluster.createService(Master) + if err != nil { + return nil, err + } + err = cluster.createPodDisruptionBudgets() + if err != nil { + return nil, err + } + _, err = cluster.createConnectionPooler(mockInstallLookupFunction) + if err != nil { + return nil, err + } + err = cluster.createLogicalBackupJob() + if err != nil { + return nil, err + } + pvcList := CreatePVCs(namespace, clusterName, cluster.labelsSet(false), 2, "1Gi") + for _, pvc := range pvcList.Items { + _, err = cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &pvc, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + } + podsList := createPods(cluster) + for _, pod := range podsList { + _, err = cluster.KubeClient.Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) + if err != nil { + return nil, err + } } - // check statefulset annotations - _, err := cluster.createStatefulSet() - assert.NoError(t, err) + // resources which Patroni creates + if err = createPatroniResources(cluster); err != nil { + return nil, err + } - stsList, err := client.StatefulSets(namespace).List(context.TODO(), listOptions) - assert.NoError(t, err) + return cluster, nil +} + +func createPatroniResources(cluster *Cluster) error { + patroniService := cluster.generateService(Replica, &pg.Spec) + patroniService.ObjectMeta.Name = cluster.serviceName(Patroni) + _, err := cluster.KubeClient.Services(namespace).Create(context.TODO(), patroniService, metav1.CreateOptions{}) + if err != nil { + return err + } + + for _, suffix := range patroniObjectSuffixes { + metadata := metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", clusterName, suffix), + Namespace: namespace, + Annotations: map[string]string{ + "initialize": "123456789", + }, + Labels: cluster.labelsSet(false), + } + + if cluster.OpConfig.KubernetesUseConfigMaps { + configMap := v1.ConfigMap{ + ObjectMeta: metadata, + } + _, err := cluster.KubeClient.ConfigMaps(namespace).Create(context.TODO(), &configMap, metav1.CreateOptions{}) + if err != nil { + return err + } + } else { + endpoints := v1.Endpoints{ + ObjectMeta: metadata, + } + _, err := cluster.KubeClient.Endpoints(namespace).Create(context.TODO(), &endpoints, metav1.CreateOptions{}) + if err != nil { + return err + } + } + } + + return nil +} + +func annotateResources(cluster *Cluster) error { + clusterOptions := clusterLabelsOptions(cluster) + patchData, err := metaAnnotationsPatch(externalAnnotations) + if err != nil { + return err + } + + stsList, err := cluster.KubeClient.StatefulSets(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } for _, sts := range stsList.Items { - if !(util.MapContains(sts.ObjectMeta.Annotations, inheritedAnnotations)) { - t.Errorf("%s: StatefulSet %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations) - } - // pod template - if !(util.MapContains(sts.Spec.Template.ObjectMeta.Annotations, inheritedAnnotations)) { - t.Errorf("%s: pod template %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations) - } - // pvc template - if util.MapContains(sts.Spec.VolumeClaimTemplates[0].Annotations, inheritedAnnotations) { - t.Errorf("%s: PVC template %v not expected to have inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations) + sts.Annotations = externalAnnotations + if _, err = cluster.KubeClient.StatefulSets(namespace).Patch(context.TODO(), sts.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}); err != nil { + return err } } - // check service annotations - cluster.createService(Master) - svcList, err := client.Services(namespace).List(context.TODO(), listOptions) - assert.NoError(t, err) + podList, err := cluster.KubeClient.Pods(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, pod := range podList.Items { + pod.Annotations = externalAnnotations + if _, err = cluster.KubeClient.Pods(namespace).Patch(context.TODO(), pod.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}); err != nil { + return err + } + } + + svcList, err := cluster.KubeClient.Services(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } for _, svc := range svcList.Items { - if !(util.MapContains(svc.ObjectMeta.Annotations, inheritedAnnotations)) { - t.Errorf("%s: Service %v not inherited annotations %#v, got %#v", testName, svc.ObjectMeta.Name, inheritedAnnotations, svc.ObjectMeta.Annotations) + svc.Annotations = externalAnnotations + if _, err = cluster.KubeClient.Services(namespace).Patch(context.TODO(), svc.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}); err != nil { + return err } } - // check pod disruption budget annotations - cluster.createPodDisruptionBudget() - pdbList, err := client.PodDisruptionBudgets(namespace).List(context.TODO(), listOptions) - assert.NoError(t, err) + pdbList, err := cluster.KubeClient.PodDisruptionBudgets(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } for _, pdb := range pdbList.Items { - if !(util.MapContains(pdb.ObjectMeta.Annotations, inheritedAnnotations)) { - t.Errorf("%s: Pod Disruption Budget %v not inherited annotations %#v, got %#v", testName, pdb.ObjectMeta.Name, inheritedAnnotations, pdb.ObjectMeta.Annotations) + pdb.Annotations = externalAnnotations + _, err = cluster.KubeClient.PodDisruptionBudgets(namespace).Patch(context.TODO(), pdb.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return err } } - // check pooler deployment annotations - cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{} - cluster.ConnectionPooler[role] = &ConnectionPoolerObjects{ - Name: cluster.connectionPoolerName(role), - ClusterName: cluster.ClusterName, - Namespace: cluster.Namespace, - Role: role, + cronJobList, err := cluster.KubeClient.CronJobs(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err } - deploy, err := cluster.generateConnectionPoolerDeployment(cluster.ConnectionPooler[role]) + for _, cronJob := range cronJobList.Items { + cronJob.Annotations = externalAnnotations + _, err = cluster.KubeClient.CronJobs(namespace).Patch(context.TODO(), cronJob.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return err + } + } + + pvcList, err := cluster.KubeClient.PersistentVolumeClaims(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, pvc := range pvcList.Items { + pvc.Annotations = externalAnnotations + if _, err = cluster.KubeClient.PersistentVolumeClaims(namespace).Patch(context.TODO(), pvc.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}); err != nil { + return err + } + } + + for _, role := range []PostgresRole{Master, Replica} { + deploy, err := cluster.KubeClient.Deployments(namespace).Get(context.TODO(), cluster.connectionPoolerName(role), metav1.GetOptions{}) + if err != nil { + return err + } + deploy.Annotations = externalAnnotations + if _, err = cluster.KubeClient.Deployments(namespace).Patch(context.TODO(), deploy.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}); err != nil { + return err + } + } + + secrets, err := cluster.KubeClient.Secrets(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, secret := range secrets.Items { + secret.Annotations = externalAnnotations + if _, err = cluster.KubeClient.Secrets(namespace).Patch(context.TODO(), secret.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}); err != nil { + return err + } + } + + endpoints, err := cluster.KubeClient.Endpoints(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, ep := range endpoints.Items { + ep.Annotations = externalAnnotations + if _, err = cluster.KubeClient.Endpoints(namespace).Patch(context.TODO(), ep.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}); err != nil { + return err + } + } + + configMaps, err := cluster.KubeClient.ConfigMaps(namespace).List(context.TODO(), clusterOptions) + if err != nil { + return err + } + for _, cm := range configMaps.Items { + cm.Annotations = externalAnnotations + if _, err = cluster.KubeClient.ConfigMaps(namespace).Patch(context.TODO(), cm.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}); err != nil { + return err + } + } + + return nil +} + +func TestInheritedAnnotations(t *testing.T) { + // mocks + ctrl := gomock.NewController(t) + defer ctrl.Finish() + client, _ := newFakeK8sAnnotationsClient() + mockClient := mocks.NewMockHTTPClient(ctrl) + + cluster, err := newInheritedAnnotationsCluster(client) assert.NoError(t, err) - if !(util.MapContains(deploy.ObjectMeta.Annotations, inheritedAnnotations)) { - t.Errorf("%s: Deployment %v not inherited annotations %#v, got %#v", testName, deploy.ObjectMeta.Name, inheritedAnnotations, deploy.ObjectMeta.Annotations) + configJson := `{"postgresql": {"parameters": {"log_min_duration_statement": 200, "max_connections": 50}}}, "ttl": 20}` + response := http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader([]byte(configJson))), } + mockClient.EXPECT().Do(gomock.Any()).Return(&response, nil).AnyTimes() + cluster.patroni = patroni.New(patroniLogger, mockClient) + err = cluster.Sync(&cluster.Postgresql) + assert.NoError(t, err) + + filterLabels := cluster.labelsSet(false) + + // Finally, tests! + result := map[string]string{"owned-by": "acid"} + assert.True(t, reflect.DeepEqual(result, cluster.annotationsSet(nil))) + + // 1. Check initial state + err = checkResourcesInheritedAnnotations(cluster, result) + assert.NoError(t, err) + + // 2. Check annotation value change + + // 2.1 Sync event + newSpec := cluster.Postgresql.DeepCopy() + newSpec.Annotations["owned-by"] = "fooSync" + result["owned-by"] = "fooSync" + + err = cluster.Sync(newSpec) + assert.NoError(t, err) + err = checkResourcesInheritedAnnotations(cluster, result) + assert.NoError(t, err) + + // + existing PVC without annotations + cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &CreatePVCs(namespace, clusterName, filterLabels, 3, "1Gi").Items[2], metav1.CreateOptions{}) + err = cluster.Sync(newSpec) + assert.NoError(t, err) + err = checkResourcesInheritedAnnotations(cluster, result) + assert.NoError(t, err) + + // 2.2 Update event + newSpec = cluster.Postgresql.DeepCopy() + newSpec.Annotations["owned-by"] = "fooUpdate" + result["owned-by"] = "fooUpdate" + // + new PVC + cluster.KubeClient.PersistentVolumeClaims(namespace).Create(context.TODO(), &CreatePVCs(namespace, clusterName, filterLabels, 4, "1Gi").Items[3], metav1.CreateOptions{}) + + err = cluster.Update(cluster.Postgresql.DeepCopy(), newSpec) + assert.NoError(t, err) + + err = checkResourcesInheritedAnnotations(cluster, result) + assert.NoError(t, err) + + // 3. Change from ConfigMaps to Endpoints + err = cluster.deletePatroniResources() + assert.NoError(t, err) + cluster.OpConfig.KubernetesUseConfigMaps = false + err = createPatroniResources(cluster) + assert.NoError(t, err) + err = cluster.Sync(newSpec.DeepCopy()) + assert.NoError(t, err) + err = checkResourcesInheritedAnnotations(cluster, result) + assert.NoError(t, err) + + // 4. Existing annotations (should not be removed) + err = annotateResources(cluster) + assert.NoError(t, err) + maps.Copy(result, externalAnnotations) + err = cluster.Sync(newSpec.DeepCopy()) + assert.NoError(t, err) + err = checkResourcesInheritedAnnotations(cluster, result) + assert.NoError(t, err) } func Test_trimCronjobName(t *testing.T) { @@ -179,3 +655,127 @@ func Test_trimCronjobName(t *testing.T) { }) } } + +func TestIsInMaintenanceWindow(t *testing.T) { + cluster := New( + Config{ + OpConfig: config.Config{ + EnableMaintenanceWindows: util.True(), + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + cluster.Name = clusterName + cluster.Namespace = namespace + + now := time.Now() + futureTimeStart := now.Add(1 * time.Hour) + futureTimeStartFormatted := futureTimeStart.Format("15:04") + futureTimeEnd := now.Add(2 * time.Hour) + futureTimeEndFormatted := futureTimeEnd.Format("15:04") + + tests := []struct { + name string + windows []acidv1.MaintenanceWindow + configWindows []string + windowsFlag bool + expected bool + }{ + { + name: "no maintenance windows", + windows: nil, + configWindows: nil, + windowsFlag: true, + expected: true, + }, + { + name: "maintenance windows diabled", + windows: []acidv1.MaintenanceWindow{ + { + Everyday: true, + StartTime: mustParseTime("00:00"), + EndTime: mustParseTime("23:59"), + }, + }, + configWindows: nil, + windowsFlag: false, + expected: true, + }, + { + name: "maintenance windows with everyday", + windows: []acidv1.MaintenanceWindow{ + { + Everyday: true, + StartTime: mustParseTime("00:00"), + EndTime: mustParseTime("23:59"), + }, + }, + configWindows: nil, + windowsFlag: true, + expected: true, + }, + { + name: "maintenance windows with weekday", + windows: []acidv1.MaintenanceWindow{ + { + Weekday: now.Weekday(), + StartTime: mustParseTime("00:00"), + EndTime: mustParseTime("23:59"), + }, + }, + configWindows: nil, + windowsFlag: true, + expected: true, + }, + { + name: "maintenance windows with future interval time", + windows: []acidv1.MaintenanceWindow{ + { + Weekday: now.Weekday(), + StartTime: mustParseTime(futureTimeStartFormatted), + EndTime: mustParseTime(futureTimeEndFormatted), + }, + }, + windowsFlag: true, + expected: false, + }, + { + name: "global maintenance windows with future interval time", + windows: nil, + configWindows: []string{fmt.Sprintf("%s-%s", futureTimeStartFormatted, futureTimeEndFormatted)}, + windowsFlag: true, + expected: false, + }, + { + name: "global maintenance windows all day", + windows: nil, + configWindows: []string{"00:00-02:00", "02:00-23:59"}, + windowsFlag: true, + expected: true, + }, + { + name: "global maintenance windows ignored", + windows: nil, + configWindows: []string{"00:00-02:00", "02:00-23:59"}, + windowsFlag: false, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cluster.OpConfig.EnableMaintenanceWindows = &tt.windowsFlag + cluster.OpConfig.MaintenanceWindows = tt.configWindows + cluster.Spec.MaintenanceWindows = tt.windows + if cluster.isInMaintenanceWindow(cluster.Spec.MaintenanceWindows) != tt.expected { + t.Errorf("Expected isInMaintenanceWindow to return %t", tt.expected) + } + }) + } +} diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go index 9a41f5f05..e32e558e6 100644 --- a/pkg/cluster/volumes.go +++ b/pkg/cluster/volumes.go @@ -9,13 +9,13 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" - "github.com/aws/aws-sdk-go/aws" - acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/zalando/postgres-operator/pkg/spec" - "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/filesystems" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/volumes" ) @@ -35,25 +35,21 @@ func (c *Cluster) syncVolumes() error { err = c.populateVolumeMetaData() if err != nil { - c.logger.Errorf("populating EBS meta data failed, skipping potential adjustements: %v", err) + c.logger.Errorf("populating EBS meta data failed, skipping potential adjustments: %v", err) } else { err = c.syncUnderlyingEBSVolume() if err != nil { - c.logger.Errorf("errors occured during EBS volume adjustments: %v", err) + c.logger.Errorf("errors occurred during EBS volume adjustments: %v", err) } } + } - // resize pvc to adjust filesystem size until better K8s support - if err = c.syncVolumeClaims(); err != nil { - err = fmt.Errorf("could not sync persistent volume claims: %v", err) - return err - } - } else if c.OpConfig.StorageResizeMode == "pvc" { - if err = c.syncVolumeClaims(); err != nil { - err = fmt.Errorf("could not sync persistent volume claims: %v", err) - return err - } - } else if c.OpConfig.StorageResizeMode == "ebs" { + if err = c.syncVolumeClaims(); err != nil { + err = fmt.Errorf("could not sync persistent volume claims: %v", err) + return err + } + + if c.OpConfig.StorageResizeMode == "ebs" { // potentially enlarge volumes before changing the statefulset. By doing that // in this order we make sure the operator is not stuck waiting for a pod that // cannot start because it ran out of disk space. @@ -64,15 +60,13 @@ func (c *Cluster) syncVolumes() error { err = fmt.Errorf("could not sync persistent volumes: %v", err) return err } - } else { - c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.") } return nil } func (c *Cluster) syncUnderlyingEBSVolume() error { - c.logger.Infof("starting to sync EBS volumes: type, iops, throughput, and size") + c.logger.Debug("starting to sync EBS volumes: type, iops, throughput, and size") var ( err error @@ -88,7 +82,7 @@ func (c *Cluster) syncUnderlyingEBSVolume() error { awsGp3 := aws.String("gp3") awsIo2 := aws.String("io2") - errors := []string{} + errors := make([]string, 0) for _, volume := range c.EBSVolumes { var modifyIops *int64 @@ -96,19 +90,19 @@ func (c *Cluster) syncUnderlyingEBSVolume() error { var modifySize *int64 var modifyType *string - if targetValue.Iops != nil { - if volume.Iops != *targetValue.Iops { + if targetValue.Iops != nil && *targetValue.Iops >= int64(3000) { + if volume.Iops != int64(*targetValue.Iops) { modifyIops = targetValue.Iops } } - if targetValue.Throughput != nil { - if volume.Throughput != *targetValue.Throughput { + if targetValue.Throughput != nil && *targetValue.Throughput >= int64(125) { + if volume.Throughput != int64(*targetValue.Throughput) { modifyThroughput = targetValue.Throughput } } - if targetSize > volume.Size { + if targetSize > int64(volume.Size) { modifySize = &targetSize } @@ -128,33 +122,36 @@ func (c *Cluster) syncUnderlyingEBSVolume() error { err = c.VolumeResizer.ModifyVolume(volume.VolumeID, modifyType, modifySize, modifyIops, modifyThroughput) if err != nil { - errors = append(errors, fmt.Sprintf("modify volume failed: volume=%s size=%d iops=%d throughput=%d", volume.VolumeID, volume.Size, volume.Iops, volume.Throughput)) + errors = append(errors, fmt.Sprintf("modify failed: %v, showing current EBS values: volume-id=%s size=%d iops=%d throughput=%d", err, volume.VolumeID, volume.Size, volume.Iops, volume.Throughput)) } } } if len(errors) > 0 { for _, s := range errors { - c.logger.Warningf(s) + c.logger.Warningf("%s", s) } - // c.logger.Errorf("failed to modify %d of %d volumes", len(c.EBSVolumes), len(errors)) } return nil } func (c *Cluster) populateVolumeMetaData() error { - c.logger.Infof("starting reading ebs meta data") + c.logger.Debug("starting reading ebs meta data") pvs, err := c.listPersistentVolumes() if err != nil { return fmt.Errorf("could not list persistent volumes: %v", err) } - c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes)) + if len(pvs) == 0 { + c.EBSVolumes = make(map[string]volumes.VolumeProperties) + return fmt.Errorf("no persistent volumes found") + } + c.logger.Debugf("found %d persistent volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes)) volumeIds := []string{} var volumeID string for _, pv := range pvs { - volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) + volumeID, err = c.VolumeResizer.GetProviderVolumeID(pv) if err != nil { continue } @@ -167,8 +164,8 @@ func (c *Cluster) populateVolumeMetaData() error { return err } - if len(currentVolumes) != len(c.EBSVolumes) { - c.logger.Debugf("number of ebs volumes (%d) discovered differs from already known volumes (%d)", len(currentVolumes), len(c.EBSVolumes)) + if len(currentVolumes) != len(c.EBSVolumes) && len(c.EBSVolumes) > 0 { + c.logger.Infof("number of ebs volumes (%d) discovered differs from already known volumes (%d)", len(currentVolumes), len(c.EBSVolumes)) } // reset map, operator is not responsible for dangling ebs volumes @@ -184,28 +181,71 @@ func (c *Cluster) populateVolumeMetaData() error { func (c *Cluster) syncVolumeClaims() error { c.setProcessName("syncing volume claims") - needsResizing, err := c.volumeClaimsNeedResizing(c.Spec.Volume) + ignoreResize := false + + if c.OpConfig.StorageResizeMode == "off" || c.OpConfig.StorageResizeMode == "ebs" { + ignoreResize = true + c.logger.Debugf("Storage resize mode is set to %q. Skipping volume size sync of persistent volume claims.", c.OpConfig.StorageResizeMode) + } + + newSize, err := resource.ParseQuantity(c.Spec.Volume.Size) if err != nil { - return fmt.Errorf("could not compare size of the volume claims: %v", err) + return fmt.Errorf("could not parse volume size from the manifest: %v", err) + } + manifestSize := quantityToGigabyte(newSize) + + pvcs, err := c.listPersistentVolumeClaims() + if err != nil { + return fmt.Errorf("could not list persistent volume claims: %v", err) + } + for _, pvc := range pvcs { + c.VolumeClaims[pvc.UID] = &pvc + needsUpdate := false + currentSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage]) + if !ignoreResize && currentSize != manifestSize { + if currentSize < manifestSize { + pvc.Spec.Resources.Requests[v1.ResourceStorage] = newSize + needsUpdate = true + c.logger.Infof("persistent volume claim for volume %q needs to be resized", pvc.Name) + } else { + c.logger.Warningf("cannot shrink persistent volume") + } + } + + if needsUpdate { + c.logger.Infof("updating persistent volume claim definition for volume %q", pvc.Name) + updatedPvc, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Update(context.TODO(), &pvc, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update persistent volume claim: %q", err) + } + c.VolumeClaims[pvc.UID] = updatedPvc + c.logger.Infof("successfully updated persistent volume claim %q", pvc.Name) + } else { + c.logger.Debugf("volume claim for volume %q do not require updates", pvc.Name) + } + + newAnnotations := c.annotationsSet(nil) + if changed, _ := c.compareAnnotations(pvc.Annotations, newAnnotations, nil); changed { + patchData, err := metaAnnotationsPatch(newAnnotations) + if err != nil { + return fmt.Errorf("could not form patch for the persistent volume claim for volume %q: %v", pvc.Name, err) + } + patchedPvc, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Patch(context.TODO(), pvc.Name, types.MergePatchType, []byte(patchData), metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("could not patch annotations of the persistent volume claim for volume %q: %v", pvc.Name, err) + } + c.VolumeClaims[pvc.UID] = patchedPvc + } } - if !needsResizing { - c.logger.Infof("volume claims do not require changes") - return nil - } - - if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil { - return fmt.Errorf("could not sync volume claims: %v", err) - } - - c.logger.Infof("volume claims have been synced successfully") + c.logger.Debug("volume claims have been synced successfully") return nil } // syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset. func (c *Cluster) syncEbsVolumes() error { - c.setProcessName("syncing EBS and Claims volumes") + c.setProcessName("syncing EBS volumes") act, err := c.volumesNeedResizing() if err != nil { @@ -219,7 +259,7 @@ func (c *Cluster) syncEbsVolumes() error { return fmt.Errorf("could not sync volumes: %v", err) } - c.logger.Infof("volumes have been synced successfully") + c.logger.Debug("volumes have been synced successfully") return nil } @@ -232,58 +272,41 @@ func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, erro pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(context.TODO(), listOptions) if err != nil { - return nil, fmt.Errorf("could not list of PersistentVolumeClaims: %v", err) + return nil, fmt.Errorf("could not list of persistent volume claims: %v", err) } return pvcs.Items, nil } func (c *Cluster) deletePersistentVolumeClaims() error { - c.logger.Debugln("deleting PVCs") - pvcs, err := c.listPersistentVolumeClaims() - if err != nil { - return err - } - for _, pvc := range pvcs { - c.logger.Debugf("deleting PVC %q", util.NameFromMeta(pvc.ObjectMeta)) - if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, c.deleteOptions); err != nil { - c.logger.Warningf("could not delete PersistentVolumeClaim: %v", err) + c.setProcessName("deleting persistent volume claims") + errors := make([]string, 0) + for uid := range c.VolumeClaims { + err := c.deletePersistentVolumeClaim(uid) + if err != nil { + errors = append(errors, fmt.Sprintf("%v", err)) } } - if len(pvcs) > 0 { - c.logger.Debugln("PVCs have been deleted") - } else { - c.logger.Debugln("no PVCs to delete") + + if len(errors) > 0 { + c.logger.Warningf("could not delete all persistent volume claims: %v", strings.Join(errors, `', '`)) } return nil } -func (c *Cluster) resizeVolumeClaims(newVolume acidv1.Volume) error { - c.logger.Debugln("resizing PVCs") - pvcs, err := c.listPersistentVolumeClaims() - if err != nil { - return err - } - newQuantity, err := resource.ParseQuantity(newVolume.Size) - if err != nil { - return fmt.Errorf("could not parse volume size: %v", err) - } - newSize := quantityToGigabyte(newQuantity) - for _, pvc := range pvcs { - volumeSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage]) - if volumeSize >= newSize { - if volumeSize > newSize { - c.logger.Warningf("cannot shrink persistent volume") - } - continue - } - pvc.Spec.Resources.Requests[v1.ResourceStorage] = newQuantity - c.logger.Debugf("updating persistent volume claim definition for volume %q", pvc.Name) - if _, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Update(context.TODO(), &pvc, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("could not update persistent volume claim: %q", err) - } - c.logger.Debugf("successfully updated persistent volume claim %q", pvc.Name) +func (c *Cluster) deletePersistentVolumeClaim(uid types.UID) error { + c.setProcessName("deleting persistent volume claim") + pvc := c.VolumeClaims[uid] + c.logger.Debugf("deleting persistent volume claim %q", pvc.Name) + err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, c.deleteOptions) + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("persistent volume claim %q has already been deleted", pvc.Name) + } else if err != nil { + return fmt.Errorf("could not delete persistent volume claim %q: %v", pvc.Name, err) } + c.logger.Infof("persistent volume claim %q has been deleted", pvc.Name) + delete(c.VolumeClaims, uid) + return nil } @@ -292,7 +315,7 @@ func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) { pvcs, err := c.listPersistentVolumeClaims() if err != nil { - return nil, fmt.Errorf("could not list cluster's PersistentVolumeClaims: %v", err) + return nil, fmt.Errorf("could not list cluster's persistent volume claims: %v", err) } pods, err := c.listPods() @@ -375,22 +398,22 @@ func (c *Cluster) resizeVolumes() error { if err != nil { return err } - c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize) + c.logger.Infof("updating persistent volume %q to %d", pv.Name, newSize) if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil { return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err) } - c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name) + c.logger.Infof("resizing the filesystem on the volume %q", pv.Name) podName := getPodNameFromPersistentVolume(pv) if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil { return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err) } - c.logger.Debugf("filesystem resize successful on volume %q", pv.Name) + c.logger.Infof("filesystem resize successful on volume %q", pv.Name) pv.Spec.Capacity[v1.ResourceStorage] = newQuantity - c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name) + c.logger.Infof("updating persistent volume definition for volume %q", pv.Name) if _, err := c.KubeClient.PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("could not update persistent volume: %q", err) } - c.logger.Debugf("successfully updated persistent volume %q", pv.Name) + c.logger.Infof("successfully updated persistent volume %q", pv.Name) if !compatible { c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name) @@ -403,25 +426,6 @@ func (c *Cluster) resizeVolumes() error { return nil } -func (c *Cluster) volumeClaimsNeedResizing(newVolume acidv1.Volume) (bool, error) { - newSize, err := resource.ParseQuantity(newVolume.Size) - manifestSize := quantityToGigabyte(newSize) - if err != nil { - return false, fmt.Errorf("could not parse volume size from the manifest: %v", err) - } - pvcs, err := c.listPersistentVolumeClaims() - if err != nil { - return false, fmt.Errorf("could not receive persistent volume claims: %v", err) - } - for _, pvc := range pvcs { - currentSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage]) - if currentSize != manifestSize { - return true, nil - } - } - return false, nil -} - func (c *Cluster) volumesNeedResizing() (bool, error) { newQuantity, _ := resource.ParseQuantity(c.Spec.Volume.Size) newSize := quantityToGigabyte(newQuantity) @@ -451,29 +455,17 @@ func quantityToGigabyte(q resource.Quantity) int64 { } func (c *Cluster) executeEBSMigration() error { - if !c.OpConfig.EnableEBSGp3Migration { - return nil - } - c.logger.Infof("starting EBS gp2 to gp3 migration") - pvs, err := c.listPersistentVolumes() if err != nil { return fmt.Errorf("could not list persistent volumes: %v", err) } + if len(pvs) == 0 { + c.logger.Warningf("no persistent volumes found - skipping EBS migration") + return nil + } c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes)) - volumeIds := []string{} - var volumeID string - for _, pv := range pvs { - volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) - if err != nil { - continue - } - - volumeIds = append(volumeIds, volumeID) - } - - if len(volumeIds) == len(c.EBSVolumes) { + if len(pvs) == len(c.EBSVolumes) { hasGp2 := false for _, v := range c.EBSVolumes { if v.VolumeType == "gp2" { @@ -482,20 +474,15 @@ func (c *Cluster) executeEBSMigration() error { } if !hasGp2 { - c.logger.Infof("no EBS gp2 volumes left to migrate") + c.logger.Debugf("no EBS gp2 volumes left to migrate") return nil } } - awsVolumes, err := c.VolumeResizer.DescribeVolumes(volumeIds) - if nil != err { - return err - } - var i3000 int64 = 3000 var i125 int64 = 125 - for _, volume := range awsVolumes { + for _, volume := range c.EBSVolumes { if volume.VolumeType == "gp2" && volume.Size < c.OpConfig.EnableEBSGp3MigrationMaxSize { c.logger.Infof("modifying EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size) err = c.VolumeResizer.ModifyVolume(volume.VolumeID, aws.String("gp3"), &volume.Size, &i3000, &i125) diff --git a/pkg/cluster/volumes_test.go b/pkg/cluster/volumes_test.go index 204ea8aab..d6472539b 100644 --- a/pkg/cluster/volumes_test.go +++ b/pkg/cluster/volumes_test.go @@ -11,7 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -74,6 +74,7 @@ func TestResizeVolumeClaim(t *testing.T) { cluster.Name = clusterName cluster.Namespace = namespace filterLabels := cluster.labelsSet(false) + cluster.Spec.Volume.Size = newVolumeSize // define and create PVCs for 1Gi volumes pvcList := CreatePVCs(namespace, clusterName, filterLabels, 2, "1Gi") @@ -85,14 +86,14 @@ func TestResizeVolumeClaim(t *testing.T) { } // test resizing - cluster.resizeVolumeClaims(acidv1.Volume{Size: newVolumeSize}) + cluster.syncVolumes() pvcs, err := cluster.listPersistentVolumeClaims() assert.NoError(t, err) // check if listPersistentVolumeClaims returns only the PVCs matching the filter if len(pvcs) != len(pvcList.Items)-1 { - t.Errorf("%s: could not find all PVCs, got %v, expected %v", testName, len(pvcs), len(pvcList.Items)-1) + t.Errorf("%s: could not find all persistent volume claims, got %v, expected %v", testName, len(pvcs), len(pvcList.Items)-1) } // check if PVCs were correctly resized @@ -164,7 +165,7 @@ func CreatePVCs(namespace string, clusterName string, labels labels.Set, n int, Labels: labels, }, Spec: v1.PersistentVolumeClaimSpec{ - Resources: v1.ResourceRequirements{ + Resources: v1.VolumeResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceStorage: storage1Gi, }, @@ -215,6 +216,12 @@ func TestMigrateEBS(t *testing.T) { resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil) resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil) + resizer.EXPECT().GetProviderVolumeID(gomock.Any()). + DoAndReturn(func(pv *v1.PersistentVolume) (string, error) { + return resizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) + }). + Times(2) + resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return( []volumes.VolumeProperties{ {VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 100}, @@ -224,7 +231,10 @@ func TestMigrateEBS(t *testing.T) { resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq(aws.String("gp3")), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) cluster.VolumeResizer = resizer - cluster.executeEBSMigration() + err := cluster.populateVolumeMetaData() + assert.NoError(t, err) + err = cluster.executeEBSMigration() + assert.NoError(t, err) } func initTestVolumesAndPods(client k8sutil.KubernetesClient, namespace, clustername string, labels labels.Set, volumes []testVolume) { @@ -252,7 +262,7 @@ func initTestVolumesAndPods(client k8sutil.KubernetesClient, namespace, clustern Labels: labels, }, Spec: v1.PersistentVolumeClaimSpec{ - Resources: v1.ResourceRequirements{ + Resources: v1.VolumeResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceStorage: storage1Gi, }, @@ -318,6 +328,12 @@ func TestMigrateGp3Support(t *testing.T) { resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil) resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-3")).Return("ebs-volume-3", nil) + resizer.EXPECT().GetProviderVolumeID(gomock.Any()). + DoAndReturn(func(pv *v1.PersistentVolume) (string, error) { + return resizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) + }). + Times(3) + resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2", "ebs-volume-3"})).Return( []volumes.VolumeProperties{ {VolumeID: "ebs-volume-1", VolumeType: "gp3", Size: 100, Iops: 3000}, @@ -373,6 +389,12 @@ func TestManualGp2Gp3Support(t *testing.T) { resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil) resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil) + resizer.EXPECT().GetProviderVolumeID(gomock.Any()). + DoAndReturn(func(pv *v1.PersistentVolume) (string, error) { + return resizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) + }). + Times(2) + resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return( []volumes.VolumeProperties{ {VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 150, Iops: 3000}, @@ -432,6 +454,12 @@ func TestDontTouchType(t *testing.T) { resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-1")).Return("ebs-volume-1", nil) resizer.EXPECT().ExtractVolumeID(gomock.Eq("aws://eu-central-1b/ebs-volume-2")).Return("ebs-volume-2", nil) + resizer.EXPECT().GetProviderVolumeID(gomock.Any()). + DoAndReturn(func(pv *v1.PersistentVolume) (string, error) { + return resizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) + }). + Times(2) + resizer.EXPECT().DescribeVolumes(gomock.Eq([]string{"ebs-volume-1", "ebs-volume-2"})).Return( []volumes.VolumeProperties{ {VolumeID: "ebs-volume-1", VolumeType: "gp2", Size: 150, Iops: 3000}, diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index f992ff782..879e44ad4 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -26,6 +26,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + informers_core_v1 "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes/scheme" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" @@ -64,6 +65,7 @@ type Controller struct { nodesInformer cache.SharedIndexInformer podCh chan cluster.PodEvent + clusterEventStores []cache.Store // [workerID]Store clusterEventQueues []*cache.FIFO // [workerID]Queue lastClusterSyncTime int64 lastClusterRepairTime int64 @@ -95,7 +97,9 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string) // disabling the sending of events also to the logoutput // the operator currently duplicates a lot of log entries with this setup // eventBroadcaster.StartLogging(logger.Infof) - recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName}) + scheme := scheme.Scheme + acidv1.AddToScheme(scheme) + recorder := eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: myComponentName}) c := &Controller{ config: *controllerConfig, @@ -275,7 +279,7 @@ func (c *Controller) initRoleBinding() { }`, c.PodServiceAccount.Name, c.PodServiceAccount.Name, c.PodServiceAccount.Name) c.opConfig.PodServiceAccountRoleBindingDefinition = compactValue(stringValue) } - c.logger.Info("Parse role bindings") + // re-uses k8s internal parsing. See k8s client-go issue #193 for explanation decode := scheme.Codecs.UniversalDeserializer().Decode obj, groupVersionKind, err := decode([]byte(c.opConfig.PodServiceAccountRoleBindingDefinition), nil, nil) @@ -307,8 +311,10 @@ func (c *Controller) initController() { c.controllerID = os.Getenv("CONTROLLER_ID") if configObjectName := os.Getenv("POSTGRES_OPERATOR_CONFIGURATION_OBJECT"); configObjectName != "" { - if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil { - c.logger.Fatalf("could not register Operator Configuration CustomResourceDefinition: %v", err) + if c.opConfig.EnableCRDRegistration != nil && *c.opConfig.EnableCRDRegistration { + if err := c.createConfigurationCRD(); err != nil { + c.logger.Fatalf("could not register Operator Configuration CustomResourceDefinition: %v", err) + } } if cfg, err := c.readOperatorConfigurationFromCRD(spec.GetOperatorNamespace(), configObjectName); err != nil { c.logger.Fatalf("unable to read operator configuration: %v", err) @@ -323,8 +329,10 @@ func (c *Controller) initController() { c.modifyConfigFromEnvironment() - if err := c.createPostgresCRD(c.opConfig.EnableCRDValidation); err != nil { - c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err) + if c.opConfig.EnableCRDRegistration != nil && *c.opConfig.EnableCRDRegistration { + if err := c.createPostgresCRD(); err != nil { + c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err) + } } c.initSharedInformers() @@ -341,23 +349,27 @@ func (c *Controller) initController() { logMultiLineConfig(c.logger, c.opConfig.MustMarshal()) roleDefs := c.getInfrastructureRoleDefinitions() - if infraRoles, err := c.getInfrastructureRoles(roleDefs); err != nil { - c.logger.Warningf("could not get infrastructure roles: %v", err) - } else { + infraRoles, err := c.getInfrastructureRoles(roleDefs) + if err != nil { + c.logger.Warningf("could not get all infrastructure roles: %v", err) + } + if len(infraRoles) > 0 { c.config.InfrastructureRoles = infraRoles } + c.clusterEventStores = make([]cache.Store, c.opConfig.Workers) c.clusterEventQueues = make([]*cache.FIFO, c.opConfig.Workers) c.workerLogs = make(map[uint32]ringlog.RingLogger, c.opConfig.Workers) for i := range c.clusterEventQueues { - c.clusterEventQueues[i] = cache.NewFIFO(func(obj interface{}) (string, error) { + keyFn := func(obj interface{}) (string, error) { e, ok := obj.(ClusterEvent) if !ok { - return "", fmt.Errorf("could not cast to ClusterEvent") + return "", fmt.Errorf("could not cast to cluster event") } - return queueClusterKey(e.EventType, e.UID), nil - }) + } + c.clusterEventStores[i] = cache.NewStore(keyFn) + c.clusterEventQueues[i] = cache.NewFIFO(keyFn) } c.apiserver = apiserver.New(c, c.opConfig.APIPort, c.logger.Logger) @@ -393,16 +405,8 @@ func (c *Controller) initSharedInformers() { } // Pods - podLw := &cache.ListWatch{ - ListFunc: c.podListFunc, - WatchFunc: c.podWatchFunc, - } - - c.podInformer = cache.NewSharedIndexInformer( - podLw, - &v1.Pod{}, - constants.QueueResyncPeriodPod, - cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + c.podInformer = informers_core_v1.NewPodInformer(c.KubeClient.Clientset, + c.opConfig.WatchedNamespace, constants.QueueResyncPeriodPod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) c.podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.podAdd, @@ -411,15 +415,7 @@ func (c *Controller) initSharedInformers() { }) // Kubernetes Nodes - nodeLw := &cache.ListWatch{ - ListFunc: c.nodeListFunc, - WatchFunc: c.nodeWatchFunc, - } - - c.nodesInformer = cache.NewSharedIndexInformer( - nodeLw, - &v1.Node{}, - constants.QueueResyncPeriodNode, + c.nodesInformer = informers_core_v1.NewNodeInformer(c.KubeClient.Clientset, constants.QueueResyncPeriodNode, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) c.nodesInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -445,7 +441,7 @@ func (c *Controller) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) { panic("could not acquire initial list of clusters") } - wg.Add(5) + wg.Add(5 + util.Bool2Int(c.opConfig.EnablePostgresTeamCRD)) go c.runPodInformer(stopCh, wg) go c.runPostgresqlInformer(stopCh, wg) go c.clusterResync(stopCh, wg) diff --git a/pkg/controller/logs_and_api.go b/pkg/controller/logs_and_api.go index 24e73fabe..24881f9d7 100644 --- a/pkg/controller/logs_and_api.go +++ b/pkg/controller/logs_and_api.go @@ -15,11 +15,11 @@ import ( ) // ClusterStatus provides status of the cluster -func (c *Controller) ClusterStatus(team, namespace, cluster string) (*cluster.ClusterStatus, error) { +func (c *Controller) ClusterStatus(namespace, cluster string) (*cluster.ClusterStatus, error) { clusterName := spec.NamespacedName{ Namespace: namespace, - Name: team + "-" + cluster, + Name: cluster, } c.clustersMu.RLock() @@ -80,8 +80,8 @@ func (c *Controller) GetStatus() *spec.ControllerStatus { c.clustersMu.RUnlock() queueSizes := make(map[int]int, c.opConfig.Workers) - for workerID, queue := range c.clusterEventQueues { - queueSizes[workerID] = len(queue.ListKeys()) + for workerID, store := range c.clusterEventStores { + queueSizes[workerID] = len(store.ListKeys()) } return &spec.ControllerStatus{ @@ -92,11 +92,11 @@ func (c *Controller) GetStatus() *spec.ControllerStatus { } // ClusterLogs dumps cluster ring logs -func (c *Controller) ClusterLogs(team, namespace, name string) ([]*spec.LogEntry, error) { +func (c *Controller) ClusterLogs(namespace, name string) ([]*spec.LogEntry, error) { clusterName := spec.NamespacedName{ Namespace: namespace, - Name: team + "-" + name, + Name: name, } c.clustersMu.RLock() @@ -180,11 +180,11 @@ func (c *Controller) Fire(e *logrus.Entry) error { // ListQueue dumps cluster event queue of the provided worker func (c *Controller) ListQueue(workerID uint32) (*spec.QueueDump, error) { - if workerID >= uint32(len(c.clusterEventQueues)) { + if workerID >= uint32(len(c.clusterEventStores)) { return nil, fmt.Errorf("could not find worker") } - q := c.clusterEventQueues[workerID] + q := c.clusterEventStores[workerID] return &spec.QueueDump{ Keys: q.ListKeys(), List: q.List(), @@ -196,7 +196,7 @@ func (c *Controller) GetWorkersCnt() uint32 { return c.opConfig.Workers } -//WorkerStatus provides status of the worker +// WorkerStatus provides status of the worker func (c *Controller) WorkerStatus(workerID uint32) (*cluster.WorkerStatus, error) { obj, ok := c.curWorkerCluster.Load(workerID) if !ok || obj == nil { @@ -215,11 +215,11 @@ func (c *Controller) WorkerStatus(workerID uint32) (*cluster.WorkerStatus, error } // ClusterHistory dumps history of cluster changes -func (c *Controller) ClusterHistory(team, namespace, name string) ([]*spec.Diff, error) { +func (c *Controller) ClusterHistory(namespace, name string) ([]*spec.Diff, error) { clusterName := spec.NamespacedName{ Namespace: namespace, - Name: team + "-" + name, + Name: name, } c.clustersMu.RLock() diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 2836b4f7f..730a9133b 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -9,33 +9,11 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" "github.com/zalando/postgres-operator/pkg/cluster" "github.com/zalando/postgres-operator/pkg/util" ) -func (c *Controller) nodeListFunc(options metav1.ListOptions) (runtime.Object, error) { - opts := metav1.ListOptions{ - Watch: options.Watch, - ResourceVersion: options.ResourceVersion, - TimeoutSeconds: options.TimeoutSeconds, - } - - return c.KubeClient.Nodes().List(context.TODO(), opts) -} - -func (c *Controller) nodeWatchFunc(options metav1.ListOptions) (watch.Interface, error) { - opts := metav1.ListOptions{ - Watch: options.Watch, - ResourceVersion: options.ResourceVersion, - TimeoutSeconds: options.TimeoutSeconds, - } - - return c.KubeClient.Nodes().Watch(context.TODO(), opts) -} - func (c *Controller) nodeAdd(obj interface{}) { node, ok := obj.(*v1.Node) if !ok { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 761cf1b60..66fc7a731 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -10,6 +10,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -24,42 +25,59 @@ func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, con return config, nil } -func int32ToPointer(value int32) *int32 { - return &value -} - // importConfigurationFromCRD is a transitional function that converts CRD configuration to the one based on the configmap func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigurationData) *config.Config { result := &config.Config{} // general config - result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True()) + result.EnableCRDRegistration = util.CoalesceBool(fromCRD.EnableCRDRegistration, util.True()) + result.CRDCategories = util.CoalesceStrArr(fromCRD.CRDCategories, []string{"all"}) result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EnablePgVersionEnvVar = fromCRD.EnablePgVersionEnvVar result.EnableSpiloWalPathCompat = fromCRD.EnableSpiloWalPathCompat + result.EnableTeamIdClusternamePrefix = fromCRD.EnableTeamIdClusternamePrefix result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps - result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-13:2.0-p7") + result.DockerImage = util.Coalesce(fromCRD.DockerImage, "ghcr.io/zalando/spilo-18:4.1-p1") result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8) result.MinInstances = fromCRD.MinInstances result.MaxInstances = fromCRD.MaxInstances + result.IgnoreInstanceLimitsAnnotationKey = fromCRD.IgnoreInstanceLimitsAnnotationKey + result.IgnoreResourcesLimitsAnnotationKey = fromCRD.IgnoreResourcesLimitsAnnotationKey result.ResyncPeriod = util.CoalesceDuration(time.Duration(fromCRD.ResyncPeriod), "30m") result.RepairPeriod = util.CoalesceDuration(time.Duration(fromCRD.RepairPeriod), "5m") result.SetMemoryRequestToLimit = fromCRD.SetMemoryRequestToLimit result.ShmVolume = util.CoalesceBool(fromCRD.ShmVolume, util.True()) result.SidecarImages = fromCRD.SidecarImages result.SidecarContainers = fromCRD.SidecarContainers + result.EnableMaintenanceWindows = util.CoalesceBool(fromCRD.EnableMaintenanceWindows, util.True()) + if len(fromCRD.MaintenanceWindows) > 0 { + result.MaintenanceWindows = make([]string, 0, len(fromCRD.MaintenanceWindows)) + for _, window := range fromCRD.MaintenanceWindows { + w, err := window.MarshalJSON() + if err != nil { + panic(fmt.Errorf("could not marshal configured maintenance window: %v", err)) + } + result.MaintenanceWindows = append(result.MaintenanceWindows, string(w)) + } + } // user config result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres") result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby") + result.AdditionalOwnerRoles = fromCRD.PostgresUsersConfiguration.AdditionalOwnerRoles + result.EnablePasswordRotation = fromCRD.PostgresUsersConfiguration.EnablePasswordRotation + result.PasswordRotationInterval = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.PasswordRotationInterval, 90) + result.PasswordRotationUserRetention = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.DeepCopy().PasswordRotationUserRetention, 180) // major version upgrade config - result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "off") - result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "9.5") - result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "13") + result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "manual") + result.MajorVersionUpgradeTeamAllowList = fromCRD.MajorVersionUpgrade.MajorVersionUpgradeTeamAllowList + result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "14") + result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "18") // kubernetes config + result.EnableOwnerReferences = util.CoalesceBool(fromCRD.Kubernetes.EnableOwnerReferences, util.False()) result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations result.PodServiceAccountName = util.Coalesce(fromCRD.Kubernetes.PodServiceAccountName, "postgres-pod") result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition @@ -67,6 +85,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodEnvironmentConfigMap = fromCRD.Kubernetes.PodEnvironmentConfigMap result.PodEnvironmentSecret = fromCRD.Kubernetes.PodEnvironmentSecret result.PodTerminateGracePeriod = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod), "5m") + result.LivenessProbe = fromCRD.Kubernetes.LivenessProbe result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged result.SpiloAllowPrivilegeEscalation = util.CoalesceBool(fromCRD.Kubernetes.SpiloAllowPrivilegeEscalation, util.True()) result.SpiloRunAsUser = fromCRD.Kubernetes.SpiloRunAsUser @@ -76,12 +95,16 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local") result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat + result.PDBMasterLabelSelector = util.CoalesceBool(fromCRD.Kubernetes.PDBMasterLabelSelector, util.True()) result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True()) result.StorageResizeMode = util.Coalesce(fromCRD.Kubernetes.StorageResizeMode, "pvc") result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True()) result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True()) + result.SharePgSocketWithSidecars = util.CoalesceBool(fromCRD.Kubernetes.SharePgSocketWithSidecars, util.False()) result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName + result.EnableCrossNamespaceSecret = fromCRD.Kubernetes.EnableCrossNamespaceSecret + result.EnableFinalizers = util.CoalesceBool(fromCRD.Kubernetes.EnableFinalizers, util.False()) result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName if fromCRD.Kubernetes.InfrastructureRolesDefs != nil { @@ -103,23 +126,33 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels result.InheritedAnnotations = fromCRD.Kubernetes.InheritedAnnotations result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations + result.IgnoredAnnotations = fromCRD.Kubernetes.IgnoredAnnotations result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey result.DeleteAnnotationNameKey = fromCRD.Kubernetes.DeleteAnnotationNameKey result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel + result.NodeReadinessLabelMerge = fromCRD.Kubernetes.NodeReadinessLabelMerge result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready") + result.PersistentVolumeClaimRetentionPolicy = fromCRD.Kubernetes.PersistentVolumeClaimRetentionPolicy + result.EnableSecretsDeletion = util.CoalesceBool(fromCRD.Kubernetes.EnableSecretsDeletion, util.True()) + result.EnablePersistentVolumeClaimDeletion = util.CoalesceBool(fromCRD.Kubernetes.EnablePersistentVolumeClaimDeletion, util.True()) + result.EnableReadinessProbe = fromCRD.Kubernetes.EnableReadinessProbe result.MasterPodMoveTimeout = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout), "10m") result.EnablePodAntiAffinity = fromCRD.Kubernetes.EnablePodAntiAffinity result.PodAntiAffinityTopologyKey = util.Coalesce(fromCRD.Kubernetes.PodAntiAffinityTopologyKey, "kubernetes.io/hostname") + result.PodAntiAffinityPreferredDuringScheduling = fromCRD.Kubernetes.PodAntiAffinityPreferredDuringScheduling + result.PodToleration = fromCRD.Kubernetes.PodToleration // Postgres Pod resources - result.DefaultCPURequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPURequest, "100m") - result.DefaultMemoryRequest = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryRequest, "100Mi") - result.DefaultCPULimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultCPULimit, "1") - result.DefaultMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.DefaultMemoryLimit, "500Mi") - result.MinCPULimit = util.Coalesce(fromCRD.PostgresPodResources.MinCPULimit, "250m") - result.MinMemoryLimit = util.Coalesce(fromCRD.PostgresPodResources.MinMemoryLimit, "250Mi") + result.DefaultCPURequest = fromCRD.PostgresPodResources.DefaultCPURequest + result.DefaultMemoryRequest = fromCRD.PostgresPodResources.DefaultMemoryRequest + result.DefaultCPULimit = fromCRD.PostgresPodResources.DefaultCPULimit + result.DefaultMemoryLimit = fromCRD.PostgresPodResources.DefaultMemoryLimit + result.MinCPULimit = fromCRD.PostgresPodResources.MinCPULimit + result.MinMemoryLimit = fromCRD.PostgresPodResources.MinMemoryLimit + result.MaxCPURequest = fromCRD.PostgresPodResources.MaxCPURequest + result.MaxMemoryRequest = fromCRD.PostgresPodResources.MaxMemoryRequest // timeout config result.ResourceCheckInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ResourceCheckInterval), "3s") @@ -128,14 +161,24 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodDeletionWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PodDeletionWaitTimeout), "10m") result.ReadyWaitInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitInterval), "4s") result.ReadyWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitTimeout), "30s") + result.PatroniAPICheckInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PatroniAPICheckInterval), "1s") + result.PatroniAPICheckTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PatroniAPICheckTimeout), "5s") // load balancer config result.DbHostedZone = util.Coalesce(fromCRD.LoadBalancer.DbHostedZone, "db.example.com") result.EnableMasterLoadBalancer = fromCRD.LoadBalancer.EnableMasterLoadBalancer + result.EnableMasterPoolerLoadBalancer = fromCRD.LoadBalancer.EnableMasterPoolerLoadBalancer result.EnableReplicaLoadBalancer = fromCRD.LoadBalancer.EnableReplicaLoadBalancer + result.EnableReplicaPoolerLoadBalancer = fromCRD.LoadBalancer.EnableReplicaPoolerLoadBalancer + result.EnableMasterNodePort = fromCRD.LoadBalancer.EnableMasterNodePort + result.EnableMasterPoolerNodePort = fromCRD.LoadBalancer.EnableMasterPoolerNodePort + result.EnableReplicaNodePort = fromCRD.LoadBalancer.EnableReplicaNodePort + result.EnableReplicaPoolerNodePort = fromCRD.LoadBalancer.EnableReplicaPoolerNodePort result.CustomServiceAnnotations = fromCRD.LoadBalancer.CustomServiceAnnotations result.MasterDNSNameFormat = fromCRD.LoadBalancer.MasterDNSNameFormat + result.MasterLegacyDNSNameFormat = fromCRD.LoadBalancer.MasterLegacyDNSNameFormat result.ReplicaDNSNameFormat = fromCRD.LoadBalancer.ReplicaDNSNameFormat + result.ReplicaLegacyDNSNameFormat = fromCRD.LoadBalancer.ReplicaLegacyDNSNameFormat result.ExternalTrafficPolicy = util.Coalesce(fromCRD.LoadBalancer.ExternalTrafficPolicy, "Cluster") // AWS or GCP config @@ -145,27 +188,41 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials + result.WALAZStorageAccount = fromCRD.AWSGCP.WALAZStorageAccount result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount - result.AdditionalSecretMountPath = util.Coalesce(fromCRD.AWSGCP.AdditionalSecretMountPath, "/meta/credentials") + result.AdditionalSecretMountPath = fromCRD.AWSGCP.AdditionalSecretMountPath result.EnableEBSGp3Migration = fromCRD.AWSGCP.EnableEBSGp3Migration result.EnableEBSGp3MigrationMaxSize = util.CoalesceInt64(fromCRD.AWSGCP.EnableEBSGp3MigrationMaxSize, 1000) // logical backup config result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") - result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup:v1.6.3") + result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1") result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3") + result.LogicalBackupAzureStorageAccountName = fromCRD.LogicalBackup.AzureStorageAccountName + result.LogicalBackupAzureStorageAccountKey = fromCRD.LogicalBackup.AzureStorageAccountKey + result.LogicalBackupAzureStorageContainer = fromCRD.LogicalBackup.AzureStorageContainer result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket + result.LogicalBackupS3BucketPrefix = util.Coalesce(fromCRD.LogicalBackup.S3BucketPrefix, "spilo") result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE + result.LogicalBackupS3RetentionTime = fromCRD.LogicalBackup.RetentionTime result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials result.LogicalBackupJobPrefix = util.Coalesce(fromCRD.LogicalBackup.JobPrefix, "logical-backup-") + result.LogicalBackupCronjobEnvironmentSecret = fromCRD.LogicalBackup.CronjobEnvironmentSecret + result.LogicalBackupCPURequest = fromCRD.LogicalBackup.CPURequest + result.LogicalBackupMemoryRequest = fromCRD.LogicalBackup.MemoryRequest + result.LogicalBackupCPULimit = fromCRD.LogicalBackup.CPULimit + result.LogicalBackupMemoryLimit = fromCRD.LogicalBackup.MemoryLimit + result.LogicalBackupSuccessfulJobsHistoryLimit = util.CoalesceInt32(fromCRD.LogicalBackup.SuccessfulJobsHistoryLimit, k8sutil.Int32ToPointer(3)) + result.LogicalBackupFailedJobsHistoryLimit = util.CoalesceInt32(fromCRD.LogicalBackup.FailedJobsHistoryLimit, k8sutil.Int32ToPointer(3)) + result.LogicalBackupTTLSecondsAfterFinished = fromCRD.LogicalBackup.TTLSecondsAfterFinished // debug config - result.DebugLogging = fromCRD.OperatorDebug.DebugLogging - result.EnableDBAccess = fromCRD.OperatorDebug.EnableDBAccess + result.DebugLogging = *util.CoalesceBool(fromCRD.OperatorDebug.DebugLogging, util.True()) + result.EnableDBAccess = *util.CoalesceBool(fromCRD.OperatorDebug.EnableDBAccess, util.True()) // Teams API config result.EnableTeamsAPI = fromCRD.TeamsAPI.EnableTeamsAPI @@ -176,7 +233,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.TeamAdminRole = fromCRD.TeamsAPI.TeamAdminRole result.PamRoleName = util.Coalesce(fromCRD.TeamsAPI.PamRoleName, "zalandos") result.PamConfiguration = util.Coalesce(fromCRD.TeamsAPI.PamConfiguration, "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees") - result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin"}) + result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin", "cron_admin"}) result.PostgresSuperuserTeams = fromCRD.TeamsAPI.PostgresSuperuserTeams result.EnablePostgresTeamCRD = fromCRD.TeamsAPI.EnablePostgresTeamCRD result.EnablePostgresTeamCRDSuperusers = fromCRD.TeamsAPI.EnablePostgresTeamCRDSuperusers @@ -197,15 +254,18 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit + // Patroni config + result.EnablePatroniFailsafeMode = util.CoalesceBool(fromCRD.Patroni.FailsafeMode, util.False()) + // Connection pooler. Looks like we can't use defaulting in CRD before 1.17, // so ensure default values here. result.ConnectionPooler.NumberOfInstances = util.CoalesceInt32( fromCRD.ConnectionPooler.NumberOfInstances, - int32ToPointer(2)) + k8sutil.Int32ToPointer(2)) result.ConnectionPooler.NumberOfInstances = util.MaxInt32( result.ConnectionPooler.NumberOfInstances, - int32ToPointer(2)) + k8sutil.Int32ToPointer(2)) result.ConnectionPooler.Schema = util.Coalesce( fromCRD.ConnectionPooler.Schema, @@ -216,37 +276,26 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur constants.ConnectionPoolerUserName) if result.ConnectionPooler.User == result.SuperUsername { - msg := "Connection pool user is not allowed to be the same as super user, username: %s" + msg := "connection pool user is not allowed to be the same as super user, username: %s" panic(fmt.Errorf(msg, result.ConnectionPooler.User)) } result.ConnectionPooler.Image = util.Coalesce( fromCRD.ConnectionPooler.Image, - "registry.opensource.zalan.do/acid/pgbouncer") + "ghcr.io/zalando/postgres-operator/pgbouncer:latest") result.ConnectionPooler.Mode = util.Coalesce( fromCRD.ConnectionPooler.Mode, constants.ConnectionPoolerDefaultMode) - result.ConnectionPooler.ConnectionPoolerDefaultCPURequest = util.Coalesce( - fromCRD.ConnectionPooler.DefaultCPURequest, - constants.ConnectionPoolerDefaultCpuRequest) - - result.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest = util.Coalesce( - fromCRD.ConnectionPooler.DefaultMemoryRequest, - constants.ConnectionPoolerDefaultMemoryRequest) - - result.ConnectionPooler.ConnectionPoolerDefaultCPULimit = util.Coalesce( - fromCRD.ConnectionPooler.DefaultCPULimit, - constants.ConnectionPoolerDefaultCpuLimit) - - result.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit = util.Coalesce( - fromCRD.ConnectionPooler.DefaultMemoryLimit, - constants.ConnectionPoolerDefaultMemoryLimit) + result.ConnectionPooler.ConnectionPoolerDefaultCPURequest = fromCRD.ConnectionPooler.DefaultCPURequest + result.ConnectionPooler.ConnectionPoolerDefaultMemoryRequest = fromCRD.ConnectionPooler.DefaultMemoryRequest + result.ConnectionPooler.ConnectionPoolerDefaultCPULimit = fromCRD.ConnectionPooler.DefaultCPULimit + result.ConnectionPooler.ConnectionPoolerDefaultMemoryLimit = fromCRD.ConnectionPooler.DefaultMemoryLimit result.ConnectionPooler.MaxDBConnections = util.CoalesceInt32( fromCRD.ConnectionPooler.MaxDBConnections, - int32ToPointer(constants.ConnectionPoolerMaxDBConnections)) + k8sutil.Int32ToPointer(constants.ConnectionPoolerMaxDBConnections)) return result } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 0defe88b1..1aaa307ea 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1,12 +1,7 @@ package controller import ( - "context" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" "github.com/zalando/postgres-operator/pkg/cluster" "github.com/zalando/postgres-operator/pkg/spec" @@ -14,26 +9,6 @@ import ( "k8s.io/apimachinery/pkg/types" ) -func (c *Controller) podListFunc(options metav1.ListOptions) (runtime.Object, error) { - opts := metav1.ListOptions{ - Watch: options.Watch, - ResourceVersion: options.ResourceVersion, - TimeoutSeconds: options.TimeoutSeconds, - } - - return c.KubeClient.Pods(c.opConfig.WatchedNamespace).List(context.TODO(), opts) -} - -func (c *Controller) podWatchFunc(options metav1.ListOptions) (watch.Interface, error) { - opts := metav1.ListOptions{ - Watch: options.Watch, - ResourceVersion: options.ResourceVersion, - TimeoutSeconds: options.TimeoutSeconds, - } - - return c.KubeClient.Pods(c.opConfig.WatchedNamespace).Watch(context.TODO(), opts) -} - func (c *Controller) dispatchPodEvent(clusterName spec.NamespacedName, event cluster.PodEvent) { c.clustersMu.RLock() cluster, ok := c.clusters[clusterName] diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index b3c4d8414..0933f7823 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -143,7 +143,7 @@ func (c *Controller) acquireInitialListOfClusters() error { if list, err = c.listClusters(metav1.ListOptions{ResourceVersion: "0"}); err != nil { return err } - c.logger.Debugf("acquiring initial list of clusters") + c.logger.Debug("acquiring initial list of clusters") for _, pg := range list.Items { // XXX: check the cluster status field instead if pg.Error != "" { @@ -158,7 +158,15 @@ func (c *Controller) acquireInitialListOfClusters() error { return nil } -func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedName, pgSpec *acidv1.Postgresql) *cluster.Cluster { +func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedName, pgSpec *acidv1.Postgresql) (*cluster.Cluster, error) { + if c.opConfig.EnableTeamIdClusternamePrefix { + if _, err := acidv1.ExtractClusterName(clusterName.Name, pgSpec.Spec.TeamID); err != nil { + pgSpec.Status.PostgresClusterStatus = acidv1.ClusterStatusInvalid + c.KubeClient.SetPostgresCRDStatus(clusterName, pgSpec) + return nil, err + } + } + cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg, c.eventRecorder) cl.Run(c.stopCh) teamName := strings.ToLower(cl.Spec.TeamID) @@ -171,12 +179,13 @@ func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedNam c.clusterLogs[clusterName] = ringlog.New(c.opConfig.RingLogLines) c.clusterHistory[clusterName] = ringlog.New(c.opConfig.ClusterHistoryEntries) - return cl + return cl, nil } -func (c *Controller) processEvent(event ClusterEvent) { +func (c *Controller) processEvent(event ClusterEvent, isInInitialList bool) { var clusterName spec.NamespacedName var clHistory ringlog.RingLogger + var err error lg := c.logger.WithField("worker", event.WorkerID) @@ -216,7 +225,7 @@ func (c *Controller) processEvent(event ClusterEvent) { c.mergeDeprecatedPostgreSQLSpecParameters(&event.NewSpec.Spec) } - if err := c.submitRBACCredentials(event); err != nil { + if err = c.submitRBACCredentials(event); err != nil { c.logger.Warnf("pods and/or Patroni may misfunction due to the lack of permissions: %v", err) } @@ -225,34 +234,53 @@ func (c *Controller) processEvent(event ClusterEvent) { switch event.EventType { case EventAdd: if clusterFound { - lg.Infof("recieved add event for already existing Postgres cluster") + lg.Infof("received add event for already existing Postgres cluster") return } lg.Infof("creating a new Postgres cluster") - cl = c.addCluster(lg, clusterName, event.NewSpec) + cl, err = c.addCluster(lg, clusterName, event.NewSpec) + if err != nil { + lg.Errorf("creation of cluster is blocked: %v", err) + return + } c.curWorkerCluster.Store(event.WorkerID, cl) - if err := cl.Create(); err != nil { + err = cl.Create() + if err != nil { + cl.Status = acidv1.PostgresStatus{PostgresClusterStatus: acidv1.ClusterStatusInvalid} cl.Error = fmt.Sprintf("could not create cluster: %v", err) lg.Error(cl.Error) c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Create", "%v", cl.Error) - return } lg.Infoln("cluster has been created") case EventUpdate: - lg.Infoln("update of the cluster started") - if !clusterFound { lg.Warningln("cluster does not exist") return } c.curWorkerCluster.Store(event.WorkerID, cl) - if err := cl.Update(event.OldSpec, event.NewSpec); err != nil { + + // Check if this cluster has been marked for deletion + if !event.NewSpec.ObjectMeta.DeletionTimestamp.IsZero() { + lg.Infof("cluster has a DeletionTimestamp of %s, starting deletion now.", event.NewSpec.ObjectMeta.DeletionTimestamp.Format(time.RFC3339)) + if err = cl.Delete(); err != nil { + cl.Error = fmt.Sprintf("error deleting cluster and its resources: %v", err) + c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Delete", "%v", cl.Error) + lg.Error(cl.Error) + return + } + lg.Infoln("cluster has been deleted via update event") + return + } + + lg.Infoln("update of the cluster started") + err = cl.Update(event.OldSpec, event.NewSpec) + if err != nil { cl.Error = fmt.Sprintf("could not update cluster: %v", err) lg.Error(cl.Error) @@ -271,14 +299,18 @@ func (c *Controller) processEvent(event ClusterEvent) { lg.Errorf("unknown cluster: %q", clusterName) return } - lg.Infoln("deletion of the cluster started") teamName := strings.ToLower(cl.Spec.TeamID) - c.curWorkerCluster.Store(event.WorkerID, cl) - cl.Delete() - // Fixme - no error handling for delete ? - // c.eventRecorder.Eventf(cl.GetReference, v1.EventTypeWarning, "Delete", "%v", cl.Error) + + // when using finalizers the deletion already happened + if c.opConfig.EnableFinalizers == nil || !*c.opConfig.EnableFinalizers { + lg.Infoln("deletion of the cluster started") + if err := cl.Delete(); err != nil { + cl.Error = fmt.Sprintf("could not delete cluster: %v", err) + c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Delete", "%v", cl.Error) + } + } func() { defer c.clustersMu.Unlock() @@ -303,19 +335,34 @@ func (c *Controller) processEvent(event ClusterEvent) { // no race condition because a cluster is always processed by single worker if !clusterFound { - cl = c.addCluster(lg, clusterName, event.NewSpec) + cl, err = c.addCluster(lg, clusterName, event.NewSpec) + if err != nil { + lg.Errorf("syncing of cluster is blocked: %v", err) + return + } } c.curWorkerCluster.Store(event.WorkerID, cl) - if err := cl.Sync(event.NewSpec); err != nil { - cl.Error = fmt.Sprintf("could not sync cluster: %v", err) - c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Sync", "%v", cl.Error) - lg.Error(cl.Error) - return + + // has this cluster been marked as deleted already, then we shall start cleaning up + if !cl.ObjectMeta.DeletionTimestamp.IsZero() { + lg.Infof("cluster has a DeletionTimestamp of %s, starting deletion now.", cl.ObjectMeta.DeletionTimestamp.Format(time.RFC3339)) + if err = cl.Delete(); err != nil { + cl.Error = fmt.Sprintf("error deleting cluster and its resources: %v", err) + c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Delete", "%v", cl.Error) + lg.Error(cl.Error) + return + } + } else { + if err = cl.Sync(event.NewSpec); err != nil { + cl.Error = fmt.Sprintf("could not sync cluster: %v", err) + c.eventRecorder.Eventf(cl.GetReference(), v1.EventTypeWarning, "Sync", "%v", cl.Error) + lg.Error(cl.Error) + return + } + lg.Infof("cluster has been synced") } cl.Error = "" - - lg.Infof("cluster has been synced") } } @@ -324,11 +371,22 @@ func (c *Controller) processClusterEventsQueue(idx int, stopCh <-chan struct{}, go func() { <-stopCh - c.clusterEventQueues[idx].Close() + (*c.clusterEventQueues[idx]).Close() }() for { - obj, err := c.clusterEventQueues[idx].Pop(cache.PopProcessFunc(func(interface{}) error { return nil })) + _, err := (*c.clusterEventQueues[idx]).Pop(cache.PopProcessFunc(func(obj interface{}, isInitialList bool) error { + event, ok := obj.(ClusterEvent) + if !ok { + c.logger.Errorf("could not cast to cluster event") + return nil // skip event to keep processing + } + c.processEvent(event, isInitialList) + if err := c.clusterEventStores[idx].Delete(obj); err != nil { + c.logger.Errorf("failed to delete key from lookup store: %v", err) + } + return nil + })) if err != nil { if err == cache.ErrFIFOClosed { return @@ -336,25 +394,14 @@ func (c *Controller) processClusterEventsQueue(idx int, stopCh <-chan struct{}, c.logger.Errorf("error when processing cluster events queue: %v", err) continue } - event, ok := obj.(ClusterEvent) - if !ok { - c.logger.Errorf("could not cast to ClusterEvent") - } - - c.processEvent(event) } } func (c *Controller) warnOnDeprecatedPostgreSQLSpecParameters(spec *acidv1.PostgresSpec) { - deprecate := func(deprecated, replacement string) { c.logger.Warningf("parameter %q is deprecated. Consider setting %q instead", deprecated, replacement) } - noeffect := func(param string, explanation string) { - c.logger.Warningf("parameter %q takes no effect. %s", param, explanation) - } - if spec.UseLoadBalancer != nil { deprecate("useLoadBalancer", "enableMasterLoadBalancer") } @@ -362,10 +409,6 @@ func (c *Controller) warnOnDeprecatedPostgreSQLSpecParameters(spec *acidv1.Postg deprecate("replicaLoadBalancer", "enableReplicaLoadBalancer") } - if len(spec.MaintenanceWindows) > 0 { - noeffect("maintenanceWindows", "Not implemented.") - } - if (spec.UseLoadBalancer != nil || spec.ReplicaLoadBalancer != nil) && (spec.EnableReplicaLoadBalancer != nil || spec.EnableMasterLoadBalancer != nil) { c.logger.Warnf("both old and new load balancer parameters are present in the manifest, ignoring old ones") @@ -400,7 +443,7 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. clusterError string ) - if informerOldSpec != nil { //update, delete + if informerOldSpec != nil { // update, delete uid = informerOldSpec.GetUID() clusterName = util.NameFromMeta(informerOldSpec.ObjectMeta) @@ -415,25 +458,28 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. } else { clusterError = informerOldSpec.Error } - } else { //add, sync + } else { // add, sync uid = informerNewSpec.GetUID() clusterName = util.NameFromMeta(informerNewSpec.ObjectMeta) clusterError = informerNewSpec.Error } - // only allow deletion if delete annotations are set and conditions are met if eventType == EventDelete { - if err := c.meetsClusterDeleteAnnotations(informerOldSpec); err != nil { - c.logger.WithField("cluster-name", clusterName).Warnf( - "ignoring %q event for cluster %q - manifest does not fulfill delete requirements: %s", eventType, clusterName, err) - c.logger.WithField("cluster-name", clusterName).Warnf( - "please, recreate Postgresql resource %q and set annotations to delete properly", clusterName) - if currentManifest, marshalErr := json.Marshal(informerOldSpec); marshalErr != nil { - c.logger.WithField("cluster-name", clusterName).Warnf("could not marshal current manifest:\n%+v", informerOldSpec) - } else { - c.logger.WithField("cluster-name", clusterName).Warnf("%s\n", string(currentManifest)) + // when owner references are used operator cannot block deletion + if c.opConfig.EnableOwnerReferences == nil || !*c.opConfig.EnableOwnerReferences { + // only allow deletion if delete annotations are set and conditions are met + if err := c.meetsClusterDeleteAnnotations(informerOldSpec); err != nil { + c.logger.WithField("cluster-name", clusterName).Warnf( + "ignoring %q event for cluster %q - manifest does not fulfill delete requirements: %s", eventType, clusterName, err) + c.logger.WithField("cluster-name", clusterName).Warnf( + "please, recreate Postgresql resource %q and set annotations to delete properly", clusterName) + if currentManifest, marshalErr := json.Marshal(informerOldSpec); marshalErr != nil { + c.logger.WithField("cluster-name", clusterName).Warnf("could not marshal current manifest:\n%+v", informerOldSpec) + } else { + c.logger.WithField("cluster-name", clusterName).Warnf("%s\n", string(currentManifest)) + } + return } - return } } @@ -442,13 +488,25 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. switch eventType { case EventAdd: - c.KubeClient.SetPostgresCRDStatus(clusterName, acidv1.ClusterStatusAddFailed) + informerNewSpec.Status.PostgresClusterStatus = acidv1.ClusterStatusAddFailed + _, err := c.KubeClient.SetPostgresCRDStatus(clusterName, informerNewSpec) + if err != nil { + c.logger.WithField("cluster-name", clusterName).Errorf("could not set PostgresCRD status: %v", err) + } c.eventRecorder.Eventf(c.GetReference(informerNewSpec), v1.EventTypeWarning, "Create", "%v", clusterError) case EventUpdate: - c.KubeClient.SetPostgresCRDStatus(clusterName, acidv1.ClusterStatusUpdateFailed) + informerNewSpec.Status.PostgresClusterStatus = acidv1.ClusterStatusUpdateFailed + _, err := c.KubeClient.SetPostgresCRDStatus(clusterName, informerNewSpec) + if err != nil { + c.logger.WithField("cluster-name", clusterName).Errorf("could not set PostgresCRD status: %v", err) + } c.eventRecorder.Eventf(c.GetReference(informerNewSpec), v1.EventTypeWarning, "Update", "%v", clusterError) default: - c.KubeClient.SetPostgresCRDStatus(clusterName, acidv1.ClusterStatusSyncFailed) + informerNewSpec.Status.PostgresClusterStatus = acidv1.ClusterStatusSyncFailed + _, err := c.KubeClient.SetPostgresCRDStatus(clusterName, informerNewSpec) + if err != nil { + c.logger.WithField("cluster-name", clusterName).Errorf("could not set PostgresCRD status: %v", err) + } c.eventRecorder.Eventf(c.GetReference(informerNewSpec), v1.EventTypeWarning, "Sync", "%v", clusterError) } @@ -470,7 +528,10 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. } lg := c.logger.WithField("worker", workerID).WithField("cluster-name", clusterName) - if err := c.clusterEventQueues[workerID].Add(clusterEvent); err != nil { + if err := c.clusterEventStores[workerID].Add(clusterEvent); err != nil { + lg.Errorf("error while storing cluster event for lookup: %v", clusterEvent) + } + if err := (*c.clusterEventQueues[workerID]).Add(clusterEvent); err != nil { lg.Errorf("error while queueing cluster event: %v", clusterEvent) } lg.Infof("%s event has been queued", eventType) @@ -480,9 +541,9 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. } // A delete event discards all prior requests for that cluster. for _, evType := range []EventType{EventAdd, EventSync, EventUpdate, EventRepair} { - obj, exists, err := c.clusterEventQueues[workerID].GetByKey(queueClusterKey(evType, uid)) + obj, exists, err := c.clusterEventStores[workerID].GetByKey(queueClusterKey(evType, uid)) if err != nil { - lg.Warningf("could not get event from the queue: %v", err) + lg.Warningf("could not get event from the lookup store: %v", err) continue } @@ -490,12 +551,18 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. continue } - err = c.clusterEventQueues[workerID].Delete(obj) + err = (*c.clusterEventQueues[workerID]).Delete(obj) if err != nil { lg.Warningf("could not delete event from the queue: %v", err) } else { lg.Debugf("event %s has been discarded for the cluster", evType) } + err = c.clusterEventStores[workerID].Delete(obj) + if err != nil { + lg.Warningf("could not delete event from the lookup store: %v", err) + } else { + lg.Debugf("event %s has been deleted from the lookup store", evType) + } } } @@ -505,15 +572,25 @@ func (c *Controller) postgresqlAdd(obj interface{}) { // We will not get multiple Add events for the same cluster c.queueClusterEvent(nil, pg, EventAdd) } - - return } func (c *Controller) postgresqlUpdate(prev, cur interface{}) { pgOld := c.postgresqlCheck(prev) pgNew := c.postgresqlCheck(cur) if pgOld != nil && pgNew != nil { - // Avoid the inifinite recursion for status updates + clusterName := util.NameFromMeta(pgNew.ObjectMeta) + + // Check if DeletionTimestamp was set (resource marked for deletion) + deletionTimestampChanged := pgOld.ObjectMeta.DeletionTimestamp.IsZero() && !pgNew.ObjectMeta.DeletionTimestamp.IsZero() + if deletionTimestampChanged { + c.logger.WithField("cluster-name", clusterName).Infof( + "UPDATE event: DeletionTimestamp set to %s, queueing event", + pgNew.ObjectMeta.DeletionTimestamp.Format(time.RFC3339)) + c.queueClusterEvent(pgOld, pgNew, EventUpdate) + return + } + + // Avoid the infinite recursion for status updates if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) { if reflect.DeepEqual(pgNew.Annotations, pgOld.Annotations) { return @@ -521,8 +598,6 @@ func (c *Controller) postgresqlUpdate(prev, cur interface{}) { } c.queueClusterEvent(pgOld, pgNew, EventUpdate) } - - return } func (c *Controller) postgresqlDelete(obj interface{}) { @@ -530,8 +605,6 @@ func (c *Controller) postgresqlDelete(obj interface{}) { if pg != nil { c.queueClusterEvent(pg, nil, EventDelete) } - - return } func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql { @@ -547,15 +620,15 @@ func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql { } /* - Ensures the pod service account and role bindings exists in a namespace - before a PG cluster is created there so that a user does not have to deploy - these credentials manually. StatefulSets require the service account to - create pods; Patroni requires relevant RBAC bindings to access endpoints. +Ensures the pod service account and role bindings exists in a namespace +before a PG cluster is created there so that a user does not have to deploy +these credentials manually. StatefulSets require the service account to +create pods; Patroni requires relevant RBAC bindings to access endpoints +or config maps. - The operator does not sync accounts/role bindings after creation. +The operator does not sync accounts/role bindings after creation. */ func (c *Controller) submitRBACCredentials(event ClusterEvent) error { - namespace := event.NewSpec.GetNamespace() if err := c.createPodServiceAccount(namespace); err != nil { @@ -569,12 +642,11 @@ func (c *Controller) submitRBACCredentials(event ClusterEvent) error { } func (c *Controller) createPodServiceAccount(namespace string) error { - podServiceAccountName := c.opConfig.PodServiceAccountName _, err := c.KubeClient.ServiceAccounts(namespace).Get(context.TODO(), podServiceAccountName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { - c.logger.Infof(fmt.Sprintf("creating pod service account %q in the %q namespace", podServiceAccountName, namespace)) + c.logger.Infof("creating pod service account %q in the %q namespace", podServiceAccountName, namespace) // get a separate copy of service account // to prevent a race condition when setting a namespace for many clusters @@ -592,7 +664,6 @@ func (c *Controller) createPodServiceAccount(namespace string) error { } func (c *Controller) createRoleBindings(namespace string) error { - podServiceAccountName := c.opConfig.PodServiceAccountName podServiceAccountRoleBindingName := c.PodServiceAccountRoleBinding.Name diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 8aa891c09..6296e5341 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -2,14 +2,12 @@ package controller import ( "context" - "encoding/json" "fmt" "strings" v1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -18,7 +16,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func (c *Controller) makeClusterConfig() cluster.Config { @@ -53,28 +51,28 @@ func (c *Controller) clusterWorkerID(clusterName spec.NamespacedName) uint32 { return c.clusterWorkers[clusterName] } -func (c *Controller) createOperatorCRD(crd *apiextv1.CustomResourceDefinition) error { - if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil { - if k8sutil.ResourceAlreadyExists(err) { - c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name) - - patch, err := json.Marshal(crd) - if err != nil { - return fmt.Errorf("could not marshal new customResourceDefintion: %v", err) - } - if _, err := c.KubeClient.CustomResourceDefinitions().Patch( - context.TODO(), crd.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { - return fmt.Errorf("could not update customResourceDefinition: %v", err) - } - } else { - c.logger.Errorf("could not create customResourceDefinition %q: %v", crd.Name, err) +func (c *Controller) createOperatorCRD(desiredCrd *apiextv1.CustomResourceDefinition) error { + crd, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), desiredCrd.Name, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), desiredCrd, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("could not create customResourceDefinition %q: %v", desiredCrd.Name, err) } - } else { - c.logger.Infof("customResourceDefinition %q has been registered", crd.Name) } + if err != nil { + c.logger.Errorf("could not get customResourceDefinition %q: %v", desiredCrd.Name, err) + } + if crd != nil { + c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name) + crd.Spec = desiredCrd.Spec + _, err := c.KubeClient.CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update customResourceDefinition %q: %v", crd.Name, err) + } + } + c.logger.Infof("customResourceDefinition %q is registered", crd.Name) - return wait.Poll(c.config.CRDReadyWaitInterval, c.config.CRDReadyWaitTimeout, func() (bool, error) { - c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{}) + return wait.PollUntilContextTimeout(context.TODO(), c.config.CRDReadyWaitInterval, c.config.CRDReadyWaitTimeout, false, func(ctx context.Context) (bool, error) { + c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), desiredCrd.Name, metav1.GetOptions{}) if err != nil { return false, err } @@ -96,12 +94,20 @@ func (c *Controller) createOperatorCRD(crd *apiextv1.CustomResourceDefinition) e }) } -func (c *Controller) createPostgresCRD(enableValidation *bool) error { - return c.createOperatorCRD(acidv1.PostgresCRD(enableValidation)) +func (c *Controller) createPostgresCRD() error { + crd, err := acidv1.PostgresCRD(c.opConfig.CRDCategories) + if err != nil { + return fmt.Errorf("could not create Postgres CRD object: %v", err) + } + return c.createOperatorCRD(crd) } -func (c *Controller) createConfigurationCRD(enableValidation *bool) error { - return c.createOperatorCRD(acidv1.ConfigurationCRD(enableValidation)) +func (c *Controller) createConfigurationCRD() error { + crd, err := acidv1.OperatorConfigurationCRD(c.opConfig.CRDCategories) + if err != nil { + return fmt.Errorf("could not create OperatorConfiguration CRD object: %v", err) + } + return c.createOperatorCRD(crd) } func readDecodedRole(s string) (*spec.PgUser, error) { @@ -195,13 +201,12 @@ func (c *Controller) getInfrastructureRoleDefinitions() []*config.Infrastructure func (c *Controller) getInfrastructureRoles( rolesSecrets []*config.InfrastructureRole) ( - map[string]spec.PgUser, []error) { - - var errors []error - var noRolesProvided = true + map[string]spec.PgUser, error) { + errors := make([]string, 0) + noRolesProvided := true roles := []spec.PgUser{} - uniqRoles := map[string]spec.PgUser{} + uniqRoles := make(map[string]spec.PgUser) // To be compatible with the legacy implementation we need to return nil if // the provided secret name is empty. The equivalent situation in the @@ -214,37 +219,39 @@ func (c *Controller) getInfrastructureRoles( } if noRolesProvided { - return nil, nil + return uniqRoles, nil } for _, secret := range rolesSecrets { infraRoles, err := c.getInfrastructureRole(secret) if err != nil || infraRoles == nil { - c.logger.Debugf("Cannot get infrastructure role: %+v", *secret) + c.logger.Debugf("cannot get infrastructure role: %+v", *secret) if err != nil { - errors = append(errors, err) + errors = append(errors, fmt.Sprintf("%v", err)) } continue } - for _, r := range infraRoles { - roles = append(roles, r) - } + roles = append(roles, infraRoles...) } for _, r := range roles { if _, exists := uniqRoles[r.Name]; exists { - msg := "Conflicting infrastructure roles: roles[%s] = (%q, %q)" + msg := "conflicting infrastructure roles: roles[%s] = (%q, %q)" c.logger.Debugf(msg, r.Name, uniqRoles[r.Name], r) } uniqRoles[r.Name] = r } - return uniqRoles, errors + if len(errors) > 0 { + return uniqRoles, fmt.Errorf("%s", strings.Join(errors, `', '`)) + } + + return uniqRoles, nil } // Generate list of users representing one infrastructure role based on its @@ -407,6 +414,7 @@ func (c *Controller) postgresTeamAdd(obj interface{}) { pgTeam, ok := obj.(*acidv1.PostgresTeam) if !ok { c.logger.Errorf("could not cast to PostgresTeam spec") + return } c.logger.Debugf("PostgreTeam %q added. Reloading postgres team CRDs and overwriting cached map", pgTeam.Name) c.loadPostgresTeams() @@ -416,6 +424,7 @@ func (c *Controller) postgresTeamUpdate(prev, obj interface{}) { pgTeam, ok := obj.(*acidv1.PostgresTeam) if !ok { c.logger.Errorf("could not cast to PostgresTeam spec") + return } c.logger.Debugf("PostgreTeam %q updated. Reloading postgres team CRDs and overwriting cached map", pgTeam.Name) c.loadPostgresTeams() diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go index d8e4c3782..4c3a9b356 100644 --- a/pkg/controller/util_test.go +++ b/pkg/controller/util_test.go @@ -7,6 +7,7 @@ import ( b64 "encoding/base64" + "github.com/stretchr/testify/assert" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" @@ -90,21 +91,21 @@ func TestClusterWorkerID(t *testing.T) { // not exist, or empty) and the old format. func TestOldInfrastructureRoleFormat(t *testing.T) { var testTable = []struct { - secretName spec.NamespacedName - expectedRoles map[string]spec.PgUser - expectedErrors []error + secretName spec.NamespacedName + expectedRoles map[string]spec.PgUser + expectedError error }{ { // empty secret name spec.NamespacedName{}, - nil, + map[string]spec.PgUser{}, nil, }, { // secret does not exist spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: "null"}, map[string]spec.PgUser{}, - []error{fmt.Errorf(`could not get infrastructure roles secret default/null: NotFound`)}, + fmt.Errorf(`could not get infrastructure roles secret default/null: NotFound`), }, { spec.NamespacedName{ @@ -129,9 +130,9 @@ func TestOldInfrastructureRoleFormat(t *testing.T) { }, } for _, test := range testTable { - roles, errors := utilTestController.getInfrastructureRoles( + roles, err := utilTestController.getInfrastructureRoles( []*config.InfrastructureRole{ - &config.InfrastructureRole{ + { SecretName: test.secretName, UserKey: "user", PasswordKey: "password", @@ -140,22 +141,9 @@ func TestOldInfrastructureRoleFormat(t *testing.T) { }, }) - if len(errors) != len(test.expectedErrors) { + if err != nil && err.Error() != test.expectedError.Error() { t.Errorf("expected error '%v' does not match the actual error '%v'", - test.expectedErrors, errors) - } - - for idx := range errors { - err := errors[idx] - expectedErr := test.expectedErrors[idx] - - if err != expectedErr { - if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() { - continue - } - t.Errorf("expected error '%v' does not match the actual error '%v'", - expectedErr, err) - } + test.expectedError, err) } if !reflect.DeepEqual(roles, test.expectedRoles) { @@ -169,14 +157,13 @@ func TestOldInfrastructureRoleFormat(t *testing.T) { // corresponding secrets. Here we test the new format. func TestNewInfrastructureRoleFormat(t *testing.T) { var testTable = []struct { - secrets []spec.NamespacedName - expectedRoles map[string]spec.PgUser - expectedErrors []error + secrets []spec.NamespacedName + expectedRoles map[string]spec.PgUser }{ // one secret with one configmap { []spec.NamespacedName{ - spec.NamespacedName{ + { Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, }, @@ -196,16 +183,15 @@ func TestNewInfrastructureRoleFormat(t *testing.T) { Flags: []string{"createdb"}, }, }, - nil, }, // multiple standalone secrets { []spec.NamespacedName{ - spec.NamespacedName{ + { Namespace: v1.NamespaceDefault, Name: "infrastructureroles-new-test1", }, - spec.NamespacedName{ + { Namespace: v1.NamespaceDefault, Name: "infrastructureroles-new-test2", }, @@ -224,7 +210,6 @@ func TestNewInfrastructureRoleFormat(t *testing.T) { MemberOf: []string{"new-test-inrole2"}, }, }, - nil, }, } for _, test := range testTable { @@ -239,27 +224,8 @@ func TestNewInfrastructureRoleFormat(t *testing.T) { }) } - roles, errors := utilTestController.getInfrastructureRoles(definitions) - if len(errors) != len(test.expectedErrors) { - t.Errorf("expected error does not match the actual error:\n%+v\n%+v", - test.expectedErrors, errors) - - // Stop and do not do any further checks - return - } - - for idx := range errors { - err := errors[idx] - expectedErr := test.expectedErrors[idx] - - if err != expectedErr { - if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() { - continue - } - t.Errorf("expected error '%v' does not match the actual error '%v'", - expectedErr, err) - } - } + roles, err := utilTestController.getInfrastructureRoles(definitions) + assert.NoError(t, err) if !reflect.DeepEqual(roles, test.expectedRoles) { t.Errorf("expected roles output/the actual:\n%#v\n%#v", @@ -282,7 +248,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { // only new CRD format { []*config.InfrastructureRole{ - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, @@ -296,7 +262,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { spec.NamespacedName{}, "", []*config.InfrastructureRole{ - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, @@ -314,7 +280,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { spec.NamespacedName{}, "secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, rolekey: test-role", []*config.InfrastructureRole{ - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, @@ -332,7 +298,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { spec.NamespacedName{}, "secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, defaultrolevalue: test-role", []*config.InfrastructureRole{ - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, @@ -353,7 +319,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { }, "", []*config.InfrastructureRole{ - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesOldSecretName, @@ -368,7 +334,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { // both formats for CRD { []*config.InfrastructureRole{ - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, @@ -385,7 +351,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { }, "", []*config.InfrastructureRole{ - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, @@ -395,7 +361,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { RoleKey: "test-role", Template: false, }, - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesOldSecretName, @@ -416,7 +382,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { }, "secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, rolekey: test-role", []*config.InfrastructureRole{ - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, @@ -426,7 +392,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { RoleKey: "test-role", Template: false, }, - &config.InfrastructureRole{ + { SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesOldSecretName, diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index ab4a88735..e2a14718a 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,9 +25,11 @@ SOFTWARE. package versioned import ( - "fmt" + fmt "fmt" + http "net/http" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" @@ -36,13 +38,14 @@ import ( type Interface interface { Discovery() discovery.DiscoveryInterface AcidV1() acidv1.AcidV1Interface + ZalandoV1() zalandov1.ZalandoV1Interface } -// Clientset contains the clients for groups. Each group has exactly one -// version included in a Clientset. +// Clientset contains the clients for groups. type Clientset struct { *discovery.DiscoveryClient - acidV1 *acidv1.AcidV1Client + acidV1 *acidv1.AcidV1Client + zalandoV1 *zalandov1.ZalandoV1Client } // AcidV1 retrieves the AcidV1Client @@ -50,6 +53,11 @@ func (c *Clientset) AcidV1() acidv1.AcidV1Interface { return c.acidV1 } +// ZalandoV1 retrieves the ZalandoV1Client +func (c *Clientset) ZalandoV1() zalandov1.ZalandoV1Interface { + return c.zalandoV1 +} + // Discovery retrieves the DiscoveryClient func (c *Clientset) Discovery() discovery.DiscoveryInterface { if c == nil { @@ -61,22 +69,49 @@ func (c *Clientset) Discovery() discovery.DiscoveryInterface { // NewForConfig creates a new Clientset for the given config. // If config's RateLimiter is not set and QPS and Burst are acceptable, // NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*Clientset, error) { configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { if configShallowCopy.Burst <= 0 { return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") } configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) } + var cs Clientset var err error - cs.acidV1, err = acidv1.NewForConfig(&configShallowCopy) + cs.acidV1, err = acidv1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + cs.zalandoV1, err = zalandov1.NewForConfigAndClient(&configShallowCopy, httpClient) if err != nil { return nil, err } - cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) if err != nil { return nil, err } @@ -86,17 +121,18 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { // NewForConfigOrDie creates a new Clientset for the given config and // panics if there is an error in the config. func NewForConfigOrDie(c *rest.Config) *Clientset { - var cs Clientset - cs.acidV1 = acidv1.NewForConfigOrDie(c) - - cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) - return &cs + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs } // New creates a new Clientset for the given RESTClient. func New(c rest.Interface) *Clientset { var cs Clientset cs.acidV1 = acidv1.New(c) + cs.zalandoV1 = zalandov1.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) return &cs diff --git a/pkg/generated/clientset/versioned/doc.go b/pkg/generated/clientset/versioned/doc.go index ae87609f6..34b48f910 100644 --- a/pkg/generated/clientset/versioned/doc.go +++ b/pkg/generated/clientset/versioned/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2025 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 6ae5db2d3..f7f61ddea 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,9 @@ import ( clientset "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake" + zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" + fakezalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" @@ -37,7 +40,7 @@ import ( // NewSimpleClientset returns a clientset that will respond with the provided objects. // It's backed by a very simple object tracker that processes creates, updates and deletions as-is, -// without applying any validations and/or defaults. It shouldn't be considered a replacement +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement // for a real clientset and is mostly useful in simple unit tests. func NewSimpleClientset(objects ...runtime.Object) *Clientset { o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) @@ -51,9 +54,13 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset { cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} cs.AddReactor("*", "*", testing.ObjectReaction(o)) cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + var opts metav1.ListOptions + if watchAction, ok := action.(testing.WatchActionImpl); ok { + opts = watchAction.ListOptions + } gvr := action.GetResource() ns := action.GetNamespace() - watch, err := o.Watch(gvr, ns) + watch, err := o.Watch(gvr, ns, opts) if err != nil { return false, nil, err } @@ -80,9 +87,28 @@ func (c *Clientset) Tracker() testing.ObjectTracker { return c.tracker } -var _ clientset.Interface = &Clientset{} +// IsWatchListSemanticsUnSupported informs the reflector that this client +// doesn't support WatchList semantics. +// +// This is a synthetic method whose sole purpose is to satisfy the optional +// interface check performed by the reflector. +// Returning true signals that WatchList can NOT be used. +// No additional logic is implemented here. +func (c *Clientset) IsWatchListSemanticsUnSupported() bool { + return true +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) // AcidV1 retrieves the AcidV1Client func (c *Clientset) AcidV1() acidv1.AcidV1Interface { return &fakeacidv1.FakeAcidV1{Fake: &c.Fake} } + +// ZalandoV1 retrieves the ZalandoV1Client +func (c *Clientset) ZalandoV1() zalandov1.ZalandoV1Interface { + return &fakezalandov1.FakeZalandoV1{Fake: &c.Fake} +} diff --git a/pkg/generated/clientset/versioned/fake/doc.go b/pkg/generated/clientset/versioned/fake/doc.go index bc1c91a11..bd1f71a60 100644 --- a/pkg/generated/clientset/versioned/fake/doc.go +++ b/pkg/generated/clientset/versioned/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index c4d383aab..d6bf8f312 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ package fake import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,19 +39,20 @@ var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ acidv1.AddToScheme, + zalandov1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) // -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. diff --git a/pkg/generated/clientset/versioned/scheme/doc.go b/pkg/generated/clientset/versioned/scheme/doc.go index cd594164b..987597d94 100644 --- a/pkg/generated/clientset/versioned/scheme/doc.go +++ b/pkg/generated/clientset/versioned/scheme/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go index 8be969eb5..45a31ab41 100644 --- a/pkg/generated/clientset/versioned/scheme/register.go +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ package scheme import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,19 +39,20 @@ var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ acidv1.AddToScheme, + zalandov1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) // -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go index 5666201d4..93564ec3b 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,8 +25,10 @@ SOFTWARE. package v1 import ( - v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" + http "net/http" + + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) @@ -55,12 +57,24 @@ func (c *AcidV1Client) Postgresqls(namespace string) PostgresqlInterface { } // NewForConfig creates a new AcidV1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*AcidV1Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { + setConfigDefaults(&config) + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { return nil, err } - client, err := rest.RESTClientFor(&config) + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new AcidV1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*AcidV1Client, error) { + config := *c + setConfigDefaults(&config) + client, err := rest.RESTClientForConfigAndClient(&config, h) if err != nil { return nil, err } @@ -82,17 +96,15 @@ func New(c rest.Interface) *AcidV1Client { return &AcidV1Client{c} } -func setConfigDefaults(config *rest.Config) error { - gv := v1.SchemeGroupVersion +func setConfigDefaults(config *rest.Config) { + gv := acidzalandov1.SchemeGroupVersion config.GroupVersion = &gv config.APIPath = "/apis" - config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + config.NegotiatedSerializer = rest.CodecFactoryForGeneratedClient(scheme.Scheme, scheme.Codecs).WithoutConversion() if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - - return nil } // RESTClient returns a RESTClient that is used to communicate diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go index eb8fcf1f4..5d2970a5f 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go index c5fd1c04b..5f237162a 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go index 03e7dda94..643b8fd26 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -35,15 +35,15 @@ type FakeAcidV1 struct { } func (c *FakeAcidV1) OperatorConfigurations(namespace string) v1.OperatorConfigurationInterface { - return &FakeOperatorConfigurations{c, namespace} + return newFakeOperatorConfigurations(c, namespace) } func (c *FakeAcidV1) PostgresTeams(namespace string) v1.PostgresTeamInterface { - return &FakePostgresTeams{c, namespace} + return newFakePostgresTeams(c, namespace) } func (c *FakeAcidV1) Postgresqls(namespace string) v1.PostgresqlInterface { - return &FakePostgresqls{c, namespace} + return newFakePostgresqls(c, namespace) } // RESTClient returns a RESTClient that is used to communicate diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go index c03ea7d94..66db8ec40 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,31 +25,34 @@ SOFTWARE. package fake import ( - "context" - - acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - schema "k8s.io/apimachinery/pkg/runtime/schema" - testing "k8s.io/client-go/testing" + v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + gentype "k8s.io/client-go/gentype" ) -// FakeOperatorConfigurations implements OperatorConfigurationInterface -type FakeOperatorConfigurations struct { +// fakeOperatorConfigurations implements OperatorConfigurationInterface +type fakeOperatorConfigurations struct { + *gentype.FakeClientWithList[*v1.OperatorConfiguration, *v1.OperatorConfigurationList] Fake *FakeAcidV1 - ns string } -var operatorconfigurationsResource = schema.GroupVersionResource{Group: "acid.zalan.do", Version: "v1", Resource: "operatorconfigurations"} - -var operatorconfigurationsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "OperatorConfiguration"} - -// Get takes name of the operatorConfiguration, and returns the corresponding operatorConfiguration object, and an error if there is any. -func (c *FakeOperatorConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(operatorconfigurationsResource, c.ns, name), &acidzalandov1.OperatorConfiguration{}) - - if obj == nil { - return nil, err +func newFakeOperatorConfigurations(fake *FakeAcidV1, namespace string) acidzalandov1.OperatorConfigurationInterface { + return &fakeOperatorConfigurations{ + gentype.NewFakeClientWithList[*v1.OperatorConfiguration, *v1.OperatorConfigurationList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("operatorconfigurations"), + v1.SchemeGroupVersion.WithKind("OperatorConfiguration"), + func() *v1.OperatorConfiguration { return &v1.OperatorConfiguration{} }, + func() *v1.OperatorConfigurationList { return &v1.OperatorConfigurationList{} }, + func(dst, src *v1.OperatorConfigurationList) { dst.ListMeta = src.ListMeta }, + func(list *v1.OperatorConfigurationList) []*v1.OperatorConfiguration { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1.OperatorConfigurationList, items []*v1.OperatorConfiguration) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, } - return obj.(*acidzalandov1.OperatorConfiguration), err } diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go index 01a0ed7a4..b2ac1e9e1 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,124 +25,30 @@ SOFTWARE. package fake import ( - "context" - - acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" + v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + gentype "k8s.io/client-go/gentype" ) -// FakePostgresqls implements PostgresqlInterface -type FakePostgresqls struct { +// fakePostgresqls implements PostgresqlInterface +type fakePostgresqls struct { + *gentype.FakeClientWithList[*v1.Postgresql, *v1.PostgresqlList] Fake *FakeAcidV1 - ns string } -var postgresqlsResource = schema.GroupVersionResource{Group: "acid.zalan.do", Version: "v1", Resource: "postgresqls"} - -var postgresqlsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "Postgresql"} - -// Get takes name of the postgresql, and returns the corresponding postgresql object, and an error if there is any. -func (c *FakePostgresqls) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.Postgresql, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(postgresqlsResource, c.ns, name), &acidzalandov1.Postgresql{}) - - if obj == nil { - return nil, err +func newFakePostgresqls(fake *FakeAcidV1, namespace string) acidzalandov1.PostgresqlInterface { + return &fakePostgresqls{ + gentype.NewFakeClientWithList[*v1.Postgresql, *v1.PostgresqlList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("postgresqls"), + v1.SchemeGroupVersion.WithKind("Postgresql"), + func() *v1.Postgresql { return &v1.Postgresql{} }, + func() *v1.PostgresqlList { return &v1.PostgresqlList{} }, + func(dst, src *v1.PostgresqlList) { dst.ListMeta = src.ListMeta }, + func(list *v1.PostgresqlList) []*v1.Postgresql { return gentype.ToPointerSlice(list.Items) }, + func(list *v1.PostgresqlList, items []*v1.Postgresql) { list.Items = gentype.FromPointerSlice(items) }, + ), + fake, } - return obj.(*acidzalandov1.Postgresql), err -} - -// List takes label and field selectors, and returns the list of Postgresqls that match those selectors. -func (c *FakePostgresqls) List(ctx context.Context, opts v1.ListOptions) (result *acidzalandov1.PostgresqlList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(postgresqlsResource, postgresqlsKind, c.ns, opts), &acidzalandov1.PostgresqlList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &acidzalandov1.PostgresqlList{ListMeta: obj.(*acidzalandov1.PostgresqlList).ListMeta} - for _, item := range obj.(*acidzalandov1.PostgresqlList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested postgresqls. -func (c *FakePostgresqls) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(postgresqlsResource, c.ns, opts)) - -} - -// Create takes the representation of a postgresql and creates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *FakePostgresqls) Create(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.CreateOptions) (result *acidzalandov1.Postgresql, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(postgresqlsResource, c.ns, postgresql), &acidzalandov1.Postgresql{}) - - if obj == nil { - return nil, err - } - return obj.(*acidzalandov1.Postgresql), err -} - -// Update takes the representation of a postgresql and updates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *FakePostgresqls) Update(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.UpdateOptions) (result *acidzalandov1.Postgresql, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(postgresqlsResource, c.ns, postgresql), &acidzalandov1.Postgresql{}) - - if obj == nil { - return nil, err - } - return obj.(*acidzalandov1.Postgresql), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakePostgresqls) UpdateStatus(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts v1.UpdateOptions) (*acidzalandov1.Postgresql, error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceAction(postgresqlsResource, "status", c.ns, postgresql), &acidzalandov1.Postgresql{}) - - if obj == nil { - return nil, err - } - return obj.(*acidzalandov1.Postgresql), err -} - -// Delete takes name of the postgresql and deletes it. Returns an error if one occurs. -func (c *FakePostgresqls) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteAction(postgresqlsResource, c.ns, name), &acidzalandov1.Postgresql{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakePostgresqls) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(postgresqlsResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &acidzalandov1.PostgresqlList{}) - return err -} - -// Patch applies the patch and returns the patched postgresql. -func (c *FakePostgresqls) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *acidzalandov1.Postgresql, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(postgresqlsResource, c.ns, name, pt, data, subresources...), &acidzalandov1.Postgresql{}) - - if obj == nil { - return nil, err - } - return obj.(*acidzalandov1.Postgresql), err } diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go index b333ae046..3067aac18 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,112 +25,32 @@ SOFTWARE. package fake import ( - "context" - - acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" + v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + gentype "k8s.io/client-go/gentype" ) -// FakePostgresTeams implements PostgresTeamInterface -type FakePostgresTeams struct { +// fakePostgresTeams implements PostgresTeamInterface +type fakePostgresTeams struct { + *gentype.FakeClientWithList[*v1.PostgresTeam, *v1.PostgresTeamList] Fake *FakeAcidV1 - ns string } -var postgresteamsResource = schema.GroupVersionResource{Group: "acid.zalan.do", Version: "v1", Resource: "postgresteams"} - -var postgresteamsKind = schema.GroupVersionKind{Group: "acid.zalan.do", Version: "v1", Kind: "PostgresTeam"} - -// Get takes name of the postgresTeam, and returns the corresponding postgresTeam object, and an error if there is any. -func (c *FakePostgresTeams) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.PostgresTeam, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(postgresteamsResource, c.ns, name), &acidzalandov1.PostgresTeam{}) - - if obj == nil { - return nil, err +func newFakePostgresTeams(fake *FakeAcidV1, namespace string) acidzalandov1.PostgresTeamInterface { + return &fakePostgresTeams{ + gentype.NewFakeClientWithList[*v1.PostgresTeam, *v1.PostgresTeamList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("postgresteams"), + v1.SchemeGroupVersion.WithKind("PostgresTeam"), + func() *v1.PostgresTeam { return &v1.PostgresTeam{} }, + func() *v1.PostgresTeamList { return &v1.PostgresTeamList{} }, + func(dst, src *v1.PostgresTeamList) { dst.ListMeta = src.ListMeta }, + func(list *v1.PostgresTeamList) []*v1.PostgresTeam { return gentype.ToPointerSlice(list.Items) }, + func(list *v1.PostgresTeamList, items []*v1.PostgresTeam) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, } - return obj.(*acidzalandov1.PostgresTeam), err -} - -// List takes label and field selectors, and returns the list of PostgresTeams that match those selectors. -func (c *FakePostgresTeams) List(ctx context.Context, opts v1.ListOptions) (result *acidzalandov1.PostgresTeamList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(postgresteamsResource, postgresteamsKind, c.ns, opts), &acidzalandov1.PostgresTeamList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &acidzalandov1.PostgresTeamList{ListMeta: obj.(*acidzalandov1.PostgresTeamList).ListMeta} - for _, item := range obj.(*acidzalandov1.PostgresTeamList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested postgresTeams. -func (c *FakePostgresTeams) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(postgresteamsResource, c.ns, opts)) - -} - -// Create takes the representation of a postgresTeam and creates it. Returns the server's representation of the postgresTeam, and an error, if there is any. -func (c *FakePostgresTeams) Create(ctx context.Context, postgresTeam *acidzalandov1.PostgresTeam, opts v1.CreateOptions) (result *acidzalandov1.PostgresTeam, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(postgresteamsResource, c.ns, postgresTeam), &acidzalandov1.PostgresTeam{}) - - if obj == nil { - return nil, err - } - return obj.(*acidzalandov1.PostgresTeam), err -} - -// Update takes the representation of a postgresTeam and updates it. Returns the server's representation of the postgresTeam, and an error, if there is any. -func (c *FakePostgresTeams) Update(ctx context.Context, postgresTeam *acidzalandov1.PostgresTeam, opts v1.UpdateOptions) (result *acidzalandov1.PostgresTeam, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(postgresteamsResource, c.ns, postgresTeam), &acidzalandov1.PostgresTeam{}) - - if obj == nil { - return nil, err - } - return obj.(*acidzalandov1.PostgresTeam), err -} - -// Delete takes name of the postgresTeam and deletes it. Returns an error if one occurs. -func (c *FakePostgresTeams) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteAction(postgresteamsResource, c.ns, name), &acidzalandov1.PostgresTeam{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakePostgresTeams) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(postgresteamsResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &acidzalandov1.PostgresTeamList{}) - return err -} - -// Patch applies the patch and returns the patched postgresTeam. -func (c *FakePostgresTeams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *acidzalandov1.PostgresTeam, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(postgresteamsResource, c.ns, name, pt, data, subresources...), &acidzalandov1.PostgresTeam{}) - - if obj == nil { - return nil, err - } - return obj.(*acidzalandov1.PostgresTeam), err } diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go index b4e99cbc8..228ef2924 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go index be22e075d..292fa2fce 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,12 +25,14 @@ SOFTWARE. package v1 import ( - "context" + context "context" acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - rest "k8s.io/client-go/rest" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" ) // OperatorConfigurationsGetter has a method to return a OperatorConfigurationInterface. @@ -41,33 +43,32 @@ type OperatorConfigurationsGetter interface { // OperatorConfigurationInterface has methods to work with OperatorConfiguration resources. type OperatorConfigurationInterface interface { - Get(ctx context.Context, name string, opts v1.GetOptions) (*acidzalandov1.OperatorConfiguration, error) + Create(ctx context.Context, operatorConfiguration *acidzalandov1.OperatorConfiguration, opts metav1.CreateOptions) (*acidzalandov1.OperatorConfiguration, error) + Update(ctx context.Context, operatorConfiguration *acidzalandov1.OperatorConfiguration, opts metav1.UpdateOptions) (*acidzalandov1.OperatorConfiguration, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*acidzalandov1.OperatorConfiguration, error) + List(ctx context.Context, opts metav1.ListOptions) (*acidzalandov1.OperatorConfigurationList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *acidzalandov1.OperatorConfiguration, err error) OperatorConfigurationExpansion } // operatorConfigurations implements OperatorConfigurationInterface type operatorConfigurations struct { - client rest.Interface - ns string + *gentype.ClientWithList[*acidzalandov1.OperatorConfiguration, *acidzalandov1.OperatorConfigurationList] } // newOperatorConfigurations returns a OperatorConfigurations func newOperatorConfigurations(c *AcidV1Client, namespace string) *operatorConfigurations { return &operatorConfigurations{ - client: c.RESTClient(), - ns: namespace, + gentype.NewClientWithList[*acidzalandov1.OperatorConfiguration, *acidzalandov1.OperatorConfigurationList]( + "operatorconfigurations", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *acidzalandov1.OperatorConfiguration { return &acidzalandov1.OperatorConfiguration{} }, + func() *acidzalandov1.OperatorConfigurationList { return &acidzalandov1.OperatorConfigurationList{} }, + ), } } - -// Get takes name of the operatorConfiguration, and returns the corresponding operatorConfiguration object, and an error if there is any. -func (c *operatorConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *acidzalandov1.OperatorConfiguration, err error) { - result = &acidzalandov1.OperatorConfiguration{} - err = c.client.Get(). - Namespace(c.ns). - Resource("operatorconfigurations"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go index 5241cfb54..75793c57a 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,15 +25,14 @@ SOFTWARE. package v1 import ( - "context" - "time" + context "context" - v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" + gentype "k8s.io/client-go/gentype" ) // PostgresqlsGetter has a method to return a PostgresqlInterface. @@ -44,158 +43,34 @@ type PostgresqlsGetter interface { // PostgresqlInterface has methods to work with Postgresql resources. type PostgresqlInterface interface { - Create(ctx context.Context, postgresql *v1.Postgresql, opts metav1.CreateOptions) (*v1.Postgresql, error) - Update(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (*v1.Postgresql, error) - UpdateStatus(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (*v1.Postgresql, error) + Create(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts metav1.CreateOptions) (*acidzalandov1.Postgresql, error) + Update(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts metav1.UpdateOptions) (*acidzalandov1.Postgresql, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, postgresql *acidzalandov1.Postgresql, opts metav1.UpdateOptions) (*acidzalandov1.Postgresql, error) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error - Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Postgresql, error) - List(ctx context.Context, opts metav1.ListOptions) (*v1.PostgresqlList, error) + Get(ctx context.Context, name string, opts metav1.GetOptions) (*acidzalandov1.Postgresql, error) + List(ctx context.Context, opts metav1.ListOptions) (*acidzalandov1.PostgresqlList, error) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Postgresql, err error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *acidzalandov1.Postgresql, err error) PostgresqlExpansion } // postgresqls implements PostgresqlInterface type postgresqls struct { - client rest.Interface - ns string + *gentype.ClientWithList[*acidzalandov1.Postgresql, *acidzalandov1.PostgresqlList] } // newPostgresqls returns a Postgresqls func newPostgresqls(c *AcidV1Client, namespace string) *postgresqls { return &postgresqls{ - client: c.RESTClient(), - ns: namespace, + gentype.NewClientWithList[*acidzalandov1.Postgresql, *acidzalandov1.PostgresqlList]( + "postgresqls", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *acidzalandov1.Postgresql { return &acidzalandov1.Postgresql{} }, + func() *acidzalandov1.PostgresqlList { return &acidzalandov1.PostgresqlList{} }, + ), } } - -// Get takes name of the postgresql, and returns the corresponding postgresql object, and an error if there is any. -func (c *postgresqls) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.Postgresql, err error) { - result = &v1.Postgresql{} - err = c.client.Get(). - Namespace(c.ns). - Resource("postgresqls"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of Postgresqls that match those selectors. -func (c *postgresqls) List(ctx context.Context, opts metav1.ListOptions) (result *v1.PostgresqlList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1.PostgresqlList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("postgresqls"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested postgresqls. -func (c *postgresqls) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("postgresqls"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a postgresql and creates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *postgresqls) Create(ctx context.Context, postgresql *v1.Postgresql, opts metav1.CreateOptions) (result *v1.Postgresql, err error) { - result = &v1.Postgresql{} - err = c.client.Post(). - Namespace(c.ns). - Resource("postgresqls"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(postgresql). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a postgresql and updates it. Returns the server's representation of the postgresql, and an error, if there is any. -func (c *postgresqls) Update(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (result *v1.Postgresql, err error) { - result = &v1.Postgresql{} - err = c.client.Put(). - Namespace(c.ns). - Resource("postgresqls"). - Name(postgresql.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(postgresql). - Do(ctx). - Into(result) - return -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *postgresqls) UpdateStatus(ctx context.Context, postgresql *v1.Postgresql, opts metav1.UpdateOptions) (result *v1.Postgresql, err error) { - result = &v1.Postgresql{} - err = c.client.Put(). - Namespace(c.ns). - Resource("postgresqls"). - Name(postgresql.Name). - SubResource("status"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(postgresql). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the postgresql and deletes it. Returns an error if one occurs. -func (c *postgresqls) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("postgresqls"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *postgresqls) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("postgresqls"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched postgresql. -func (c *postgresqls) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Postgresql, err error) { - result = &v1.Postgresql{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("postgresqls"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go index 96fbb882a..99df6b6cc 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,15 +25,14 @@ SOFTWARE. package v1 import ( - "context" - "time" + context "context" - v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" + gentype "k8s.io/client-go/gentype" ) // PostgresTeamsGetter has a method to return a PostgresTeamInterface. @@ -44,141 +43,32 @@ type PostgresTeamsGetter interface { // PostgresTeamInterface has methods to work with PostgresTeam resources. type PostgresTeamInterface interface { - Create(ctx context.Context, postgresTeam *v1.PostgresTeam, opts metav1.CreateOptions) (*v1.PostgresTeam, error) - Update(ctx context.Context, postgresTeam *v1.PostgresTeam, opts metav1.UpdateOptions) (*v1.PostgresTeam, error) + Create(ctx context.Context, postgresTeam *acidzalandov1.PostgresTeam, opts metav1.CreateOptions) (*acidzalandov1.PostgresTeam, error) + Update(ctx context.Context, postgresTeam *acidzalandov1.PostgresTeam, opts metav1.UpdateOptions) (*acidzalandov1.PostgresTeam, error) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error - Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.PostgresTeam, error) - List(ctx context.Context, opts metav1.ListOptions) (*v1.PostgresTeamList, error) + Get(ctx context.Context, name string, opts metav1.GetOptions) (*acidzalandov1.PostgresTeam, error) + List(ctx context.Context, opts metav1.ListOptions) (*acidzalandov1.PostgresTeamList, error) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.PostgresTeam, err error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *acidzalandov1.PostgresTeam, err error) PostgresTeamExpansion } // postgresTeams implements PostgresTeamInterface type postgresTeams struct { - client rest.Interface - ns string + *gentype.ClientWithList[*acidzalandov1.PostgresTeam, *acidzalandov1.PostgresTeamList] } // newPostgresTeams returns a PostgresTeams func newPostgresTeams(c *AcidV1Client, namespace string) *postgresTeams { return &postgresTeams{ - client: c.RESTClient(), - ns: namespace, + gentype.NewClientWithList[*acidzalandov1.PostgresTeam, *acidzalandov1.PostgresTeamList]( + "postgresteams", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *acidzalandov1.PostgresTeam { return &acidzalandov1.PostgresTeam{} }, + func() *acidzalandov1.PostgresTeamList { return &acidzalandov1.PostgresTeamList{} }, + ), } } - -// Get takes name of the postgresTeam, and returns the corresponding postgresTeam object, and an error if there is any. -func (c *postgresTeams) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.PostgresTeam, err error) { - result = &v1.PostgresTeam{} - err = c.client.Get(). - Namespace(c.ns). - Resource("postgresteams"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of PostgresTeams that match those selectors. -func (c *postgresTeams) List(ctx context.Context, opts metav1.ListOptions) (result *v1.PostgresTeamList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1.PostgresTeamList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("postgresteams"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested postgresTeams. -func (c *postgresTeams) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("postgresteams"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a postgresTeam and creates it. Returns the server's representation of the postgresTeam, and an error, if there is any. -func (c *postgresTeams) Create(ctx context.Context, postgresTeam *v1.PostgresTeam, opts metav1.CreateOptions) (result *v1.PostgresTeam, err error) { - result = &v1.PostgresTeam{} - err = c.client.Post(). - Namespace(c.ns). - Resource("postgresteams"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(postgresTeam). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a postgresTeam and updates it. Returns the server's representation of the postgresTeam, and an error, if there is any. -func (c *postgresTeams) Update(ctx context.Context, postgresTeam *v1.PostgresTeam, opts metav1.UpdateOptions) (result *v1.PostgresTeam, err error) { - result = &v1.PostgresTeam{} - err = c.client.Put(). - Namespace(c.ns). - Resource("postgresteams"). - Name(postgresTeam.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(postgresTeam). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the postgresTeam and deletes it. Returns an error if one occurs. -func (c *postgresTeams) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("postgresteams"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *postgresTeams) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("postgresteams"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched postgresTeam. -func (c *postgresTeams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.PostgresTeam, err error) { - result = &v1.PostgresTeam{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("postgresteams"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go new file mode 100644 index 000000000..5d2970a5f --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go @@ -0,0 +1,26 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1 diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go new file mode 100644 index 000000000..2681ac467 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go @@ -0,0 +1,74 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// FabricEventStreamsGetter has a method to return a FabricEventStreamInterface. +// A group's client should implement this interface. +type FabricEventStreamsGetter interface { + FabricEventStreams(namespace string) FabricEventStreamInterface +} + +// FabricEventStreamInterface has methods to work with FabricEventStream resources. +type FabricEventStreamInterface interface { + Create(ctx context.Context, fabricEventStream *zalandoorgv1.FabricEventStream, opts metav1.CreateOptions) (*zalandoorgv1.FabricEventStream, error) + Update(ctx context.Context, fabricEventStream *zalandoorgv1.FabricEventStream, opts metav1.UpdateOptions) (*zalandoorgv1.FabricEventStream, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*zalandoorgv1.FabricEventStream, error) + List(ctx context.Context, opts metav1.ListOptions) (*zalandoorgv1.FabricEventStreamList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *zalandoorgv1.FabricEventStream, err error) + FabricEventStreamExpansion +} + +// fabricEventStreams implements FabricEventStreamInterface +type fabricEventStreams struct { + *gentype.ClientWithList[*zalandoorgv1.FabricEventStream, *zalandoorgv1.FabricEventStreamList] +} + +// newFabricEventStreams returns a FabricEventStreams +func newFabricEventStreams(c *ZalandoV1Client, namespace string) *fabricEventStreams { + return &fabricEventStreams{ + gentype.NewClientWithList[*zalandoorgv1.FabricEventStream, *zalandoorgv1.FabricEventStreamList]( + "fabriceventstreams", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *zalandoorgv1.FabricEventStream { return &zalandoorgv1.FabricEventStream{} }, + func() *zalandoorgv1.FabricEventStreamList { return &zalandoorgv1.FabricEventStreamList{} }, + ), + } +} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go new file mode 100644 index 000000000..5f237162a --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go @@ -0,0 +1,26 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go new file mode 100644 index 000000000..c3ae49663 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go @@ -0,0 +1,58 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" + gentype "k8s.io/client-go/gentype" +) + +// fakeFabricEventStreams implements FabricEventStreamInterface +type fakeFabricEventStreams struct { + *gentype.FakeClientWithList[*v1.FabricEventStream, *v1.FabricEventStreamList] + Fake *FakeZalandoV1 +} + +func newFakeFabricEventStreams(fake *FakeZalandoV1, namespace string) zalandoorgv1.FabricEventStreamInterface { + return &fakeFabricEventStreams{ + gentype.NewFakeClientWithList[*v1.FabricEventStream, *v1.FabricEventStreamList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("fabriceventstreams"), + v1.SchemeGroupVersion.WithKind("FabricEventStream"), + func() *v1.FabricEventStream { return &v1.FabricEventStream{} }, + func() *v1.FabricEventStreamList { return &v1.FabricEventStreamList{} }, + func(dst, src *v1.FabricEventStreamList) { dst.ListMeta = src.ListMeta }, + func(list *v1.FabricEventStreamList) []*v1.FabricEventStream { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1.FabricEventStreamList, items []*v1.FabricEventStream) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/kubectl-pg/cmd/root.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go similarity index 54% rename from kubectl-pg/cmd/root.go rename to pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go index 916852998..588a2bb94 100644 --- a/kubectl-pg/cmd/root.go +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go @@ -1,5 +1,5 @@ /* -Copyright © 2019 Vineeth Pothulapati +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -8,43 +8,39 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. */ -package cmd +// Code generated by client-gen. DO NOT EDIT. + +package fake import ( - "fmt" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "os" + v1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" ) -var rootCmd = &cobra.Command{ - Use: "kubectl-pg", - Short: "kubectl plugin for the Zalando Postgres operator.", - Long: `kubectl pg plugin for interaction with Zalando postgres operator.`, +type FakeZalandoV1 struct { + *testing.Fake } -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } +func (c *FakeZalandoV1) FabricEventStreams(namespace string) v1.FabricEventStreamInterface { + return newFakeFabricEventStreams(c, namespace) } -func init() { - viper.SetDefault("author", "Vineeth Pothulapati ") - viper.SetDefault("license", "mit") +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeZalandoV1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret } diff --git a/kubectl-pg/main.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go similarity index 77% rename from kubectl-pg/main.go rename to pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go index bfcd5eb29..014071418 100644 --- a/kubectl-pg/main.go +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright © 2019 Vineeth Pothulapati +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -8,24 +8,20 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. */ -package main +// Code generated by client-gen. DO NOT EDIT. -import ( - "github.com/zalando/postgres-operator/kubectl-pg/cmd" -) +package v1 -func main() { - cmd.Execute() -} +type FabricEventStreamExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go new file mode 100644 index 000000000..803aee179 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go @@ -0,0 +1,107 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + http "net/http" + + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type ZalandoV1Interface interface { + RESTClient() rest.Interface + FabricEventStreamsGetter +} + +// ZalandoV1Client is used to interact with features provided by the zalando.org group. +type ZalandoV1Client struct { + restClient rest.Interface +} + +func (c *ZalandoV1Client) FabricEventStreams(namespace string) FabricEventStreamInterface { + return newFabricEventStreams(c, namespace) +} + +// NewForConfig creates a new ZalandoV1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*ZalandoV1Client, error) { + config := *c + setConfigDefaults(&config) + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new ZalandoV1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*ZalandoV1Client, error) { + config := *c + setConfigDefaults(&config) + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &ZalandoV1Client{client}, nil +} + +// NewForConfigOrDie creates a new ZalandoV1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *ZalandoV1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new ZalandoV1Client for the given RESTClient. +func New(c rest.Interface) *ZalandoV1Client { + return &ZalandoV1Client{c} +} + +func setConfigDefaults(config *rest.Config) { + gv := zalandoorgv1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = rest.CodecFactoryForGeneratedClient(scheme.Scheme, scheme.Codecs).WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *ZalandoV1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go index 6f77564fa..a75e6ae2b 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go index 5c05e6d68..d176a2b35 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -30,6 +30,8 @@ import ( // Interface provides access to all the informers in this group version. type Interface interface { + // OperatorConfigurations returns a OperatorConfigurationInformer. + OperatorConfigurations() OperatorConfigurationInformer // PostgresTeams returns a PostgresTeamInformer. PostgresTeams() PostgresTeamInformer // Postgresqls returns a PostgresqlInformer. @@ -47,6 +49,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } +// OperatorConfigurations returns a OperatorConfigurationInformer. +func (v *version) OperatorConfigurations() OperatorConfigurationInformer { + return &operatorConfigurationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // PostgresTeams returns a PostgresTeamInformer. func (v *version) PostgresTeams() PostgresTeamInformer { return &postgresTeamInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/operatorconfiguration.go new file mode 100644 index 000000000..e2fee1a4e --- /dev/null +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/operatorconfiguration.go @@ -0,0 +1,122 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + apisacidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/generated/listers/acid.zalan.do/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// OperatorConfigurationInformer provides access to a shared informer and lister for +// OperatorConfigurations. +type OperatorConfigurationInformer interface { + Informer() cache.SharedIndexInformer + Lister() acidzalandov1.OperatorConfigurationLister +} + +type operatorConfigurationInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewOperatorConfigurationInformer constructs a new informer for OperatorConfiguration type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewOperatorConfigurationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewOperatorConfigurationInformerWithOptions(client, namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: indexers}) +} + +// NewFilteredOperatorConfigurationInformer constructs a new informer for OperatorConfiguration type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredOperatorConfigurationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return NewOperatorConfigurationInformerWithOptions(client, namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: indexers, TweakListOptions: tweakListOptions}) +} + +// NewOperatorConfigurationInformerWithOptions constructs a new informer for OperatorConfiguration type with additional options. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewOperatorConfigurationInformerWithOptions(client versioned.Interface, namespace string, options internalinterfaces.InformerOptions) cache.SharedIndexInformer { + gvr := schema.GroupVersionResource{Group: "acid.zalan.do", Version: "v1", Resource: "operatorconfigurations"} + identifier := options.InformerName.WithResource(gvr) + tweakListOptions := options.TweakListOptions + return cache.NewSharedIndexInformerWithOptions( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.AcidV1().OperatorConfigurations(namespace).List(context.Background(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.AcidV1().OperatorConfigurations(namespace).Watch(context.Background(), opts) + }, + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.AcidV1().OperatorConfigurations(namespace).List(ctx, opts) + }, + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.AcidV1().OperatorConfigurations(namespace).Watch(ctx, opts) + }, + }, client), + &apisacidzalandov1.OperatorConfiguration{}, + cache.SharedIndexInformerOptions{ + ResyncPeriod: options.ResyncPeriod, + Indexers: options.Indexers, + Identifier: identifier, + }, + ) +} + +func (f *operatorConfigurationInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewOperatorConfigurationInformerWithOptions(client, f.namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, InformerName: f.factory.InformerName(), TweakListOptions: f.tweakListOptions}) +} + +func (f *operatorConfigurationInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisacidzalandov1.OperatorConfiguration{}, f.defaultInformer) +} + +func (f *operatorConfigurationInformer) Lister() acidzalandov1.OperatorConfigurationLister { + return acidzalandov1.NewOperatorConfigurationLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go index 1453af276..c1b58ff05 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,15 +25,16 @@ SOFTWARE. package v1 import ( - "context" + context "context" time "time" - acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + apisacidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" - v1 "github.com/zalando/postgres-operator/pkg/generated/listers/acid.zalan.do/v1" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/generated/listers/acid.zalan.do/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" ) @@ -42,7 +43,7 @@ import ( // Postgresqls. type PostgresqlInformer interface { Informer() cache.SharedIndexInformer - Lister() v1.PostgresqlLister + Lister() acidzalandov1.PostgresqlLister } type postgresqlInformer struct { @@ -55,42 +56,67 @@ type postgresqlInformer struct { // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. func NewPostgresqlInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredPostgresqlInformer(client, namespace, resyncPeriod, indexers, nil) + return NewPostgresqlInformerWithOptions(client, namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: indexers}) } // NewFilteredPostgresqlInformer constructs a new informer for Postgresql type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. func NewFilteredPostgresqlInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return NewPostgresqlInformerWithOptions(client, namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: indexers, TweakListOptions: tweakListOptions}) +} + +// NewPostgresqlInformerWithOptions constructs a new informer for Postgresql type with additional options. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewPostgresqlInformerWithOptions(client versioned.Interface, namespace string, options internalinterfaces.InformerOptions) cache.SharedIndexInformer { + gvr := schema.GroupVersionResource{Group: "acid.zalan.do", Version: "v1", Resource: "postgresqls"} + identifier := options.InformerName.WithResource(gvr) + tweakListOptions := options.TweakListOptions + return cache.NewSharedIndexInformerWithOptions( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { - tweakListOptions(&options) + tweakListOptions(&opts) } - return client.AcidV1().Postgresqls(namespace).List(context.TODO(), options) + return client.AcidV1().Postgresqls(namespace).List(context.Background(), opts) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { - tweakListOptions(&options) + tweakListOptions(&opts) } - return client.AcidV1().Postgresqls(namespace).Watch(context.TODO(), options) + return client.AcidV1().Postgresqls(namespace).Watch(context.Background(), opts) }, + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.AcidV1().Postgresqls(namespace).List(ctx, opts) + }, + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.AcidV1().Postgresqls(namespace).Watch(ctx, opts) + }, + }, client), + &apisacidzalandov1.Postgresql{}, + cache.SharedIndexInformerOptions{ + ResyncPeriod: options.ResyncPeriod, + Indexers: options.Indexers, + Identifier: identifier, }, - &acidzalandov1.Postgresql{}, - resyncPeriod, - indexers, ) } func (f *postgresqlInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredPostgresqlInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) + return NewPostgresqlInformerWithOptions(client, f.namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, InformerName: f.factory.InformerName(), TweakListOptions: f.tweakListOptions}) } func (f *postgresqlInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&acidzalandov1.Postgresql{}, f.defaultInformer) + return f.factory.InformerFor(&apisacidzalandov1.Postgresql{}, f.defaultInformer) } -func (f *postgresqlInformer) Lister() v1.PostgresqlLister { - return v1.NewPostgresqlLister(f.Informer().GetIndexer()) +func (f *postgresqlInformer) Lister() acidzalandov1.PostgresqlLister { + return acidzalandov1.NewPostgresqlLister(f.Informer().GetIndexer()) } diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go index a19e4726f..954dfd19f 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,15 +25,16 @@ SOFTWARE. package v1 import ( - "context" + context "context" time "time" - acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + apisacidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" - v1 "github.com/zalando/postgres-operator/pkg/generated/listers/acid.zalan.do/v1" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/generated/listers/acid.zalan.do/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" ) @@ -42,7 +43,7 @@ import ( // PostgresTeams. type PostgresTeamInformer interface { Informer() cache.SharedIndexInformer - Lister() v1.PostgresTeamLister + Lister() acidzalandov1.PostgresTeamLister } type postgresTeamInformer struct { @@ -55,42 +56,67 @@ type postgresTeamInformer struct { // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. func NewPostgresTeamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredPostgresTeamInformer(client, namespace, resyncPeriod, indexers, nil) + return NewPostgresTeamInformerWithOptions(client, namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: indexers}) } // NewFilteredPostgresTeamInformer constructs a new informer for PostgresTeam type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. func NewFilteredPostgresTeamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return NewPostgresTeamInformerWithOptions(client, namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: indexers, TweakListOptions: tweakListOptions}) +} + +// NewPostgresTeamInformerWithOptions constructs a new informer for PostgresTeam type with additional options. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewPostgresTeamInformerWithOptions(client versioned.Interface, namespace string, options internalinterfaces.InformerOptions) cache.SharedIndexInformer { + gvr := schema.GroupVersionResource{Group: "acid.zalan.do", Version: "v1", Resource: "postgresteams"} + identifier := options.InformerName.WithResource(gvr) + tweakListOptions := options.TweakListOptions + return cache.NewSharedIndexInformerWithOptions( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { - tweakListOptions(&options) + tweakListOptions(&opts) } - return client.AcidV1().PostgresTeams(namespace).List(context.TODO(), options) + return client.AcidV1().PostgresTeams(namespace).List(context.Background(), opts) }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { - tweakListOptions(&options) + tweakListOptions(&opts) } - return client.AcidV1().PostgresTeams(namespace).Watch(context.TODO(), options) + return client.AcidV1().PostgresTeams(namespace).Watch(context.Background(), opts) }, + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.AcidV1().PostgresTeams(namespace).List(ctx, opts) + }, + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.AcidV1().PostgresTeams(namespace).Watch(ctx, opts) + }, + }, client), + &apisacidzalandov1.PostgresTeam{}, + cache.SharedIndexInformerOptions{ + ResyncPeriod: options.ResyncPeriod, + Indexers: options.Indexers, + Identifier: identifier, }, - &acidzalandov1.PostgresTeam{}, - resyncPeriod, - indexers, ) } func (f *postgresTeamInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredPostgresTeamInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) + return NewPostgresTeamInformerWithOptions(client, f.namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, InformerName: f.factory.InformerName(), TweakListOptions: f.tweakListOptions}) } func (f *postgresTeamInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&acidzalandov1.PostgresTeam{}, f.defaultInformer) + return f.factory.InformerFor(&apisacidzalandov1.PostgresTeam{}, f.defaultInformer) } -func (f *postgresTeamInformer) Lister() v1.PostgresTeamLister { - return v1.NewPostgresTeamLister(f.Informer().GetIndexer()) +func (f *postgresTeamInformer) Lister() acidzalandov1.PostgresTeamLister { + return acidzalandov1.NewPostgresTeamLister(f.Informer().GetIndexer()) } diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go index e4b1efdc6..fb9ba76ac 100644 --- a/pkg/generated/informers/externalversions/factory.go +++ b/pkg/generated/informers/externalversions/factory.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,6 +25,7 @@ SOFTWARE. package externalversions import ( + context "context" reflect "reflect" sync "sync" time "time" @@ -32,9 +33,11 @@ import ( versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidzalando "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do" internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + zalandoorg "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" + wait "k8s.io/apimachinery/pkg/util/wait" cache "k8s.io/client-go/tools/cache" ) @@ -48,11 +51,18 @@ type sharedInformerFactory struct { lock sync.Mutex defaultResync time.Duration customResync map[reflect.Type]time.Duration + transform cache.TransformFunc + informerName *cache.InformerName informers map[reflect.Type]cache.SharedIndexInformer // startedInformers is used for tracking which informers have been started. // This allows Start() to be called multiple times safely. startedInformers map[reflect.Type]bool + // wg tracks how many goroutines were started. + wg sync.WaitGroup + // shuttingDown is true when Shutdown has been called. It may still be running + // because it needs to wait for goroutines. + shuttingDown bool } // WithCustomResyncConfig sets a custom resync period for the specified informer types. @@ -81,6 +91,29 @@ func WithNamespace(namespace string) SharedInformerOption { } } +// WithTransform sets a transform on all informers. +func WithTransform(transform cache.TransformFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.transform = transform + return factory + } +} + +// WithInformerName sets the InformerName for informer identity used in metrics. +// The InformerName must be created via cache.NewInformerName() at startup, +// which validates global uniqueness. Each informer type will register its +// GVR under this name. +func WithInformerName(informerName *cache.InformerName) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.informerName = informerName + return factory + } +} + +func (f *sharedInformerFactory) InformerName() *cache.InformerName { + return f.informerName +} + // NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { return NewSharedInformerFactoryWithOptions(client, defaultResync) @@ -89,6 +122,7 @@ func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Dur // NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. // Listers obtained via this SharedInformerFactory will be subject to the same filters // as specified here. +// // Deprecated: Please use NewSharedInformerFactoryWithOptions instead func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) @@ -113,21 +147,44 @@ func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResy return factory } -// Start initializes all requested informers. func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.StartWithContext(wait.ContextForChannel(stopCh)) +} + +func (f *sharedInformerFactory) StartWithContext(ctx context.Context) { f.lock.Lock() defer f.lock.Unlock() + if f.shuttingDown { + return + } + for informerType, informer := range f.informers { if !f.startedInformers[informerType] { - go informer.Run(stopCh) + f.wg.Go(func() { + informer.RunWithContext(ctx) + }) f.startedInformers[informerType] = true } } } -// WaitForCacheSync waits for all started informers' cache were synced. +func (f *sharedInformerFactory) Shutdown() { + f.lock.Lock() + f.shuttingDown = true + f.lock.Unlock() + + // Will return immediately if there is nothing to wait for. + f.wg.Wait() + f.informerName.Release() +} + func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + result := f.WaitForCacheSyncWithContext(wait.ContextForChannel(stopCh)) + return result.Synced +} + +func (f *sharedInformerFactory) WaitForCacheSyncWithContext(ctx context.Context) cache.SyncResult { informers := func() map[reflect.Type]cache.SharedIndexInformer { f.lock.Lock() defer f.lock.Unlock() @@ -141,14 +198,35 @@ func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[ref return informers }() - res := map[reflect.Type]bool{} - for informType, informer := range informers { - res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + // Wait for informers to sync, without polling. + cacheSyncs := make([]cache.DoneChecker, 0, len(informers)) + for _, informer := range informers { + cacheSyncs = append(cacheSyncs, informer.HasSyncedChecker()) } + cache.WaitFor(ctx, "" /* no logging */, cacheSyncs...) + + res := cache.SyncResult{ + Synced: make(map[reflect.Type]bool, len(informers)), + } + failed := false + for informType, informer := range informers { + hasSynced := informer.HasSynced() + if !hasSynced { + failed = true + } + res.Synced[informType] = hasSynced + } + if failed { + // context.Cause is more informative than ctx.Err(). + // This must be non-nil, otherwise WaitFor wouldn't have stopped + // prematurely. + res.Err = context.Cause(ctx) + } + return res } -// InternalInformerFor returns the SharedIndexInformer for obj using an internal +// InformerFor returns the SharedIndexInformer for obj using an internal // client. func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { f.lock.Lock() @@ -166,6 +244,9 @@ func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internal } informer = newFunc(f.client, resyncPeriod) + if f.transform != nil { + informer.SetTransform(f.transform) + } f.informers[informerType] = informer return informer @@ -173,14 +254,92 @@ func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internal // SharedInformerFactory provides shared informers for resources in all known // API group versions. +// +// It is typically used like this: +// +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// handle, err := typeInformer.Informer().AddEventHandler(...) +// if err != nil { +// return fmt.Errorf("register event handler: %v", err) +// } +// defer typeInformer.Informer().RemoveEventHandler(handle) // Avoids leaking goroutines. +// factory.StartWithContext(ctx) // Start processing these informers. +// synced := factory.WaitForCacheSyncWithContext(ctx) +// if err := synced.AsError(); err != nil { +// return err +// } +// for v := range synced { +// // Only if desired log some information similar to this. +// fmt.Fprintf(os.Stdout, "cache synced: %s", v) +// } +// +// // Also make sure that all of the initial cache events have been delivered. +// if !WaitFor(ctx, "event handler sync", handle.HasSyncedChecker()) { +// // Must have failed because of context. +// return fmt.Errorf("sync event handler: %w", context.Cause(ctx)) +// } +// +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.StartWithContext(ctx) type SharedInformerFactory interface { internalinterfaces.SharedInformerFactory - ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // Start initializes all requested informers. They are handled in goroutines + // which run until the stop channel gets closed. + // Warning: Start does not block. When run in a go-routine, it will race with a later WaitForCacheSync. + // + // Contextual logging: StartWithContext should be used instead of Start in code which supports contextual logging. + Start(stopCh <-chan struct{}) + + // StartWithContext initializes all requested informers. They are handled in goroutines + // which run until the context gets canceled. + // Warning: StartWithContext does not block. When run in a go-routine, it will race with a later WaitForCacheSync. + StartWithContext(ctx context.Context) + + // Shutdown marks a factory as shutting down. At that point no new + // informers can be started anymore and Start will return without + // doing anything. + // + // In addition, Shutdown blocks until all goroutines have terminated. For that + // to happen, the close channel(s) that they were started with must be closed, + // either before Shutdown gets called or while it is waiting. + // + // Shutdown may be called multiple times, even concurrently. All such calls will + // block until all goroutines have terminated. + Shutdown() + + // WaitForCacheSync blocks until all started informers' caches were synced + // or the stop channel gets closed. + // + // Contextual logging: WaitForCacheSync should be used instead of WaitForCacheSync in code which supports contextual logging. It also returns a more useful result. WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + // WaitForCacheSyncWithContext blocks until all started informers' caches were synced + // or the context gets canceled. + WaitForCacheSyncWithContext(ctx context.Context) cache.SyncResult + + // ForResource gives generic access to a shared informer of the matching type. + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // InformerFor returns the SharedIndexInformer for obj using an internal + // client. + InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer + Acid() acidzalando.Interface + Zalando() zalandoorg.Interface } func (f *sharedInformerFactory) Acid() acidzalando.Interface { return acidzalando.New(f, f.namespace, f.tweakListOptions) } + +func (f *sharedInformerFactory) Zalando() zalandoorg.Interface { + return zalandoorg.New(f, f.namespace, f.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 5fd693558..f5953bde6 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,9 +25,10 @@ SOFTWARE. package externalversions import ( - "fmt" + fmt "fmt" v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) @@ -59,11 +60,17 @@ func (f *genericInformer) Lister() cache.GenericLister { func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=acid.zalan.do, Version=v1 + case v1.SchemeGroupVersion.WithResource("operatorconfigurations"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().OperatorConfigurations().Informer()}, nil case v1.SchemeGroupVersion.WithResource("postgresteams"): return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().PostgresTeams().Informer()}, nil case v1.SchemeGroupVersion.WithResource("postgresqls"): return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().Postgresqls().Informer()}, nil + // Group=zalando.org, Version=v1 + case zalandoorgv1.SchemeGroupVersion.WithResource("fabriceventstreams"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Zalando().V1().FabricEventStreams().Informer()}, nil + } return nil, fmt.Errorf("no informer found for %v", resource) diff --git a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go index 6d1b334bf..b8c3e7e51 100644 --- a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -40,7 +40,26 @@ type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexI type SharedInformerFactory interface { Start(stopCh <-chan struct{}) InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer + InformerName() *cache.InformerName } // TweakListOptionsFunc is a function that transforms a v1.ListOptions. type TweakListOptionsFunc func(*v1.ListOptions) + +// InformerOptions holds the options for creating an informer. +type InformerOptions struct { + // ResyncPeriod is the resync period for this informer. + // If not set, defaults to 0 (no resync). + ResyncPeriod time.Duration + + // Indexers are the indexers for this informer. + Indexers cache.Indexers + + // InformerName is used to uniquely identify this informer for metrics. + // If not set, metrics will not be published for this informer. + // Use cache.NewInformerName() to create an InformerName at startup. + InformerName *cache.InformerName + + // TweakListOptions is an optional function to modify the list options. + TweakListOptions TweakListOptionsFunc +} diff --git a/pkg/generated/informers/externalversions/zalando.org/interface.go b/pkg/generated/informers/externalversions/zalando.org/interface.go new file mode 100644 index 000000000..a05b7ea7d --- /dev/null +++ b/pkg/generated/informers/externalversions/zalando.org/interface.go @@ -0,0 +1,52 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package zalando + +import ( + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + v1 "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org/v1" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1 provides access to shared informers for resources in V1. + V1() v1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1 returns a new v1.Interface. +func (g *group) V1() v1.Interface { + return v1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go b/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go new file mode 100644 index 000000000..675058f80 --- /dev/null +++ b/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go @@ -0,0 +1,122 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + apiszalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/generated/listers/zalando.org/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// FabricEventStreamInformer provides access to a shared informer and lister for +// FabricEventStreams. +type FabricEventStreamInformer interface { + Informer() cache.SharedIndexInformer + Lister() zalandoorgv1.FabricEventStreamLister +} + +type fabricEventStreamInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewFabricEventStreamInformer constructs a new informer for FabricEventStream type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFabricEventStreamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFabricEventStreamInformerWithOptions(client, namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: indexers}) +} + +// NewFilteredFabricEventStreamInformer constructs a new informer for FabricEventStream type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredFabricEventStreamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return NewFabricEventStreamInformerWithOptions(client, namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: indexers, TweakListOptions: tweakListOptions}) +} + +// NewFabricEventStreamInformerWithOptions constructs a new informer for FabricEventStream type with additional options. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFabricEventStreamInformerWithOptions(client versioned.Interface, namespace string, options internalinterfaces.InformerOptions) cache.SharedIndexInformer { + gvr := schema.GroupVersionResource{Group: "zalando.org", Version: "v1", Resource: "fabriceventstreams"} + identifier := options.InformerName.WithResource(gvr) + tweakListOptions := options.TweakListOptions + return cache.NewSharedIndexInformerWithOptions( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.ZalandoV1().FabricEventStreams(namespace).List(context.Background(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.ZalandoV1().FabricEventStreams(namespace).Watch(context.Background(), opts) + }, + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.ZalandoV1().FabricEventStreams(namespace).List(ctx, opts) + }, + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&opts) + } + return client.ZalandoV1().FabricEventStreams(namespace).Watch(ctx, opts) + }, + }, client), + &apiszalandoorgv1.FabricEventStream{}, + cache.SharedIndexInformerOptions{ + ResyncPeriod: options.ResyncPeriod, + Indexers: options.Indexers, + Identifier: identifier, + }, + ) +} + +func (f *fabricEventStreamInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFabricEventStreamInformerWithOptions(client, f.namespace, internalinterfaces.InformerOptions{ResyncPeriod: resyncPeriod, Indexers: cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, InformerName: f.factory.InformerName(), TweakListOptions: f.tweakListOptions}) +} + +func (f *fabricEventStreamInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiszalandoorgv1.FabricEventStream{}, f.defaultInformer) +} + +func (f *fabricEventStreamInformer) Lister() zalandoorgv1.FabricEventStreamLister { + return zalandoorgv1.NewFabricEventStreamLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/zalando.org/v1/interface.go b/pkg/generated/informers/externalversions/zalando.org/v1/interface.go new file mode 100644 index 000000000..825e5f4a4 --- /dev/null +++ b/pkg/generated/informers/externalversions/zalando.org/v1/interface.go @@ -0,0 +1,51 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // FabricEventStreams returns a FabricEventStreamInformer. + FabricEventStreams() FabricEventStreamInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// FabricEventStreams returns a FabricEventStreamInformer. +func (v *version) FabricEventStreams() FabricEventStreamInformer { + return &fabricEventStreamInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go index cc3e578b2..f9b4c79e7 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go +++ b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,14 @@ SOFTWARE. package v1 +// OperatorConfigurationListerExpansion allows custom methods to be added to +// OperatorConfigurationLister. +type OperatorConfigurationListerExpansion interface{} + +// OperatorConfigurationNamespaceListerExpansion allows custom methods to be added to +// OperatorConfigurationNamespaceLister. +type OperatorConfigurationNamespaceListerExpansion interface{} + // PostgresTeamListerExpansion allows custom methods to be added to // PostgresTeamLister. type PostgresTeamListerExpansion interface{} diff --git a/pkg/generated/listers/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/listers/acid.zalan.do/v1/operatorconfiguration.go new file mode 100644 index 000000000..c00599718 --- /dev/null +++ b/pkg/generated/listers/acid.zalan.do/v1/operatorconfiguration.go @@ -0,0 +1,76 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// OperatorConfigurationLister helps list OperatorConfigurations. +// All objects returned here must be treated as read-only. +type OperatorConfigurationLister interface { + // List lists all OperatorConfigurations in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*acidzalandov1.OperatorConfiguration, err error) + // OperatorConfigurations returns an object that can list and get OperatorConfigurations. + OperatorConfigurations(namespace string) OperatorConfigurationNamespaceLister + OperatorConfigurationListerExpansion +} + +// operatorConfigurationLister implements the OperatorConfigurationLister interface. +type operatorConfigurationLister struct { + listers.ResourceIndexer[*acidzalandov1.OperatorConfiguration] +} + +// NewOperatorConfigurationLister returns a new OperatorConfigurationLister. +func NewOperatorConfigurationLister(indexer cache.Indexer) OperatorConfigurationLister { + return &operatorConfigurationLister{listers.New[*acidzalandov1.OperatorConfiguration](indexer, acidzalandov1.Resource("operatorconfiguration"))} +} + +// OperatorConfigurations returns an object that can list and get OperatorConfigurations. +func (s *operatorConfigurationLister) OperatorConfigurations(namespace string) OperatorConfigurationNamespaceLister { + return operatorConfigurationNamespaceLister{listers.NewNamespaced[*acidzalandov1.OperatorConfiguration](s.ResourceIndexer, namespace)} +} + +// OperatorConfigurationNamespaceLister helps list and get OperatorConfigurations. +// All objects returned here must be treated as read-only. +type OperatorConfigurationNamespaceLister interface { + // List lists all OperatorConfigurations in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*acidzalandov1.OperatorConfiguration, err error) + // Get retrieves the OperatorConfiguration from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*acidzalandov1.OperatorConfiguration, error) + OperatorConfigurationNamespaceListerExpansion +} + +// operatorConfigurationNamespaceLister implements the OperatorConfigurationNamespaceLister +// interface. +type operatorConfigurationNamespaceLister struct { + listers.ResourceIndexer[*acidzalandov1.OperatorConfiguration] +} diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go index d2258bd01..b2fe09749 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,10 +25,10 @@ SOFTWARE. package v1 import ( - v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/cache" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" ) // PostgresqlLister helps list Postgresqls. @@ -36,7 +36,7 @@ import ( type PostgresqlLister interface { // List lists all Postgresqls in the indexer. // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.Postgresql, err error) + List(selector labels.Selector) (ret []*acidzalandov1.Postgresql, err error) // Postgresqls returns an object that can list and get Postgresqls. Postgresqls(namespace string) PostgresqlNamespaceLister PostgresqlListerExpansion @@ -44,25 +44,17 @@ type PostgresqlLister interface { // postgresqlLister implements the PostgresqlLister interface. type postgresqlLister struct { - indexer cache.Indexer + listers.ResourceIndexer[*acidzalandov1.Postgresql] } // NewPostgresqlLister returns a new PostgresqlLister. func NewPostgresqlLister(indexer cache.Indexer) PostgresqlLister { - return &postgresqlLister{indexer: indexer} -} - -// List lists all Postgresqls in the indexer. -func (s *postgresqlLister) List(selector labels.Selector) (ret []*v1.Postgresql, err error) { - err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v1.Postgresql)) - }) - return ret, err + return &postgresqlLister{listers.New[*acidzalandov1.Postgresql](indexer, acidzalandov1.Resource("postgresql"))} } // Postgresqls returns an object that can list and get Postgresqls. func (s *postgresqlLister) Postgresqls(namespace string) PostgresqlNamespaceLister { - return postgresqlNamespaceLister{indexer: s.indexer, namespace: namespace} + return postgresqlNamespaceLister{listers.NewNamespaced[*acidzalandov1.Postgresql](s.ResourceIndexer, namespace)} } // PostgresqlNamespaceLister helps list and get Postgresqls. @@ -70,36 +62,15 @@ func (s *postgresqlLister) Postgresqls(namespace string) PostgresqlNamespaceList type PostgresqlNamespaceLister interface { // List lists all Postgresqls in the indexer for a given namespace. // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.Postgresql, err error) + List(selector labels.Selector) (ret []*acidzalandov1.Postgresql, err error) // Get retrieves the Postgresql from the indexer for a given namespace and name. // Objects returned here must be treated as read-only. - Get(name string) (*v1.Postgresql, error) + Get(name string) (*acidzalandov1.Postgresql, error) PostgresqlNamespaceListerExpansion } // postgresqlNamespaceLister implements the PostgresqlNamespaceLister // interface. type postgresqlNamespaceLister struct { - indexer cache.Indexer - namespace string -} - -// List lists all Postgresqls in the indexer for a given namespace. -func (s postgresqlNamespaceLister) List(selector labels.Selector) (ret []*v1.Postgresql, err error) { - err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v1.Postgresql)) - }) - return ret, err -} - -// Get retrieves the Postgresql from the indexer for a given namespace and name. -func (s postgresqlNamespaceLister) Get(name string) (*v1.Postgresql, error) { - obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.NewNotFound(v1.Resource("postgresql"), name) - } - return obj.(*v1.Postgresql), nil + listers.ResourceIndexer[*acidzalandov1.Postgresql] } diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go index 38073e92d..74fcc81d0 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2026 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,10 +25,10 @@ SOFTWARE. package v1 import ( - v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/cache" + acidzalandov1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" ) // PostgresTeamLister helps list PostgresTeams. @@ -36,7 +36,7 @@ import ( type PostgresTeamLister interface { // List lists all PostgresTeams in the indexer. // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.PostgresTeam, err error) + List(selector labels.Selector) (ret []*acidzalandov1.PostgresTeam, err error) // PostgresTeams returns an object that can list and get PostgresTeams. PostgresTeams(namespace string) PostgresTeamNamespaceLister PostgresTeamListerExpansion @@ -44,25 +44,17 @@ type PostgresTeamLister interface { // postgresTeamLister implements the PostgresTeamLister interface. type postgresTeamLister struct { - indexer cache.Indexer + listers.ResourceIndexer[*acidzalandov1.PostgresTeam] } // NewPostgresTeamLister returns a new PostgresTeamLister. func NewPostgresTeamLister(indexer cache.Indexer) PostgresTeamLister { - return &postgresTeamLister{indexer: indexer} -} - -// List lists all PostgresTeams in the indexer. -func (s *postgresTeamLister) List(selector labels.Selector) (ret []*v1.PostgresTeam, err error) { - err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v1.PostgresTeam)) - }) - return ret, err + return &postgresTeamLister{listers.New[*acidzalandov1.PostgresTeam](indexer, acidzalandov1.Resource("postgresteam"))} } // PostgresTeams returns an object that can list and get PostgresTeams. func (s *postgresTeamLister) PostgresTeams(namespace string) PostgresTeamNamespaceLister { - return postgresTeamNamespaceLister{indexer: s.indexer, namespace: namespace} + return postgresTeamNamespaceLister{listers.NewNamespaced[*acidzalandov1.PostgresTeam](s.ResourceIndexer, namespace)} } // PostgresTeamNamespaceLister helps list and get PostgresTeams. @@ -70,36 +62,15 @@ func (s *postgresTeamLister) PostgresTeams(namespace string) PostgresTeamNamespa type PostgresTeamNamespaceLister interface { // List lists all PostgresTeams in the indexer for a given namespace. // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1.PostgresTeam, err error) + List(selector labels.Selector) (ret []*acidzalandov1.PostgresTeam, err error) // Get retrieves the PostgresTeam from the indexer for a given namespace and name. // Objects returned here must be treated as read-only. - Get(name string) (*v1.PostgresTeam, error) + Get(name string) (*acidzalandov1.PostgresTeam, error) PostgresTeamNamespaceListerExpansion } // postgresTeamNamespaceLister implements the PostgresTeamNamespaceLister // interface. type postgresTeamNamespaceLister struct { - indexer cache.Indexer - namespace string -} - -// List lists all PostgresTeams in the indexer for a given namespace. -func (s postgresTeamNamespaceLister) List(selector labels.Selector) (ret []*v1.PostgresTeam, err error) { - err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v1.PostgresTeam)) - }) - return ret, err -} - -// Get retrieves the PostgresTeam from the indexer for a given namespace and name. -func (s postgresTeamNamespaceLister) Get(name string) (*v1.PostgresTeam, error) { - obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.NewNotFound(v1.Resource("postgresteam"), name) - } - return obj.(*v1.PostgresTeam), nil + listers.ResourceIndexer[*acidzalandov1.PostgresTeam] } diff --git a/pkg/generated/listers/zalando.org/v1/expansion_generated.go b/pkg/generated/listers/zalando.org/v1/expansion_generated.go new file mode 100644 index 000000000..95dfc74cf --- /dev/null +++ b/pkg/generated/listers/zalando.org/v1/expansion_generated.go @@ -0,0 +1,33 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +// FabricEventStreamListerExpansion allows custom methods to be added to +// FabricEventStreamLister. +type FabricEventStreamListerExpansion interface{} + +// FabricEventStreamNamespaceListerExpansion allows custom methods to be added to +// FabricEventStreamNamespaceLister. +type FabricEventStreamNamespaceListerExpansion interface{} diff --git a/pkg/generated/listers/zalando.org/v1/fabriceventstream.go b/pkg/generated/listers/zalando.org/v1/fabriceventstream.go new file mode 100644 index 000000000..25e57e56e --- /dev/null +++ b/pkg/generated/listers/zalando.org/v1/fabriceventstream.go @@ -0,0 +1,76 @@ +/* +Copyright 2026 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// FabricEventStreamLister helps list FabricEventStreams. +// All objects returned here must be treated as read-only. +type FabricEventStreamLister interface { + // List lists all FabricEventStreams in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*zalandoorgv1.FabricEventStream, err error) + // FabricEventStreams returns an object that can list and get FabricEventStreams. + FabricEventStreams(namespace string) FabricEventStreamNamespaceLister + FabricEventStreamListerExpansion +} + +// fabricEventStreamLister implements the FabricEventStreamLister interface. +type fabricEventStreamLister struct { + listers.ResourceIndexer[*zalandoorgv1.FabricEventStream] +} + +// NewFabricEventStreamLister returns a new FabricEventStreamLister. +func NewFabricEventStreamLister(indexer cache.Indexer) FabricEventStreamLister { + return &fabricEventStreamLister{listers.New[*zalandoorgv1.FabricEventStream](indexer, zalandoorgv1.Resource("fabriceventstream"))} +} + +// FabricEventStreams returns an object that can list and get FabricEventStreams. +func (s *fabricEventStreamLister) FabricEventStreams(namespace string) FabricEventStreamNamespaceLister { + return fabricEventStreamNamespaceLister{listers.NewNamespaced[*zalandoorgv1.FabricEventStream](s.ResourceIndexer, namespace)} +} + +// FabricEventStreamNamespaceLister helps list and get FabricEventStreams. +// All objects returned here must be treated as read-only. +type FabricEventStreamNamespaceLister interface { + // List lists all FabricEventStreams in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*zalandoorgv1.FabricEventStream, err error) + // Get retrieves the FabricEventStream from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*zalandoorgv1.FabricEventStream, error) + FabricEventStreamNamespaceListerExpansion +} + +// fabricEventStreamNamespaceLister implements the FabricEventStreamNamespaceLister +// interface. +type fabricEventStreamNamespaceLister struct { + listers.ResourceIndexer[*zalandoorgv1.FabricEventStream] +} diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 5d7794b42..3f4abb70f 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -4,7 +4,6 @@ import ( "database/sql" "encoding/json" "fmt" - "io/ioutil" "log" "os" "strings" @@ -15,8 +14,39 @@ import ( "k8s.io/client-go/rest" ) -// NamespacedName describes the namespace/name pairs used in Kubernetes names. -type NamespacedName types.NamespacedName +// NamespacedName comprises a resource name, with a mandatory namespace, +// rendered as "/". Being a type captures intent and +// helps make sure that UIDs, namespaced names and non-namespaced names +// do not get conflated in code. For most use cases, namespace and name +// will already have been format validated at the API entry point, so we +// don't do that here. Where that's not the case (e.g. in testing), +// consider using NamespacedNameOrDie() in testing.go in this package. +// +// from: https://github.com/kubernetes/apimachinery/blob/master/pkg/types/namespacedname.go +type NamespacedName struct { + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` +} + +const ( + Separator = '/' +) + +// String returns the general purpose string representation +func (n NamespacedName) String() string { + return n.Namespace + string(Separator) + n.Name +} + +// MarshalLog emits a struct containing required key/value pair +func (n NamespacedName) MarshalLog() interface{} { + return struct { + Name string `json:"name"` + Namespace string `json:"namespace,omitempty"` + }{ + Name: n.Name, + Namespace: n.Namespace, + } +} const fileWithNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" @@ -32,7 +62,8 @@ const ( RoleOriginTeamsAPI RoleOriginSystem RoleOriginBootstrap - RoleConnectionPooler + RoleOriginConnectionPooler + RoleOriginStream ) type syncUserOperation int @@ -49,12 +80,16 @@ const ( type PgUser struct { Origin RoleOrigin `yaml:"-"` Name string `yaml:"-"` + Namespace string `yaml:"-"` Password string `yaml:"-"` Flags []string `yaml:"user_flags"` MemberOf []string `yaml:"inrole"` Parameters map[string]string `yaml:"db_parameters"` AdminRole string `yaml:"admin_role"` + IsDbOwner bool `yaml:"is_db_owner"` Deleted bool `yaml:"deleted"` + Rotated bool `yaml:"rotated"` + Degraded bool `yaml:"degraded"` } func (user *PgUser) Valid() bool { @@ -116,17 +151,17 @@ type ControllerConfig struct { CRDReadyWaitTimeout time.Duration ConfigMapName NamespacedName Namespace string + IgnoredAnnotations []string EnableJsonLogging bool + + KubeQPS int + KubeBurst int } // cached value for the GetOperatorNamespace var operatorNamespace string -func (n NamespacedName) String() string { - return types.NamespacedName(n).String() -} - // MarshalJSON defines marshaling rule for the namespaced name type. func (n NamespacedName) MarshalJSON() ([]byte, error) { return []byte("\"" + n.String() + "\""), nil @@ -191,7 +226,7 @@ func (r RoleOrigin) String() string { return "system role" case RoleOriginBootstrap: return "bootstrapped role" - case RoleConnectionPooler: + case RoleOriginConnectionPooler: return "connection pooler role" default: panic(fmt.Sprintf("bogus role origin value %d", r)) @@ -205,7 +240,7 @@ func GetOperatorNamespace() string { if namespaceFromEnvironment := os.Getenv("OPERATOR_NAMESPACE"); namespaceFromEnvironment != "" { return namespaceFromEnvironment } - operatorNamespaceBytes, err := ioutil.ReadFile(fileWithNamespace) + operatorNamespaceBytes, err := os.ReadFile(fileWithNamespace) if err != nil { log.Fatalf("Unable to detect operator namespace from within its pod due to: %v", err) } diff --git a/pkg/teams/postgres_team.go b/pkg/teams/postgres_team.go index 6e9a825e5..b682585ab 100644 --- a/pkg/teams/postgres_team.go +++ b/pkg/teams/postgres_team.go @@ -44,6 +44,14 @@ func (ths *teamHashSet) toMap() map[string][]string { return newTeamMap } +func mapStringSliceToStringSliceMap[T ~[]string](input map[string]T) map[string][]string { + output := make(map[string][]string) + for k, v := range input { + output[k] = []string(v) + } + return output +} + func (ths *teamHashSet) mergeCrdMap(crdTeamMap map[string][]string) { for t, at := range crdTeamMap { ths.add(t, at) @@ -66,13 +74,20 @@ func (ptm *PostgresTeamMap) fetchAdditionalTeams(team string, superuserTeams boo teams = (*ptm)[team].AdditionalTeams } if transitive { - exclude = append(exclude, team) for _, additionalTeam := range teams { if !(util.SliceContains(exclude, additionalTeam)) { + // remember to not check team and additionalTeam again + exclude = append(exclude, additionalTeam) transitiveTeams := (*ptm).fetchAdditionalTeams(additionalTeam, superuserTeams, transitive, exclude) for _, transitiveTeam := range transitiveTeams { - if !(util.SliceContains(exclude, transitiveTeam)) && !(util.SliceContains(teams, transitiveTeam)) { - teams = append(teams, transitiveTeam) + if !(util.SliceContains(exclude, transitiveTeam)) { + // remember to not check transitive team again in case + // it is one of the next additional teams of the outer loop + exclude = append(exclude, transitiveTeam) + if !(util.SliceContains(teams, transitiveTeam)) { + // found a new transitive additional team + teams = append(teams, transitiveTeam) + } } } } @@ -84,12 +99,12 @@ func (ptm *PostgresTeamMap) fetchAdditionalTeams(team string, superuserTeams boo // GetAdditionalTeams function to retrieve list of additional teams func (ptm *PostgresTeamMap) GetAdditionalTeams(team string, transitive bool) []string { - return ptm.fetchAdditionalTeams(team, false, transitive, []string{}) + return ptm.fetchAdditionalTeams(team, false, transitive, []string{team}) } // GetAdditionalSuperuserTeams function to retrieve list of additional superuser teams func (ptm *PostgresTeamMap) GetAdditionalSuperuserTeams(team string, transitive bool) []string { - return ptm.fetchAdditionalTeams(team, true, transitive, []string{}) + return ptm.fetchAdditionalTeams(team, true, transitive, []string{team}) } // Load function to import data from PostgresTeam CRD @@ -103,9 +118,9 @@ func (ptm *PostgresTeamMap) Load(pgTeams *acidv1.PostgresTeamList) { teamIDs := make(map[string]struct{}) for _, pgTeam := range pgTeams.Items { - superuserTeamSet.mergeCrdMap(pgTeam.Spec.AdditionalSuperuserTeams) - teamSet.mergeCrdMap(pgTeam.Spec.AdditionalTeams) - teamMemberSet.mergeCrdMap(pgTeam.Spec.AdditionalMembers) + superuserTeamSet.mergeCrdMap(mapStringSliceToStringSliceMap(pgTeam.Spec.AdditionalSuperuserTeams)) + teamSet.mergeCrdMap(mapStringSliceToStringSliceMap(pgTeam.Spec.AdditionalTeams)) + teamMemberSet.mergeCrdMap(mapStringSliceToStringSliceMap(pgTeam.Spec.AdditionalMembers)) } fetchTeams(&teamIDs, superuserTeamSet) fetchTeams(&teamIDs, teamSet) diff --git a/pkg/teams/postgres_team_test.go b/pkg/teams/postgres_team_test.go index dec020c7d..fe45ade05 100644 --- a/pkg/teams/postgres_team_test.go +++ b/pkg/teams/postgres_team_test.go @@ -24,9 +24,9 @@ var ( Name: "teamAB", }, Spec: acidv1.PostgresTeamSpec{ - AdditionalSuperuserTeams: map[string][]string{"teamA": []string{"teamB", "team24x7"}, "teamB": []string{"teamA", "teamC", "team24x7"}}, - AdditionalTeams: map[string][]string{"teamA": []string{"teamC"}, "teamB": []string{}}, - AdditionalMembers: map[string][]string{"team24x7": []string{"optimusprime"}, "teamB": []string{"drno"}}, + AdditionalSuperuserTeams: map[string]acidv1.SuperUserTeams{"teamA": []string{"teamB", "team24x7"}, "teamB": []string{"teamA", "teamC", "team24x7"}}, + AdditionalTeams: map[string]acidv1.Teams{"teamA": []string{"teamC"}, "teamB": []string{}}, + AdditionalMembers: map[string]acidv1.Users{"team24x7": []string{"optimusprime"}, "teamB": []string{"drno"}}, }, }, { TypeMeta: metav1.TypeMeta{ @@ -37,9 +37,23 @@ var ( Name: "teamC", }, Spec: acidv1.PostgresTeamSpec{ - AdditionalSuperuserTeams: map[string][]string{"teamC": []string{"team24x7"}}, - AdditionalTeams: map[string][]string{"teamA": []string{"teamC"}, "teamC": []string{"teamA", "teamB", "acid"}}, - AdditionalMembers: map[string][]string{"acid": []string{"batman"}}, + AdditionalSuperuserTeams: map[string]acidv1.SuperUserTeams{"teamC": []string{"team24x7"}}, + AdditionalTeams: map[string]acidv1.Teams{"teamA": []string{"teamC"}, "teamC": []string{"teamA", "teamB", "acid"}}, + AdditionalMembers: map[string]acidv1.Users{"acid": []string{"batman"}}, + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "PostgresTeam", + APIVersion: "acid.zalan.do/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "teamD", + }, + Spec: acidv1.PostgresTeamSpec{ + AdditionalSuperuserTeams: map[string]acidv1.SuperUserTeams{}, + AdditionalTeams: map[string]acidv1.Teams{"teamA": []string{"teamD"}, "teamC": []string{"teamD"}, "teamD": []string{"teamA", "teamB", "teamC"}}, + AdditionalMembers: map[string]acidv1.Users{"acid": []string{"batman"}}, }, }, }, @@ -47,7 +61,7 @@ var ( pgTeamMap = PostgresTeamMap{ "teamA": { AdditionalSuperuserTeams: []string{"teamB", "team24x7"}, - AdditionalTeams: []string{"teamC"}, + AdditionalTeams: []string{"teamC", "teamD"}, AdditionalMembers: []string{}, }, "teamB": { @@ -57,7 +71,12 @@ var ( }, "teamC": { AdditionalSuperuserTeams: []string{"team24x7"}, - AdditionalTeams: []string{"teamA", "teamB", "acid"}, + AdditionalTeams: []string{"teamA", "teamB", "teamD", "acid"}, + AdditionalMembers: []string{}, + }, + "teamD": { + AdditionalSuperuserTeams: []string{}, + AdditionalTeams: []string{"teamA", "teamB", "teamC"}, AdditionalMembers: []string{}, }, "team24x7": { @@ -119,14 +138,14 @@ func TestGetAdditionalTeams(t *testing.T) { "Check that additional teams are returned", "teamA", false, - []string{"teamC"}, + []string{"teamC", "teamD"}, "GetAdditionalTeams returns wrong list", }, { "Check that additional teams are returned incl. transitive teams", "teamA", true, - []string{"teamC", "teamB", "acid"}, + []string{"teamC", "teamD", "teamB", "acid"}, "GetAdditionalTeams returns wrong list", }, { diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 52530fb12..43fa37a33 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -14,20 +14,23 @@ import ( // CRD describes CustomResourceDefinition specific configuration parameters type CRD struct { - ReadyWaitInterval time.Duration `name:"ready_wait_interval" default:"4s"` - ReadyWaitTimeout time.Duration `name:"ready_wait_timeout" default:"30s"` - ResyncPeriod time.Duration `name:"resync_period" default:"30m"` - RepairPeriod time.Duration `name:"repair_period" default:"5m"` - EnableCRDValidation *bool `name:"enable_crd_validation" default:"true"` + ReadyWaitInterval time.Duration `name:"ready_wait_interval" default:"4s"` + ReadyWaitTimeout time.Duration `name:"ready_wait_timeout" default:"30s"` + ResyncPeriod time.Duration `name:"resync_period" default:"30m"` + RepairPeriod time.Duration `name:"repair_period" default:"5m"` + EnableCRDRegistration *bool `name:"enable_crd_registration" default:"true"` + CRDCategories []string `name:"crd_categories" default:"all"` } // Resources describes kubernetes resource specific configuration parameters type Resources struct { + EnableOwnerReferences *bool `name:"enable_owner_references" default:"false"` ResourceCheckInterval time.Duration `name:"resource_check_interval" default:"3s"` ResourceCheckTimeout time.Duration `name:"resource_check_timeout" default:"10m"` PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"` PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"` PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` + LivenessProbe *v1.Probe `name:"-"` SpiloRunAsUser *int64 `name:"spilo_runasuser"` SpiloRunAsGroup *int64 `name:"spilo_runasgroup"` SpiloFSGroup *int64 `name:"spilo_fsgroup"` @@ -40,39 +43,46 @@ type Resources struct { InheritedLabels []string `name:"inherited_labels" default:""` InheritedAnnotations []string `name:"inherited_annotations" default:""` DownscalerAnnotations []string `name:"downscaler_annotations"` + IgnoredAnnotations []string `name:"ignored_annotations"` ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` DeleteAnnotationDateKey string `name:"delete_annotation_date_key"` DeleteAnnotationNameKey string `name:"delete_annotation_name_key"` PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` PodToleration map[string]string `name:"toleration" default:""` - DefaultCPURequest string `name:"default_cpu_request" default:"100m"` - DefaultMemoryRequest string `name:"default_memory_request" default:"100Mi"` - DefaultCPULimit string `name:"default_cpu_limit" default:"1"` - DefaultMemoryLimit string `name:"default_memory_limit" default:"500Mi"` - MinCPULimit string `name:"min_cpu_limit" default:"250m"` - MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"` + DefaultCPURequest string `name:"default_cpu_request"` + DefaultMemoryRequest string `name:"default_memory_request"` + DefaultCPULimit string `name:"default_cpu_limit"` + DefaultMemoryLimit string `name:"default_memory_limit"` + MinCPULimit string `name:"min_cpu_limit"` + MinMemoryLimit string `name:"min_memory_limit"` + MaxCPURequest string `name:"max_cpu_request"` + MaxMemoryRequest string `name:"max_memory_request"` PodEnvironmentConfigMap spec.NamespacedName `name:"pod_environment_configmap"` PodEnvironmentSecret string `name:"pod_environment_secret"` NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""` - MaxInstances int32 `name:"max_instances" default:"-1"` - MinInstances int32 `name:"min_instances" default:"-1"` + NodeReadinessLabelMerge string `name:"node_readiness_label_merge" default:"OR"` ShmVolume *bool `name:"enable_shm_volume" default:"true"` + + MaxInstances int32 `name:"max_instances" default:"-1"` + MinInstances int32 `name:"min_instances" default:"-1"` + IgnoreInstanceLimitsAnnotationKey string `name:"ignore_instance_limits_annotation_key"` + IgnoreResourcesLimitsAnnotationKey string `name:"ignore_resources_limits_annotation_key"` } type InfrastructureRole struct { // Name of a secret which describes the role, and optionally name of a // configmap with an extra information - SecretName spec.NamespacedName + SecretName spec.NamespacedName `json:"secretname,omitempty"` - UserKey string - PasswordKey string - RoleKey string + UserKey string `json:"userkey,omitempty"` + PasswordKey string `json:"passwordkey,omitempty"` + RoleKey string `json:"rolekey,omitempty"` - DefaultUserValue string - DefaultRoleValue string + DefaultUserValue string `json:"defaultuservalue,omitempty"` + DefaultRoleValue string `json:"defaultrolevalue,omitempty"` // This field point out the detailed yaml definition of the role, if exists - Details string + Details string `json:"details,omitempty"` // Specify if a secret contains multiple fields in the following format: // @@ -83,7 +93,7 @@ type InfrastructureRole struct { // If it does, Name/Password/Role are interpreted not as unique field // names, but as a template. - Template bool + Template bool `json:"template,omitempty"` } // Auth describes authentication specific configuration parameters @@ -98,6 +108,10 @@ type Auth struct { InfrastructureRolesDefs string `name:"infrastructure_roles_secrets"` SuperUsername string `name:"super_username" default:"postgres"` ReplicationUsername string `name:"replication_username" default:"standby"` + AdditionalOwnerRoles []string `name:"additional_owner_roles" default:""` + EnablePasswordRotation bool `name:"enable_password_rotation" default:"false"` + PasswordRotationInterval uint32 `name:"password_rotation_interval" default:"90"` + PasswordRotationUserRetention uint32 `name:"password_rotation_user_retention" default:"180"` } // Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping: @@ -114,16 +128,29 @@ type Scalyr struct { // LogicalBackup defines configuration for logical backup type LogicalBackup struct { LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` - LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup:v1.6.3"` + LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"ghcr.io/zalando/postgres-operator/logical-backup:v1.15.1"` LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"` + LogicalBackupAzureStorageAccountName string `name:"logical_backup_azure_storage_account_name" default:""` + LogicalBackupAzureStorageContainer string `name:"logical_backup_azure_storage_container" default:""` + LogicalBackupAzureStorageAccountKey string `name:"logical_backup_azure_storage_account_key" default:""` LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` + LogicalBackupS3BucketPrefix string `name:"logical_backup_s3_bucket_prefix" default:"spilo"` LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` LogicalBackupS3Endpoint string `name:"logical_backup_s3_endpoint" default:""` LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` + LogicalBackupS3RetentionTime string `name:"logical_backup_s3_retention_time" default:""` LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""` LogicalBackupJobPrefix string `name:"logical_backup_job_prefix" default:"logical-backup-"` + LogicalBackupCronjobEnvironmentSecret string `name:"logical_backup_cronjob_environment_secret" default:""` + LogicalBackupCPURequest string `name:"logical_backup_cpu_request"` + LogicalBackupMemoryRequest string `name:"logical_backup_memory_request"` + LogicalBackupCPULimit string `name:"logical_backup_cpu_limit"` + LogicalBackupMemoryLimit string `name:"logical_backup_memory_limit"` + LogicalBackupSuccessfulJobsHistoryLimit *int32 `name:"logical_backup_successful_jobs_history_limit" default:"3"` + LogicalBackupFailedJobsHistoryLimit *int32 `name:"logical_backup_failed_jobs_history_limit" default:"3"` + LogicalBackupTTLSecondsAfterFinished *int32 `name:"logical_backup_ttl_seconds_after_finished" default:"86400"` } // Operator options for connection pooler @@ -131,13 +158,13 @@ type ConnectionPooler struct { NumberOfInstances *int32 `name:"connection_pooler_number_of_instances" default:"2"` Schema string `name:"connection_pooler_schema" default:"pooler"` User string `name:"connection_pooler_user" default:"pooler"` - Image string `name:"connection_pooler_image" default:"registry.opensource.zalan.do/acid/pgbouncer"` + Image string `name:"connection_pooler_image" default:"ghcr.io/zalando/postgres-operator/pgbouncer:latest"` Mode string `name:"connection_pooler_mode" default:"transaction"` MaxDBConnections *int32 `name:"connection_pooler_max_db_connections" default:"60"` - ConnectionPoolerDefaultCPURequest string `name:"connection_pooler_default_cpu_request" default:"500m"` - ConnectionPoolerDefaultMemoryRequest string `name:"connection_pooler_default_memory_request" default:"100Mi"` - ConnectionPoolerDefaultCPULimit string `name:"connection_pooler_default_cpu_limit" default:"1"` - ConnectionPoolerDefaultMemoryLimit string `name:"connection_pooler_default_memory_limit" default:"100Mi"` + ConnectionPoolerDefaultCPURequest string `name:"connection_pooler_default_cpu_request"` + ConnectionPoolerDefaultMemoryRequest string `name:"connection_pooler_default_memory_request"` + ConnectionPoolerDefaultCPULimit string `name:"connection_pooler_default_cpu_limit"` + ConnectionPoolerDefaultMemoryLimit string `name:"connection_pooler_default_memory_limit"` } // Config describes operator config @@ -149,69 +176,94 @@ type Config struct { LogicalBackup ConnectionPooler - WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' - KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` - EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-13:2.0-p7"` - SidecarImages map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers - SidecarContainers []v1.Container `name:"sidecars"` - PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` + WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' + KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` + EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS + EnableMaintenanceWindows *bool `name:"enable_maintenance_windows" default:"true"` + MaintenanceWindows []string `name:"maintenance_windows"` + DockerImage string `name:"docker_image" default:"ghcr.io/zalando/spilo-18:4.1-p1"` + SidecarImages map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers + SidecarContainers []v1.Container `name:"sidecars"` + PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` // value of this string must be valid JSON or YAML; see initPodServiceAccount - PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` - PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` - MasterPodMoveTimeout time.Duration `name:"master_pod_move_timeout" default:"20m"` - DbHostedZone string `name:"db_hosted_zone" default:"db.example.com"` - AWSRegion string `name:"aws_region" default:"eu-central-1"` - WALES3Bucket string `name:"wal_s3_bucket"` - LogS3Bucket string `name:"log_s3_bucket"` - KubeIAMRole string `name:"kube_iam_role"` - WALGSBucket string `name:"wal_gs_bucket"` - GCPCredentials string `name:"gcp_credentials"` - AdditionalSecretMount string `name:"additional_secret_mount"` - AdditionalSecretMountPath string `name:"additional_secret_mount_path" default:"/meta/credentials"` - EnableEBSGp3Migration bool `name:"enable_ebs_gp3_migration" default:"false"` - EnableEBSGp3MigrationMaxSize int64 `name:"enable_ebs_gp3_migration_max_size" default:"1000"` - DebugLogging bool `name:"debug_logging" default:"true"` - EnableDBAccess bool `name:"enable_database_access" default:"true"` - EnableTeamsAPI bool `name:"enable_teams_api" default:"true"` - EnableTeamSuperuser bool `name:"enable_team_superuser" default:"false"` - TeamAdminRole string `name:"team_admin_role" default:"admin"` - RoleDeletionSuffix string `name:"role_deletion_suffix" default:"_deleted"` - EnableTeamMemberDeprecation bool `name:"enable_team_member_deprecation" default:"false"` - EnableAdminRoleForUsers bool `name:"enable_admin_role_for_users" default:"true"` - EnablePostgresTeamCRD bool `name:"enable_postgres_team_crd" default:"false"` - EnablePostgresTeamCRDSuperusers bool `name:"enable_postgres_team_crd_superusers" default:"false"` - EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` - EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` - CustomServiceAnnotations map[string]string `name:"custom_service_annotations"` - CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` - EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` - PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` - StorageResizeMode string `name:"storage_resize_mode" default:"pvc"` - EnableLoadBalancer *bool `name:"enable_load_balancer"` // deprecated and kept for backward compatibility - ExternalTrafficPolicy string `name:"external_traffic_policy" default:"Cluster"` - MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` - ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"` - PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"` - EnablePodDisruptionBudget *bool `name:"enable_pod_disruption_budget" default:"true"` - EnableInitContainers *bool `name:"enable_init_containers" default:"true"` - EnableSidecars *bool `name:"enable_sidecars" default:"true"` - Workers uint32 `name:"workers" default:"8"` - APIPort int `name:"api_port" default:"8080"` - RingLogLines int `name:"ring_log_lines" default:"100"` - ClusterHistoryEntries int `name:"cluster_history_entries" default:"1000"` - TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"` - PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` - PodManagementPolicy string `name:"pod_management_policy" default:"ordered_ready"` - ProtectedRoles []string `name:"protected_role_names" default:"admin"` - PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` - SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` - EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` - EnablePgVersionEnvVar bool `name:"enable_pgversion_env_var" default:"true"` - EnableSpiloWalPathCompat bool `name:"enable_spilo_wal_path_compat" default:"false"` - MajorVersionUpgradeMode string `name:"major_version_upgrade_mode" default:"off"` - MinimalMajorVersion string `name:"minimal_major_version" default:"9.5"` - TargetMajorVersion string `name:"target_major_version" default:"13"` + PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` + PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` + MasterPodMoveTimeout time.Duration `name:"master_pod_move_timeout" default:"20m"` + DbHostedZone string `name:"db_hosted_zone" default:"db.example.com"` + AWSRegion string `name:"aws_region" default:"eu-central-1"` + WALES3Bucket string `name:"wal_s3_bucket"` + LogS3Bucket string `name:"log_s3_bucket"` + KubeIAMRole string `name:"kube_iam_role"` + WALGSBucket string `name:"wal_gs_bucket"` + GCPCredentials string `name:"gcp_credentials"` + WALAZStorageAccount string `name:"wal_az_storage_account"` + AdditionalSecretMount string `name:"additional_secret_mount"` + AdditionalSecretMountPath string `name:"additional_secret_mount_path"` + EnableEBSGp3Migration bool `name:"enable_ebs_gp3_migration" default:"false"` + EnableEBSGp3MigrationMaxSize int64 `name:"enable_ebs_gp3_migration_max_size" default:"1000"` + DebugLogging bool `name:"debug_logging" default:"true"` + EnableDBAccess bool `name:"enable_database_access" default:"true"` + EnableTeamsAPI bool `name:"enable_teams_api" default:"true"` + EnableTeamSuperuser bool `name:"enable_team_superuser" default:"false"` + TeamAdminRole string `name:"team_admin_role" default:"admin"` + RoleDeletionSuffix string `name:"role_deletion_suffix" default:"_deleted"` + EnableTeamMemberDeprecation bool `name:"enable_team_member_deprecation" default:"false"` + EnableAdminRoleForUsers bool `name:"enable_admin_role_for_users" default:"true"` + EnablePostgresTeamCRD bool `name:"enable_postgres_team_crd" default:"false"` + EnablePostgresTeamCRDSuperusers bool `name:"enable_postgres_team_crd_superusers" default:"false"` + EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` + EnableMasterPoolerLoadBalancer bool `name:"enable_master_pooler_load_balancer" default:"false"` + EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` + EnableReplicaPoolerLoadBalancer bool `name:"enable_replica_pooler_load_balancer" default:"false"` + EnableMasterNodePort bool `name:"enable_master_node_port" default:"false"` + EnableMasterPoolerNodePort bool `name:"enable_master_pooler_node_port" default:"false"` + EnableReplicaNodePort bool `name:"enable_replica_node_port" default:"false"` + EnableReplicaPoolerNodePort bool `name:"enable_replica_pooler_node_port" default:"false"` + CustomServiceAnnotations map[string]string `name:"custom_service_annotations"` + CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` + EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` + PodAntiAffinityPreferredDuringScheduling bool `name:"pod_antiaffinity_preferred_during_scheduling" default:"false"` + PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` + StorageResizeMode string `name:"storage_resize_mode" default:"pvc"` + EnableLoadBalancer *bool `name:"enable_load_balancer"` // deprecated and kept for backward compatibility + ExternalTrafficPolicy string `name:"external_traffic_policy" default:"Cluster"` + MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{namespace}.{hostedzone}"` + MasterLegacyDNSNameFormat StringTemplate `name:"master_legacy_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` + ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{namespace}.{hostedzone}"` + ReplicaLegacyDNSNameFormat StringTemplate `name:"replica_legacy_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"` + PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"` + PDBMasterLabelSelector *bool `name:"pdb_master_label_selector" default:"true"` + EnablePodDisruptionBudget *bool `name:"enable_pod_disruption_budget" default:"true"` + EnableInitContainers *bool `name:"enable_init_containers" default:"true"` + EnableSidecars *bool `name:"enable_sidecars" default:"true"` + SharePgSocketWithSidecars *bool `name:"share_pgsocket_with_sidecars" default:"false"` + Workers uint32 `name:"workers" default:"8"` + APIPort int `name:"api_port" default:"8080"` + RingLogLines int `name:"ring_log_lines" default:"100"` + ClusterHistoryEntries int `name:"cluster_history_entries" default:"1000"` + TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"` + PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` + PodManagementPolicy string `name:"pod_management_policy" default:"ordered_ready"` + EnableReadinessProbe bool `name:"enable_readiness_probe" default:"false"` + ProtectedRoles []string `name:"protected_role_names" default:"admin,cron_admin"` + PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` + SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` + EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` + EnableCrossNamespaceSecret bool `name:"enable_cross_namespace_secret" default:"false"` + EnableFinalizers *bool `name:"enable_finalizers" default:"false"` + EnablePgVersionEnvVar bool `name:"enable_pgversion_env_var" default:"true"` + EnableSpiloWalPathCompat bool `name:"enable_spilo_wal_path_compat" default:"false"` + EnableTeamIdClusternamePrefix bool `name:"enable_team_id_clustername_prefix" default:"false"` + MajorVersionUpgradeMode string `name:"major_version_upgrade_mode" default:"manual"` + MajorVersionUpgradeTeamAllowList []string `name:"major_version_upgrade_team_allow_list" default:""` + MinimalMajorVersion string `name:"minimal_major_version" default:"14"` + TargetMajorVersion string `name:"target_major_version" default:"18"` + PatroniAPICheckInterval time.Duration `name:"patroni_api_check_interval" default:"1s"` + PatroniAPICheckTimeout time.Duration `name:"patroni_api_check_timeout" default:"5s"` + EnablePatroniFailsafeMode *bool `name:"enable_patroni_failsafe_mode" default:"false"` + EnableSecretsDeletion *bool `name:"enable_secrets_deletion" default:"true"` + EnablePersistentVolumeClaimDeletion *bool `name:"enable_persistent_volume_claim_deletion" default:"true"` + PersistentVolumeClaimRetentionPolicy map[string]string `name:"persistent_volume_claim_retention_policy" default:"when_deleted:retain,when_scaled:retain"` } // MustMarshal marshals the config or panics @@ -278,7 +330,7 @@ func validate(cfg *Config) (err error) { } if cfg.ConnectionPooler.User == cfg.SuperUsername { - msg := "Connection pool user is not allowed to be the same as super user, username: %s" + msg := "connection pool user is not allowed to be the same as super user, username: %s" err = fmt.Errorf(msg, cfg.ConnectionPooler.User) } diff --git a/pkg/util/config/config_test.go b/pkg/util/config/config_test.go index eb27d1bda..c5fd48d93 100644 --- a/pkg/util/config/config_test.go +++ b/pkg/util/config/config_test.go @@ -2,10 +2,19 @@ package config import ( "fmt" + "os" "reflect" + "strings" "testing" ) +func TestMain(m *testing.M) { + // Set OPERATOR_NAMESPACE to avoid log.Fatal in GetOperatorNamespace + // when running tests outside a Kubernetes pod + os.Setenv("OPERATOR_NAMESPACE", "default") + os.Exit(m.Run()) +} + var getMapPairsFromStringTest = []struct { in string expected []string @@ -29,3 +38,300 @@ func TestGetMapPairsFromString(t *testing.T) { } } } + +func int32Ptr(i int32) *int32 { + return &i +} + +func boolPtr(b bool) *bool { + return &b +} + +var validateTests = []struct { + description string + cfg Config + expectError bool + errorMsg string +}{ + { + description: "valid config", + cfg: Config{ + Resources: Resources{ + MinInstances: 1, + MaxInstances: 5, + }, + Auth: Auth{ + SuperUsername: "postgres", + }, + ConnectionPooler: ConnectionPooler{ + NumberOfInstances: int32Ptr(2), + User: "pooler", + }, + Workers: 4, + }, + expectError: false, + }, + { + description: "min instances greater than max instances", + cfg: Config{ + Resources: Resources{ + MinInstances: 10, + MaxInstances: 5, + }, + Auth: Auth{ + SuperUsername: "postgres", + }, + ConnectionPooler: ConnectionPooler{ + NumberOfInstances: int32Ptr(2), + User: "pooler", + }, + Workers: 4, + }, + expectError: true, + errorMsg: "minimum number of instances", + }, + { + description: "workers set to zero", + cfg: Config{ + Resources: Resources{ + MinInstances: 1, + MaxInstances: 5, + }, + Auth: Auth{ + SuperUsername: "postgres", + }, + ConnectionPooler: ConnectionPooler{ + NumberOfInstances: int32Ptr(2), + User: "pooler", + }, + Workers: 0, + }, + expectError: true, + errorMsg: "number of workers should be higher than 0", + }, + { + description: "connection pooler instances below minimum", + cfg: Config{ + Resources: Resources{ + MinInstances: 1, + MaxInstances: 5, + }, + Auth: Auth{ + SuperUsername: "postgres", + }, + ConnectionPooler: ConnectionPooler{ + NumberOfInstances: int32Ptr(0), + User: "pooler", + }, + Workers: 4, + }, + expectError: true, + errorMsg: "number of connection pooler instances", + }, + { + description: "connection pooler user same as super user", + cfg: Config{ + Resources: Resources{ + MinInstances: 1, + MaxInstances: 5, + }, + Auth: Auth{ + SuperUsername: "postgres", + }, + ConnectionPooler: ConnectionPooler{ + NumberOfInstances: int32Ptr(2), + User: "postgres", + }, + Workers: 4, + }, + expectError: true, + errorMsg: "connection pool user is not allowed to be the same as super user", + }, + { + description: "min and max instances both negative (disabled)", + cfg: Config{ + Resources: Resources{ + MinInstances: -1, + MaxInstances: -1, + }, + Auth: Auth{ + SuperUsername: "postgres", + }, + ConnectionPooler: ConnectionPooler{ + NumberOfInstances: int32Ptr(2), + User: "pooler", + }, + Workers: 4, + }, + expectError: false, + }, +} + +func TestValidate(t *testing.T) { + for _, tt := range validateTests { + t.Run(tt.description, func(t *testing.T) { + err := validate(&tt.cfg) + if tt.expectError { + if err == nil { + t.Errorf("expected error containing %q, got nil", tt.errorMsg) + return + } + if !strings.Contains(err.Error(), tt.errorMsg) { + t.Errorf("expected error containing %q, got %q", tt.errorMsg, err.Error()) + } + } else { + if err != nil { + t.Errorf("expected no error, got %v", err) + } + } + }) + } +} + +var newFromMapTests = []struct { + description string + input map[string]string + expectPanic bool + panicMsg string + validateFunc func(t *testing.T, cfg *Config) +}{ + { + description: "empty map uses defaults", + input: map[string]string{}, + expectPanic: false, + validateFunc: func(t *testing.T, cfg *Config) { + if cfg.Workers != 8 { + t.Errorf("expected default Workers=8, got %d", cfg.Workers) + } + if cfg.SuperUsername != "postgres" { + t.Errorf("expected default SuperUsername=postgres, got %s", cfg.SuperUsername) + } + if cfg.ReplicationUsername != "standby" { + t.Errorf("expected default ReplicationUsername=standby, got %s", cfg.ReplicationUsername) + } + }, + }, + { + description: "custom values override defaults", + input: map[string]string{ + "workers": "16", + "super_username": "admin", + }, + expectPanic: false, + validateFunc: func(t *testing.T, cfg *Config) { + if cfg.Workers != 16 { + t.Errorf("expected Workers=16, got %d", cfg.Workers) + } + if cfg.SuperUsername != "admin" { + t.Errorf("expected SuperUsername=admin, got %s", cfg.SuperUsername) + } + }, + }, + { + description: "duration parsing", + input: map[string]string{ + "ready_wait_interval": "10s", + "ready_wait_timeout": "1m", + }, + expectPanic: false, + validateFunc: func(t *testing.T, cfg *Config) { + if cfg.ReadyWaitInterval.Seconds() != 10 { + t.Errorf("expected ReadyWaitInterval=10s, got %v", cfg.ReadyWaitInterval) + } + if cfg.ReadyWaitTimeout.Minutes() != 1 { + t.Errorf("expected ReadyWaitTimeout=1m, got %v", cfg.ReadyWaitTimeout) + } + }, + }, + { + description: "boolean parsing", + input: map[string]string{ + "enable_teams_api": "false", + "debug_logging": "false", + }, + expectPanic: false, + validateFunc: func(t *testing.T, cfg *Config) { + if cfg.EnableTeamsAPI != false { + t.Errorf("expected EnableTeamsAPI=false, got %v", cfg.EnableTeamsAPI) + } + if cfg.DebugLogging != false { + t.Errorf("expected DebugLogging=false, got %v", cfg.DebugLogging) + } + }, + }, + { + description: "map parsing", + input: map[string]string{ + "cluster_labels": "app:myapp,env:prod", + }, + expectPanic: false, + validateFunc: func(t *testing.T, cfg *Config) { + if cfg.ClusterLabels["app"] != "myapp" { + t.Errorf("expected ClusterLabels[app]=myapp, got %s", cfg.ClusterLabels["app"]) + } + if cfg.ClusterLabels["env"] != "prod" { + t.Errorf("expected ClusterLabels[env]=prod, got %s", cfg.ClusterLabels["env"]) + } + }, + }, + { + description: "slice parsing", + input: map[string]string{ + "inherited_labels": "label1,label2,label3", + }, + expectPanic: false, + validateFunc: func(t *testing.T, cfg *Config) { + expected := []string{"label1", "label2", "label3"} + if !reflect.DeepEqual(cfg.InheritedLabels, expected) { + t.Errorf("expected InheritedLabels=%v, got %v", expected, cfg.InheritedLabels) + } + }, + }, + { + description: "invalid workers triggers validation panic", + input: map[string]string{ + "workers": "0", + }, + expectPanic: true, + panicMsg: "number of workers should be higher than 0", + }, + { + description: "invalid integer causes panic", + input: map[string]string{ + "workers": "invalid", + }, + expectPanic: true, + panicMsg: "invalid syntax", + }, +} + +func TestNewFromMap(t *testing.T) { + for _, tt := range newFromMapTests { + t.Run(tt.description, func(t *testing.T) { + if tt.expectPanic { + defer func() { + r := recover() + if r == nil { + t.Errorf("expected panic with message containing %q, but no panic occurred", tt.panicMsg) + return + } + errMsg := fmt.Sprintf("%v", r) + if !strings.Contains(errMsg, tt.panicMsg) { + t.Errorf("expected panic message containing %q, got %q", tt.panicMsg, errMsg) + } + }() + } + + cfg := NewFromMap(tt.input) + + if tt.expectPanic { + t.Errorf("expected panic but NewFromMap returned successfully") + return + } + + if tt.validateFunc != nil { + tt.validateFunc(t, cfg) + } + }) + } +} diff --git a/pkg/util/constants/annotations.go b/pkg/util/constants/annotations.go index fc5a84fa5..0330ddcb8 100644 --- a/pkg/util/constants/annotations.go +++ b/pkg/util/constants/annotations.go @@ -3,8 +3,6 @@ package constants // Names and values in Kubernetes annotation for services, statefulsets and volumes const ( ZalandoDNSNameAnnotation = "external-dns.alpha.kubernetes.io/hostname" - ElbTimeoutAnnotationName = "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout" - ElbTimeoutAnnotationValue = "3600" KubeIAmAnnotation = "iam.amazonaws.com/role" VolumeStorateProvisionerAnnotation = "pv.kubernetes.io/provisioned-by" PostgresqlControllerAnnotationKey = "acid.zalan.do/controller" diff --git a/pkg/util/constants/aws.go b/pkg/util/constants/aws.go index f1cfd5975..147e58889 100644 --- a/pkg/util/constants/aws.go +++ b/pkg/util/constants/aws.go @@ -7,6 +7,7 @@ const ( // EBS related constants EBSVolumeIDStart = "/vol-" EBSProvisioner = "kubernetes.io/aws-ebs" + EBSDriver = "ebs.csi.aws.com" //https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_VolumeModification.html EBSVolumeStateModifying = "modifying" EBSVolumeStateOptimizing = "optimizing" diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index ded795bbe..14c137e74 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -2,6 +2,7 @@ package constants // Connection pooler specific constants const ( + ConnectionPoolerResourceSuffix = "pooler" ConnectionPoolerUserName = "pooler" ConnectionPoolerSchemaName = "pooler" ConnectionPoolerDefaultType = "pgbouncer" diff --git a/pkg/util/constants/postgresql.go b/pkg/util/constants/postgresql.go index e39fd423f..8bd7508a7 100644 --- a/pkg/util/constants/postgresql.go +++ b/pkg/util/constants/postgresql.go @@ -8,9 +8,14 @@ const ( PostgresDataMount = "/home/postgres/pgdata" PostgresDataPath = PostgresDataMount + "/pgroot" + PatroniPGParametersParameterName = "parameters" + PostgresConnectRetryTimeout = 2 * time.Minute PostgresConnectTimeout = 15 * time.Second ShmVolumeName = "dshm" ShmVolumePath = "/dev/shm" + + RunVolumeName = "postgresql-run" + RunVolumePath = "/var/run/postgresql" ) diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index dd906fe80..34f1d0737 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -4,8 +4,9 @@ package constants const ( PasswordLength = 64 SuperuserKeyName = "superuser" - ConnectionPoolerUserKeyName = "pooler" ReplicationUserKeyName = "replication" + ConnectionPoolerUserKeyName = "pooler" + EventStreamUserKeyName = "streamer" RoleFlagSuperuser = "SUPERUSER" RoleFlagInherit = "INHERIT" RoleFlagLogin = "LOGIN" @@ -19,4 +20,5 @@ const ( WriterRoleNameSuffix = "_writer" UserRoleNameSuffix = "_user" DefaultSearchPath = "\"$user\"" + RotationUserDateFormat = "060102" ) diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go new file mode 100644 index 000000000..cb4bb6a3f --- /dev/null +++ b/pkg/util/constants/streams.go @@ -0,0 +1,20 @@ +package constants + +// PostgreSQL specific constants +const ( + EventStreamCRDApiVersion = "zalando.org/v1" + EventStreamCRDKind = "FabricEventStream" + EventStreamCRDName = "fabriceventstreams.zalando.org" + EventStreamSourcePGType = "PostgresLogicalReplication" + EventStreamSourceSlotPrefix = "fes" + EventStreamSourcePluginType = "pgoutput" + EventStreamSourceAuthType = "DatabaseAuthenticationSecret" + EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" + EventStreamSinkNakadiType = "Nakadi" + EventStreamRecoveryDLQType = "DeadLetter" + EventStreamRecoveryIgnoreType = "Ignore" + EventStreamRecoveryNoneType = "None" + EventStreamRecoverySuffix = "dead-letter-queue" + EventStreamCpuAnnotationKey = "fes.zalando.org/FES_CPU" + EventStreamMemoryAnnotationKey = "fes.zalando.org/FES_MEMORY" +) diff --git a/pkg/util/httpclient/httpclient.go b/pkg/util/httpclient/httpclient.go index e7022f1a3..80716c548 100644 --- a/pkg/util/httpclient/httpclient.go +++ b/pkg/util/httpclient/httpclient.go @@ -1,6 +1,6 @@ package httpclient -//go:generate mockgen -package mocks -destination=$PWD/mocks/$GOFILE -source=$GOFILE -build_flags=-mod=vendor +//go:generate go tool mockgen -package mocks -destination=../../../mocks/$GOFILE -source=$GOFILE import "net/http" diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index dd6ec1e8b..8515027ab 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -3,30 +3,28 @@ package k8sutil import ( "context" "fmt" - "reflect" b64 "encoding/base64" "encoding/json" - batchv1beta1 "k8s.io/api/batch/v1beta1" - clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" - apiacidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - acidv1client "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" + zalandoclient "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" "github.com/zalando/postgres-operator/pkg/spec" apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - policybeta1 "k8s.io/api/policy/v1beta1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - apiextv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" + apiextv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" + batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" - policyv1beta1 "k8s.io/client-go/kubernetes/typed/policy/v1beta1" + policyv1 "k8s.io/client-go/kubernetes/typed/policy/v1" rbacv1 "k8s.io/client-go/kubernetes/typed/rbac/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -36,6 +34,14 @@ func Int32ToPointer(value int32) *int32 { return &value } +func UInt32ToPointer(value uint32) *uint32 { + return &value +} + +func StringToPointer(str string) *string { + return &str +} + // KubernetesClient describes getters for Kubernetes objects type KubernetesClient struct { corev1.SecretsGetter @@ -52,15 +58,25 @@ type KubernetesClient struct { appsv1.StatefulSetsGetter appsv1.DeploymentsGetter rbacv1.RoleBindingsGetter - policyv1beta1.PodDisruptionBudgetsGetter - apiextv1.CustomResourceDefinitionsGetter - clientbatchv1beta1.CronJobsGetter + batchv1.CronJobsGetter + policyv1.PodDisruptionBudgetsGetter + apiextv1client.CustomResourceDefinitionsGetter acidv1.OperatorConfigurationsGetter acidv1.PostgresTeamsGetter acidv1.PostgresqlsGetter + zalandov1.FabricEventStreamsGetter - RESTClient rest.Interface - AcidV1ClientSet *acidv1client.Clientset + RESTClient rest.Interface + Clientset *kubernetes.Clientset + AcidV1ClientSet *zalandoclient.Clientset + Zalandov1ClientSet *zalandoclient.Clientset +} + +type mockCustomResourceDefinition struct { + apiextv1client.CustomResourceDefinitionInterface +} + +type MockCustomResourceDefinitionsGetter struct { } type mockSecret struct { @@ -133,6 +149,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { return kubeClient, fmt.Errorf("could not get clientset: %v", err) } + kubeClient.Clientset = client kubeClient.PodsGetter = client.CoreV1() kubeClient.ServicesGetter = client.CoreV1() kubeClient.EndpointsGetter = client.CoreV1() @@ -145,10 +162,10 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.NamespacesGetter = client.CoreV1() kubeClient.StatefulSetsGetter = client.AppsV1() kubeClient.DeploymentsGetter = client.AppsV1() - kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1() + kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1() kubeClient.RESTClient = client.CoreV1().RESTClient() kubeClient.RoleBindingsGetter = client.RbacV1() - kubeClient.CronJobsGetter = client.BatchV1beta1() + kubeClient.CronJobsGetter = client.BatchV1() kubeClient.EventsGetter = client.CoreV1() apiextClient, err := apiextclient.NewForConfig(cfg) @@ -158,127 +175,74 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.CustomResourceDefinitionsGetter = apiextClient.ApiextensionsV1() - kubeClient.AcidV1ClientSet = acidv1client.NewForConfigOrDie(cfg) + kubeClient.AcidV1ClientSet = zalandoclient.NewForConfigOrDie(cfg) if err != nil { return kubeClient, fmt.Errorf("could not create acid.zalan.do clientset: %v", err) } + kubeClient.Zalandov1ClientSet = zalandoclient.NewForConfigOrDie(cfg) + if err != nil { + return kubeClient, fmt.Errorf("could not create zalando.org clientset: %v", err) + } kubeClient.OperatorConfigurationsGetter = kubeClient.AcidV1ClientSet.AcidV1() kubeClient.PostgresTeamsGetter = kubeClient.AcidV1ClientSet.AcidV1() kubeClient.PostgresqlsGetter = kubeClient.AcidV1ClientSet.AcidV1() + kubeClient.FabricEventStreamsGetter = kubeClient.Zalandov1ClientSet.ZalandoV1() return kubeClient, nil } // SetPostgresCRDStatus of Postgres cluster -func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.NamespacedName, status string) (*apiacidv1.Postgresql, error) { - var pg *apiacidv1.Postgresql - var pgStatus apiacidv1.PostgresStatus - pgStatus.PostgresClusterStatus = status - - patch, err := json.Marshal(struct { - PgStatus interface{} `json:"status"` - }{&pgStatus}) - - if err != nil { - return pg, fmt.Errorf("could not marshal status: %v", err) - } - - // we cannot do a full scale update here without fetching the previous manifest (as the resourceVersion may differ), - // however, we could do patch without it. In the future, once /status subresource is there (starting Kubernetes 1.11) - // we should take advantage of it. - pg, err = client.PostgresqlsGetter.Postgresqls(clusterName.Namespace).Patch( - context.TODO(), clusterName.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "status") +func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.NamespacedName, pg *apiacidv1.Postgresql) (*apiacidv1.Postgresql, error) { + pg, err := client.PostgresqlsGetter.Postgresqls(clusterName.Namespace).UpdateStatus(context.TODO(), pg, metav1.UpdateOptions{}) if err != nil { return pg, fmt.Errorf("could not update status: %v", err) } - // update the spec, maintaining the new resourceVersion. return pg, nil } -// SameService compares the Services -func SameService(cur, new *v1.Service) (match bool, reason string) { - //TODO: improve comparison - if cur.Spec.Type != new.Spec.Type { - return false, fmt.Sprintf("new service's type %q does not match the current one %q", - new.Spec.Type, cur.Spec.Type) - } +// SetFinalizer of Postgres cluster +func (client *KubernetesClient) SetFinalizer(clusterName spec.NamespacedName, pg *apiacidv1.Postgresql, finalizers []string) (*apiacidv1.Postgresql, error) { + var ( + updatedPg *apiacidv1.Postgresql + patch []byte + err error + ) + pg.ObjectMeta.Finalizers = finalizers - oldSourceRanges := cur.Spec.LoadBalancerSourceRanges - newSourceRanges := new.Spec.LoadBalancerSourceRanges - - /* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */ - if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) { - if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) { - return false, "new service's LoadBalancerSourceRange does not match the current one" + if len(finalizers) > 0 { + patch, err = json.Marshal(struct { + PgMetadata interface{} `json:"metadata"` + }{&pg.ObjectMeta}) + if err != nil { + return pg, fmt.Errorf("could not marshal ObjectMeta: %v", err) } + + updatedPg, err = client.PostgresqlsGetter.Postgresqls(clusterName.Namespace).Patch( + context.TODO(), clusterName.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + } else { + // in case finalizers are empty and update is needed to remove + updatedPg, err = client.PostgresqlsGetter.Postgresqls(clusterName.Namespace).Update( + context.TODO(), pg, metav1.UpdateOptions{}) + } + if err != nil { + return updatedPg, fmt.Errorf("could not set finalizer: %v", err) } - match = true - - reasonPrefix := "new service's annotations does not match the current one:" - for ann := range cur.Annotations { - if _, ok := new.Annotations[ann]; !ok { - match = false - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" Removed '%s'.", ann) - } - } - - for ann := range new.Annotations { - v, ok := cur.Annotations[ann] - if !ok { - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" Added '%s' with value '%s'.", ann, new.Annotations[ann]) - match = false - } else if v != new.Annotations[ann] { - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", ann, v, new.Annotations[ann]) - match = false - } - } - - return match, reason + return updatedPg, nil } -// SamePDB compares the PodDisruptionBudgets -func SamePDB(cur, new *policybeta1.PodDisruptionBudget) (match bool, reason string) { - //TODO: improve comparison - match = reflect.DeepEqual(new.Spec, cur.Spec) - if !match { - reason = "new PDB spec does not match the current one" - } - - return +func (c *mockCustomResourceDefinition) Get(ctx context.Context, name string, options metav1.GetOptions) (*apiextv1.CustomResourceDefinition, error) { + return &apiextv1.CustomResourceDefinition{}, nil } -func getJobImage(cronJob *batchv1beta1.CronJob) string { - return cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image +func (c *mockCustomResourceDefinition) Create(ctx context.Context, crd *apiextv1.CustomResourceDefinition, options metav1.CreateOptions) (*apiextv1.CustomResourceDefinition, error) { + return &apiextv1.CustomResourceDefinition{}, nil } -// SameLogicalBackupJob compares Specs of logical backup cron jobs -func SameLogicalBackupJob(cur, new *batchv1beta1.CronJob) (match bool, reason string) { - - if cur.Spec.Schedule != new.Spec.Schedule { - return false, fmt.Sprintf("new job's schedule %q does not match the current one %q", - new.Spec.Schedule, cur.Spec.Schedule) - } - - newImage := getJobImage(new) - curImage := getJobImage(cur) - if newImage != curImage { - return false, fmt.Sprintf("new job's image %q does not match the current one %q", - newImage, curImage) - } - - return true, "" +func (mock *MockCustomResourceDefinitionsGetter) CustomResourceDefinitions() apiextv1client.CustomResourceDefinitionInterface { + return &mockCustomResourceDefinition{} } func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { @@ -394,7 +358,7 @@ func (mock *mockDeployment) Get(ctx context.Context, name string, opts metav1.Ge Template: v1.PodTemplateSpec{ Spec: v1.PodSpec{ Containers: []v1.Container{ - v1.Container{ + { Image: "pooler:1.0", }, }, @@ -485,6 +449,8 @@ func NewMockKubernetesClient() KubernetesClient { ConfigMapsGetter: &MockConfigMapsGetter{}, DeploymentsGetter: &MockDeploymentGetter{}, ServicesGetter: &MockServiceGetter{}, + + CustomResourceDefinitionsGetter: &MockCustomResourceDefinitionsGetter{}, } } diff --git a/pkg/util/k8sutil/k8sutil_test.go b/pkg/util/k8sutil/k8sutil_test.go deleted file mode 100644 index b3e768501..000000000 --- a/pkg/util/k8sutil/k8sutil_test.go +++ /dev/null @@ -1,311 +0,0 @@ -package k8sutil - -import ( - "strings" - "testing" - - "github.com/zalando/postgres-operator/pkg/util/constants" - - v1 "k8s.io/api/core/v1" -) - -func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service { - svc := &v1.Service{ - Spec: v1.ServiceSpec{ - Type: svcT, - LoadBalancerSourceRanges: lbSr, - }, - } - svc.Annotations = ann - return svc -} - -func TestSameService(t *testing.T) { - tests := []struct { - about string - current *v1.Service - new *v1.Service - reason string - match bool - }{ - { - about: "two equal services", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: true, - }, - { - about: "services differ on service type", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`, - }, - { - about: "services differ on lb source ranges", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"185.249.56.0/22"}), - match: false, - reason: `new service's LoadBalancerSourceRange does not match the current one`, - }, - { - about: "new service doesn't have lb source ranges", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{}), - match: false, - reason: `new service's LoadBalancerSourceRange does not match the current one`, - }, - { - about: "services differ on DNS annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`, - }, - { - about: "services differ on AWS ELB annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: "1800", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`, - }, - { - about: "service changes existing annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "baz", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: 'foo' changed from 'bar' to 'baz'.`, - }, - { - about: "service changes multiple existing annotations", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - "bar": "foo", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "baz", - "bar": "fooz", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations does not match the current one:`, - }, - { - about: "service adds a new custom annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: Added 'foo' with value 'bar'.`, - }, - { - about: "service removes a custom annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: Removed 'foo'.`, - }, - { - about: "service removes a custom annotation and adds a new one", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "bar": "foo", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`, - }, - { - about: "service removes a custom annotation, adds a new one and change another", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - "zalan": "do", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "bar": "foo", - "zalan": "do.com", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations does not match the current one: Removed 'foo'.`, - }, - { - about: "service add annotations", - current: newsService( - map[string]string{}, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations does not match the current one: Added `, - }, - } - for _, tt := range tests { - t.Run(tt.about, func(t *testing.T) { - match, reason := SameService(tt.current, tt.new) - if match && !tt.match { - t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new) - return - } - if !match && tt.match { - t.Errorf("expected services to be the same: '%q' and '%q'", tt.current, tt.new) - return - } - if !match && !tt.match { - if !strings.HasPrefix(reason, tt.reason) { - t.Errorf("expected reason prefix '%s', found '%s'", tt.reason, reason) - return - } - } - }) - } -} diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go index b63912e55..2129f1acc 100644 --- a/pkg/util/patroni/patroni.go +++ b/pkg/util/patroni/patroni.go @@ -4,30 +4,41 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" + "math" "net" "net/http" "strconv" "time" + "github.com/zalando/postgres-operator/pkg/util/constants" httpclient "github.com/zalando/postgres-operator/pkg/util/httpclient" "github.com/sirupsen/logrus" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" v1 "k8s.io/api/core/v1" ) const ( - failoverPath = "/failover" - configPath = "/config" - apiPort = 8008 - timeout = 30 * time.Second + switchoverPath = "/switchover" + configPath = "/config" + clusterPath = "/cluster" + statusPath = "/patroni" + restartPath = "/restart" + ApiPort = 8008 + timeout = 30 * time.Second ) // Interface describe patroni methods type Interface interface { - Switchover(master *v1.Pod, candidate string) error + GetClusterMembers(master *v1.Pod) ([]ClusterMember, error) + Switchover(master *v1.Pod, candidate string, scheduled_at string) error SetPostgresParameters(server *v1.Pod, options map[string]string) error + SetStandbyClusterParameters(server *v1.Pod, options map[string]interface{}) error GetMemberData(server *v1.Pod) (MemberData, error) + Restart(server *v1.Pod) error + GetConfig(server *v1.Pod) (acidv1.Patroni, map[string]string, error) + SetConfig(server *v1.Pod, config map[string]interface{}) error } // Patroni API client @@ -64,7 +75,7 @@ func apiURL(masterPod *v1.Pod) (string, error) { return "", fmt.Errorf("%s is not a valid IPv4/IPv6 address", masterPod.Status.PodIP) } } - return fmt.Sprintf("http://%s", net.JoinHostPort(ip.String(), strconv.Itoa(apiPort))), nil + return fmt.Sprintf("http://%s", net.JoinHostPort(ip.String(), strconv.Itoa(ApiPort))), nil } func (p *Patroni) httpPostOrPatch(method string, url string, body *bytes.Buffer) (err error) { @@ -92,8 +103,8 @@ func (p *Patroni) httpPostOrPatch(method string, url string, body *bytes.Buffer) } }() - if resp.StatusCode != http.StatusOK { - bodyBytes, err := ioutil.ReadAll(resp.Body) + if resp.StatusCode < http.StatusOK || resp.StatusCode >= 300 { + bodyBytes, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("could not read response: %v", err) } @@ -103,10 +114,31 @@ func (p *Patroni) httpPostOrPatch(method string, url string, body *bytes.Buffer) return nil } +func (p *Patroni) httpGet(url string) (string, error) { + p.logger.Debugf("making GET http request: %s", url) + + response, err := p.httpClient.Get(url) + if err != nil { + return "", fmt.Errorf("could not make request: %v", err) + } + defer response.Body.Close() + + bodyBytes, err := io.ReadAll(response.Body) + if err != nil { + return "", fmt.Errorf("could not read response: %v", err) + } + + if response.StatusCode < http.StatusOK || response.StatusCode >= 300 { + return string(bodyBytes), fmt.Errorf("patroni returned '%d'", response.StatusCode) + } + + return string(bodyBytes), nil +} + // Switchover by calling Patroni REST API -func (p *Patroni) Switchover(master *v1.Pod, candidate string) error { +func (p *Patroni) Switchover(master *v1.Pod, candidate string, scheduled_at string) error { buf := &bytes.Buffer{} - err := json.NewEncoder(buf).Encode(map[string]string{"leader": master.Name, "member": candidate}) + err := json.NewEncoder(buf).Encode(map[string]string{"leader": master.Name, "member": candidate, "scheduled_at": scheduled_at}) if err != nil { return fmt.Errorf("could not encode json: %v", err) } @@ -114,12 +146,12 @@ func (p *Patroni) Switchover(master *v1.Pod, candidate string) error { if err != nil { return err } - return p.httpPostOrPatch(http.MethodPost, apiURLString+failoverPath, buf) + return p.httpPostOrPatch(http.MethodPost, apiURLString+switchoverPath, buf) } //TODO: add an option call /patroni to check if it is necessary to restart the server -//SetPostgresParameters sets Postgres options via Patroni patch API call. +// SetPostgresParameters sets Postgres options via Patroni patch API call. func (p *Patroni) SetPostgresParameters(server *v1.Pod, parameters map[string]string) error { buf := &bytes.Buffer{} err := json.NewEncoder(buf).Encode(map[string]map[string]interface{}{"postgresql": {"parameters": parameters}}) @@ -133,6 +165,55 @@ func (p *Patroni) SetPostgresParameters(server *v1.Pod, parameters map[string]st return p.httpPostOrPatch(http.MethodPatch, apiURLString+configPath, buf) } +// SetStandbyClusterParameters sets StandbyCluster options via Patroni patch API call. +func (p *Patroni) SetStandbyClusterParameters(server *v1.Pod, parameters map[string]interface{}) error { + return p.SetConfig(server, map[string]interface{}{"standby_cluster": parameters}) +} + +// SetConfig sets Patroni options via Patroni patch API call. +func (p *Patroni) SetConfig(server *v1.Pod, config map[string]interface{}) error { + buf := &bytes.Buffer{} + err := json.NewEncoder(buf).Encode(config) + if err != nil { + return fmt.Errorf("could not encode json: %v", err) + } + apiURLString, err := apiURL(server) + if err != nil { + return err + } + return p.httpPostOrPatch(http.MethodPatch, apiURLString+configPath, buf) +} + +// ClusterMembers array of cluster members from Patroni API +type ClusterMembers struct { + Members []ClusterMember `json:"members"` +} + +// ClusterMember cluster member data from Patroni API +type ClusterMember struct { + Name string `json:"name"` + Role string `json:"role"` + State string `json:"state"` + Timeline int `json:"timeline"` + Lag ReplicationLag `json:"lag,omitempty"` +} + +type ReplicationLag uint64 + +// UnmarshalJSON converts member lag (can be int or string) into uint64 +func (rl *ReplicationLag) UnmarshalJSON(data []byte) error { + var lagUInt64 uint64 + if data[0] == '"' { + *rl = math.MaxUint64 + return nil + } + if err := json.Unmarshal(data, &lagUInt64); err != nil { + return err + } + *rl = ReplicationLag(lagUInt64) + return nil +} + // MemberDataPatroni child element type MemberDataPatroni struct { Version string `json:"version"` @@ -149,6 +230,82 @@ type MemberData struct { Patroni MemberDataPatroni `json:"patroni"` } +func (p *Patroni) GetConfig(server *v1.Pod) (acidv1.Patroni, map[string]string, error) { + var ( + patroniConfig acidv1.Patroni + pgConfig map[string]interface{} + ) + apiURLString, err := apiURL(server) + if err != nil { + return patroniConfig, nil, err + } + body, err := p.httpGet(apiURLString + configPath) + if err != nil { + return patroniConfig, nil, err + } + err = json.Unmarshal([]byte(body), &patroniConfig) + if err != nil { + return patroniConfig, nil, err + } + + // unmarshalling postgresql parameters needs a detour + err = json.Unmarshal([]byte(body), &pgConfig) + if err != nil { + return patroniConfig, nil, err + } + pgParameters := make(map[string]string) + if _, exists := pgConfig["postgresql"]; exists { + effectivePostgresql := pgConfig["postgresql"].(map[string]interface{}) + effectivePgParameters := effectivePostgresql[constants.PatroniPGParametersParameterName].(map[string]interface{}) + for parameter, value := range effectivePgParameters { + strValue := fmt.Sprintf("%v", value) + pgParameters[parameter] = strValue + } + } + + return patroniConfig, pgParameters, err +} + +// Restart method restarts instance via Patroni POST API call. +func (p *Patroni) Restart(server *v1.Pod) error { + buf := &bytes.Buffer{} + err := json.NewEncoder(buf).Encode(map[string]interface{}{"restart_pending": true}) + if err != nil { + return fmt.Errorf("could not encode json: %v", err) + } + apiURLString, err := apiURL(server) + if err != nil { + return err + } + if err := p.httpPostOrPatch(http.MethodPost, apiURLString+restartPath, buf); err != nil { + return err + } + p.logger.Infof("Postgres server successfuly restarted in pod %s", server.Name) + + return nil +} + +// GetClusterMembers read cluster data from patroni API +func (p *Patroni) GetClusterMembers(server *v1.Pod) ([]ClusterMember, error) { + + apiURLString, err := apiURL(server) + if err != nil { + return []ClusterMember{}, err + } + body, err := p.httpGet(apiURLString + clusterPath) + if err != nil { + return []ClusterMember{}, err + } + + data := ClusterMembers{} + err = json.Unmarshal([]byte(body), &data) + if err != nil { + return []ClusterMember{}, err + } + + return data.Members, nil +} + // GetMemberData read member data from patroni API func (p *Patroni) GetMemberData(server *v1.Pod) (MemberData, error) { @@ -156,19 +313,13 @@ func (p *Patroni) GetMemberData(server *v1.Pod) (MemberData, error) { if err != nil { return MemberData{}, err } - response, err := p.httpClient.Get(apiURLString) + body, err := p.httpGet(apiURLString + statusPath) if err != nil { - return MemberData{}, fmt.Errorf("could not perform Get request: %v", err) - } - defer response.Body.Close() - - body, err := ioutil.ReadAll(response.Body) - if err != nil { - return MemberData{}, fmt.Errorf("could not read response: %v", err) + return MemberData{}, err } data := MemberData{} - err = json.Unmarshal(body, &data) + err = json.Unmarshal([]byte(body), &data) if err != nil { return MemberData{}, err } diff --git a/pkg/util/patroni/patroni_test.go b/pkg/util/patroni/patroni_test.go index 939270453..39b498d2e 100644 --- a/pkg/util/patroni/patroni_test.go +++ b/pkg/util/patroni/patroni_test.go @@ -4,16 +4,22 @@ import ( "bytes" "errors" "fmt" - "io/ioutil" + "io" + "math" "net/http" + "reflect" "testing" "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" "github.com/zalando/postgres-operator/mocks" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" v1 "k8s.io/api/core/v1" ) +var logger = logrus.New().WithField("test", "patroni") + func newMockPod(ip string) *v1.Pod { return &v1.Pod{ Status: v1.PodStatus{ @@ -30,17 +36,17 @@ func TestApiURL(t *testing.T) { }{ { "127.0.0.1", - fmt.Sprintf("http://127.0.0.1:%d", apiPort), + fmt.Sprintf("http://127.0.0.1:%d", ApiPort), nil, }, { "0000:0000:0000:0000:0000:0000:0000:0001", - fmt.Sprintf("http://[::1]:%d", apiPort), + fmt.Sprintf("http://[::1]:%d", ApiPort), nil, }, { "::1", - fmt.Sprintf("http://[::1]:%d", apiPort), + fmt.Sprintf("http://[::1]:%d", ApiPort), nil, }, { @@ -80,31 +86,200 @@ func TestApiURL(t *testing.T) { } } -func TestPatroniAPI(t *testing.T) { +func TestGetClusterMembers(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - json := `{"state": "running", "postmaster_start_time": "2021-02-19 14:31:50.053 CET", "role": "master", "server_version": 90621, "cluster_unlocked": false, "xlog": {"location": 55978296057856}, "timeline": 6, "database_system_identifier": "6462555844314089962", "pending_restart": true, "patroni": {"version": "2.0.1", "scope": "acid-rest92-standby"}}` - r := ioutil.NopCloser(bytes.NewReader([]byte(json))) + expectedClusterMemberData := []ClusterMember{ + { + Name: "acid-test-cluster-0", + Role: "leader", + State: "running", + Timeline: 1, + }, { + Name: "acid-test-cluster-1", + Role: "sync_standby", + State: "streaming", + Timeline: 1, + Lag: 0, + }, { + Name: "acid-test-cluster-2", + Role: "replica", + State: "streaming", + Timeline: 1, + Lag: math.MaxUint64, + }, { + Name: "acid-test-cluster-3", + Role: "replica", + State: "in archive recovery", + Timeline: 1, + Lag: 3000000000, + }} + + json := `{"members": [ + {"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, + {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "streaming", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, + {"name": "acid-test-cluster-2", "role": "replica", "state": "streaming", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": "unknown"}, + {"name": "acid-test-cluster-3", "role": "replica", "state": "in archive recovery", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 3000000000} + ]}` + r := io.NopCloser(bytes.NewReader([]byte(json))) response := http.Response{ - Status: "200", - Body: r, + StatusCode: 200, + Body: r, } mockClient := mocks.NewMockHTTPClient(ctrl) mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil) - p := New(nil, mockClient) + p := New(logger, mockClient) - pod := v1.Pod{ - Status: v1.PodStatus{ - PodIP: "192.168.100.1", - }, + clusterMemberData, err := p.GetClusterMembers(newMockPod("192.168.100.1")) + + if !reflect.DeepEqual(expectedClusterMemberData, clusterMemberData) { + t.Errorf("Patroni cluster members differ: expected: %#v, got: %#v", expectedClusterMemberData, clusterMemberData) } - _, err := p.GetMemberData(&pod) if err != nil { t.Errorf("Could not read Patroni data: %v", err) } } + +func TestGetMemberData(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + expectedMemberData := MemberData{ + State: "running", + Role: "master", + ServerVersion: 130004, + PendingRestart: true, + Patroni: MemberDataPatroni{ + Version: "2.1.1", + Scope: "acid-test-cluster", + }, + } + + json := `{"state": "running", "postmaster_start_time": "2021-02-19 14:31:50.053 CET", "role": "master", "server_version": 130004, "cluster_unlocked": false, "xlog": {"location": 123456789}, "timeline": 1, "database_system_identifier": "6462555844314089962", "pending_restart": true, "patroni": {"version": "2.1.1", "scope": "acid-test-cluster"}}` + r := io.NopCloser(bytes.NewReader([]byte(json))) + + response := http.Response{ + StatusCode: 200, + Body: r, + } + + mockClient := mocks.NewMockHTTPClient(ctrl) + mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil) + + p := New(logger, mockClient) + + memberData, err := p.GetMemberData(newMockPod("192.168.100.1")) + + if !reflect.DeepEqual(expectedMemberData, memberData) { + t.Errorf("Patroni member data differs: expected: %#v, got: %#v", expectedMemberData, memberData) + } + + if err != nil { + t.Errorf("Could not read Patroni data: %v", err) + } +} + +func TestGetConfig(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + expectedPatroniConfig := acidv1.Patroni{ + TTL: 30, + LoopWait: 10, + RetryTimeout: 10, + MaximumLagOnFailover: 33554432, + Slots: map[string]map[string]string{ + "cdc": { + "database": "foo", + "plugin": "pgoutput", + "type": "logical", + }, + }, + } + + expectedPgParameters := map[string]string{ + "archive_mode": "on", + "archive_timeout": "1800s", + "autovacuum_analyze_scale_factor": "0.02", + "autovacuum_max_workers": "5", + "autovacuum_vacuum_scale_factor": "0.05", + "checkpoint_completion_target": "0.9", + "hot_standby": "on", + "log_autovacuum_min_duration": "0", + "log_checkpoints": "on", + "log_connections": "on", + "log_disconnections": "on", + "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", + "log_lock_waits": "on", + "log_min_duration_statement": "500", + "log_statement": "ddl", + "log_temp_files": "0", + "max_connections": "100", + "max_replication_slots": "10", + "max_wal_senders": "10", + "tcp_keepalives_idle": "900", + "tcp_keepalives_interval": "100", + "track_functions": "all", + "wal_level": "hot_standby", + "wal_log_hints": "on", + } + + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` + r := io.NopCloser(bytes.NewReader([]byte(configJson))) + + response := http.Response{ + StatusCode: 200, + Body: r, + } + + mockClient := mocks.NewMockHTTPClient(ctrl) + mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil) + + p := New(logger, mockClient) + + patroniConfig, pgParameters, err := p.GetConfig(newMockPod("192.168.100.1")) + if err != nil { + t.Errorf("Could not read Patroni config endpoint: %v", err) + } + + if !reflect.DeepEqual(expectedPatroniConfig, patroniConfig) { + t.Errorf("Patroni config differs: expected: %#v, got: %#v", expectedPatroniConfig, patroniConfig) + } + if !reflect.DeepEqual(expectedPgParameters, pgParameters) { + t.Errorf("Postgre parameters differ: expected: %#v, got: %#v", expectedPgParameters, pgParameters) + } +} + +func TestSetPostgresParameters(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + parametersToSet := map[string]string{ + "max_connections": "50", + "wal_level": "logical", + } + + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` + r := io.NopCloser(bytes.NewReader([]byte(configJson))) + + response := http.Response{ + StatusCode: 200, + Body: r, + } + + mockClient := mocks.NewMockHTTPClient(ctrl) + mockClient.EXPECT().Do(gomock.Any()).Return(&response, nil) + + p := New(logger, mockClient) + + err := p.SetPostgresParameters(newMockPod("192.168.100.1"), parametersToSet) + if err != nil { + t.Errorf("could not call patch Patroni config: %v", err) + } + +} diff --git a/pkg/util/teams/teams.go b/pkg/util/teams/teams.go index d7413ab9c..bf1260d6e 100644 --- a/pkg/util/teams/teams.go +++ b/pkg/util/teams/teams.go @@ -45,7 +45,7 @@ type httpClient interface { // Interface to the TeamsAPIClient type Interface interface { - TeamInfo(teamID, token string) (tm *Team, err error) + TeamInfo(teamID, token string) (tm *Team, statusCode int, err error) } // API describes teams API @@ -67,7 +67,7 @@ func NewTeamsAPI(url string, log *logrus.Entry) *API { } // TeamInfo returns information about a given team using its ID and a token to authenticate to the API service. -func (t *API) TeamInfo(teamID, token string) (tm *Team, err error) { +func (t *API) TeamInfo(teamID, token string) (tm *Team, statusCode int, err error) { var ( req *http.Request resp *http.Response @@ -77,37 +77,38 @@ func (t *API) TeamInfo(teamID, token string) (tm *Team, err error) { t.logger.Debugf("request url: %s", url) req, err = http.NewRequest("GET", url, nil) if err != nil { - return nil, err + return nil, http.StatusBadRequest, err } req.Header.Add("Authorization", "Bearer "+token) if resp, err = t.httpClient.Do(req); err != nil { - return nil, err + return nil, http.StatusUnauthorized, err } defer func() { if closeErr := resp.Body.Close(); closeErr != nil { err = fmt.Errorf("error when closing response: %v", closeErr) } }() - if resp.StatusCode != 200 { + statusCode = resp.StatusCode + if statusCode != http.StatusOK { var raw map[string]json.RawMessage d := json.NewDecoder(resp.Body) err = d.Decode(&raw) if err != nil { - return nil, fmt.Errorf("team API query failed with status code %d and malformed response: %v", resp.StatusCode, err) + return nil, statusCode, fmt.Errorf("team API query failed with status code %d and malformed response: %v", statusCode, err) } if errMessage, ok := raw["error"]; ok { - return nil, fmt.Errorf("team API query failed with status code %d and message: '%v'", resp.StatusCode, string(errMessage)) + return nil, statusCode, fmt.Errorf("team API query failed with status code %d and message: '%v'", statusCode, string(errMessage)) } - return nil, fmt.Errorf("team API query failed with status code %d", resp.StatusCode) + return nil, statusCode, fmt.Errorf("team API query failed with status code %d", statusCode) } tm = &Team{} d := json.NewDecoder(resp.Body) if err = d.Decode(tm); err != nil { - return nil, fmt.Errorf("could not parse team API response: %v", err) + return nil, statusCode, fmt.Errorf("could not parse team API response: %v", err) } - return tm, nil + return tm, statusCode, nil } diff --git a/pkg/util/teams/teams_test.go b/pkg/util/teams/teams_test.go index 33d01b75b..da9f497c1 100644 --- a/pkg/util/teams/teams_test.go +++ b/pkg/util/teams/teams_test.go @@ -1,225 +1,243 @@ -package teams - -import ( - "fmt" - "net/http" - "net/http/httptest" - "reflect" - "testing" - - "github.com/sirupsen/logrus" -) - -var ( - logger = logrus.New().WithField("pkg", "teamsapi") - token = "ec45b1cfbe7100c6315d183a3eb6cec0M2U1LWJkMzEtZDgzNzNmZGQyNGM3IiwiYXV0aF90aW1lIjoxNDkzNzMwNzQ1LCJpc3MiOiJodHRwcz" -) - -var teamsAPItc = []struct { - in string - inCode int - out *Team - err error -}{ - {`{ -"dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", -"id": "acid", -"id_name": "acid", -"team_id": "111222", -"type": "official", -"name": "Acid team name", -"mail": [ -"email1@example.com", -"email2@example.com" -], -"alias": [ -"acid" -], -"member": [ - "member1", - "member2", - "member3" -], -"infrastructure-accounts": [ -{ - "id": "1234512345", - "name": "acid", - "provider": "aws", - "type": "aws", - "description": "", - "owner": "acid", - "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - "disabled": false -}, -{ - "id": "5432154321", - "name": "db", - "provider": "aws", - "type": "aws", - "description": "", - "owner": "acid", - "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - "disabled": false -} -], -"cost_center": "00099999", -"delivery_lead": "member4", -"parent_team_id": "111221" -}`, - 200, - &Team{ - Dn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - ID: "acid", - TeamName: "acid", - TeamID: "111222", - Type: "official", - FullName: "Acid team name", - Aliases: []string{"acid"}, - Mails: []string{"email1@example.com", "email2@example.com"}, - Members: []string{"member1", "member2", "member3"}, - CostCenter: "00099999", - DeliveryLead: "member4", - ParentTeamID: "111221", - InfrastructureAccounts: []infrastructureAccount{ - { - ID: "1234512345", - Name: "acid", - Provider: "aws", - Type: "aws", - Description: "", - Owner: "acid", - OwnerDn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - Disabled: false}, - { - ID: "5432154321", - Name: "db", - Provider: "aws", - Type: "aws", - Description: "", - Owner: "acid", - OwnerDn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - Disabled: false}, - }, - }, - nil}, { - `{"error": "Access Token not valid"}`, - 401, - nil, - fmt.Errorf(`team API query failed with status code 401 and message: '"Access Token not valid"'`), - }, - { - `{"status": "I'm a teapot'"}`, - 418, - nil, - fmt.Errorf(`team API query failed with status code 418`), - }, - { - `{"status": "I'm a teapot`, - 418, - nil, - fmt.Errorf(`team API query failed with status code 418 and malformed response: unexpected EOF`), - }, - { - `{"status": "I'm a teapot`, - 200, - nil, - fmt.Errorf(`could not parse team API response: unexpected EOF`), - }, -} - -var requestsURLtc = []struct { - url string - err error -}{ - { - "coffee://localhost/", - fmt.Errorf(`Get "coffee://localhost/teams/acid": unsupported protocol scheme "coffee"`), - }, - { - "http://192.168.0.%31/", - fmt.Errorf(`parse "http://192.168.0.%%31/teams/acid": invalid URL escape "%%31"`), - }, -} - -func TestInfo(t *testing.T) { - for _, tc := range teamsAPItc { - func() { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Authorization") != "Bearer "+token { - t.Errorf("authorization token is wrong or not provided") - } - w.WriteHeader(tc.inCode) - if _, err := fmt.Fprint(w, tc.in); err != nil { - t.Errorf("error writing teams api response %v", err) - } - })) - defer ts.Close() - api := NewTeamsAPI(ts.URL, logger) - - actual, err := api.TeamInfo("acid", token) - if err != nil && err.Error() != tc.err.Error() { - t.Errorf("expected error: %v, got: %v", tc.err, err) - return - } - - if !reflect.DeepEqual(actual, tc.out) { - t.Errorf("expected %#v, got: %#v", tc.out, actual) - } - }() - } -} - -type mockHTTPClient struct { -} - -type mockBody struct { -} - -func (b *mockBody) Read(p []byte) (n int, err error) { - return 2, nil -} - -func (b *mockBody) Close() error { - return fmt.Errorf("close error") -} - -func (c *mockHTTPClient) Do(req *http.Request) (*http.Response, error) { - resp := http.Response{ - Status: "200 OK", - StatusCode: 200, - ContentLength: 2, - Close: false, - Request: req, - } - resp.Body = &mockBody{} - - return &resp, nil -} - -func TestHttpClientClose(t *testing.T) { - ts := httptest.NewServer(nil) - - api := NewTeamsAPI(ts.URL, logger) - api.httpClient = &mockHTTPClient{} - - _, err := api.TeamInfo("acid", token) - expError := fmt.Errorf("error when closing response: close error") - if err.Error() != expError.Error() { - t.Errorf("expected error: %v, got: %v", expError, err) - } -} - -func TestRequest(t *testing.T) { - for _, tc := range requestsURLtc { - api := NewTeamsAPI(tc.url, logger) - resp, err := api.TeamInfo("acid", token) - if resp != nil { - t.Errorf("response expected to be nil") - continue - } - - if err.Error() != tc.err.Error() { - t.Errorf("expected error: %v, got: %v", tc.err, err) - } - } -} +package teams + +import ( + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/sirupsen/logrus" +) + +var ( + logger = logrus.New().WithField("pkg", "teamsapi") + token = "ec45b1cfbe7100c6315d183a3eb6cec0M2U1LWJkMzEtZDgzNzNmZGQyNGM3IiwiYXV0aF90aW1lIjoxNDkzNzMwNzQ1LCJpc3MiOiJodHRwcz" + input = `{ + "dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + "id": "acid", + "id_name": "acid", + "team_id": "111222", + "type": "official", + "name": "Acid team name", + "mail": [ + "email1@example.com", + "email2@example.com" + ], + "alias": [ + "acid" + ], + "member": [ + "member1", + "member2", + "member3" + ], + "infrastructure-accounts": [ + { + "id": "1234512345", + "name": "acid", + "provider": "aws", + "type": "aws", + "description": "", + "owner": "acid", + "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + "disabled": false + }, + { + "id": "5432154321", + "name": "db", + "provider": "aws", + "type": "aws", + "description": "", + "owner": "acid", + "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + "disabled": false + } + ], + "cost_center": "00099999", + "delivery_lead": "member4", + "parent_team_id": "111221" + }` +) +var teamsAPItc = []struct { + in string + inCode int + inTeam string + out *Team + err error +}{ + { + input, + 200, + "acid", + &Team{ + Dn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + ID: "acid", + TeamName: "acid", + TeamID: "111222", + Type: "official", + FullName: "Acid team name", + Aliases: []string{"acid"}, + Mails: []string{"email1@example.com", "email2@example.com"}, + Members: []string{"member1", "member2", "member3"}, + CostCenter: "00099999", + DeliveryLead: "member4", + ParentTeamID: "111221", + InfrastructureAccounts: []infrastructureAccount{ + { + ID: "1234512345", + Name: "acid", + Provider: "aws", + Type: "aws", + Description: "", + Owner: "acid", + OwnerDn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + Disabled: false}, + { + ID: "5432154321", + Name: "db", + Provider: "aws", + Type: "aws", + Description: "", + Owner: "acid", + OwnerDn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + Disabled: false}, + }, + }, + nil}, { + `{"error": "Access Token not valid"}`, + 401, + "acid", + nil, + fmt.Errorf(`team API query failed with status code 401 and message: '"Access Token not valid"'`), + }, + { + `{"status": "I'm a teapot'"}`, + 418, + "acid", + nil, + fmt.Errorf(`team API query failed with status code 418`), + }, + { + `{"status": "I'm a teapot`, + 418, + "acid", + nil, + fmt.Errorf(`team API query failed with status code 418 and malformed response: unexpected EOF`), + }, + { + `{"status": "I'm a teapot`, + 200, + "acid", + nil, + fmt.Errorf(`could not parse team API response: unexpected EOF`), + }, + { + input, + 404, + "banana", + nil, + fmt.Errorf(`team API query failed with status code 404`), + }, +} + +var requestsURLtc = []struct { + url string + err error +}{ + { + "coffee://localhost/", + fmt.Errorf(`Get "coffee://localhost/teams/acid": unsupported protocol scheme "coffee"`), + }, + { + "http://192.168.0.%31/", + fmt.Errorf(`parse "http://192.168.0.%%31/teams/acid": invalid URL escape "%%31"`), + }, +} + +func TestInfo(t *testing.T) { + for _, tc := range teamsAPItc { + func() { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+token { + t.Errorf("authorization token is wrong or not provided") + } + w.WriteHeader(tc.inCode) + if _, err := fmt.Fprint(w, tc.in); err != nil { + t.Errorf("error writing teams api response %v", err) + } + })) + defer ts.Close() + api := NewTeamsAPI(ts.URL, logger) + + actual, statusCode, err := api.TeamInfo(tc.inTeam, token) + if err != nil && err.Error() != tc.err.Error() { + t.Errorf("expected error: %v, got: %v", tc.err, err) + return + } + + if !reflect.DeepEqual(actual, tc.out) { + t.Errorf("expected %#v, got: %#v", tc.out, actual) + } + + if statusCode != tc.inCode { + t.Errorf("expected %d, got: %d", tc.inCode, statusCode) + } + }() + } +} + +type mockHTTPClient struct { +} + +type mockBody struct { +} + +func (b *mockBody) Read(p []byte) (n int, err error) { + return 2, nil +} + +func (b *mockBody) Close() error { + return fmt.Errorf("close error") +} + +func (c *mockHTTPClient) Do(req *http.Request) (*http.Response, error) { + resp := http.Response{ + Status: "200 OK", + StatusCode: 200, + ContentLength: 2, + Close: false, + Request: req, + } + resp.Body = &mockBody{} + + return &resp, nil +} + +func TestHttpClientClose(t *testing.T) { + ts := httptest.NewServer(nil) + + api := NewTeamsAPI(ts.URL, logger) + api.httpClient = &mockHTTPClient{} + + _, _, err := api.TeamInfo("acid", token) + expError := fmt.Errorf("error when closing response: close error") + if err.Error() != expError.Error() { + t.Errorf("expected error: %v, got: %v", expError, err) + } +} + +func TestRequest(t *testing.T) { + for _, tc := range requestsURLtc { + api := NewTeamsAPI(tc.url, logger) + resp, _, err := api.TeamInfo("acid", token) + if resp != nil { + t.Errorf("response expected to be nil") + continue + } + + if err.Error() != tc.err.Error() { + t.Errorf("expected error: %v, got: %v", tc.err, err) + } + } +} diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go index 3da933644..b3b60df04 100644 --- a/pkg/util/users/users.go +++ b/pkg/util/users/users.go @@ -18,11 +18,13 @@ const ( alterUserRenameSQL = `ALTER ROLE "%s" RENAME TO "%s%s"` alterRoleResetAllSQL = `ALTER ROLE "%s" RESET ALL` alterRoleSetSQL = `ALTER ROLE "%s" SET %s TO %s` + dropUserSQL = `SET LOCAL synchronous_commit = 'local'; DROP ROLE "%s";` grantToUserSQL = `GRANT %s TO "%s"` + revokeFromUserSQL = `REVOKE "%s" FROM "%s"` doBlockStmt = `SET LOCAL synchronous_commit = 'local'; DO $$ BEGIN %s; END;$$;` passwordTemplate = "ENCRYPTED PASSWORD '%s'" inRoleTemplate = `IN ROLE %s` - adminTemplate = `ADMIN %s` + adminTemplate = `ADMIN "%s"` ) // DefaultUserSyncStrategy implements a user sync strategy that merges already existing database users @@ -30,8 +32,9 @@ const ( // an existing roles of another role membership, nor it removes the already assigned flag // (except for the NOLOGIN). TODO: process other NOflags, i.e. NOSUPERUSER correctly. type DefaultUserSyncStrategy struct { - PasswordEncryption string - RoleDeletionSuffix string + PasswordEncryption string + RoleDeletionSuffix string + AdditionalOwnerRoles []string } // ProduceSyncRequests figures out the types of changes that need to happen with the given users. @@ -40,10 +43,15 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM var reqs []spec.PgSyncUserRequest for name, newUser := range newUsers { - // do not create user that exists in DB with deletion suffix + // do not create user when there exists a user with the same name plus deletion suffix + // instead request a renaming of the deleted user back to the original name (see * below) if newUser.Deleted { continue } + // when the secret of the user could not be created or updated skip any database actions + if newUser.Degraded { + continue + } dbUser, exists := dbUsers[name] if !exists { reqs = append(reqs, spec.PgSyncUserRequest{Kind: spec.PGSyncUserAdd, User: newUser}) @@ -54,12 +62,14 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM r := spec.PgSyncUserRequest{} newMD5Password := util.NewEncryptor(strategy.PasswordEncryption).PGUserPassword(newUser) + // do not compare for roles coming from docker image if dbUser.Password != newMD5Password { r.User.Password = newMD5Password r.Kind = spec.PGsyncUserAlter } if addNewRoles, equal := util.SubstractStringSlices(newUser.MemberOf, dbUser.MemberOf); !equal { r.User.MemberOf = addNewRoles + r.User.IsDbOwner = newUser.IsDbOwner r.Kind = spec.PGsyncUserAlter } if addNewFlags, equal := util.SubstractStringSlices(newUser.Flags, dbUser.Flags); !equal { @@ -70,28 +80,35 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM r.User.Name = newUser.Name reqs = append(reqs, r) } - if len(newUser.Parameters) > 0 && !reflect.DeepEqual(dbUser.Parameters, newUser.Parameters) { + if len(newUser.Parameters) > 0 && + !reflect.DeepEqual(dbUser.Parameters, newUser.Parameters) { reqs = append(reqs, spec.PgSyncUserRequest{Kind: spec.PGSyncAlterSet, User: newUser}) } } } - // No existing roles are deleted or stripped of role membership/flags + // no existing roles are deleted or stripped of role membership/flags // but team roles will be renamed and denied from LOGIN for name, dbUser := range dbUsers { if _, exists := newUsers[name]; !exists { - // toggle LOGIN flag based on role deletion - userFlags := make([]string, len(dbUser.Flags)) - userFlags = append(userFlags, dbUser.Flags...) if dbUser.Deleted { - dbUser.Flags = util.StringSliceReplaceElement(dbUser.Flags, constants.RoleFlagNoLogin, constants.RoleFlagLogin) + // * user with deletion suffix and NOLOGIN found in database + // grant back LOGIN and rename only if original user is wanted and does not exist in database + originalName := strings.TrimSuffix(name, strategy.RoleDeletionSuffix) + _, originalUserWanted := newUsers[originalName] + _, originalUserAlreadyExists := dbUsers[originalName] + if !originalUserWanted || originalUserAlreadyExists { + continue + } + // a deleted dbUser has no NOLOGIN flag, so we can add the LOGIN flag + dbUser.Flags = append(dbUser.Flags, constants.RoleFlagLogin) } else { + // user found in database and not wanted in newUsers - replace LOGIN flag with NOLOGIN dbUser.Flags = util.StringSliceReplaceElement(dbUser.Flags, constants.RoleFlagLogin, constants.RoleFlagNoLogin) } - if !util.IsEqualIgnoreOrder(userFlags, dbUser.Flags) { - reqs = append(reqs, spec.PgSyncUserRequest{Kind: spec.PGsyncUserAlter, User: dbUser}) - } - + // request ALTER ROLE to grant or revoke LOGIN + reqs = append(reqs, spec.PgSyncUserRequest{Kind: spec.PGsyncUserAlter, User: dbUser}) + // request RENAME which will happen on behalf of the pgUser.Deleted field reqs = append(reqs, spec.PgSyncUserRequest{Kind: spec.PGSyncUserRename, User: dbUser}) } } @@ -101,7 +118,7 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM // ExecuteSyncRequests makes actual database changes from the requests passed in its arguments. func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSyncUserRequest, db *sql.DB) error { var reqretries []spec.PgSyncUserRequest - var errors []string + errors := make([]string, 0) for _, request := range requests { switch request.Kind { case spec.PGSyncUserAdd: @@ -113,6 +130,15 @@ func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSy if err := strategy.alterPgUser(request.User, db); err != nil { reqretries = append(reqretries, request) errors = append(errors, fmt.Sprintf("could not alter user %q: %v", request.User.Name, err)) + // XXX: we do not allow additional owner roles to be members of database owners + // if ALTER fails it could be because of the wrong memberhip (check #1862 for details) + // so in any case try to revoke the database owner from the additional owner roles + // the initial ALTER statement will be retried once and should work then + if request.User.IsDbOwner && len(strategy.AdditionalOwnerRoles) > 0 { + if err := resolveOwnerMembership(request.User, strategy.AdditionalOwnerRoles, db); err != nil { + errors = append(errors, fmt.Sprintf("could not resolve owner membership for %q: %v", request.User.Name, err)) + } + } } case spec.PGSyncAlterSet: if err := strategy.alterPgUserSet(request.User, db); err != nil { @@ -138,13 +164,28 @@ func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSy return err } } else { - return fmt.Errorf("could not execute sync requests for users: %v", errors) + return fmt.Errorf("could not execute sync requests for users: %v", strings.Join(errors, `', '`)) } } return nil } +func resolveOwnerMembership(dbOwner spec.PgUser, additionalOwners []string, db *sql.DB) error { + errors := make([]string, 0) + for _, additionalOwner := range additionalOwners { + if err := revokeRole(dbOwner.Name, additionalOwner, db); err != nil { + errors = append(errors, fmt.Sprintf("could not revoke %q from %q: %v", dbOwner.Name, additionalOwner, err)) + } + } + + if len(errors) > 0 { + return fmt.Errorf("could not resolve membership between %q and additional owner roles: %v", dbOwner.Name, strings.Join(errors, `', '`)) + } + + return nil +} + func (strategy DefaultUserSyncStrategy) alterPgUserSet(user spec.PgUser, db *sql.DB) error { queries := produceAlterRoleSetStmts(user) query := fmt.Sprintf(doBlockStmt, strings.Join(queries, ";")) @@ -265,6 +306,16 @@ func quoteMemberList(user spec.PgUser) string { return strings.Join(memberof, ",") } +func revokeRole(groupRole, role string, db *sql.DB) error { + revokeStmt := fmt.Sprintf(revokeFromUserSQL, groupRole, role) + + if _, err := db.Exec(fmt.Sprintf(doBlockStmt, revokeStmt)); err != nil { + return err + } + + return nil +} + // quoteVal quotes values to be used at ALTER ROLE SET param = value if necessary func quoteParameterValue(name, val string) string { start := val[0] @@ -288,3 +339,13 @@ func quoteParameterValue(name, val string) string { } return fmt.Sprintf(`'%s'`, strings.Trim(val, " ")) } + +// DropPgUser to remove user created by the operator e.g. for password rotation +func DropPgUser(user string, db *sql.DB) error { + query := fmt.Sprintf(dropUserSQL, user) + if _, err := db.Exec(query); err != nil { // TODO: Try several times + return err + } + + return nil +} diff --git a/pkg/util/util.go b/pkg/util/util.go index a52925583..993a773c4 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -19,6 +19,7 @@ import ( "github.com/motomux/pretty" resource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "github.com/zalando/postgres-operator/pkg/spec" "golang.org/x/crypto/pbkdf2" @@ -34,7 +35,7 @@ const ( var passwordChars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") func init() { - rand.Seed(time.Now().Unix()) + rand.New(rand.NewSource(time.Now().Unix())) } // helper function to get bool pointers @@ -86,7 +87,7 @@ func NewEncryptor(encryption string) *Encryptor { } hasher, ok := m[encryption] if !ok { - hasher = e.PGUserPasswordMD5 + hasher = e.PGUserPasswordScramSHA256 } e.encrypt = hasher return &e @@ -151,6 +152,17 @@ func IsEqualIgnoreOrder(a, b []string) bool { return reflect.DeepEqual(a_copy, b_copy) } +// Iterate through slice and remove certain string, then return cleaned slice +func RemoveString(slice []string, s string) (result []string) { + for _, item := range slice { + if item == s { + continue + } + result = append(result, item) + } + return result +} + // SliceReplaceElement func StringSliceReplaceElement(s []string, a, b string) (result []string) { tmp := make([]string, 0, len(s)) @@ -322,6 +334,20 @@ func testNil(values ...*int32) bool { return false } +// ToIntStr converts int to IntOrString type +func ToIntStr(val int) *intstr.IntOrString { + b := intstr.FromInt(val) + return &b +} + +// Bool2Int converts bool to int +func Bool2Int(flag bool) int { + if flag { + return 1 + } + return 0 +} + // MaxInt32 : Return maximum of two integers provided via pointers. If one value // is not defined, return the other one. If both are not defined, result is also // undefined, caller needs to check for that. @@ -352,3 +378,21 @@ func IsSmallerQuantity(requestStr, limitStr string) (bool, error) { return request.Cmp(limit) == -1, nil } + +func MinResource(maxRequestStr, requestStr string) (resource.Quantity, error) { + + isSmaller, err := IsSmallerQuantity(maxRequestStr, requestStr) + if isSmaller && err == nil { + maxRequest, err := resource.ParseQuantity(maxRequestStr) + if err != nil { + return maxRequest, err + } + return maxRequest, nil + } + + request, err := resource.ParseQuantity(requestStr) + if err != nil { + return request, err + } + return request, nil +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 75853c3d6..37e41f1cf 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -62,6 +62,19 @@ var substractTest = []struct { }{ {[]string{"a", "b", "c", "d"}, []string{"a", "b", "c", "d"}, []string{}, true}, {[]string{"a", "b", "c", "d"}, []string{"a", "bb", "c", "d"}, []string{"b"}, false}, + {[]string{""}, []string{"b"}, []string{""}, false}, + {[]string{"a"}, []string{""}, []string{"a"}, false}, +} + +var removeStringTest = []struct { + slice []string + item string + result []string +}{ + {[]string{"a", "b", "c"}, "b", []string{"a", "c"}}, + {[]string{"a"}, "b", []string{"a"}}, + {[]string{"a"}, "a", []string{}}, + {[]string{}, "a", []string{}}, } var sliceContaintsTest = []struct { @@ -198,6 +211,15 @@ func TestFindNamedStringSubmatch(t *testing.T) { } } +func TestRemoveString(t *testing.T) { + for _, tt := range removeStringTest { + res := RemoveString(tt.slice, tt.item) + if !IsEqualIgnoreOrder(res, tt.result) { + t.Errorf("RemoveString expected: %#v, got: %#v", tt.result, res) + } + } +} + func TestSliceContains(t *testing.T) { for _, tt := range sliceContaintsTest { res := SliceContains(tt.slice, tt.item) diff --git a/pkg/util/volumes/ebs.go b/pkg/util/volumes/ebs.go index 8f998b4cb..bb7506d93 100644 --- a/pkg/util/volumes/ebs.go +++ b/pkg/util/volumes/ebs.go @@ -1,12 +1,13 @@ package volumes import ( + "context" "fmt" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" v1 "k8s.io/api/core/v1" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -15,17 +16,17 @@ import ( // EBSVolumeResizer implements volume resizing interface for AWS EBS volumes. type EBSVolumeResizer struct { - connection *ec2.EC2 + connection *ec2.Client AWSRegion string } // ConnectToProvider connects to AWS. func (r *EBSVolumeResizer) ConnectToProvider() error { - sess, err := session.NewSession(&aws.Config{Region: aws.String(r.AWSRegion)}) + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(r.AWSRegion)) if err != nil { return fmt.Errorf("could not establish AWS session: %v", err) } - r.connection = ec2.New(sess) + r.connection = ec2.NewFromConfig(cfg) return nil } @@ -36,11 +37,16 @@ func (r *EBSVolumeResizer) IsConnectedToProvider() bool { // VolumeBelongsToProvider checks if the given persistent volume is backed by EBS. func (r *EBSVolumeResizer) VolumeBelongsToProvider(pv *v1.PersistentVolume) bool { - return pv.Spec.AWSElasticBlockStore != nil && pv.Annotations[constants.VolumeStorateProvisionerAnnotation] == constants.EBSProvisioner + return (pv.Spec.AWSElasticBlockStore != nil && pv.Annotations[constants.VolumeStorateProvisionerAnnotation] == constants.EBSProvisioner) || + (pv.Spec.CSI != nil && pv.Spec.CSI.Driver == constants.EBSDriver) } -// ExtractVolumeID extracts volumeID +// ExtractVolumeID extracts volumeID from "aws://eu-central-1a/vol-075ddfc4a127d0bd4" +// or return only the vol-075ddfc4a127d0bd4 when it doesn't have "aws://" func (r *EBSVolumeResizer) ExtractVolumeID(volumeID string) (string, error) { + if (strings.HasPrefix(volumeID, "vol-")) && !(strings.HasPrefix(volumeID, "aws://")) { + return volumeID, nil + } idx := strings.LastIndex(volumeID, constants.EBSVolumeIDStart) + 1 if idx == 0 { return "", fmt.Errorf("malformed EBS volume id %q", volumeID) @@ -50,7 +56,12 @@ func (r *EBSVolumeResizer) ExtractVolumeID(volumeID string) (string, error) { // GetProviderVolumeID converts aws://eu-central-1b/vol-00f93d4827217c629 to vol-00f93d4827217c629 for EBS volumes func (r *EBSVolumeResizer) GetProviderVolumeID(pv *v1.PersistentVolume) (string, error) { - volumeID := pv.Spec.AWSElasticBlockStore.VolumeID + var volumeID string = "" + if pv.Spec.CSI != nil { + volumeID = pv.Spec.CSI.VolumeHandle + } else if pv.Spec.AWSElasticBlockStore != nil { + volumeID = pv.Spec.AWSElasticBlockStore.VolumeID + } if volumeID == "" { return "", fmt.Errorf("got empty volume id for volume %v", pv) } @@ -67,7 +78,7 @@ func (r *EBSVolumeResizer) DescribeVolumes(volumeIds []string) ([]VolumeProperti } } - volumeOutput, err := r.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: aws.StringSlice((volumeIds))}) + volumeOutput, err := r.connection.DescribeVolumes(context.TODO(), &ec2.DescribeVolumesInput{VolumeIds: volumeIds}) if err != nil { return nil, err } @@ -78,12 +89,13 @@ func (r *EBSVolumeResizer) DescribeVolumes(volumeIds []string) ([]VolumeProperti } for _, v := range volumeOutput.Volumes { - if *v.VolumeType == "gp3" { - p = append(p, VolumeProperties{VolumeID: *v.VolumeId, Size: *v.Size, VolumeType: *v.VolumeType, Iops: *v.Iops, Throughput: *v.Throughput}) - } else if *v.VolumeType == "gp2" { - p = append(p, VolumeProperties{VolumeID: *v.VolumeId, Size: *v.Size, VolumeType: *v.VolumeType}) - } else { - return nil, fmt.Errorf("Discovered unexpected volume type %s %s", *v.VolumeId, *v.VolumeType) + switch v.VolumeType { + case "gp3": + p = append(p, VolumeProperties{VolumeID: *v.VolumeId, Size: int64(*v.Size), VolumeType: string(v.VolumeType), Iops: int64(*v.Iops), Throughput: int64(*v.Throughput)}) + case "gp2": + p = append(p, VolumeProperties{VolumeID: *v.VolumeId, Size: int64(*v.Size), VolumeType: string(v.VolumeType)}) + default: + return nil, fmt.Errorf("discovered unexpected volume type %s %s", *v.VolumeId, v.VolumeType) } } @@ -93,7 +105,7 @@ func (r *EBSVolumeResizer) DescribeVolumes(volumeIds []string) ([]VolumeProperti // ResizeVolume actually calls AWS API to resize the EBS volume if necessary. func (r *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { /* first check if the volume is already of a requested size */ - volumeOutput, err := r.connection.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeID}}) + volumeOutput, err := r.connection.DescribeVolumes(context.TODO(), &ec2.DescribeVolumesInput{VolumeIds: []string{volumeID}}) if err != nil { return fmt.Errorf("could not get information about the volume: %v", err) } @@ -101,17 +113,18 @@ func (r *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { if *vol.VolumeId != volumeID { return fmt.Errorf("describe volume %q returned information about a non-matching volume %q", volumeID, *vol.VolumeId) } - if *vol.Size == newSize { + sizeInt32 := int32(newSize) + if *vol.Size == sizeInt32 { // nothing to do return nil } - input := ec2.ModifyVolumeInput{Size: &newSize, VolumeId: &volumeID} - output, err := r.connection.ModifyVolume(&input) + input := ec2.ModifyVolumeInput{Size: &sizeInt32, VolumeId: &volumeID} + output, err := r.connection.ModifyVolume(context.TODO(), &input) if err != nil { return fmt.Errorf("could not modify persistent volume: %v", err) } - state := *output.VolumeModification.ModificationState + state := output.VolumeModification.ModificationState if state == constants.EBSVolumeStateFailed { return fmt.Errorf("could not modify persistent volume %q: modification state failed", volumeID) } @@ -122,10 +135,10 @@ func (r *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { return nil } // wait until the volume reaches the "optimizing" or "completed" state - in := ec2.DescribeVolumesModificationsInput{VolumeIds: []*string{&volumeID}} + in := ec2.DescribeVolumesModificationsInput{VolumeIds: []string{volumeID}} return retryutil.Retry(constants.EBSVolumeResizeWaitInterval, constants.EBSVolumeResizeWaitTimeout, func() (bool, error) { - out, err := r.connection.DescribeVolumesModifications(&in) + out, err := r.connection.DescribeVolumesModifications(context.TODO(), &in) if err != nil { return false, fmt.Errorf("could not describe volume modification: %v", err) } @@ -136,20 +149,35 @@ func (r *EBSVolumeResizer) ResizeVolume(volumeID string, newSize int64) error { return false, fmt.Errorf("non-matching volume id when describing modifications: %q is different from %q", *out.VolumesModifications[0].VolumeId, volumeID) } - return *out.VolumesModifications[0].ModificationState != constants.EBSVolumeStateModifying, nil + return out.VolumesModifications[0].ModificationState != constants.EBSVolumeStateModifying, nil }) } // ModifyVolume Modify EBS volume func (r *EBSVolumeResizer) ModifyVolume(volumeID string, newType *string, newSize *int64, iops *int64, throughput *int64) error { /* first check if the volume is already of a requested size */ - input := ec2.ModifyVolumeInput{Size: newSize, VolumeId: &volumeID, VolumeType: newType, Iops: iops, Throughput: throughput} - output, err := r.connection.ModifyVolume(&input) + var sizeInt32 *int32 + var iopsInt32 *int32 + var throughputInt32 *int32 + if newSize != nil { + s := int32(*newSize) + sizeInt32 = &s + } + if iops != nil { + i := int32(*iops) + iopsInt32 = &i + } + if throughput != nil { + t := int32(*throughput) + throughputInt32 = &t + } + input := ec2.ModifyVolumeInput{Size: sizeInt32, VolumeId: &volumeID, VolumeType: types.VolumeType(*newType), Iops: iopsInt32, Throughput: throughputInt32} + output, err := r.connection.ModifyVolume(context.TODO(), &input) if err != nil { return fmt.Errorf("could not modify persistent volume: %v", err) } - state := *output.VolumeModification.ModificationState + state := output.VolumeModification.ModificationState if state == constants.EBSVolumeStateFailed { return fmt.Errorf("could not modify persistent volume %q: modification state failed", volumeID) } @@ -160,10 +188,10 @@ func (r *EBSVolumeResizer) ModifyVolume(volumeID string, newType *string, newSiz return nil } // wait until the volume reaches the "optimizing" or "completed" state - in := ec2.DescribeVolumesModificationsInput{VolumeIds: []*string{&volumeID}} + in := ec2.DescribeVolumesModificationsInput{VolumeIds: []string{volumeID}} return retryutil.Retry(constants.EBSVolumeResizeWaitInterval, constants.EBSVolumeResizeWaitTimeout, func() (bool, error) { - out, err := r.connection.DescribeVolumesModifications(&in) + out, err := r.connection.DescribeVolumesModifications(context.TODO(), &in) if err != nil { return false, fmt.Errorf("could not describe volume modification: %v", err) } @@ -174,7 +202,7 @@ func (r *EBSVolumeResizer) ModifyVolume(volumeID string, newType *string, newSiz return false, fmt.Errorf("non-matching volume id when describing modifications: %q is different from %q", *out.VolumesModifications[0].VolumeId, volumeID) } - return *out.VolumesModifications[0].ModificationState != constants.EBSVolumeStateModifying, nil + return out.VolumesModifications[0].ModificationState != constants.EBSVolumeStateModifying, nil }) } diff --git a/pkg/util/volumes/ebs_test.go b/pkg/util/volumes/ebs_test.go new file mode 100644 index 000000000..6f722ff7b --- /dev/null +++ b/pkg/util/volumes/ebs_test.go @@ -0,0 +1,123 @@ +package volumes + +import ( + "fmt" + "testing" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGetProviderVolumeID(t *testing.T) { + tests := []struct { + name string + pv *v1.PersistentVolume + expected string + err error + }{ + { + name: "CSI volume handle", + pv: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + VolumeHandle: "vol-075ddfc4a127d0bd5", + }, + }, + }, + }, + expected: "vol-075ddfc4a127d0bd5", + err: nil, + }, + { + name: "AWS EBS volume handle", + pv: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{ + VolumeID: "aws://eu-central-1a/vol-075ddfc4a127d0bd4", + }, + }, + }, + }, + expected: "vol-075ddfc4a127d0bd4", + err: nil, + }, + { + name: "Empty volume handle", + pv: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{}, + }, + expected: "", + err: fmt.Errorf("got empty volume id for volume %v", &v1.PersistentVolume{}), + }, + } + + resizer := EBSVolumeResizer{} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + volumeID, err := resizer.GetProviderVolumeID(tt.pv) + if volumeID != tt.expected || (err != nil && err.Error() != tt.err.Error()) { + t.Errorf("expected %v, got %v, expected err %v, got %v", tt.expected, volumeID, tt.err, err) + } + }) + } +} + +func TestVolumeBelongsToProvider(t *testing.T) { + tests := []struct { + name string + pv *v1.PersistentVolume + expected bool + }{ + { + name: "CSI volume handle", + pv: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: "ebs.csi.aws.com", + VolumeHandle: "vol-075ddfc4a127d0bd5", + }, + }, + }, + }, + expected: true, + }, + { + name: "AWS EBS volume handle", + pv: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string { + "pv.kubernetes.io/provisioned-by": "kubernetes.io/aws-ebs", + }, + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{ + VolumeID: "aws://eu-central-1a/vol-075ddfc4a127d0bd4", + }, + }, + }, + }, + expected: true, + }, + { + name: "Empty volume source", + pv: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{}, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resizer := EBSVolumeResizer{} + isProvider := resizer.VolumeBelongsToProvider(tt.pv) + if isProvider != tt.expected { + t.Errorf("expected %v, got %v", tt.expected, isProvider) + } + }) + } +} diff --git a/pkg/util/volumes/volumes.go b/pkg/util/volumes/volumes.go index 556729dc4..32f68c65e 100644 --- a/pkg/util/volumes/volumes.go +++ b/pkg/util/volumes/volumes.go @@ -1,6 +1,6 @@ package volumes -//go:generate mockgen -package mocks -destination=$PWD/mocks/$GOFILE -source=$GOFILE -build_flags=-mod=vendor +//go:generate go tool mockgen -package mocks -destination=../../../mocks/$GOFILE -source=$GOFILE import v1 "k8s.io/api/core/v1" diff --git a/pkg/util/volumes/volumes_test.go b/pkg/util/volumes/volumes_test.go new file mode 100644 index 000000000..6bf39924e --- /dev/null +++ b/pkg/util/volumes/volumes_test.go @@ -0,0 +1,49 @@ +package volumes + +import ( + "fmt" + "testing" +) + +func TestExtractVolumeID(t *testing.T) { + var tests = []struct { + input string + expectedResult string + expectedErr error + }{ + { + input: "aws://eu-central-1c/vol-01234a5b6c78df9gh", + expectedResult: "vol-01234a5b6c78df9gh", + expectedErr: nil, + }, + { + input: "vol-0g9fd87c6b5a43210", + expectedResult: "vol-0g9fd87c6b5a43210", + expectedErr: nil, + }, + { + input: "aws://eu-central-1c/01234a5b6c78df9g0", + expectedResult: "", + expectedErr: fmt.Errorf("malformed EBS volume id %q", "aws://eu-central-1c/01234a5b6c78df9g0"), + }, + { + input: "hg9fd87c6b5a43210", + expectedResult: "", + expectedErr: fmt.Errorf("malformed EBS volume id %q", "hg9fd87c6b5a43210"), + }, + } + + resizer := EBSVolumeResizer{} + + for _, tt := range tests { + volumeId, err := resizer.ExtractVolumeID(tt.input) + if volumeId != tt.expectedResult { + t.Errorf("%s expected: %s, got %s", t.Name(), tt.expectedResult, volumeId) + } + if err != tt.expectedErr { + if tt.expectedErr != nil && err.Error() != tt.expectedErr.Error() { + t.Errorf("%s unexpected error: got %v", t.Name(), err) + } + } + } +} diff --git a/pooler/Dockerfile b/pooler/Dockerfile new file mode 100644 index 000000000..e7836e0f2 --- /dev/null +++ b/pooler/Dockerfile @@ -0,0 +1,54 @@ +ARG BASE_IMAGE=alpine:3.22 +FROM ${BASE_IMAGE} AS build_stage + +RUN apk add -U --no-cache \ + autoconf \ + automake \ + curl \ + gcc \ + libc-dev \ + libevent \ + libevent-dev \ + libtool \ + make \ + openssl-dev \ + pkgconfig \ + git + +WORKDIR /src + +RUN git clone --single-branch --depth 1 https://github.com/pgbouncer/pgbouncer.git . && \ + git checkout $(git describe --tags $(git rev-list --tags --max-count=1)) + +RUN git submodule init && git submodule update + +RUN ./autogen.sh && \ + ./configure --prefix=/pgbouncer --with-libevent=/usr/lib && \ + sed -i '/dist_man_MANS/d' Makefile && \ + make && \ + make install + +FROM ${BASE_IMAGE} + +RUN apk -U upgrade --no-cache \ + && apk --no-cache add bash c-ares ca-certificates gettext libevent openssl postgresql-client + +RUN addgroup -g 101 -S pgbouncer && \ + adduser -u 100 -S pgbouncer -G pgbouncer && \ + mkdir -p /etc/pgbouncer /var/log/pgbouncer /var/run/pgbouncer /etc/ssl/certs + +COPY --from=build_stage /pgbouncer/bin/pgbouncer /bin/pgbouncer +COPY pgbouncer.ini.tmpl /etc/pgbouncer/ +COPY entrypoint.sh /entrypoint.sh + +RUN chown -R pgbouncer:pgbouncer \ + /var/log/pgbouncer \ + /var/run/pgbouncer \ + /etc/pgbouncer \ + /etc/ssl/certs \ + && chmod +x /entrypoint.sh + +USER pgbouncer:pgbouncer +WORKDIR /etc/pgbouncer + +ENTRYPOINT ["/bin/sh", "/entrypoint.sh"] diff --git a/pooler/entrypoint.sh b/pooler/entrypoint.sh new file mode 100755 index 000000000..326d63fbe --- /dev/null +++ b/pooler/entrypoint.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -ex + +if [ -z "${CONNECTION_POOLER_CLIENT_TLS_CRT}" ]; then + openssl req -nodes -new -x509 -subj /CN=spilo.dummy.org \ + -keyout /etc/ssl/certs/pgbouncer.key \ + -out /etc/ssl/certs/pgbouncer.crt +else + ln -s ${CONNECTION_POOLER_CLIENT_TLS_CRT} /etc/ssl/certs/pgbouncer.crt + ln -s ${CONNECTION_POOLER_CLIENT_TLS_KEY} /etc/ssl/certs/pgbouncer.key + if [ ! -z "${CONNECTION_POOLER_CLIENT_CA_FILE}" ]; then + ln -s ${CONNECTION_POOLER_CLIENT_CA_FILE} /etc/ssl/certs/ca.crt + fi +fi + +envsubst < /etc/pgbouncer/pgbouncer.ini.tmpl > /etc/pgbouncer/pgbouncer.ini + +exec /bin/pgbouncer /etc/pgbouncer/pgbouncer.ini diff --git a/pooler/pgbouncer.ini.tmpl b/pooler/pgbouncer.ini.tmpl new file mode 100644 index 000000000..c26cf1453 --- /dev/null +++ b/pooler/pgbouncer.ini.tmpl @@ -0,0 +1,70 @@ +# vim: set ft=dosini: + +[databases] +* = host=$PGHOST port=$PGPORT auth_user=$PGUSER +postgres = host=$PGHOST port=$PGPORT auth_user=$PGUSER + +[pgbouncer] +pool_mode = $CONNECTION_POOLER_MODE +listen_port = $CONNECTION_POOLER_PORT +listen_addr = * +admin_users = $PGUSER +stats_users = $INFRASTRUCTURE_ROLES +auth_dbname = postgres +auth_file = /etc/pgbouncer/userlist.txt +auth_query = SELECT * FROM $PGSCHEMA.user_lookup($1) +auth_type = md5 +logfile = /var/log/pgbouncer/pgbouncer.log +pidfile = /var/run/pgbouncer/pgbouncer.pid + +server_tls_sslmode = require +server_tls_ca_file = /etc/ssl/certs/pgbouncer.crt +server_tls_protocols = secure +client_tls_sslmode = require +client_tls_key_file = /etc/ssl/certs/pgbouncer.key +client_tls_cert_file = /etc/ssl/certs/pgbouncer.crt + +log_connections = 0 +log_disconnections = 0 + +# Number of prepared statements to cache on a server connection (zero value +# disables support of prepared statements). +max_prepared_statements = 200 + +# How many server connections to allow per user/database pair. +default_pool_size = $CONNECTION_POOLER_DEFAULT_SIZE + +# Add more server connections to pool if below this number. Improves behavior +# when usual load comes suddenly back after period of total inactivity. +# +# NOTE: This value is per pool, i.e. a pair of (db, user), not a global one. +# Which means on the higher level it has to be calculated from the max allowed +# database connections and number of databases and users. If not taken into +# account, then for too many users or databases PgBouncer will go crazy +# opening/evicting connections. For now disable it. +# +# min_pool_size = $CONNECTION_POOLER_MIN_SIZE + +# How many additional connections to allow to a pool +reserve_pool_size = $CONNECTION_POOLER_RESERVE_SIZE + +# Maximum number of client connections allowed. +max_client_conn = $CONNECTION_POOLER_MAX_CLIENT_CONN + +# Do not allow more than this many connections per database (regardless of +# pool, i.e. user) +max_db_connections = $CONNECTION_POOLER_MAX_DB_CONN + +# If a client has been in "idle in transaction" state longer, it will be +# disconnected. [seconds] +idle_transaction_timeout = 600 + +# If login failed, because of failure from connect() or authentication that +# pooler waits this much before retrying to connect. Default is 15. [seconds] +server_login_retry = 5 + +# To ignore extra parameter in startup packet. By default only 'database' and +# 'user' are allowed, all others raise error. This is needed to tolerate +# overenthusiastic JDBC wanting to unconditionally set 'extra_float_digits=2' +# in startup packet. +ignore_startup_parameters = extra_float_digits,options diff --git a/run_operator_locally.sh b/run_operator_locally.sh index 9e3e082da..47dbb6071 100755 --- a/run_operator_locally.sh +++ b/run_operator_locally.sh @@ -30,8 +30,8 @@ function retry(){ local -r retry_cmd="$1" local -r retry_msg="$2" - # times out after 1 minute - for i in {1..20}; do + # Time out after three minutes. + for i in {1..60}; do if eval "$retry_cmd"; then return 0 fi @@ -98,7 +98,7 @@ function build_operator_binary(){ # redirecting stderr greatly reduces non-informative output during normal builds echo "Build operator binary (stderr redirected to /dev/null)..." - make clean deps local test > /dev/null 2>&1 + make clean local test > /dev/null 2>&1 } @@ -165,11 +165,63 @@ function forward_ports(){ local operator_pod operator_pod=$(kubectl get pod -l name=postgres-operator -o jsonpath={.items..metadata.name}) - # runs in the background to keep current terminal responsive - # stdout redirect removes the info message about forwarded ports; the message sometimes garbles the cli prompt - kubectl port-forward "$operator_pod" "$LOCAL_PORT":"$OPERATOR_PORT" &> /dev/null & + # Spawn `kubectl port-forward` in the background to keep current terminal + # responsive. Hide stdout because otherwise there is a note about each TCP + # connection. Do not hide stderr so port-forward setup errors can be + # debugged. Sometimes the port-forward setup fails because expected k8s + # state isn't achieved yet. Try to detect that case and then run the + # command again (in a finite loop). + for _attempt in {1..20}; do + # Delay between retry attempts. First attempt should already be + # delayed. + echo "soon: invoke kubectl port-forward command (attempt $_attempt)" + sleep 5 - echo $! > "$PATH_TO_PORT_FORWARED_KUBECTL_PID" + # With the --pod-running-timeout=4s argument the process is expected + # to terminate within about that time if the pod isn't ready yet. + kubectl port-forward --pod-running-timeout=4s "$operator_pod" "$LOCAL_PORT":"$OPERATOR_PORT" 1> /dev/null & + _kubectl_pid=$! + _pf_success=true + + # A successful `kubectl port-forward` setup can pragmatically be + # detected with a time-based criterion: it is a long-running process if + # successfully set up. If it does not terminate within deadline then + # consider the setup successful. Overall, observe the process for + # roughly 7 seconds. If it terminates before that it's certainly an + # error. If it did not terminate within that time frame then consider + # setup successful. + for ib in {1..7}; do + sleep 1 + # Portable and non-blocking test: is process still running? + if kill -s 0 -- "${_kubectl_pid}" >/dev/null 2>&1; then + echo "port-forward process is still running" + else + # port-forward process seems to have terminated, reap zombie + set +e + # `wait` is now expected to be non-blocking, and exits with the + # exit code of pid (first arg). + wait $_kubectl_pid + _kubectl_rc=$? + set -e + echo "port-forward process terminated with exit code ${_kubectl_rc}" + _pf_success=false + break + fi + done + + if [ ${_pf_success} = true ]; then + echo "port-forward setup seems successful. leave retry loop." + break + fi + + done + + if [ "${_pf_success}" = false ]; then + echo "port-forward setup failed after retrying. exit." + exit 1 + fi + + echo "${_kubectl_pid}" > "$PATH_TO_PORT_FORWARED_KUBECTL_PID" } diff --git a/ui/.dockerignore b/ui/.dockerignore index a53cb76a3..2bc7915f1 100644 --- a/ui/.dockerignore +++ b/ui/.dockerignore @@ -5,6 +5,8 @@ .git __pycache__ +.npm/ + app/node_modules operator_ui/static/build/*.hot-update.js operator_ui/static/build/*.hot-update.json diff --git a/ui/Dockerfile b/ui/Dockerfile index ad775ece2..63170a24b 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,41 +1,34 @@ -FROM registry.opensource.zalan.do/library/alpine-3.12:latest +ARG BASE_IMAGE=python:3.11-slim +ARG NODE_IMAGE=node:lts-alpine + +FROM $NODE_IMAGE AS build + +COPY . /workdir +WORKDIR /workdir/app + +RUN npm install \ + && npm run build + +FROM $BASE_IMAGE LABEL maintainer="Team ACID @ Zalando " EXPOSE 8081 +WORKDIR /app -RUN \ - apk add --no-cache \ - alpine-sdk \ - autoconf \ - automake \ - ca-certificates \ - libffi-dev \ - libtool \ - python3 \ - python3-dev \ - zlib-dev \ - && \ - python3 -m ensurepip && \ - rm -r /usr/lib/python*/ensurepip && \ - pip3 install --upgrade \ - gevent \ - jq \ - pip \ - setuptools \ - && \ - rm -rf \ - /root/.cache \ - /tmp/* \ - /var/cache/apk/* +RUN apt-get -qq -y update \ + # https://www.psycopg.org/docs/install.html#psycopg-vs-psycopg-binary + && apt-get -qq -y install --no-install-recommends g++ libpq-dev python3-dev \ + && apt-get -qq -y clean \ + && rm -rf /var/lib/apt/lists/* -COPY requirements.txt / -COPY start_server.sh / -RUN pip3 install -r /requirements.txt +COPY requirements.txt . +COPY start_server.sh . +RUN pip install -r requirements.txt -COPY operator_ui /operator_ui +COPY operator_ui operator_ui/ +COPY --from=build /workdir/operator_ui/static/build/ operator_ui/static/build/ ARG VERSION=dev -RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" /operator_ui/__init__.py +RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" operator_ui/__init__.py -WORKDIR / -CMD ["/usr/bin/python3", "-m", "operator_ui"] +CMD ["python", "-m", "operator_ui"] diff --git a/ui/Makefile b/ui/Makefile index 29c8d9409..ff4cf94f6 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -1,6 +1,7 @@ .PHONY: clean test appjs docker push mock -IMAGE ?= registry.opensource.zalan.do/acid/postgres-operator-ui +IMAGE ?= postgres-operator-ui +BASE_IMAGE ?= python:3.11-slim VERSION ?= $(shell git describe --tags --always --dirty) TAG ?= $(VERSION) GITHEAD = $(shell git rev-parse --short HEAD) @@ -21,8 +22,8 @@ test: tox appjs: - docker run $(TTYFLAGS) -u $$(id -u) -v $$(pwd):/workdir -w /workdir/app node:10.1.0-alpine npm install - docker run $(TTYFLAGS) -u $$(id -u) -v $$(pwd):/workdir -w /workdir/app node:10.1.0-alpine npm run build + docker run $(TTYFLAGS) -u $$(id -u) -v $$(pwd):/workdir -w /workdir/app node:lts-alpine npm install --cache /workdir/.npm + docker run $(TTYFLAGS) -u $$(id -u) -v $$(pwd):/workdir -w /workdir/app node:lts-alpine npm run build --cache /workdir/.npm docker: appjs echo `(env)` @@ -30,10 +31,7 @@ docker: appjs echo "Version ${VERSION}" echo "CDP tag ${CDP_TAG}" echo "git describe $(shell git describe --tags --always --dirty)" - docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)" -f Dockerfile . - -push: - docker push "$(IMAGE):$(TAG)$(CDP_TAG)" + docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)" -f Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" . mock: docker run -it -p 8081:8081 "$(IMAGE):$(TAG)" --mock diff --git a/ui/app/package.json b/ui/app/package.json index d0528e0bd..7fa1ef373 100644 --- a/ui/app/package.json +++ b/ui/app/package.json @@ -1,6 +1,6 @@ { "name": "postgres-operator-ui", - "version": "1.3.0", + "version": "1.15.1", "description": "PostgreSQL Operator UI", "main": "src/app.js", "config": { @@ -26,29 +26,28 @@ }, "homepage": "https://github.com/zalando/postgres-operator.git#readme", "dependencies": { - "@babel/core": "^7.0.0-beta.46", - "@babel/polyfill": "^7.0.0-beta.46", - "@babel/runtime": "^7.0.0-beta.46", - "pixi.js": "^4.7.3" + "@babel/core": "^7.20.12", + "@babel/polyfill": "^7.12.1", + "@babel/runtime": "^7.20.13", + "pixi.js": "^7.1.1" }, "devDependencies": { - "@babel/plugin-transform-runtime": "^7.0.0-beta.46", - "@babel/preset-env": "^7.0.0-beta.46", - "babel-loader": "^8.0.0-beta.2", - "brfs": "^1.6.1", + "@babel/plugin-transform-runtime": "^7.19.6", + "@babel/preset-env": "^7.20.2", + "babel-loader": "^8.2.5", + "brfs": "^2.0.2", "dedent-js": "1.0.1", - "eslint": "^4.19.1", - "eslint-loader": "^1.6.1", - "js-yaml": "3.13.1", - "pug": "^2.0.3", - "rimraf": "^2.5.4", - "riot": "^3.9.5", + "eslint": "^8.32.0", + "js-yaml": "4.2.0", + "pug": "^3.0.2", + "rimraf": "^4.1.2", + "riot": "^3.13.2", "riot-hot-reload": "1.0.0", - "riot-route": "^3.1.3", - "riot-tag-loader": "2.0.2", + "riot-route": "^3.1.4", + "riot-tag-loader": "2.1.0", "sort-by": "^1.2.0", - "transform-loader": "^0.2.3", - "webpack": "^4.28.2", - "webpack-cli": "^3.1.2" + "transform-loader": "^0.2.4", + "webpack": "^4.46.0", + "webpack-cli": "^4.10.0" } } diff --git a/ui/app/src/app.js b/ui/app/src/app.js index 9eb7569e6..20c92e63e 100644 --- a/ui/app/src/app.js +++ b/ui/app/src/app.js @@ -214,13 +214,13 @@ const delete_cluster = (namespace, clustername) => { jQuery.ajax({ type: 'DELETE', url: ( - '/postgresqls/' + './postgresqls/' + encodeURI(namespace) + '/' + encodeURI(clustername) ), dataType: 'text', - success: () => location.assign('/#/list'), - error: (r, status, error) => location.assign('/#/list'), // TODO: show error + success: () => location.assign('./#/list'), + error: (r, status, error) => location.assign('./#/list'), // TODO: show error }) }, }, diff --git a/ui/app/src/app.tag.pug b/ui/app/src/app.tag.pug index 365371e63..f53d425da 100644 --- a/ui/app/src/app.tag.pug +++ b/ui/app/src/app.tag.pug @@ -4,23 +4,23 @@ app .container .navbar-header - a.navbar-brand(href='/') + a.navbar-brand(href='./') | PostgreSQL Operator UI #navbar.navbar-collapse.collapse ul.nav.navbar-nav li(class='{ active: ["EDIT", "LIST", "LOGS", "STATUS"].includes(activenav) }') - a(href='/#/list') PostgreSQL clusters + a(href='./#/list') PostgreSQL clusters li(class='{ active: "BACKUPS" === activenav }') - a(href='/#/backups') Backups + a(href='./#/backups') Backups li(class='{ active: "OPERATOR" === activenav }') - a(href='/#/operator') Status + a(href='./#/operator') Status li(class='{ active: "NEW" === activenav }') - a(href='/#/new') New cluster + a(href='./#/new') New cluster li(if='{ config }') a(href='{ config.docs_link }' target='_blank') Documentation @@ -55,7 +55,7 @@ app | | or | - a(href="/") start over + a(href="./") start over | . div(if='{ config }') @@ -152,12 +152,12 @@ app ;( jQuery - .get('/config') + .get('./config') .done(config => { this.config = config ;( jQuery - .get('/teams') + .get('./teams') .done(teams => { this.teams = teams.sort() this.team = this.teams[0] diff --git a/ui/app/src/edit.tag.pug b/ui/app/src/edit.tag.pug index c1d94e589..e51630344 100644 --- a/ui/app/src/edit.tag.pug +++ b/ui/app/src/edit.tag.pug @@ -6,18 +6,18 @@ edit ol.breadcrumb li.breadcrumb-item - a(href='/#/list') + a(href='./#/list') | PostgreSQL clusters li.breadcrumb-item(if='{ cluster_path }') - a(href='/#/status/{ cluster_path }') + a(href='./#/status/{ cluster_path }') | { qname } li.breadcrumb-item.active( aria-current='page' if='{ cluster_path }' ) - a(href='/#/edit/{ cluster_path }') + a(href='./#/edit/{ cluster_path }') | Edit .row(if='{ cluster_path }') @@ -71,8 +71,8 @@ edit this.updateEditable = e => { if (this.refs.changedProperties.value) { - this.editablePropertiesPreview = yamlParser.safeDump( - yamlParser.safeLoad( + this.editablePropertiesPreview = yamlParser.dump( + yamlParser.load( this.refs.changedProperties.value, ), ) @@ -85,14 +85,14 @@ edit this.saveMessage = '' jsonPayload = JSON.stringify( - yamlParser.safeLoad( + yamlParser.load( this.refs.changedProperties.value, ), ) jQuery.ajax({ type: 'POST', - url: '/postgresqls/' + this.cluster_path, + url: './postgresqls/' + this.cluster_path, contentType:"application/json", data: jsonPayload, processData: false, @@ -113,7 +113,7 @@ edit this.pollProgress = () => { jQuery.get( - '/postgresqls/' + this.cluster_path, + './postgresqls/' + this.cluster_path, ).then(data => { // Input data: @@ -126,9 +126,10 @@ edit if (i.metadata.selfLink) { delete i.metadata.selfLink } if (i.metadata.uid) { delete i.metadata.uid } if (i.metadata.resourceVersion) { delete i.metadata.resourceVersion } + if (i.metadata.managedFields) { delete i.metadata.managedFields } this.update() - this.refs.yamlNice.innerHTML = yamlParser.safeDump(i.postgresql, {sortKeys: true}) + this.refs.yamlNice.innerHTML = yamlParser.dump(i.postgresql, {sortKeys: true}) // Output data: const o = this.editableProperties = { spec: {} } @@ -138,7 +139,22 @@ edit o.spec.enableMasterLoadBalancer = i.spec.enableMasterLoadBalancer || false o.spec.enableReplicaLoadBalancer = i.spec.enableReplicaLoadBalancer || false o.spec.enableConnectionPooler = i.spec.enableConnectionPooler || false - o.spec.volume = { size: i.spec.volume.size } + o.spec.enableReplicaConnectionPooler = i.spec.enableReplicaConnectionPooler || false + o.spec.enableMasterPoolerLoadBalancer = i.spec.enableMasterPoolerLoadBalancer || false + o.spec.enableReplicaPoolerLoadBalancer = i.spec.enableReplicaPoolerLoadBalancer || false + o.spec.maintenanceWindows = i.spec.maintenanceWindows || [] + + o.spec.volume = { + size: i.spec.volume.size, + throughput: i.spec.volume.throughput || 125, + iops: i.spec.volume.iops || 3000 + } + if ('storageClass' in i.spec.volume) { + o.spec.volume.storageClass=i.spec.volume.storageClass + } + + o.spec.postgresql = {} + o.spec.postgresql.version = i.spec.postgresql.version if ('users' in i.spec && typeof i.spec.users === 'object') { o.spec.users = Object.mapValues(i.spec.users, roleFlags => @@ -166,7 +182,7 @@ edit ].forEach(resourceType => { if (resourceType in resources) { const resourceClaim = resources[resourceType] - if (typeof resourceClaim === '') { + if (typeof resourceClaim === 'string') { o.spec.resources[section][resourceType] = resources[resourceType] } } @@ -177,7 +193,7 @@ edit this.editablePropertiesText = ( yamlParser - .safeDump(this.editableProperties) + .dump(this.editableProperties) .slice(0, -1) ) this.editablePropertiesPreview = this.editablePropertiesText diff --git a/ui/app/src/help-general.tag.pug b/ui/app/src/help-general.tag.pug index 1af25f1a8..7771fcebd 100644 --- a/ui/app/src/help-general.tag.pug +++ b/ui/app/src/help-general.tag.pug @@ -13,6 +13,6 @@ help-general h3 Basics p. - The PostgreSQL operator will use your definition to create a new + The Postgres Operator will use your definition to create a new PostgreSQL cluster for you. You can either copy the yaml definition - to the repositiory or you can just hit create cluster. + to a repositiory or hit create cluster (not available in prod). diff --git a/ui/app/src/logs.tag.pug b/ui/app/src/logs.tag.pug index e25cc979e..be79168af 100644 --- a/ui/app/src/logs.tag.pug +++ b/ui/app/src/logs.tag.pug @@ -5,15 +5,15 @@ logs ol.breadcrumb li.breadcrumb-item - a(href='/#/list') + a(href='./#/list') | PostgreSQL clusters li.breadcrumb-item - a(href='/#/status/{ cluster_path }') + a(href='./#/status/{ cluster_path }') | { qname } li.breadcrumb-item - a(href='/#/logs/{ cluster_path }') + a(href='./#/logs/{ cluster_path }') | Logs .sk-spinner-pulse(if='{ logs === undefined }') @@ -26,7 +26,7 @@ logs | | or | - a(href="/") start over + a(href="./") start over | . .container-fluid(if='{ logs }') @@ -72,7 +72,7 @@ logs ) ;( jQuery - .get(`/operator/clusters/${cluster_path}/logs`) + .get(`./operator/clusters/${cluster_path}/logs`) .done(logs => this.logs = logs.reverse()) .fail(() => this.logs = null) .always(() => this.update()) diff --git a/ui/app/src/new.tag.pug b/ui/app/src/new.tag.pug index 6293a6c7a..0e687e929 100644 --- a/ui/app/src/new.tag.pug +++ b/ui/app/src/new.tag.pug @@ -18,7 +18,7 @@ new ol.breadcrumb li.breadcrumb-item - a(href='/#/new') + a(href='./#/new') | New PostgreSQL cluster .row.text-center(if='{ !creating }') @@ -64,7 +64,7 @@ new a.btn.btn-small.btn-warning( if='{ clusterExists }' - href='/#/status/{ namespace.state }/{ team }-{ name }' + href='./#/status/{ namespace.state }/{ name }' ) | Cluster exists (show status) @@ -137,10 +137,10 @@ new input.form-control( ref='name' type='text' - placeholder='new-cluster (can be { 53 - team.length - 1 } characters long)' + placeholder='new-cluster (can be 53 characters long)' title='Database cluster name, must be a valid hostname component' pattern='[a-z0-9]+[a-z0-9\-]+[a-z0-9]+' - maxlength='{ 53 - team.length - 1 }' + maxlength=53 required value='{ name }' onchange='{ nameChange }' @@ -216,40 +216,73 @@ new ) tr(if='{ [undefined, true].includes(config.master_load_balancer_visible) }') - td Master load balancer + td Enable load balancer td - label - input( - type='checkbox' - value='{ enableMasterLoadBalancer }' - onchange='{ toggleEnableMasterLoadBalancer }' - ) - | - | Enable master ELB + ul.ips + li + label + input( + type='checkbox' + value='{ enableMasterLoadBalancer }' + onchange='{ toggleEnableMasterLoadBalancer }' + ) + | + | Master + li(if='{ [undefined, true].includes(config.replica_load_balancer_visible) && instanceCount > 1 }') + label + input( + type='checkbox' + value='{ enableReplicaLoadBalancer }' + onchange='{ toggleEnableReplicaLoadBalancer }' + ) + | + | Replica - tr(if='{ [undefined, true].includes(config.replica_load_balancer_visible) }') - td Replica load balancer + tr(if='{ [undefined, true].includes(config.connection_pooler_visible) }') + td Enable connection pooler td - label - input( - type='checkbox' - value='{ enableReplicaLoadBalancer }' - onchange='{ toggleEnableReplicaLoadBalancer }' - ) - | - | Enable replica ELB + ul.ips + li + label + input( + type='checkbox' + value='{ enableConnectionPooler }' + onchange='{ toggleEnableConnectionPooler }' + ) + | + | Master + li(if='{ [undefined, true].includes(config.replica_connection_pooler_visible) && instanceCount > 1 }') + label + input( + type='checkbox' + value='{ enableReplicaConnectionPooler }' + onchange='{ toggleEnableReplicaConnectionPooler }' + ) + | + | Replica - tr - td Enable Connection Pool + tr(if='{ [undefined, true].includes(config.master_pooler_load_balancer_visible) }') + td Enable connection pooler load balancer td - label - input( - type='checkbox' - value='{ enableConnectionPooler }' - onchange='{ toggleEnableConnectionPooler }' - ) - | - | Enable Connection Pool (using PGBouncer) + ul.ips + li + label + input( + type='checkbox' + value='{ enableMasterPoolerLoadBalancer }' + onchange='{ toggleEnableMasterPoolerLoadBalancer }' + ) + | + | Master + li(if='{ [undefined, true].includes(config.replica_pooler_load_balancer_visible) && instanceCount > 1 }') + label + input( + type='checkbox' + value='{ enableReplicaPoolerLoadBalancer }' + onchange='{ toggleEnableReplicaPoolerLoadBalancer }' + ) + | + | Replica tr td Volume size @@ -267,6 +300,50 @@ new .input-group-addon .input-units Gi + tr + td storageClass + td + .input-group + input.form-control( + ref='volumeStorageClass' + type='text' + value='{ volumeStorageClass }' + onchange='{ storageClassChange }' + onkeyup='{ storageClassChange }' + ) + + tr + td + td Specify Iops and Throughput only if you need more than the default 3000 Iops and 125Mb/s EBS provides. + + tr + td Iops + td + .input-group + input.form-control( + ref='iops' + type='number' + value='{ iops }' + onchange='{ iopsChange }' + onkeyup='{ iopsChange }' + ) + .input-group-addon + .input-units + + tr + td Throughput + td + .input-group + input.form-control( + ref='throughput' + type='number' + value='{ throughput }' + onchange='{ throughputChange }' + onkeyup='{ throughputChange }' + ) + .input-group-addon + .input-units MB/s + tr(if='{ config.users_visible }') td button.btn.btn-success.btn-xs(onclick='{ users.add }') @@ -489,7 +566,7 @@ new apiVersion: "acid.zalan.do/v1" metadata: - name: "{{ team }}-{{ name }}" + name: "{{ name }}" namespace: "{{ namespace.state }}" labels: team: {{ team }} @@ -508,8 +585,26 @@ new {{#if enableConnectionPooler}} enableConnectionPooler: true {{/if}} + {{#if enableReplicaConnectionPooler}} + enableReplicaConnectionPooler: true + {{/if}} + {{#if enableMasterPoolerLoadBalancer}} + enableMasterPoolerLoadBalancer: true + {{/if}} + {{#if enableReplicaPoolerLoadBalancer}} + enableReplicaPoolerLoadBalancer: true + {{/if}} + {{#if maintenanceWindows}} + maintenanceWindows: + {{#each maintenanceWindows}} + - "{{ this }}" + {{/each}} + {{/if}} volume: - size: "{{ volumeSize }}Gi" + size: "{{ volumeSize }}Gi"{{#if volumeStorageClass}} + storageClass: "{{ volumeStorageClass }}"{{/if}}{{#if iops}} + iops: {{ iops }}{{/if}}{{#if throughput}} + throughput: {{ throughput }}{{/if}} {{#if users}} users:{{#each users}} {{ state }}: []{{/each}}{{/if}} @@ -559,7 +654,14 @@ new enableMasterLoadBalancer: this.enableMasterLoadBalancer, enableReplicaLoadBalancer: this.enableReplicaLoadBalancer, enableConnectionPooler: this.enableConnectionPooler, + enableReplicaConnectionPooler: this.enableReplicaConnectionPooler, + enableMasterPoolerLoadBalancer: this.enableMasterPoolerLoadBalancer, + enableReplicaPoolerLoadBalancer: this.enableReplicaPoolerLoadBalancer, + maintenanceWindows: this.maintenanceWindows, volumeSize: this.volumeSize, + volumeStorageClass: this.volumeStorageClass, + iops: this.iops, + throughput: this.throughput, users: this.users.valids, databases: this.databases.valids, ranges: this.ranges, @@ -620,20 +722,47 @@ new this.enableConnectionPooler = !this.enableConnectionPooler } + this.toggleEnableReplicaConnectionPooler = e => { + this.enableReplicaConnectionPooler = !this.enableReplicaConnectionPooler + } + + this.toggleEnableMasterPoolerLoadBalancer = e => { + this.enableMasterPoolerLoadBalancer = !this.enableMasterPoolerLoadBalancer + } + + this.toggleEnableReplicaPoolerLoadBalancer = e => { + this.enableReplicaPoolerLoadBalancer = !this.enableReplicaPoolerLoadBalancer + } + + this.maintenanceWindows = e => { + this.maintenanceWindows = e.target.value + } + this.volumeChange = e => { this.volumeSize = +e.target.value } + this.storageClassChange = e => { + this.volumeStorageClass = e.target.value + } + + this.iopsChange = e => { + this.iops = +e.target.value + } + + this.throughputChange = e => { + this.throughput = +e.target.value + } + this.updateDNSName = () => { this.dnsName = this.config.dns_format_string.format( this.name, - this.team, this.namespace.state, ) } this.updateClusterName = () => { - this.clusterName = (this.team + '-' + this.name).toLowerCase() + this.clusterName = (this.name).toLowerCase() this.checkClusterExists() this.updateDNSName() } @@ -650,12 +779,17 @@ new this.instanceCountChange = e => { this.instanceCount = +e.target.value + if (this.instanceCount < 2) { + this.enableReplicaLoadBalancer = false + this.enableReplicaConnectionPooler = false + this.enableReplicaPoolerLoadBalancer = false + } } this.checkClusterExists = () => ( jQuery .get( - '/postgresqls/' + './postgresqls/' + this.namespace.state + '/' + this.clusterName @@ -670,14 +804,14 @@ new } this.requestCreate = e => { - jsonPayload = JSON.stringify(yamlParser.safeLoad(this.getYAML())) + jsonPayload = JSON.stringify(yamlParser.load(this.getYAML())) this.creating = true this.update() jQuery.ajax({ type: 'POST', - url: '/create-cluster', + url: './create-cluster', contentType:'application/json', data: jsonPayload, processData: false, @@ -907,14 +1041,19 @@ new this.team = '' } - this.clusterName = (this.name + '-' + this.team).toLowerCase() + this.clusterName = (this.name + '-').toLowerCase() this.volumeSize = 10 + this.volumeStorageClass = '' this.instanceCount = 1 this.ranges = {} this.odd = '' this.enableMasterLoadBalancer = false this.enableReplicaLoadBalancer = false this.enableConnectionPooler = false + this.enableReplicaConnectionPooler = false + this.enableMasterPoolerLoadBalancer = false + this.enableReplicaPoolerLoadBalancer = false + this.maintenanceWindows = {} this.postgresqlVersion = this.postgresqlVersion = ( this.config.postgresql_versions[0] diff --git a/ui/app/src/postgresql.tag.pug b/ui/app/src/postgresql.tag.pug index c557e4da8..7f91ff525 100644 --- a/ui/app/src/postgresql.tag.pug +++ b/ui/app/src/postgresql.tag.pug @@ -6,11 +6,11 @@ postgresql ol.breadcrumb li.breadcrumb-item - a(href='/#/list') + a(href='./#/list') | PostgreSQL clusters li.breadcrumb-item(if='{ cluster_path }') - a(href='/#/status/{ cluster_path }') + a(href='./#/status/{ cluster_path }') | { qname } .row(if='{ cluster_path }') @@ -39,20 +39,20 @@ postgresql ) a.btn.btn-info( - href='/#/logs/{ cluster_path }' + href='./#/logs/{ cluster_path }' ) | Logs a.btn( class='btn-{ opts.read_write ? "primary" : "info" }' if='{ progress.postgresql }' - href='/#/clone/{ clustername }/{ uid }/{ encodeURI(new Date().toISOString()) }' + href='./#/clone/{ clustername }/{ uid }/{ encodeURI(new Date().toISOString()) }' ) | Clone a.btn( class='btn-{ opts.read_write ? "warning" : "info" }' - href='/#/edit/{ cluster_path }' + href='./#/edit/{ cluster_path }' ) | Edit @@ -87,14 +87,11 @@ postgresql .alert.alert-info(if='{ progress.statefulSet && !progress.containerFirst }') Waiting for 1st container to spawn .alert.alert-success(if='{ progress.containerFirst }') First PostgreSQL cluster container spawned - .alert.alert-info(if='{ !progress.postgresql }') PostgreSQL cluster manifest pending - .alert.alert-success(if='{ progress.postgresql }') PostgreSQL cluster manifest created - .alert.alert-info(if='{ progress.containerFirst && !progress.masterLabel }') Waiting for master to become available .alert.alert-success(if='{ progress.masterLabel }') PostgreSQL master available, label is attached .alert.alert-success(if='{ progress.masterLabel && progress.dnsName }') PostgreSQL ready: { progress.dnsName } - .alert.alert-success(if='{ progress.pooler }') Connection pooler deployment created + .alert.alert-success(if='{ progress.pooler && this.progress.postgresqlManifest.spec.enableConnectionPooler }') Pooler ready: { progress.poolerDnsName } .col-lg-3 help-general(config='{ opts.config }') @@ -110,6 +107,7 @@ postgresql this.progress = {} this.progress.requestStatus = 'OK' + this.progress.pooler = false this.pollProgressTimer = false @@ -124,9 +122,8 @@ postgresql this.pollProgress = () => { jQuery.get( - '/postgresqls/' + this.cluster_path, + './postgresqls/' + this.cluster_path, ).done(data => { - this.progress.pooler = false this.progress.postgresql = true this.progress.postgresqlManifest = data // copy status as we delete later for edit @@ -137,13 +134,13 @@ postgresql this.update() jQuery.get( - '/statefulsets/' + this.cluster_path, + './statefulsets/' + this.cluster_path, ).done(data => { this.progress.statefulSet = true this.update() jQuery.get( - '/statefulsets/' + this.cluster_path + '/pods', + './statefulsets/' + this.cluster_path + '/pods', ).done(data => { if (data.length > 0) { this.progress.containerFirst = true @@ -157,7 +154,7 @@ postgresql this.update() jQuery.get( - '/services/' + this.cluster_path, + './services/' + this.cluster_path, ).done(data => { if (data.metadata && data.metadata.annotations && 'zalando.org/dnsname' in data.metadata.annotations) { this.progress.dnsName = data.metadata.annotations['zalando.org/dnsname'] @@ -168,10 +165,26 @@ postgresql this.progress.dnsName = data.metadata.name + '.' + data.metadata.namespace } - jQuery.get('/pooler/' + this.cluster_path).done(data => { - this.progress.pooler = {"url": ""} - this.update() - }) + if (this.progress.poolerEnabled == true) { + jQuery.get( + './pooler/' + this.cluster_path, + ).done(data => { + this.progress.pooler = {"url": ""} + jQuery.get( + './services/' + this.cluster_path + "-pooler", + ).done(data => { + if (data.metadata && data.metadata.annotations && 'zalando.org/dnsname' in data.metadata.annotations) { + this.progress.poolerDnsName = data.metadata.annotations['zalando.org/dnsname'] + } else if (data.metadata && data.metadata.annotations && 'external-dns.alpha.kubernetes.io/hostname' in data.metadata.annotations) { + this.progress.poolerDnsName = data.metadata.annotations['external-dns.alpha.kubernetes.io/hostname'] + } else { + this.progress.poolerDnsName = data.metadata.name + '.' + data.metadata.namespace + } + this.update() + }) + this.update() + }) + } this.update() }) @@ -218,7 +231,7 @@ postgresql delete manifest.status if (this.refs.yamlNice) { - this.refs.yamlNice.innerHTML = yamlParser.safeDump( + this.refs.yamlNice.innerHTML = yamlParser.dump( this.progress.postgresqlManifest, { sortKeys: true, diff --git a/ui/app/src/postgresqls.tag.pug b/ui/app/src/postgresqls.tag.pug index 38e5fcd9d..aed0d21b0 100644 --- a/ui/app/src/postgresqls.tag.pug +++ b/ui/app/src/postgresqls.tag.pug @@ -6,7 +6,7 @@ postgresqls ol.breadcrumb li.breadcrumb-item - a(href='/#/list') + a(href='./#/list') | PostgreSQL clusters .sk-spinner-pulse( @@ -20,7 +20,7 @@ postgresqls | | or | - a(href='/') start over + a(href='./') start over | . div( @@ -51,7 +51,25 @@ postgresqls th(style='width: 140px') CPU th(style='width: 130px') Memory th(style='width: 100px') Size - th(style='width: 120px') Cost/Month + th(style='width: 100px') IOPS + th(style='width: 100px') Throughput + th(style='width: 120px') + .tooltip(style='width: 120px') + | Cost/Month + .tooltiptext + strong Cost = MAX(CPU, Memory) + rest + br + | 1 CPU core : 42.09$ + br + | 1GB memory: 10.5225$ + br + | 1GB volume: 0.0952$ + br + | IOPS (-3000 baseline): 0.006$ + br + | Throughput (-125 baseline): 0.0476$ + br + | 1 ELB: 21.96$ th(stlye='width: 120px') tbody @@ -63,13 +81,15 @@ postgresqls td(style='white-space: pre') | { namespace } td - a(href='/#/status/{ cluster_path(this) }') { name } + a(href='./#/status/{ cluster_path(this) }') { name } btn.btn-danger(if='{status.PostgresClusterStatus == "CreateFailed"}') Create Failed td { nodes } td { cpu } / { cpu_limit } td { memory } / { memory_limit } td { volume_size } - td { calcCosts(nodes, cpu, memory, volume_size) }$ + td { iops } + td { throughput } + td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput, num_elb) }$ td @@ -81,7 +101,7 @@ postgresqls ) a.btn.btn-info( - href='/#/status/{ cluster_path(this) }' + href='./#/status/{ cluster_path(this) }' ) i.fa.fa-check-circle.regular | Status @@ -94,21 +114,21 @@ postgresqls | Pgview a.btn.btn-info( - href='/#/logs/{ cluster_path(this) }' + href='./#/logs/{ cluster_path(this) }' ) i.fa.fa-align-justify | Logs a.btn( class='btn-{ opts.read_write ? "primary" : "info" }' - href='/#/clone/{ encodeURI(name) }/{ encodeURI(uid) }/{ encodeURI(new Date().toISOString()) }' + href='./#/clone/{ encodeURI(name) }/{ encodeURI(uid) }/{ encodeURI(new Date().toISOString()) }' ) i.fa.fa-clone.regular | Clone a.btn( class='btn-{ opts.read_write ? "warning" : "info" }' - href='/#/edit/{ cluster_path(this) }' + href='./#/edit/{ cluster_path(this) }' ) | Edit @@ -132,8 +152,26 @@ postgresqls th(style='width: 140px') CPU th(style='width: 130px') Memory th(style='width: 100px') Size - th(style='width: 120px') Cost/Month - th(stlye='width: 120px') + th(style='width: 100px') IOPS + th(style='width: 100px') Throughput + th(style='width: 120px') + .tooltip(style='width: 120px') + | Cost/Month + .tooltiptext + strong Cost = MAX(CPU, Memory) + rest + br + | 1 CPU core : 42.09$ + br + | 1GB memory: 10.5225$ + br + | 1GB volume: 0.0952$ + br + | IOPS (-3000 baseline): 0.006$ + br + | Throughput (-125 baseline): 0.0476$ + br + | 1 ELB: 21.96$ + th(style='width: 120px') tbody tr( @@ -145,14 +183,16 @@ postgresqls | { namespace } td a( - href='/#/status/{ cluster_path(this) }' + href='./#/status/{ cluster_path(this) }' ) | { name } td { nodes } td { cpu } / { cpu_limit } td { memory } / { memory_limit } td { volume_size } - td { calcCosts(nodes, cpu, memory, volume_size) }$ + td { iops } + td { throughput } + td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput, num_elb) }$ td @@ -163,7 +203,7 @@ postgresqls ) a.btn.btn-info( - href='/#/status/{ cluster_path(this) }' + href='./#/status/{ cluster_path(this) }' ) i.fa.fa-check-circle.regular | Status @@ -177,21 +217,21 @@ postgresqls | Pgview a.btn.btn-info( - href='/#/logs/{ cluster_path(this) }' + href='./#/logs/{ cluster_path(this) }' ) i.fa.fa-align-justify | Logs a.btn( class='btn-{ opts.read_write ? "primary" : "info" }' - href='/#/clone/{ encodeURI(name) }/{ encodeURI(uid) }/{ encodeURI(new Date().toISOString()) }' + href='./#/clone/{ encodeURI(name) }/{ encodeURI(uid) }/{ encodeURI(new Date().toISOString()) }' ) i.fa.fa-clone.regular | Clone a.btn( class='btn-{ opts.read_write ? "warning" : "info" }' - href='/#/edit/{ cluster_path(this) }' + href='./#/edit/{ cluster_path(this) }' ) | Edit @@ -227,17 +267,47 @@ postgresqls + '/' + encodeURI(cluster.name) ) - const calcCosts = this.calcCosts = (nodes, cpu, memory, disk) => { - costs = Math.max(nodes, opts.config.min_pods) * (toCores(cpu) * opts.config.cost_core + toMemory(memory) * opts.config.cost_memory + toDisk(disk) * opts.config.cost_ebs) - return costs.toFixed(2) + const calcCosts = this.calcCosts = (nodes, cpu, memory, disk, iops, throughput, num_elb) => { + podcount = Math.max(nodes, opts.config.min_pods) + corecost = toCores(cpu) * opts.config.cost_core * 30.5 * 24 + memorycost = toMemory(memory) * opts.config.cost_memory * 30.5 * 24 + elbcost = num_elb * opts.config.cost_elb * 30.5 * 24 + diskcost = toDisk(disk) * opts.config.cost_ebs + iopscost = 0 + if (iops !== undefined && iops > opts.config.free_iops) { + if (iops > opts.config.limit_iops) { + iops = opts.config.limit_iops + } + iopscost = (iops - opts.config.free_iops) * opts.config.cost_iops + } + throughputcost = 0 + if (throughput !== undefined && throughput > opts.config.free_throughput) { + if (throughput > opts.config.limit_throughput) { + throughput = opts.config.limit_throughput + } + throughputcost = (throughput - opts.config.free_throughput) * opts.config.cost_throughput + } + + costs = podcount * (Math.max(corecost, memorycost) + diskcost + iopscost + throughputcost) + elbcost + return costs.toFixed(2) } const toDisk = this.toDisk = value => { - if(value.endsWith("Gi")) { + if(value.endsWith("Mi")) { + value = value.substring(0, value.length-2) + value = Number(value) / 1000. + return value + } + else if(value.endsWith("Gi")) { value = value.substring(0, value.length-2) value = Number(value) return value } + else if(value.endsWith("Ti")) { + value = value.substring(0, value.length-2) + value = Number(value) * 1000 + return value + } return value } @@ -253,6 +323,11 @@ postgresqls value = Number(value) return value } + else if(value.endsWith("Ti")) { + value = value.substring(0, value.length-2) + value = Number(value) * 1000 + return value + } return value } @@ -268,7 +343,7 @@ postgresqls this.on('mount', () => jQuery - .get('/postgresqls') + .get('./postgresqls') .done(clusters => { this.my_clusters = [] this.other_clusters = [] diff --git a/ui/app/src/restore.tag.pug b/ui/app/src/restore.tag.pug index 3a322df3f..523dbfbed 100644 --- a/ui/app/src/restore.tag.pug +++ b/ui/app/src/restore.tag.pug @@ -10,7 +10,7 @@ restore | | or | - a(href="/") start over + a(href="./") start over | . p(if='{ stored_clusters && stored_clusters.length === 0 }') @@ -23,7 +23,7 @@ restore ol.breadcrumb li.breadcrumb-item - a(href='/#/backups') + a(href='./#/backups') | PostgreSQL cluster backups ({ stored_clusters.length }) p @@ -63,7 +63,7 @@ restore p(if='{ versions === null }') | Error loading backups. Please try again or | - a(href="/") start over + a(href="./") start over | . p(if='{ versions && versions.length === 0 }') @@ -96,7 +96,7 @@ restore p(if='{ basebackups === null }') | Error loading snapshots. Please try again or | - a(href="/") start over + a(href="./") start over | . p(if='{ basebackups && basebackups.length === 0 }') @@ -312,7 +312,7 @@ restore get_subresources_once({ parent_resource: this, key: 'stored_clusters', - url: '/stored_clusters', + url: './stored_clusters', build_subresource: stored_cluster_name => ({ id: 'stored-cluster-' + stored_cluster_name, name: stored_cluster_name, @@ -324,7 +324,7 @@ restore body: stored_cluster => get_subresources_once({ parent_resource: stored_cluster, key: 'versions', - url: '/stored_clusters/' + stored_cluster.name, + url: './stored_clusters/' + stored_cluster.name, build_subresource: version_name => ({ id: stored_cluster.id + '-version-' + version_name, name: version_name, @@ -340,7 +340,7 @@ restore parent_resource: version, key: 'basebackups', url: ( - '/stored_clusters/' + stored_cluster.name + './stored_clusters/' + stored_cluster.name + '/' + version.name ), build_subresource: basebackup => Object.assign(basebackup, { diff --git a/ui/app/src/status.tag.pug b/ui/app/src/status.tag.pug index eae63fbdb..8a35703d9 100644 --- a/ui/app/src/status.tag.pug +++ b/ui/app/src/status.tag.pug @@ -6,19 +6,19 @@ status ol.breadcrumb li.breadcrumb-item - a(href='/#/operator') + a(href='./#/operator') | Workers virtual(if='{ operatorShowLogs && logs }') li.breadcrumb-item { worker_id } li.breadcrumb-item - a(href='/#/operator/worker/{ worker_id }/logs') + a(href='./#/operator/worker/{ worker_id }/logs') | Logs virtual(if='{ operatorShowQueue && queue }') li.breadcrumb-item { worker_id } li.breadcrumb-item - a(href='/#/operator/worker/{ worker_id }/queue') + a(href='./#/operator/worker/{ worker_id }/queue') | Queue div(if='{ status }') @@ -44,12 +44,12 @@ status ) a.btn.btn-info( - href='/#/operator/worker/{ worker_id }/logs' + href='./#/operator/worker/{ worker_id }/logs' ) | Logs a.btn.btn-info( - href='/#/operator/worker/{ worker_id }/queue' + href='./#/operator/worker/{ worker_id }/queue' ) | Queue @@ -96,7 +96,7 @@ status this.pollStatus = () => { jQuery.get( - '/operator/status', + './operator/status', ).done(data => { this.update({status: data}) }) @@ -111,7 +111,7 @@ status this.pollLogs = id => { jQuery.get( - '/operator/workers/' + id + '/logs', + './operator/workers/' + id + '/logs', ).done(data => { data.reverse() this.update({logs: data}) @@ -120,7 +120,7 @@ status this.pollQueue = id => { jQuery.get( - '/operator/workers/' + id + '/queue', + './operator/workers/' + id + '/queue', ).done(data => this.update({queue: data.List}) ) diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 464168ea4..e19a850b8 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -18,7 +18,7 @@ spec: serviceAccountName: postgres-operator-ui containers: - name: "service" - image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.6.0 + image: ghcr.io/zalando/postgres-operator-ui:v1.15.1 ports: - containerPort: 8081 protocol: "TCP" @@ -45,6 +45,7 @@ spec: - name: "RESOURCES_VISIBLE" value: "False" - name: "TARGET_NAMESPACE" + # Set to "*" to allow viewing/creation of clusters in all namespaces value: "default" - name: "TEAMS" value: |- @@ -55,19 +56,47 @@ spec: value: |- { "docs_link":"https://postgres-operator.readthedocs.io/en/latest/", - "dns_format_string": "{1}-{0}.{2}", + "dns_format_string": "{0}.{1}", "databases_visible": true, "master_load_balancer_visible": true, "nat_gateways_visible": false, "replica_load_balancer_visible": true, "resources_visible": true, "users_visible": true, - "cost_ebs": 0.119, + "cost_ebs": 0.0952, + "cost_iops": 0.006, + "cost_throughput": 0.0476, "cost_core": 0.0575, "cost_memory": 0.014375, + "free_iops": 3000, + "free_throughput": 125, + "limit_iops": 16000, + "limit_throughput": 1000, "postgresql_versions": [ - "13", - "12", - "11" + "18", + "17", + "16", + "15", + "14" ] } + # Exemple of settings to make snapshot view working in the ui when using AWS + # - name: SPILO_S3_BACKUP_PREFIX + # value: spilo/ + # - name: AWS_ACCESS_KEY_ID + # valueFrom: + # secretKeyRef: + # name: + # key: AWS_ACCESS_KEY_ID + # - name: AWS_SECRET_ACCESS_KEY + # valueFrom: + # secretKeyRef: + # name: + # key: AWS_SECRET_ACCESS_KEY + # - name: AWS_DEFAULT_REGION + # valueFrom: + # secretKeyRef: + # name: + # key: AWS_DEFAULT_REGION + # - name: SPILO_S3_BACKUP_BUCKET + # value: diff --git a/ui/manifests/ingress.yaml b/ui/manifests/ingress.yaml index 4efac53ac..3d721b9b6 100644 --- a/ui/manifests/ingress.yaml +++ b/ui/manifests/ingress.yaml @@ -1,4 +1,4 @@ -apiVersion: "networking.k8s.io/v1beta1" +apiVersion: "networking.k8s.io/v1" kind: "Ingress" metadata: name: "postgres-operator-ui" @@ -6,10 +6,15 @@ metadata: labels: application: "postgres-operator-ui" spec: + # ingressClassName: "ingress-nginx" rules: - host: "ui.example.org" http: paths: - - backend: - serviceName: "postgres-operator-ui" - servicePort: 80 + - path: / + pathType: Prefix + backend: + service: + name: "postgres-operator-ui" + port: + number: 80 diff --git a/charts/postgres-operator/values-crd.yaml b/ui/operator_ui/adapters/__init__.py similarity index 100% rename from charts/postgres-operator/values-crd.yaml rename to ui/operator_ui/adapters/__init__.py diff --git a/ui/operator_ui/adapters/logger.py b/ui/operator_ui/adapters/logger.py new file mode 100644 index 000000000..99166f749 --- /dev/null +++ b/ui/operator_ui/adapters/logger.py @@ -0,0 +1,46 @@ +import logging +from logging.config import dictConfig + +dictConfig( + { + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "json": { + "class": "pythonjsonlogger.jsonlogger.JsonFormatter", + "format": "%(asctime)s %(levelname)s: %(message)s", + } + }, + "handlers": { + "stream_handler": { + "class": "logging.StreamHandler", + "formatter": "json", + "stream": "ext://flask.logging.wsgi_errors_stream", + } + }, + "root": { + "level": "DEBUG", + "handlers": ["stream_handler"] + } + } +) + + +class Logger: + def __init__(self): + self.logger = logging.getLogger(__name__) + + def debug(self, msg: str, *args, **kwargs): + self.logger.debug(msg, *args, **kwargs) + + def info(self, msg: str, *args, **kwargs): + self.logger.info(msg, *args, **kwargs) + + def error(self, msg: str, *args, **kwargs): + self.logger.error(msg, *args, **kwargs) + + def exception(self, msg: str, *args, **kwargs): + self.logger.exception(msg, *args, **kwargs) + + +logger = Logger() diff --git a/ui/operator_ui/cluster_discovery.py b/ui/operator_ui/cluster_discovery.py index 9c89735ac..6bb211646 100644 --- a/ui/operator_ui/cluster_discovery.py +++ b/ui/operator_ui/cluster_discovery.py @@ -73,7 +73,7 @@ class StaticClusterDiscoverer: cluster = Cluster(generate_cluster_id(DEFAULT_CLUSTERS), DEFAULT_CLUSTERS) else: logger.info("in cluster configuration failed") - config = kubernetes.client.configuration + config = kubernetes.client.Configuration() cluster = Cluster( generate_cluster_id(config.host), config.host, diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index 5fbb6d24e..c91450d4c 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -1,13 +1,8 @@ #!/usr/bin/env python3 # pylama:ignore=E402 -import gevent.monkey - -gevent.monkey.patch_all() - import requests import tokens -import sys from backoff import expo, on_exception from click import ParamType, command, echo, option @@ -16,29 +11,22 @@ from flask import ( Flask, Response, abort, - redirect, render_template, request, send_from_directory, - session, ) -from flask_oauthlib.client import OAuth -from functools import wraps from gevent import sleep, spawn from gevent.pywsgi import WSGIServer from jq import jq from json import dumps, loads -from logging import DEBUG, ERROR, INFO, basicConfig, exception, getLogger from os import getenv from re import X, compile from requests.exceptions import RequestException from signal import SIGTERM, signal -from urllib.parse import urljoin from . import __version__ from .cluster_discovery import DEFAULT_CLUSTERS, StaticClusterDiscoverer -from .oauth import OAuthRemoteAppWithRefresh from .spiloutils import ( apply_postgresql, @@ -62,48 +50,51 @@ from .utils import ( these, ) - -# Disable access logs from Flask -getLogger('gevent').setLevel(ERROR) - -logger = getLogger(__name__) +from operator_ui.adapters.logger import logger SERVER_STATUS = {'shutdown': False} APP_URL = getenv('APP_URL') -AUTHORIZE_URL = getenv('AUTHORIZE_URL') SPILO_S3_BACKUP_BUCKET = getenv('SPILO_S3_BACKUP_BUCKET') TEAM_SERVICE_URL = getenv('TEAM_SERVICE_URL') -ACCESS_TOKEN_URL = getenv('ACCESS_TOKEN_URL') -TOKENINFO_URL = getenv('OAUTH2_TOKEN_INFO_URL') OPERATOR_API_URL = getenv('OPERATOR_API_URL', 'http://postgres-operator') OPERATOR_CLUSTER_NAME_LABEL = getenv('OPERATOR_CLUSTER_NAME_LABEL', 'cluster-name') -OPERATOR_UI_CONFIG = getenv('OPERATOR_UI_CONFIG', '{}') +OPERATOR_UI_CONFIG = loads(getenv('OPERATOR_UI_CONFIG', '{}')) OPERATOR_UI_MAINTENANCE_CHECK = getenv('OPERATOR_UI_MAINTENANCE_CHECK', '{}') READ_ONLY_MODE = getenv('READ_ONLY_MODE', False) in [True, 'true'] -RESOURCES_VISIBLE = getenv('RESOURCES_VISIBLE', True) SPILO_S3_BACKUP_PREFIX = getenv('SPILO_S3_BACKUP_PREFIX', 'spilo/') SUPERUSER_TEAM = getenv('SUPERUSER_TEAM', 'acid') TARGET_NAMESPACE = getenv('TARGET_NAMESPACE') GOOGLE_ANALYTICS = getenv('GOOGLE_ANALYTICS', False) -MIN_PODS= getenv('MIN_PODS', 2) +MIN_PODS = getenv('MIN_PODS', 2) +RESOURCES_VISIBLE = getenv('RESOURCES_VISIBLE', True) +CUSTOM_MESSAGE_RED = getenv('CUSTOM_MESSAGE_RED', '') -# storage pricing, i.e. https://aws.amazon.com/ebs/pricing/ -COST_EBS = float(getenv('COST_EBS', 0.119)) # GB per month +APPLICATION_DEPLOYMENT_DOCS = getenv('APPLICATION_DEPLOYMENT_DOCS', '') +CONNECTION_DOCS = getenv('CONNECTION_DOCS', '') + +# storage pricing, i.e. https://aws.amazon.com/ebs/pricing/ (e.g. Europe - Franfurt) +COST_EBS = float(getenv('COST_EBS', 0.0952)) # GB per month +COST_IOPS = float(getenv('COST_IOPS', 0.006)) # IOPS per month above 3000 baseline +COST_THROUGHPUT = float(getenv('COST_THROUGHPUT', 0.0476)) # MB/s per month above 125 MB/s baseline # compute costs, i.e. https://www.ec2instances.info/?region=eu-central-1&selected=m5.2xlarge -COST_CORE = 30.5 * 24 * float(getenv('COST_CORE', 0.0575)) # Core per hour m5.2xlarge / 8. -COST_MEMORY = 30.5 * 24 * float(getenv('COST_MEMORY', 0.014375)) # Memory GB m5.2xlarge / 32. +COST_CORE = float(getenv('COST_CORE', 0.0575)) # Core per hour m5.2xlarge / 8. +COST_MEMORY = float(getenv('COST_MEMORY', 0.014375)) # Memory GB m5.2xlarge / 32. +COST_ELB = float(getenv('COST_ELB', 0.03)) # per hour -WALE_S3_ENDPOINT = getenv( - 'WALE_S3_ENDPOINT', - 'https+path://s3-eu-central-1.amazonaws.com:443', -) +# maximum and limitation of IOPS and throughput +FREE_IOPS = float(getenv('FREE_IOPS', 3000)) +LIMIT_IOPS = float(getenv('LIMIT_IOPS', 16000)) +FREE_THROUGHPUT = float(getenv('FREE_THROUGHPUT', 125)) +LIMIT_THROUGHPUT = float(getenv('LIMIT_THROUGHPUT', 1000)) +# get the default value of core and memory +DEFAULT_MEMORY = getenv('DEFAULT_MEMORY', '300Mi') +DEFAULT_MEMORY_LIMIT = getenv('DEFAULT_MEMORY_LIMIT', '300Mi') +DEFAULT_CPU = getenv('DEFAULT_CPU', '10m') +DEFAULT_CPU_LIMIT = getenv('DEFAULT_CPU_LIMIT', '300m') -USE_AWS_INSTANCE_PROFILE = ( - getenv('USE_AWS_INSTANCE_PROFILE', 'false').lower() != 'false' -) AWS_ENDPOINT = getenv('AWS_ENDPOINT') @@ -166,38 +157,6 @@ class WSGITransferEncodingChunked: return self.app(environ, start_response) -oauth = OAuth(app) - -auth = OAuthRemoteAppWithRefresh( - oauth, - 'auth', - request_token_url=None, - access_token_method='POST', - access_token_url=ACCESS_TOKEN_URL, - authorize_url=AUTHORIZE_URL, -) -oauth.remote_apps['auth'] = auth - - -def verify_token(token): - if not token: - return False - - r = requests.get(TOKENINFO_URL, headers={'Authorization': token}) - - return r.status_code == 200 - - -def authorize(f): - @wraps(f) - def wrapper(*args, **kwargs): - if AUTHORIZE_URL and 'auth_token' not in session: - return redirect(urljoin(APP_URL, '/login')) - return f(*args, **kwargs) - - return wrapper - - def ok(body={}, status=200): return ( Response( @@ -279,19 +238,16 @@ STATIC_HEADERS = { @app.route('/css/') -@authorize def send_css(path): return send_from_directory('static/', path), 200, STATIC_HEADERS @app.route('/js/') -@authorize def send_js(path): return send_from_directory('static/', path), 200, STATIC_HEADERS @app.route('/') -@authorize def index(): return render_template('index.html', google_analytics=GOOGLE_ANALYTICS) @@ -302,27 +258,34 @@ DEFAULT_UI_CONFIG = { 'nat_gateways_visible': True, 'users_visible': True, 'databases_visible': True, - 'resources_visible': True, - 'postgresql_versions': ['11','12','13'], - 'dns_format_string': '{0}.{1}.{2}', + 'resources_visible': RESOURCES_VISIBLE, + 'postgresql_versions': ['14', '15', '16', '17', '18'], + 'dns_format_string': '{0}.{1}', 'pgui_link': '', 'static_network_whitelist': {}, + 'read_only_mode': READ_ONLY_MODE, + 'superuser_team': SUPERUSER_TEAM, + 'target_namespace': TARGET_NAMESPACE, + 'connection_docs': CONNECTION_DOCS, + 'application_deployment_docs': APPLICATION_DEPLOYMENT_DOCS, 'cost_ebs': COST_EBS, + 'cost_iops': COST_IOPS, + 'cost_throughput': COST_THROUGHPUT, 'cost_core': COST_CORE, 'cost_memory': COST_MEMORY, - 'min_pods': MIN_PODS + 'cost_elb': COST_ELB, + 'min_pods': MIN_PODS, + 'free_iops': FREE_IOPS, + 'free_throughput': FREE_THROUGHPUT, + 'limit_iops': LIMIT_IOPS, + 'limit_throughput': LIMIT_THROUGHPUT } @app.route('/config') -@authorize def get_config(): - config = loads(OPERATOR_UI_CONFIG) or DEFAULT_UI_CONFIG - config['read_only_mode'] = READ_ONLY_MODE - config['resources_visible'] = RESOURCES_VISIBLE - config['superuser_team'] = SUPERUSER_TEAM - config['target_namespace'] = TARGET_NAMESPACE - config['min_pods'] = MIN_PODS + config = DEFAULT_UI_CONFIG.copy() + config.update(OPERATOR_UI_CONFIG) config['namespaces'] = ( [TARGET_NAMESPACE] @@ -381,17 +344,15 @@ def get_teams_for_user(user_name): @app.route('/teams') -@authorize def get_teams(): return ok( get_teams_for_user( - session.get('user_name', ''), + request.headers.get('X-Uid', ''), ) ) @app.route('/services//') -@authorize def get_service(namespace: str, cluster: str): if TARGET_NAMESPACE not in ['', '*', namespace]: @@ -407,7 +368,6 @@ def get_service(namespace: str, cluster: str): @app.route('/pooler//') -@authorize def get_list_poolers(namespace: str, cluster: str): if TARGET_NAMESPACE not in ['', '*', namespace]: @@ -423,7 +383,6 @@ def get_list_poolers(namespace: str, cluster: str): @app.route('/statefulsets//') -@authorize def get_list_clusters(namespace: str, cluster: str): if TARGET_NAMESPACE not in ['', '*', namespace]: @@ -439,7 +398,6 @@ def get_list_clusters(namespace: str, cluster: str): @app.route('/statefulsets///pods') -@authorize def get_list_members(namespace: str, cluster: str): if TARGET_NAMESPACE not in ['', '*', namespace]: @@ -459,7 +417,6 @@ def get_list_members(namespace: str, cluster: str): @app.route('/namespaces') -@authorize def get_namespaces(): if TARGET_NAMESPACE not in ['', '*']: @@ -477,16 +434,17 @@ def get_namespaces(): @app.route('/postgresqls') -@authorize def get_postgresqls(): postgresqls = [ { 'nodes': spec.get('numberOfInstances', ''), - 'memory': spec.get('resources', {}).get('requests', {}).get('memory', 0), - 'memory_limit': spec.get('resources', {}).get('limits', {}).get('memory', 0), - 'cpu': spec.get('resources', {}).get('requests', {}).get('cpu', 0), - 'cpu_limit': spec.get('resources', {}).get('limits', {}).get('cpu', 0), + 'memory': spec.get('resources', {}).get('requests', {}).get('memory', OPERATOR_UI_CONFIG.get("default_memory", DEFAULT_MEMORY)), + 'memory_limit': spec.get('resources', {}).get('limits', {}).get('memory', OPERATOR_UI_CONFIG.get("default_memory_limit", DEFAULT_MEMORY_LIMIT)), + 'cpu': spec.get('resources', {}).get('requests', {}).get('cpu', OPERATOR_UI_CONFIG.get("default_cpu", DEFAULT_CPU)), + 'cpu_limit': spec.get('resources', {}).get('limits', {}).get('cpu', OPERATOR_UI_CONFIG.get("default_cpu_limit", DEFAULT_CPU_LIMIT)), 'volume_size': spec.get('volume', {}).get('size', 0), + 'iops': spec.get('volume', {}).get('iops', 3000), + 'throughput': spec.get('volume', {}).get('throughput', 125), 'team': ( spec.get('teamId') or metadata.get('labels', {}).get('team', '') @@ -497,6 +455,9 @@ def get_postgresqls(): 'namespaced_name': namespace + '/' + name, 'full_name': namespace + '/' + name + ('/' + uid if uid else ''), 'status': status, + 'num_elb': spec.get('enableMasterLoadBalancer', 0) + spec.get('enableReplicaLoadBalancer', 0) + \ + spec.get('enableMasterPoolerLoadBalancer', 0) + spec.get('enableReplicaPoolerLoadBalancer', 0), + 'maintenance_windows': spec.get('maintenanceWindows', []), } for cluster in these( read_postgresqls( @@ -572,7 +533,6 @@ def read_only(handler): @app.route('/postgresqls//', methods=['POST']) -@authorize @namespaced def update_postgresql(namespace: str, cluster: str): if READ_ONLY_MODE: @@ -584,8 +544,8 @@ def update_postgresql(namespace: str, cluster: str): postgresql = request.get_json(force=True) - teams = get_teams_for_user(session.get('user_name', '')) - logger.info(f'Changes to: {cluster} by {session.get("user_name", "local-user")}/{teams} {postgresql}') # noqa + teams = get_teams_for_user(request.headers.get('X-Uid', '')) + logger.info(f'Changes to: {cluster} by {request.headers.get("X-Uid", "local-user")}/{teams} {postgresql}') # noqa if SUPERUSER_TEAM and SUPERUSER_TEAM in teams: logger.info(f'Allowing edit due to membership in superuser team {SUPERUSER_TEAM}') # noqa @@ -599,6 +559,11 @@ def update_postgresql(namespace: str, cluster: str): return fail('allowedSourceRanges invalid') spec['allowedSourceRanges'] = postgresql['spec']['allowedSourceRanges'] + if 'maintenanceWindows' in postgresql['spec']: + if not isinstance(postgresql['spec']['maintenanceWindows'], list): + return fail('maintenanceWindows invalid') + spec['maintenanceWindows'] = postgresql['spec']['maintenanceWindows'] + if 'numberOfInstances' in postgresql['spec']: if not isinstance(postgresql['spec']['numberOfInstances'], int): return fail('numberOfInstances invalid') @@ -614,49 +579,42 @@ def update_postgresql(namespace: str, cluster: str): spec['volume'] = {'size': size} - if 'enableConnectionPooler' in postgresql['spec']: - cp = postgresql['spec']['enableConnectionPooler'] - if not cp: - if 'enableConnectionPooler' in o['spec']: - del o['spec']['enableConnectionPooler'] - else: - spec['enableConnectionPooler'] = True - else: - if 'enableConnectionPooler' in o['spec']: - del o['spec']['enableConnectionPooler'] + if ( + 'volume' in postgresql['spec'] + and 'iops' in postgresql['spec']['volume'] + and postgresql['spec']['volume']['iops'] != None + ): + iops = int(postgresql['spec']['volume']['iops']) + if not 'volume' in spec: + spec['volume'] = {} - if 'enableReplicaConnectionPooler' in postgresql['spec']: - cp = postgresql['spec']['enableReplicaConnectionPooler'] - if not cp: - if 'enableReplicaConnectionPooler' in o['spec']: - del o['spec']['enableReplicaConnectionPooler'] - else: - spec['enableReplicaConnectionPooler'] = True - else: - if 'enableReplicaConnectionPooler' in o['spec']: - del o['spec']['enableReplicaConnectionPooler'] + spec['volume']['iops'] = iops - if 'enableReplicaLoadBalancer' in postgresql['spec']: - rlb = postgresql['spec']['enableReplicaLoadBalancer'] - if not rlb: - if 'enableReplicaLoadBalancer' in o['spec']: - del o['spec']['enableReplicaLoadBalancer'] - else: - spec['enableReplicaLoadBalancer'] = True - else: - if 'enableReplicaLoadBalancer' in o['spec']: - del o['spec']['enableReplicaLoadBalancer'] + if ( + 'volume' in postgresql['spec'] + and 'throughput' in postgresql['spec']['volume'] + and postgresql['spec']['volume']['throughput'] != None + ): + throughput = int(postgresql['spec']['volume']['throughput']) + if not 'volume' in spec: + spec['volume'] = {} - if 'enableMasterLoadBalancer' in postgresql['spec']: - rlb = postgresql['spec']['enableMasterLoadBalancer'] - if not rlb: - if 'enableMasterLoadBalancer' in o['spec']: - del o['spec']['enableMasterLoadBalancer'] + spec['volume']['throughput'] = throughput + + additional_specs = ['enableMasterLoadBalancer', + 'enableReplicaLoadBalancer', + 'enableConnectionPooler', + 'enableReplicaConnectionPooler', + 'enableMasterPoolerLoadBalancer', + 'enableReplicaPoolerLoadBalancer', + ] + + for var in additional_specs: + if postgresql['spec'].get(var): + spec[var] = True else: - spec['enableMasterLoadBalancer'] = True - else: - if 'enableMasterLoadBalancer' in o['spec']: - del o['spec']['enableMasterLoadBalancer'] + if var in o['spec']: + del o['spec'][var] if 'users' in postgresql['spec']: spec['users'] = postgresql['spec']['users'] @@ -758,6 +716,27 @@ def update_postgresql(namespace: str, cluster: str): owner_username=owner_username, ) + resource_types = ["cpu","memory"] + resource_constraints = ["requests","limits"] + if "resources" in postgresql["spec"]: + spec["resources"] = {} + + res = postgresql["spec"]["resources"] + for rt in resource_types: + for rc in resource_constraints: + if rc in res: + if rt in res[rc]: + if not rc in spec["resources"]: + spec["resources"][rc] = {} + spec["resources"][rc][rt] = res[rc][rt] + + if "postgresql" in postgresql["spec"]: + if "version" in postgresql["spec"]["postgresql"]: + if "postgresql" not in spec: + spec["postgresql"]={} + + spec["postgresql"]["version"] = postgresql["spec"]["postgresql"]["version"] + o['spec'].update(spec) apply_postgresql(get_cluster(), namespace, cluster, o) @@ -766,7 +745,6 @@ def update_postgresql(namespace: str, cluster: str): @app.route('/postgresqls//', methods=['GET']) -@authorize def get_postgresql(namespace: str, cluster: str): if TARGET_NAMESPACE not in ['', '*', namespace]: @@ -782,7 +760,6 @@ def get_postgresql(namespace: str, cluster: str): @app.route('/stored_clusters') -@authorize def get_stored_clusters(): return respond( read_stored_clusters( @@ -793,37 +770,30 @@ def get_stored_clusters(): @app.route('/stored_clusters/', methods=['GET']) -@authorize def get_versions(pg_cluster: str): return respond( read_versions( bucket=SPILO_S3_BACKUP_BUCKET, pg_cluster=pg_cluster, prefix=SPILO_S3_BACKUP_PREFIX, - s3_endpoint=WALE_S3_ENDPOINT, - use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE, ), ) - @app.route('/stored_clusters//', methods=['GET']) -@authorize def get_basebackups(pg_cluster: str, uid: str): return respond( read_basebackups( bucket=SPILO_S3_BACKUP_BUCKET, pg_cluster=pg_cluster, prefix=SPILO_S3_BACKUP_PREFIX, - s3_endpoint=WALE_S3_ENDPOINT, uid=uid, - use_aws_instance_profile=USE_AWS_INSTANCE_PROFILE, + postgresql_versions=OPERATOR_UI_CONFIG.get('postgresql_versions', DEFAULT_UI_CONFIG['postgresql_versions']), ), ) @app.route('/create-cluster', methods=['POST']) -@authorize def create_new_cluster(): if READ_ONLY_MODE: @@ -841,8 +811,8 @@ def create_new_cluster(): if TARGET_NAMESPACE not in ['', '*', namespace]: return wrong_namespace() - teams = get_teams_for_user(session.get('user_name', '')) - logger.info(f'Create cluster by {session.get("user_name", "local-user")}/{teams} {postgresql}') # noqa + teams = get_teams_for_user(request.headers.get('X-Uid', '')) + logger.info(f'Create cluster by {request.headers.get("X-Uid", "local-user")}/{teams} {postgresql}') # noqa if SUPERUSER_TEAM and SUPERUSER_TEAM in teams: logger.info(f'Allowing create due to membership in superuser team {SUPERUSER_TEAM}') # noqa @@ -854,7 +824,6 @@ def create_new_cluster(): @app.route('/postgresqls//', methods=['DELETE']) -@authorize def delete_postgresql(namespace: str, cluster: str): if TARGET_NAMESPACE not in ['', '*', namespace]: return wrong_namespace() @@ -866,9 +835,9 @@ def delete_postgresql(namespace: str, cluster: str): if postgresql is None: return not_found() - teams = get_teams_for_user(session.get('user_name', '')) + teams = get_teams_for_user(request.headers.get('X-Uid', '')) - logger.info(f'Delete cluster: {cluster} by {session.get("user_name", "local-user")}/{teams}') # noqa + logger.info(f'Delete cluster: {cluster} by {request.headers.get("X-Uid", "local-user")}/{teams}') # noqa if SUPERUSER_TEAM and SUPERUSER_TEAM in teams: logger.info(f'Allowing delete due to membership in superuser team {SUPERUSER_TEAM}') # noqa @@ -892,44 +861,23 @@ def proxy_operator(url: str): @app.route('/operator/status') -@authorize def get_operator_status(): return proxy_operator('/status/') @app.route('/operator/workers//queue') -@authorize def get_operator_get_queue(worker: int): return proxy_operator(f'/workers/{worker}/queue') @app.route('/operator/workers//logs') -@authorize def get_operator_get_logs(worker: int): return proxy_operator(f'/workers/{worker}/logs') @app.route('/operator/clusters///logs') -@authorize def get_operator_get_logs_per_cluster(namespace: str, cluster: str): - team, clustername = cluster.split('-', 1) - return proxy_operator(f'/clusters/{team}/{namespace}/{clustername}/logs/') - - -@app.route('/login') -def login(): - redirect = request.args.get('redirect', False) - if not redirect: - return render_template('login-deeplink.html') - - redirect_uri = urljoin(APP_URL, '/login/authorized') - return auth.authorize(callback=redirect_uri) - - -@app.route('/logout') -def logout(): - session.pop('auth_token', None) - return redirect(urljoin(APP_URL, '/')) + return proxy_operator(f'/clusters/{namespace}/{cluster}/logs/') @app.route('/favicon.png') @@ -937,34 +885,6 @@ def favicon(): return send_from_directory('static/', 'favicon-96x96.png'), 200 -@app.route('/login/authorized') -def authorized(): - resp = auth.authorized_response() - if resp is None: - return 'Access denied: reason=%s error=%s' % ( - request.args['error'], - request.args['error_description'] - ) - - if not isinstance(resp, dict): - return 'Invalid auth response' - - session['auth_token'] = (resp['access_token'], '') - - r = requests.get( - TOKENINFO_URL, - headers={ - 'Authorization': f'Bearer {session["auth_token"][0]}', - }, - ) - session['user_name'] = r.json().get('uid') - - logger.info(f'Login from: {session["user_name"]}') - - # return redirect(urljoin(APP_URL, '/')) - return render_template('login-resolve-deeplink.html') - - def shutdown(): # just wait some time to give Kubernetes time to update endpoints # this requires changing the readinessProbe's @@ -1040,28 +960,18 @@ def init_cluster(): help='Verbose logging', is_flag=True, ) -@option( - '--secret-key', - default='development', - envvar='SECRET_KEY', - help='Secret key for session cookies', -) @option( '--clusters', envvar='CLUSTERS', help=f'Comma separated list of Kubernetes API server URLs (default: {DEFAULT_CLUSTERS})', # noqa type=CommaSeparatedValues(), ) -def main(port, secret_key, debug, clusters: list): +def main(port, debug, clusters: list): global TARGET_NAMESPACE - basicConfig(stream=sys.stdout, level=(DEBUG if debug else INFO), format='%(asctime)s %(levelname)s: %(message)s',) - init_cluster() - logger.info(f'Access token URL: {ACCESS_TOKEN_URL}') logger.info(f'App URL: {APP_URL}') - logger.info(f'Authorize URL: {AUTHORIZE_URL}') logger.info(f'Operator API URL: {OPERATOR_API_URL}') logger.info(f'Operator cluster name label: {OPERATOR_CLUSTER_NAME_LABEL}') logger.info(f'Readonly mode: {"enabled" if READ_ONLY_MODE else "disabled"}') # noqa @@ -1070,9 +980,6 @@ def main(port, secret_key, debug, clusters: list): logger.info(f'Superuser team: {SUPERUSER_TEAM}') logger.info(f'Target namespace: {TARGET_NAMESPACE}') logger.info(f'Teamservice URL: {TEAM_SERVICE_URL}') - logger.info(f'Tokeninfo URL: {TOKENINFO_URL}') - logger.info(f'Use AWS instance_profile: {USE_AWS_INSTANCE_PROFILE}') - logger.info(f'WAL-E S3 endpoint: {WALE_S3_ENDPOINT}') logger.info(f'AWS S3 endpoint: {AWS_ENDPOINT}') if TARGET_NAMESPACE is None: @@ -1093,7 +1000,6 @@ def main(port, secret_key, debug, clusters: list): logger.info(f'Target namespace set to: {TARGET_NAMESPACE or "*"}') app.debug = debug - app.secret_key = secret_key signal(SIGTERM, exit_gracefully) diff --git a/ui/operator_ui/oauth.py b/ui/operator_ui/oauth.py deleted file mode 100644 index 34c07fd4f..000000000 --- a/ui/operator_ui/oauth.py +++ /dev/null @@ -1,32 +0,0 @@ -import os - -from flask_oauthlib.client import OAuthRemoteApp - - -CREDENTIALS_DIR = os.getenv('CREDENTIALS_DIR', '') - - -class OAuthRemoteAppWithRefresh(OAuthRemoteApp): - '''Same as flask_oauthlib.client.OAuthRemoteApp, but always loads client credentials from file.''' - - def __init__(self, oauth, name, **kwargs): - # constructor expects some values, so make it happy.. - kwargs['consumer_key'] = 'not-needed-here' - kwargs['consumer_secret'] = 'not-needed-here' - OAuthRemoteApp.__init__(self, oauth, name, **kwargs) - - def refresh_credentials(self): - with open(os.path.join(CREDENTIALS_DIR, 'authcode-client-id')) as fd: - self._consumer_key = fd.read().strip() - with open(os.path.join(CREDENTIALS_DIR, 'authcode-client-secret')) as fd: - self._consumer_secret = fd.read().strip() - - @property - def consumer_key(self): - self.refresh_credentials() - return self._consumer_key - - @property - def consumer_secrect(self): - self.refresh_credentials() - return self._consumer_secret diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py index 26113bd54..8d2b73967 100644 --- a/ui/operator_ui/spiloutils.py +++ b/ui/operator_ui/spiloutils.py @@ -2,17 +2,13 @@ from boto3 import client from datetime import datetime, timezone from furl import furl from json import dumps, loads -from logging import getLogger from os import environ, getenv from requests import Session from urllib.parse import urljoin from uuid import UUID -from wal_e.cmd import configure_backup_cxt -from .utils import Attrs, defaulting, these - - -logger = getLogger(__name__) +from .utils import defaulting, these +from operator_ui.adapters.logger import logger session = Session() @@ -287,10 +283,8 @@ def read_stored_clusters(bucket, prefix, delimiter='/'): def read_versions( pg_cluster, bucket, - s3_endpoint, prefix, delimiter='/', - use_aws_instance_profile=False, ): return [ 'base' if uid == 'wal' else uid @@ -308,35 +302,79 @@ def read_versions( if uid == 'wal' or defaulting(lambda: UUID(uid)) ] -BACKUP_VERSION_PREFIXES = ['','9.5/', '9.6/', '10/','11/', '12/', '13/'] +def lsn_to_wal_segment_stop(finish_lsn, start_segment, wal_segment_size=16 * 1024 * 1024): + timeline = int(start_segment[:8], 16) + log_id = finish_lsn >> 32 + seg_id = (finish_lsn & 0xFFFFFFFF) // wal_segment_size + return f"{timeline:08X}{log_id:08X}{seg_id:08X}" + +def lsn_to_offset_hex(lsn, wal_segment_size=16 * 1024 * 1024): + return f"{lsn % wal_segment_size:08X}" def read_basebackups( pg_cluster, uid, bucket, - s3_endpoint, prefix, - delimiter='/', - use_aws_instance_profile=False, + postgresql_versions, ): - environ['WALE_S3_ENDPOINT'] = s3_endpoint suffix = '' if uid == 'base' else '/' + uid backups = [] - for vp in BACKUP_VERSION_PREFIXES: + # Reuse a single S3 client configured with AWS_ENDPOINT so MinIO / + # other S3-compatible backends are hit for list+get calls too. The + # previous plain client('s3') fell back to the default AWS endpoint + # and returned empty data against a custom endpoint; read_stored_clusters + # and read_versions already pass endpoint_url=AWS_ENDPOINT (#3078). + s3_client = client('s3', endpoint_url=AWS_ENDPOINT) - backups = backups + [ - { - key: value - for key, value in basebackup.__dict__.items() - if isinstance(value, str) or isinstance(value, int) - } - for basebackup in Attrs.call( - f=configure_backup_cxt, - aws_instance_profile=use_aws_instance_profile, - s3_prefix=f's3://{bucket}/{prefix}{pg_cluster}{suffix}/wal/{vp}', - )._backup_list(detail=True) - ] + for vp in postgresql_versions: + backup_prefix = f'{prefix}{pg_cluster}{suffix}/wal/{vp}/basebackups_005/' + logger.info(f"{bucket}/{backup_prefix}") + + paginator = s3_client.get_paginator('list_objects_v2') + pages = paginator.paginate(Bucket=bucket, Prefix=backup_prefix) + + for page in pages: + for obj in page.get("Contents", []): + key = obj["Key"] + if not key.endswith("backup_stop_sentinel.json"): + continue + + response = s3_client.get_object(Bucket=bucket, Key=key) + backup_info = loads(response["Body"].read().decode("utf-8")) + last_modified = response["LastModified"].astimezone(timezone.utc).isoformat() + + backup_name = key.split("/")[-1].replace("_backup_stop_sentinel.json", "") + start_seg, start_offset = backup_name.split("_")[1], backup_name.split("_")[-1] if "_" in backup_name else None + + if "LSN" in backup_info and "FinishLSN" in backup_info: + # WAL-G + lsn = backup_info["LSN"] + finish_lsn = backup_info["FinishLSN"] + backups.append({ + "expanded_size_bytes": backup_info.get("UncompressedSize"), + "last_modified": last_modified, + "name": backup_name, + "wal_segment_backup_start": start_seg, + "wal_segment_backup_stop": lsn_to_wal_segment_stop(finish_lsn, start_seg), + "wal_segment_offset_backup_start": lsn_to_offset_hex(lsn), + "wal_segment_offset_backup_stop": lsn_to_offset_hex(finish_lsn), + }) + elif "wal_segment_backup_stop" in backup_info: + # WAL-E + stop_seg = backup_info["wal_segment_backup_stop"] + stop_offset = backup_info["wal_segment_offset_backup_stop"] + + backups.append({ + "expanded_size_bytes": backup_info.get("expanded_size_bytes"), + "last_modified": last_modified, + "name": backup_name, + "wal_segment_backup_start": start_seg, + "wal_segment_backup_stop": stop_seg, + "wal_segment_offset_backup_start": start_offset, + "wal_segment_offset_backup_stop": stop_offset, + }) return backups diff --git a/ui/operator_ui/static/styles.css b/ui/operator_ui/static/styles.css index 3f05cb290..ea1e9020b 100644 --- a/ui/operator_ui/static/styles.css +++ b/ui/operator_ui/static/styles.css @@ -64,3 +64,56 @@ label { td { vertical-align: middle !important; } + +.tooltip { + position: relative; + display: inline-block; + opacity: 1; + font-size: 14px; + font-weight: bold; + z-index: 0; +} +.tooltip:after { + content: '?'; + display: inline-block; + font-family: sans-serif; + font-weight: bold; + text-align: center; + width: 16px; + height: 16px; + font-size: 12px; + line-height: 16px; + border-radius: 12px; + padding: 0px; + color: white; + background: black; + border: 1px solid black; +} +.tooltip .tooltiptext { + visibility: hidden; + width: 250px; + background-color: white; + color: #000; + text-align: justify; + border-radius: 6px; + padding: 10px 10px; + position: absolute; + bottom: 150%; + left: 50%; + margin-left: -120px; + border: 1px solid black; + font-weight: normal; +} +.tooltip .tooltiptext::after { + content: ""; + position: absolute; + top: 100%; + left: 50%; + margin-left: -5px; + border-width: 5px; + border-style: solid; + border-color: black transparent transparent transparent; +} +.tooltip:hover .tooltiptext { + visibility: visible; +} diff --git a/ui/operator_ui/templates/index.html b/ui/operator_ui/templates/index.html index 6b4689079..60a31e190 100644 --- a/ui/operator_ui/templates/index.html +++ b/ui/operator_ui/templates/index.html @@ -3,7 +3,13 @@ PostgreSQL Operator UI - + @@ -62,9 +68,9 @@ - - - + + + @@ -168,8 +174,8 @@ - - + + {% if google_analytics %} diff --git a/ui/operator_ui/templates/login-deeplink.html b/ui/operator_ui/templates/login-deeplink.html deleted file mode 100644 index 875b8d055..000000000 --- a/ui/operator_ui/templates/login-deeplink.html +++ /dev/null @@ -1,13 +0,0 @@ - - - Storing client location ... - - - - - \ No newline at end of file diff --git a/ui/operator_ui/templates/login-resolve-deeplink.html b/ui/operator_ui/templates/login-resolve-deeplink.html deleted file mode 100644 index fac96b265..000000000 --- a/ui/operator_ui/templates/login-resolve-deeplink.html +++ /dev/null @@ -1,18 +0,0 @@ - - - Restoring client location ... - - - - - diff --git a/ui/requirements.txt b/ui/requirements.txt index 8f612d554..ace18641d 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -1,15 +1,14 @@ -Flask-OAuthlib==0.9.5 -Flask==1.1.2 -backoff==1.10.0 -boto3==1.16.52 +backoff==2.2.1 +boto3==1.34.110 boto==2.49.0 -click==7.1.2 -furl==2.1.0 -gevent==20.12.1 -jq==1.1.1 -json_delta>=2.0 -kubernetes==3.0.0 -requests==2.25.1 +click==8.1.7 +Flask==3.0.3 +furl==2.1.3 +gevent==24.2.1 +jq==1.7.0 +json_delta>=2.0.2 +kubernetes==11.0.0 +python-json-logger==2.0.7 +requests==2.32.4 stups-tokens>=1.1.19 -wal_e==1.1.1 -werkzeug==0.16.1 +werkzeug==3.1.6 diff --git a/ui/run_local.sh b/ui/run_local.sh index 79723680a..59729a92a 100755 --- a/ui/run_local.sh +++ b/ui/run_local.sh @@ -14,18 +14,28 @@ export TARGET_NAMESPACE="${TARGET_NAMESPACE-*}" default_operator_ui_config='{ "docs_link":"https://postgres-operator.readthedocs.io/en/latest/", - "dns_format_string": "{1}-{0}.{2}", + "dns_format_string": "{0}.{1}", "databases_visible": true, + "master_load_balancer_visible": true, "nat_gateways_visible": false, + "replica_load_balancer_visible": true, "resources_visible": true, "users_visible": true, - "cost_ebs": 0.119, + "cost_ebs": 0.0952, + "cost_iops": 0.006, + "cost_throughput": 0.0476, "cost_core": 0.0575, "cost_memory": 0.014375, + "free_iops": 3000, + "free_throughput": 125, + "limit_iops": 16000, + "limit_throughput": 1000, "postgresql_versions": [ - "13", - "12", - "11" + "18", + "17", + "16", + "15", + "14", ], "static_network_whitelist": { "localhost": ["172.0.0.1/32"] diff --git a/ui/setup.py b/ui/setup.py index 95ddfe182..43a1fb67d 100644 --- a/ui/setup.py +++ b/ui/setup.py @@ -69,7 +69,7 @@ setup( 'License :: OSI Approved :: MIT', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.11', 'Topic :: System :: Clustering', 'Topic :: System :: Monitoring', ],