diff --git a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md index 70d77b80a..a33b556cc 100644 --- a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md +++ b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md @@ -9,7 +9,7 @@ assignees: '' Please, answer some short questions which should help us to understand your problem / question better? -- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.7.1 +- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.8.0 - **Where do you run it - cloud or metal? Kubernetes or OpenShift?** [AWS K8s | GCP ... | Bare Metal K8s] - **Are you running Postgres Operator in production?** [yes | no] - **Type of issue?** [Bug report, question, feature request, etc.] diff --git a/.github/workflows/run_e2e.yaml b/.github/workflows/run_e2e.yaml index a8e76f0ca..5dcb195d2 100644 --- a/.github/workflows/run_e2e.yaml +++ b/.github/workflows/run_e2e.yaml @@ -14,12 +14,12 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-go@v2 with: - go-version: "^1.16.9" + go-version: "^1.17.4" - name: Make dependencies run: make deps mocks - name: Compile run: make linux - name: Run unit tests - run: go test ./... + run: make test - name: Run end-2-end tests run: make e2e diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 3ef1a348d..13810d9d2 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: "^1.16.9" + go-version: "^1.17.4" - name: Make dependencies run: make deps mocks - name: Compile diff --git a/CODEOWNERS b/CODEOWNERS index 398856c66..6d1a2ab6a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1,2 @@ # global owners -* @erthalion @sdudoladov @Jan-M @CyberDem0n @avaczi @FxKu @RafiaSabih +* @sdudoladov @Jan-M @CyberDem0n @FxKu @jopadi diff --git a/LICENSE b/LICENSE index 7c0f459a5..4a87999ea 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2021 Zalando SE +Copyright (c) 2022 Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MAINTAINERS b/MAINTAINERS index 572e6d971..b2abdf858 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1,5 +1,4 @@ -Dmitrii Dolgov Sergey Dudoladov Felix Kunde Jan Mussler -Rafia Sabih \ No newline at end of file +Jociele Padilha \ No newline at end of file diff --git a/Makefile b/Makefile index b83f7156b..030a8088c 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ docker: ${DOCKERDIR}/${DOCKERFILE} docker-context cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_FRESH)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" . indocker-race: - docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.8.1 bash -c "make linux" + docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.17.3 bash -c "make linux" push: docker push "$(IMAGE):$(TAG)$(CDP_TAG)" @@ -85,8 +85,8 @@ mocks: GO111MODULE=on go generate ./... tools: - GO111MODULE=on go get k8s.io/client-go@kubernetes-1.22.3 - GO111MODULE=on go get github.com/golang/mock/mockgen@v1.6.0 + GO111MODULE=on go get -d k8s.io/client-go@kubernetes-1.22.4 + GO111MODULE=on go install github.com/golang/mock/mockgen@v1.6.0 GO111MODULE=on go mod tidy fmt: @@ -103,5 +103,8 @@ test: hack/verify-codegen.sh GO111MODULE=on go test ./... +codegen: + hack/update-codegen.sh + e2e: docker # build operator image to be tested cd e2e; make e2etest diff --git a/README.md b/README.md index c69e0ca29..137f46a44 100644 --- a/README.md +++ b/README.md @@ -17,15 +17,15 @@ pipelines with no access to Kubernetes API directly, promoting infrastructure as * Live volume resize without pod restarts (AWS EBS, PVC) * Database connection pooling with PGBouncer * Support fast in place major version upgrade to PG13. Supports global upgrade of all clusters. -* Restore and cloning Postgres clusters (incl. major version upgrade) -* Additionally logical backups to S3 bucket can be configured -* Standby cluster from S3 WAL archive +* Restore and cloning Postgres clusters on AWS, GCS and Azure +* Additionally logical backups to S3 or GCS bucket can be configured +* Standby cluster from S3 or GCS WAL archive * Configurable for non-cloud environments * Basic credential and user management on K8s, eases application deployments * Support for custom TLS certificates * UI to create and edit Postgres cluster manifests -* Works well on Amazon AWS, Google Cloud, OpenShift and locally on Kind * Support for AWS EBS gp2 to gp3 migration, supporting iops and throughput configuration +* Compatible with OpenShift ### PostgreSQL features @@ -61,7 +61,7 @@ We introduce the major version into the backup path to smoothen the [major versi The new operator configuration can set a compatibility flag *enable_spilo_wal_path_compat* to make Spilo look for wal segments in the current path but also old format paths. This comes at potential performance costs and should be disabled after a few days. -The newest Spilo image is: `registry.opensource.zalan.do/acid/spilo-14:2.1-p3` +The newest Spilo image is: `registry.opensource.zalan.do/acid/spilo-14:2.1-p5` The last Spilo 12 image is: `registry.opensource.zalan.do/acid/spilo-12:1.6-p5` diff --git a/charts/postgres-operator-ui/Chart.yaml b/charts/postgres-operator-ui/Chart.yaml index 9e38f218a..467e19b45 100644 --- a/charts/postgres-operator-ui/Chart.yaml +++ b/charts/postgres-operator-ui/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator-ui -version: 1.7.1 -appVersion: 1.7.1 +version: 1.8.0 +appVersion: 1.8.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience keywords: diff --git a/charts/postgres-operator-ui/index.yaml b/charts/postgres-operator-ui/index.yaml index f0005e003..b7d845a73 100644 --- a/charts/postgres-operator-ui/index.yaml +++ b/charts/postgres-operator-ui/index.yaml @@ -1,9 +1,32 @@ apiVersion: v1 entries: postgres-operator-ui: + - apiVersion: v1 + appVersion: 1.8.0 + created: "2022-04-20T15:39:16.094386569+02:00" + description: Postgres Operator UI provides a graphical interface for a convenient + database-as-a-service user experience + digest: d4a7b40c23fd167841cc28342afdbd5ecc809181913a5c31061c83139187f148 + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - ui + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator-ui + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-ui-1.8.0.tgz + version: 1.8.0 - apiVersion: v1 appVersion: 1.7.1 - created: "2021-11-03T11:44:00.75040818+01:00" + created: "2022-04-20T15:39:16.093853803+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 97aed1a1d37cd5f8441eea9522f38e56cc829786ad2134c437a5e6a15c995869 @@ -26,7 +49,7 @@ entries: version: 1.7.1 - apiVersion: v1 appVersion: 1.7.0 - created: "2021-11-03T11:44:00.748544296+01:00" + created: "2022-04-20T15:39:16.093334397+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 37fba1968347daad393dbd1c6ee6e5b6a24d1095f972c0102197531c62dcada8 @@ -49,7 +72,7 @@ entries: version: 1.7.0 - apiVersion: v1 appVersion: 1.6.3 - created: "2021-11-03T11:44:00.745261041+01:00" + created: "2022-04-20T15:39:16.092419178+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 08b810aa632dcc719e4785ef184e391267f7c460caa99677f2d00719075aac78 @@ -72,7 +95,7 @@ entries: version: 1.6.3 - apiVersion: v1 appVersion: 1.6.2 - created: "2021-11-03T11:44:00.743617168+01:00" + created: "2022-04-20T15:39:16.091945123+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 14d1559bb0bd1e1e828f2daaaa6f6ac9ffc268d79824592c3589b55dd39241f6 @@ -95,7 +118,7 @@ entries: version: 1.6.2 - apiVersion: v1 appVersion: 1.6.1 - created: "2021-11-03T11:44:00.741913772+01:00" + created: "2022-04-20T15:39:16.0914401+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 3d321352f2f1e7bb7450aa8876e3d818aa9f9da9bd4250507386f0490f2c1969 @@ -118,7 +141,7 @@ entries: version: 1.6.1 - apiVersion: v1 appVersion: 1.6.0 - created: "2021-11-03T11:44:00.740290409+01:00" + created: "2022-04-20T15:39:16.090887513+02:00" description: Postgres Operator UI provides a graphical interface for a convenient database-as-a-service user experience digest: 1e0aa1e7db3c1daa96927ffbf6fdbcdb434562f961833cb5241ddbe132220ee4 @@ -139,27 +162,4 @@ entries: urls: - postgres-operator-ui-1.6.0.tgz version: 1.6.0 - - apiVersion: v1 - appVersion: 1.5.0 - created: "2021-11-03T11:44:00.738607341+01:00" - description: Postgres Operator UI provides a graphical interface for a convenient - database-as-a-service user experience - digest: c91ea39e6d51d57f4048fb1b6ec53b40823f2690eb88e4e4f1a036367b9fdd61 - home: https://github.com/zalando/postgres-operator - keywords: - - postgres - - operator - - ui - - cloud-native - - patroni - - spilo - maintainers: - - email: opensource@zalando.de - name: Zalando - name: postgres-operator-ui - sources: - - https://github.com/zalando/postgres-operator - urls: - - postgres-operator-ui-1.5.0.tgz - version: 1.5.0 -generated: "2021-11-03T11:44:00.736383918+01:00" +generated: "2022-04-20T15:39:16.0877032+02:00" diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz deleted file mode 100644 index d8527f293..000000000 Binary files a/charts/postgres-operator-ui/postgres-operator-ui-1.5.0.tgz and /dev/null differ diff --git a/charts/postgres-operator-ui/postgres-operator-ui-1.8.0.tgz b/charts/postgres-operator-ui/postgres-operator-ui-1.8.0.tgz new file mode 100644 index 000000000..e3563911c Binary files /dev/null and b/charts/postgres-operator-ui/postgres-operator-ui-1.8.0.tgz differ diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index 8942539d6..c82d38c70 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -70,6 +70,11 @@ spec: "replica_load_balancer_visible": true, "resources_visible": true, "users_visible": true, + "cost_ebs": 0.0952, + "cost_iops": 0.006, + "cost_throughput": 0.0476, + "cost_core": 0.0575, + "cost_memory": 0.014375, "postgresql_versions": [ "14", "13", diff --git a/charts/postgres-operator-ui/templates/ingress.yaml b/charts/postgres-operator-ui/templates/ingress.yaml index 21e7dbea2..37b47a6c5 100644 --- a/charts/postgres-operator-ui/templates/ingress.yaml +++ b/charts/postgres-operator-ui/templates/ingress.yaml @@ -23,6 +23,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: +{{- if .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} +{{- end }} {{- if .Values.ingress.tls }} tls: {{- range .Values.ingress.tls }} diff --git a/charts/postgres-operator-ui/values.yaml b/charts/postgres-operator-ui/values.yaml index 06004b1ca..6fa9e11cc 100644 --- a/charts/postgres-operator-ui/values.yaml +++ b/charts/postgres-operator-ui/values.yaml @@ -8,7 +8,7 @@ replicaCount: 1 image: registry: registry.opensource.zalan.do repository: acid/postgres-operator-ui - tag: v1.7.1 + tag: v1.8.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -39,7 +39,7 @@ resources: # configure UI ENVs envs: - # IMPORTANT: While operator chart and UI chart are idendependent, this is the interface between + # IMPORTANT: While operator chart and UI chart are independent, this is the interface between # UI and operator API. Insert the service name of the operator API here! operatorApiUrl: "http://postgres-operator:8080" operatorClusterNameLabel: "cluster-name" @@ -93,6 +93,7 @@ ingress: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" + ingressClassName: "" hosts: - host: ui.example.org paths: [""] diff --git a/charts/postgres-operator/Chart.yaml b/charts/postgres-operator/Chart.yaml index 7bc9ce866..7afbf2354 100644 --- a/charts/postgres-operator/Chart.yaml +++ b/charts/postgres-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: postgres-operator -version: 1.7.1 -appVersion: 1.7.1 +version: 1.8.0 +appVersion: 1.8.0 home: https://github.com/zalando/postgres-operator description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes keywords: diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 90a632fe5..4e708ace6 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -4,8 +4,6 @@ metadata: name: operatorconfigurations.acid.zalan.do labels: app.kubernetes.io/name: postgres-operator - annotations: - "helm.sh/hook": crd-install spec: group: acid.zalan.do names: @@ -63,11 +61,20 @@ spec: configuration: type: object properties: + crd_categories: + type: array + nullable: true + items: + type: string docker_image: type: string - default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p3" + default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p5" + enable_crd_registration: + type: boolean + default: true enable_crd_validation: type: boolean + description: deprecated default: true enable_lazy_spilo_upgrade: type: boolean @@ -89,11 +96,13 @@ spec: default: false max_instances: type: integer - minimum: -1 # -1 = disabled + description: "-1 = disabled" + minimum: -1 default: -1 min_instances: type: integer - minimum: -1 # -1 = disabled + description: "-1 = disabled" + minimum: -1 default: -1 resync_period: type: string @@ -121,6 +130,20 @@ spec: users: type: object properties: + additional_owner_roles: + type: array + nullable: true + items: + type: string + enable_password_rotation: + type: boolean + default: false + password_rotation_interval: + type: integer + default: 90 + password_rotation_user_retention: + type: integer + default: 180 replication_username: type: string default: standby @@ -133,6 +156,10 @@ spec: major_version_upgrade_mode: type: string default: "off" + major_version_upgrade_team_allow_list: + type: array + items: + type: string minimal_major_version: type: string default: "9.6" @@ -170,12 +197,12 @@ spec: type: array items: type: string - enable_init_containers: - type: boolean - default: true enable_cross_namespace_secret: type: boolean default: false + enable_init_containers: + type: boolean + default: true enable_pod_antiaffinity: type: boolean default: false @@ -185,6 +212,10 @@ spec: enable_sidecars: type: boolean default: true + ignored_annotations: + type: array + items: + type: string infrastructure_roles_secret_name: type: string infrastructure_roles_secrets: @@ -228,6 +259,11 @@ spec: type: object additionalProperties: type: string + node_readiness_label_merge: + type: string + enum: + - "AND" + - "OR" oauth_token_secret_name: type: string default: "postgresql-operator" @@ -322,6 +358,12 @@ spec: timeouts: type: object properties: + patroni_api_check_interval: + type: string + default: "1s" + patroni_api_check_timeout: + type: string + default: "5s" pod_label_wait_timeout: type: string default: "10m" @@ -353,9 +395,15 @@ spec: enable_master_load_balancer: type: boolean default: true + enable_master_pooler_load_balancer: + type: boolean + default: false enable_replica_load_balancer: type: boolean default: false + enable_replica_pooler_load_balancer: + type: boolean + default: false external_traffic_policy: type: string enum: @@ -391,18 +439,18 @@ spec: type: string log_s3_bucket: type: string + wal_az_storage_account: + type: string wal_gs_bucket: type: string wal_s3_bucket: type: string - wal_az_storage_account: - type: string logical_backup: type: object properties: logical_backup_docker_image: type: string - default: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1" + default: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0" logical_backup_google_application_credentials: type: string logical_backup_job_prefix: @@ -423,6 +471,8 @@ spec: type: string logical_backup_s3_sse: type: string + logical_backup_s3_retention_time: + type: string logical_backup_schedule: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' @@ -473,6 +523,7 @@ spec: type: string default: - admin + - cron_admin role_deletion_suffix: type: string default: "_deleted" @@ -537,7 +588,7 @@ spec: default: "pooler" connection_pooler_image: type: string - default: "registry.opensource.zalan.do/acid/pgbouncer:master-19" + default: "registry.opensource.zalan.do/acid/pgbouncer:master-22" connection_pooler_max_db_connections: type: integer default: 60 diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 9ac4cfb3a..8534650e1 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -4,8 +4,6 @@ metadata: name: postgresqls.acid.zalan.do labels: app.kubernetes.io/name: postgres-operator - annotations: - "helm.sh/hook": crd-install spec: group: acid.zalan.do names: @@ -149,18 +147,12 @@ spec: - "transaction" numberOfInstances: type: integer - minimum: 2 + minimum: 1 resources: type: object - required: - - requests - - limits properties: limits: type: object - required: - - cpu - - memory properties: cpu: type: string @@ -170,9 +162,6 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' requests: type: object - required: - - cpu - - memory properties: cpu: type: string @@ -199,16 +188,27 @@ spec: type: boolean enableMasterLoadBalancer: type: boolean + enableMasterPoolerLoadBalancer: + type: boolean enableReplicaLoadBalancer: type: boolean + enableReplicaPoolerLoadBalancer: + type: boolean enableShmVolume: type: boolean - init_containers: # deprecated + env: type: array nullable: true items: type: object x-kubernetes-preserve-unknown-fields: true + init_containers: + type: array + description: deprecated + nullable: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true initContainers: type: array nullable: true @@ -231,8 +231,8 @@ spec: items: type: object required: - - weight - preference + - weight properties: preference: type: object @@ -344,14 +344,17 @@ spec: type: boolean synchronous_mode_strict: type: boolean + synchronous_node_count: + type: integer ttl: type: integer podAnnotations: type: object additionalProperties: type: string - pod_priority_class_name: # deprecated + pod_priority_class_name: type: string + description: deprecated podPriorityClassName: type: string postgresql: @@ -395,19 +398,14 @@ spec: type: boolean secretNamespace: type: string - replicaLoadBalancer: # deprecated + replicaLoadBalancer: type: boolean + description: deprecated resources: type: object - required: - - requests - - limits properties: limits: type: object - required: - - cpu - - memory properties: cpu: type: string @@ -436,9 +434,6 @@ spec: # than the corresponding limit. requests: type: object - required: - - cpu - - memory properties: cpu: type: string @@ -466,11 +461,55 @@ spec: type: integer standby: type: object - required: - - s3_wal_path properties: s3_wal_path: type: string + gs_wal_path: + type: string + standby_host: + type: string + standby_port: + type: string + oneOf: + - required: + - s3_wal_path + - required: + - gs_wal_path + - required: + - standby_host + streams: + type: array + nullable: true + items: + type: object + required: + - applicationId + - database + - tables + properties: + applicationId: + type: string + batchSize: + type: integer + database: + type: string + filter: + type: object + additionalProperties: + type: string + tables: + type: object + additionalProperties: + type: object + required: + - eventType + properties: + eventType: + type: string + idColumn: + type: string + payloadColumn: + type: string teamId: type: string tls: @@ -492,10 +531,6 @@ spec: type: array items: type: object - required: - - key - - operator - - effect properties: key: type: string @@ -514,14 +549,14 @@ spec: - PreferNoSchedule tolerationSeconds: type: integer - useLoadBalancer: # deprecated + useLoadBalancer: type: boolean + description: deprecated users: type: object additionalProperties: type: array nullable: true - description: "Role flags specified here must not contradict each other" items: type: string enum: @@ -553,6 +588,16 @@ spec: - SUPERUSER - nosuperuser - NOSUPERUSER + usersWithPasswordRotation: + type: array + nullable: true + items: + type: string + usersWithInPlacePasswordRotation: + type: array + nullable: true + items: + type: string volume: type: object required: diff --git a/charts/postgres-operator/crds/postgresteams.yaml b/charts/postgres-operator/crds/postgresteams.yaml index fbf873b84..b7a36848d 100644 --- a/charts/postgres-operator/crds/postgresteams.yaml +++ b/charts/postgres-operator/crds/postgresteams.yaml @@ -4,8 +4,6 @@ metadata: name: postgresteams.acid.zalan.do labels: app.kubernetes.io/name: postgres-operator - annotations: - "helm.sh/hook": crd-install spec: group: acid.zalan.do names: diff --git a/charts/postgres-operator/index.yaml b/charts/postgres-operator/index.yaml index a5a5d4fc6..60a4fb8b9 100644 --- a/charts/postgres-operator/index.yaml +++ b/charts/postgres-operator/index.yaml @@ -1,9 +1,31 @@ apiVersion: v1 entries: postgres-operator: + - apiVersion: v1 + appVersion: 1.8.0 + created: "2022-04-14T15:02:07.818613578+02:00" + description: Postgres Operator creates and manages PostgreSQL clusters running + in Kubernetes + digest: 3ae232cf009e09aa2ad11c171484cd2f1b72e63c59735e58fbe2b6eb842f4c86 + home: https://github.com/zalando/postgres-operator + keywords: + - postgres + - operator + - cloud-native + - patroni + - spilo + maintainers: + - email: opensource@zalando.de + name: Zalando + name: postgres-operator + sources: + - https://github.com/zalando/postgres-operator + urls: + - postgres-operator-1.8.0.tgz + version: 1.8.0 - apiVersion: v1 appVersion: 1.7.1 - created: "2021-11-03T11:43:09.121092027+01:00" + created: "2022-04-14T15:02:07.817076674+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes digest: 7262563bec0b058e669ae6bcff0226e33fa9ece9c41ac46a53274046afe7700c @@ -25,7 +47,7 @@ entries: version: 1.7.1 - apiVersion: v1 appVersion: 1.7.0 - created: "2021-11-03T11:43:09.119630978+01:00" + created: "2022-04-14T15:02:07.815161671+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes digest: c3e99fb94305f81484b8b1af18eefb78681f3b5d057d5ad10565e4afb7c65ffe @@ -47,7 +69,7 @@ entries: version: 1.7.0 - apiVersion: v1 appVersion: 1.6.3 - created: "2021-11-03T11:43:09.118146221+01:00" + created: "2022-04-14T15:02:07.813087244+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes digest: ea08f991bf23c9ad114bca98ebcbe3e2fa15beab163061399394905eaee89b35 @@ -69,7 +91,7 @@ entries: version: 1.6.3 - apiVersion: v1 appVersion: 1.6.2 - created: "2021-11-03T11:43:09.115637274+01:00" + created: "2022-04-14T15:02:07.8114121+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes digest: d886f8a0879ca07d1e5246ee7bc55710e1c872f3977280fe495db6fc2057a7f4 @@ -91,7 +113,7 @@ entries: version: 1.6.2 - apiVersion: v1 appVersion: 1.6.1 - created: "2021-11-03T11:43:09.114107417+01:00" + created: "2022-04-14T15:02:07.809829808+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes digest: 4ba5972cd486dcaa2d11c5613a6f97f6b7b831822e610fe9e10a57ea1db23556 @@ -113,7 +135,7 @@ entries: version: 1.6.1 - apiVersion: v1 appVersion: 1.6.0 - created: "2021-11-03T11:43:09.112550808+01:00" + created: "2022-04-14T15:02:07.808307624+02:00" description: Postgres Operator creates and manages PostgreSQL clusters running in Kubernetes digest: f52149718ea364f46b4b9eec9a65f6253ad182bb78df541d14cd5277b9c8a8c3 @@ -133,26 +155,4 @@ entries: urls: - postgres-operator-1.6.0.tgz version: 1.6.0 - - apiVersion: v1 - appVersion: 1.5.0 - created: "2021-11-03T11:43:09.110907395+01:00" - description: Postgres Operator creates and manages PostgreSQL clusters running - in Kubernetes - digest: 198351d5db52e65cdf383d6f3e1745d91ac1e2a01121f8476f8b1be728b09531 - home: https://github.com/zalando/postgres-operator - keywords: - - postgres - - operator - - cloud-native - - patroni - - spilo - maintainers: - - email: opensource@zalando.de - name: Zalando - name: postgres-operator - sources: - - https://github.com/zalando/postgres-operator - urls: - - postgres-operator-1.5.0.tgz - version: 1.5.0 -generated: "2021-11-03T11:43:09.109237601+01:00" +generated: "2022-04-14T15:02:07.806370532+02:00" diff --git a/charts/postgres-operator/postgres-operator-1.5.0.tgz b/charts/postgres-operator/postgres-operator-1.5.0.tgz deleted file mode 100644 index 6e1a48ab7..000000000 Binary files a/charts/postgres-operator/postgres-operator-1.5.0.tgz and /dev/null differ diff --git a/charts/postgres-operator/postgres-operator-1.8.0.tgz b/charts/postgres-operator/postgres-operator-1.8.0.tgz new file mode 100644 index 000000000..caecd2929 Binary files /dev/null and b/charts/postgres-operator/postgres-operator-1.8.0.tgz differ diff --git a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml index 33c43822f..fdccf16d3 100644 --- a/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml +++ b/charts/postgres-operator/templates/clusterrole-postgres-pod.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} rules: -# Patroni needs to watch and manage endpoints +# Patroni needs to watch and manage config maps or endpoints {{- if toString .Values.configGeneral.kubernetes_use_configmaps | eq "true" }} - apiGroups: - "" @@ -24,12 +24,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get {{- else }} - apiGroups: - "" diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 885bad3f7..199086acc 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -34,16 +34,34 @@ rules: - get - list - watch +# all verbs allowed for event streams +{{- if .Values.enableStreams }} +- apiGroups: + - zalando.org + resources: + - fabriceventstreams + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +{{- end }} # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - - create - get +{{- if toString .Values.configGeneral.enable_crd_registration | eq "true" }} + - create - patch - update +{{- end }} # to send events to the CRs - apiGroups: - "" @@ -71,12 +89,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get {{- else }} # to read configuration from ConfigMaps - apiGroups: diff --git a/charts/postgres-operator/templates/crds.yaml b/charts/postgres-operator/templates/crds.yaml deleted file mode 100644 index 733830014..000000000 --- a/charts/postgres-operator/templates/crds.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{{ if .Values.crd.create }} -{{- range $path, $bytes := .Files.Glob "crds/*.yaml" }} -{{ $.Files.Get $path }} ---- -{{- end }} -{{- end }} diff --git a/charts/postgres-operator/templates/user-facing-clusterroles.yaml b/charts/postgres-operator/templates/user-facing-clusterroles.yaml new file mode 100644 index 000000000..d7db347b2 --- /dev/null +++ b/charts/postgres-operator/templates/user-facing-clusterroles.yaml @@ -0,0 +1,71 @@ +{{ if .Values.rbac.createAggregateClusterRoles }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "postgres-operator.fullname" . }}:users:admin +rules: +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + - postgresqls/status + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "postgres-operator.fullname" . }}:users:edit +rules: +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + verbs: + - create + - update + - patch + - delete + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "postgres-operator.fullname" . }}:users:view +rules: +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + - postgresqls/status + verbs: + - get + - list + - watch +{{ end }} diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 1ad096403..3878cc0cd 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -1,7 +1,7 @@ image: registry: registry.opensource.zalan.do repository: acid/postgres-operator - tag: v1.7.1 + tag: v1.8.0 pullPolicy: "IfNotPresent" # Optionally specify an array of imagePullSecrets. @@ -20,8 +20,11 @@ enableJsonLogging: false # general configuration parameters configGeneral: - # choose if deployment creates/updates CRDs with OpenAPIV3Validation - enable_crd_validation: true + # the deployment should create/update the CRDs + enable_crd_registration: true + # specify categories under which crds should be listed + crd_categories: + - "all" # update only the statefulsets without immediately doing the rolling update enable_lazy_spilo_upgrade: false # set the PGVERSION env var instead of providing the version via postgresql.bin_dir in SPILO_CONFIGURATION @@ -35,7 +38,7 @@ configGeneral: # Select if setup uses endpoints (default), or configmaps to manage leader (DCS=k8s) # kubernetes_use_configmaps: false # Spilo docker image - docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p3 + docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p5 # min number of instances in Postgres cluster. -1 = no limit min_instances: -1 # max number of instances in Postgres cluster. -1 = no limit @@ -56,6 +59,16 @@ configGeneral: # parameters describing Postgres users configUsers: + # roles to be granted to database owners + # additional_owner_roles: + # - cron_admin + + # enable password rotation for app users that are not database owners + enable_password_rotation: false + # rotation interval for updating credentials in K8s secrets of app users + password_rotation_interval: 90 + # retention interval to keep rotation users + password_rotation_user_retention: 180 # postgres username used for replication between instances replication_username: standby # postgres superuser name to be created by initdb @@ -64,6 +77,10 @@ configUsers: configMajorVersionUpgrade: # "off": no upgrade, "manual": manifest triggers action, "full": minimal version violation triggers too major_version_upgrade_mode: "off" + # upgrades will only be carried out for clusters of listed teams when mode is "off" + # major_version_upgrade_team_allow_list: + # - acid + # minimal Postgres major version that will not automatically be upgraded minimal_major_version: "9.6" # target Postgres major version when upgrading clusters automatically @@ -107,6 +124,11 @@ configKubernetes: enable_pod_disruption_budget: true # enables sidecar containers to run alongside Spilo in the same pod enable_sidecars: true + + # annotations to be ignored when comparing statefulsets, services etc. + # ignored_annotations: + # - k8s.v1.cni.cncf.io/network-status + # namespaced name of the secret containing infrastructure roles names and passwords # infrastructure_roles_secret_name: postgresql-infrastructure-roles @@ -126,6 +148,9 @@ configKubernetes: # node_readiness_label: # status: ready + # defines how nodeAffinity from manifest should be merged with node_readiness_label + # node_readiness_label_merge: "OR" + # namespaced name of the secret containing the OAuth2 token to pass to the teams API # oauth_token_secret_name: postgresql-operator @@ -194,6 +219,10 @@ configPostgresPodResources: # timeouts related to some operator actions configTimeouts: + # interval between consecutive attempts of operator calling the Patroni API + patroni_api_check_interval: 1s + # timeout when waiting for successful response from Patroni API + patroni_api_check_timeout: 5s # timeout when waiting for the Postgres pods to be deleted pod_deletion_wait_timeout: 10m # timeout when waiting for pod role and cluster labels @@ -218,8 +247,12 @@ configLoadBalancer: # toggles service type load balancer pointing to the master pod of the cluster enable_master_load_balancer: false + # toggles service type load balancer pointing to the master pooler pod of the cluster + enable_master_pooler_load_balancer: false # toggles service type load balancer pointing to the replica pod of the cluster enable_replica_load_balancer: false + # toggles service type load balancer pointing to the replica pooler pod of the cluster + enable_replica_pooler_load_balancer: false # define external traffic policy for the load balancer external_traffic_policy: "Cluster" # defines the DNS name string template for the master load balancer cluster @@ -280,7 +313,7 @@ configAwsOrGcp: # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0" # path of google cloud service account json file # logical_backup_google_application_credentials: "" @@ -300,6 +333,8 @@ configLogicalBackup: logical_backup_s3_secret_access_key: "" # S3 server side encryption logical_backup_s3_sse: "AES256" + # S3 retention time for stored backups for example "2 week" or "7 days" + logical_backup_s3_retention_time: "" # backup schedule in the cron format logical_backup_schedule: "30 00 * * *" @@ -328,6 +363,7 @@ configTeamsApi: # List of roles that cannot be overwritten by an application, team or infrastructure role protected_role_names: - admin + - cron_admin # Suffix to add if members are removed from TeamsAPI or PostgresTeam CRD role_deletion_suffix: "_deleted" # role name to grant to team members created from the Teams API @@ -345,7 +381,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-19" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-22" # max db connections the pooler should hold connection_pooler_max_db_connections: 60 # default pooling mode @@ -358,14 +394,14 @@ configConnectionPooler: connection_pooler_default_cpu_limit: "1" connection_pooler_default_memory_limit: 100Mi +# Zalando's internal CDC stream feature +enableStreams: false + rbac: # Specifies whether RBAC resources should be created create: true - -crd: - # Specifies whether custom resource definitions should be created - # When using helm3, this is ignored; instead use "--skip-crds" to skip. - create: true + # Specifies whether ClusterRoles that are aggregated into the K8s default roles should be created. (https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings) + createAggregateClusterRoles: false serviceAccount: # Specifies whether a ServiceAccount should be created diff --git a/cmd/main.go b/cmd/main.go index 376df0bad..0b48ac863 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,13 +2,14 @@ package main import ( "flag" - log "github.com/sirupsen/logrus" "os" "os/signal" "sync" "syscall" "time" + log "github.com/sirupsen/logrus" + "github.com/zalando/postgres-operator/pkg/controller" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/k8sutil" diff --git a/delivery.yaml b/delivery.yaml index 99f8f2078..15a30d42f 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -16,7 +16,7 @@ pipeline: - desc: 'Install go' cmd: | cd /tmp - wget -q https://storage.googleapis.com/golang/go1.16.9.linux-amd64.tar.gz -O go.tar.gz + wget -q https://storage.googleapis.com/golang/go1.17.4.linux-amd64.tar.gz -O go.tar.gz tar -xf go.tar.gz mv go /usr/local ln -s /usr/local/go/bin/go /usr/bin/go diff --git a/docker/DebugDockerfile b/docker/DebugDockerfile index e8f51badd..1981dfc82 100644 --- a/docker/DebugDockerfile +++ b/docker/DebugDockerfile @@ -9,7 +9,7 @@ COPY build/* / RUN addgroup -g 1000 pgo RUN adduser -D -u 1000 -G pgo -g 'Postgres Operator' pgo -RUN go get github.com/derekparker/delve/cmd/dlv +RUN go get -d github.com/derekparker/delve/cmd/dlv RUN cp /root/go/bin/dlv /dlv RUN chown -R pgo:pgo /dlv diff --git a/docker/logical-backup/dump.sh b/docker/logical-backup/dump.sh index c931dc962..2627ac8c9 100755 --- a/docker/logical-backup/dump.sh +++ b/docker/logical-backup/dump.sh @@ -12,9 +12,21 @@ DUMP_SIZE_COEFF=5 ERRORCOUNT=0 TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) -K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1 +if [ "$KUBERNETES_SERVICE_HOST" != "${KUBERNETES_SERVICE_HOST#*[0-9].[0-9]}" ]; then + echo "IPv4" + K8S_API_URL=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1 +elif [ "$KUBERNETES_SERVICE_HOST" != "${KUBERNETES_SERVICE_HOST#*:[0-9a-fA-F]}" ]; then + echo "IPv6" + K8S_API_URL=https://[$KUBERNETES_SERVICE_HOST]:$KUBERNETES_SERVICE_PORT/api/v1 +else + echo "Unrecognized IP format '$KUBERNETES_SERVICE_HOST'" +fi +echo "API Endpoint: ${K8S_API_URL}" CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt +LOGICAL_BACKUP_PROVIDER=${LOGICAL_BACKUP_PROVIDER:="s3"} +LOGICAL_BACKUP_S3_RETENTION_TIME=${LOGICAL_BACKUP_S3_RETENTION_TIME:=""} + function estimate_size { "$PG_BIN"/psql -tqAc "${ALL_DB_SIZE_QUERY}" } @@ -28,6 +40,57 @@ function compress { pigz } +function aws_delete_objects { + args=( + "--bucket=$LOGICAL_BACKUP_S3_BUCKET" + ) + + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") + [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION") + + aws s3api delete-objects "${args[@]}" --delete Objects=["$(printf {Key=%q}, "$@")"],Quiet=true +} +export -f aws_delete_objects + +function aws_delete_outdated { + if [[ -z "$LOGICAL_BACKUP_S3_RETENTION_TIME" ]] ; then + echo "no retention time configured: skip cleanup of outdated backups" + return 0 + fi + + # define cutoff date for outdated backups (day precision) + cutoff_date=$(date -d "$LOGICAL_BACKUP_S3_RETENTION_TIME ago" +%F) + + # mimic bucket setup from Spilo + prefix="spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/" + + args=( + "--no-paginate" + "--output=text" + "--prefix=$prefix" + "--bucket=$LOGICAL_BACKUP_S3_BUCKET" + ) + + [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT") + [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION") + + # list objects older than the cutoff date + aws s3api list-objects "${args[@]}" --query="Contents[?LastModified<='$cutoff_date'].[Key]" > /tmp/outdated-backups + + # spare the last backup + sed -i '$d' /tmp/outdated-backups + + count=$(wc -l < /tmp/outdated-backups) + if [[ $count == 0 ]] ; then + echo "no outdated backups to delete" + return 0 + fi + echo "deleting $count outdated backups created before $cutoff_date" + + # deleted outdated files in batches with 100 at a time + tr '\n' '\0' < /tmp/outdated-backups | xargs -0 -P1 -n100 bash -c 'aws_delete_objects "$@"' _ +} + function aws_upload { declare -r EXPECTED_SIZE="$1" @@ -59,6 +122,7 @@ function upload { ;; *) aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF)) + aws_delete_outdated ;; esac } diff --git a/docs/administrator.md b/docs/administrator.md index f03326f5a..aea4ec605 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -3,6 +3,25 @@ Learn how to configure and manage the Postgres Operator in your Kubernetes (K8s) environment. +## CRD registration and validation + +On startup, the operator will try to register the necessary +[CustomResourceDefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions) +`Postgresql` and `OperatorConfiguration`. The latter will only get created if +the `POSTGRES_OPERATOR_CONFIGURATION_OBJECT` [environment variable](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L36) +is set in the deployment yaml and is not empty. If the CRDs already exists they +will only be patched. If you do not wish the operator to create or update the +CRDs set `enable_crd_registration` config option to `false`. + +CRDs are defined with a `openAPIV3Schema` structural schema against which new +manifests of [`postgresql`](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql.crd.yaml) or [`OperatorConfiguration`](https://github.com/zalando/postgres-operator/blob/master/manifests/operatorconfiguration.crd.yaml) +resources will be validated. On creation you can bypass the validation with +`kubectl create --validate=false`. + +By default, the operator will register the CRDs in the `all` category so +that resources are listed on `kubectl get all` commands. The `crd_categories` +config option allows for customization of categories. + ## Upgrading the operator The Postgres Operator is upgraded by changing the docker image within the @@ -63,30 +82,6 @@ upgrade procedure, refer to the [corresponding PR in Spilo](https://github.com/z When `major_version_upgrade_mode` is set to `manual` the operator will run the upgrade script for you after the manifest is updated and pods are rotated. -## CRD Validation - -[CustomResourceDefinitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions) -will be registered with schema validation by default when the operator is -deployed. The `OperatorConfiguration` CRD will only get created if the -`POSTGRES_OPERATOR_CONFIGURATION_OBJECT` [environment variable](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L36) -in the deployment yaml is set and not empty. - -When submitting manifests of [`postgresql`](https://github.com/zalando/postgres-operator/blob/master/manifests/postgresql.crd.yaml) or -[`OperatorConfiguration`](https://github.com/zalando/postgres-operator/blob/master/manifests/operatorconfiguration.crd.yaml) custom -resources with kubectl, validation can be bypassed with `--validate=false`. The -operator can also be configured to not register CRDs with validation on `ADD` or -`UPDATE` events. Running instances are not affected when enabling the validation -afterwards unless the manifests is not changed then. Note, that the provided CRD -manifests contain the validation for users to understand what schema is -enforced. - -Once the validation is enabled it can only be disabled manually by editing or -patching the CRD manifest: - -```bash -kubectl patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}' -``` - ## Non-default cluster domain If your cluster uses a DNS domain other than the default `cluster.local`, this @@ -291,6 +286,86 @@ kubectl create -f manifests/user-facing-clusterroles.yaml It creates zalando-postgres-operator:user:view, :edit and :admin clusterroles that are aggregated into the K8s [default roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#default-roles-and-role-bindings). +For Helm deployments setting `rbac.createAggregateClusterRoles: true` adds these clusterroles to the deployment. + +## Password rotation in K8s secrets + +The operator regularly updates credentials in the K8s secrets if the +`enable_password_rotation` option is set to `true` in the configuration. +It happens only for `LOGIN` roles with an associated secret (manifest roles, +default users from `preparedDatabases`). Furthermore, there are the following +exceptions: + +1. Infrastructure role secrets since rotation should happen by the infrastructure. +2. Team API roles that connect via OAuth2 and JWT token (no secrets to these roles anyway). +3. Database owners since ownership on database objects can not be inherited. +4. System users such as `postgres`, `standby` and `pooler` user. + +The interval of days can be set with `password_rotation_interval` (default +`90` = 90 days, minimum 1). On each rotation the user name and password values +are replaced in the K8s secret. They belong to a newly created user named after +the original role plus rotation date in YYMMDD format. All priviliges are +inherited meaning that migration scripts should still grant and revoke rights +against the original role. The timestamp of the next rotation (in RFC 3339 +format, UTC timezone) is written to the secret as well. Note, if the rotation +interval is decreased it is reflected in the secrets only if the next rotation +date is more days away than the new length of the interval. + +Pods still using the previous secret values which they keep in memory continue +to connect to the database since the password of the corresponding user is not +replaced. However, a retention policy can be configured for users created by +the password rotation feature with `password_rotation_user_retention`. The +operator will ensure that this period is at least twice as long as the +configured rotation interval, hence the default of `180` = 180 days. When +the creation date of a rotated user is older than the retention period it +might not get removed immediately. Only on the next user rotation it is checked +if users can get removed. Therefore, you might want to configure the retention +to be a multiple of the rotation interval. + +### Password rotation for single users + +From the configuration, password rotation is enabled for all secrets with the +mentioned exceptions. If you wish to first test rotation for a single user (or +just have it enabled only for a few secrets) you can specify it in the cluster +manifest. The rotation and retention intervals can only be configured globally. + +``` +spec: + usersWithSecretRotation: + - foo_user + - bar_reader_user +``` + +### Password replacement without extra users + +For some use cases where the secret is only used rarely - think of a `flyway` +user running a migration script on pod start - we do not need to create extra +database users but can replace only the password in the K8s secret. This type +of rotation cannot be configured globally but specified in the cluster +manifest: + +``` +spec: + usersWithInPlaceSecretRotation: + - flyway + - bar_owner_user +``` + +This would be the recommended option to enable rotation in secrets of database +owners, but only if they are not used as application users for regular read +and write operations. + +### Turning off password rotation + +When password rotation is turned off again the operator will check if the +`username` value in the secret matches the original username and replace it +with the latter. A new password is assigned and the `nextRotation` field is +cleared. A final lookup for child (rotation) users to be removed is done but +they will only be dropped if the retention policy allows for it. This is to +avoid sudden connection issues in pods which still use credentials of these +users in memory. You have to remove these child users manually or re-enable +password rotation with smaller interval so they get cleaned up. + ## Use taints and tolerations for dedicated PostgreSQL nodes To ensure Postgres pods are running on nodes without any other application pods, @@ -337,6 +412,81 @@ master pods from being evicted by the K8s runtime. To prevent eviction completely, specify the toleration by leaving out the `tolerationSeconds` value (similar to how Kubernetes' own DaemonSets are configured) +## Node readiness labels + +The operator can watch on certain node labels to detect e.g. the start of a +Kubernetes cluster upgrade procedure and move master pods off the nodes to be +decommissioned. Key-value pairs for these node readiness labels can be +specified in the configuration (option name is in singular form): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-operator +data: + node_readiness_label: "status1:ready,status2:ready" +``` + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-configuration +configuration: + kubernetes: + node_readiness_label: + status1: ready + status2: ready +``` + +The operator will create a `nodeAffinity` on the pods. This makes the +`node_readiness_label` option the global configuration for defining node +affinities for all Postgres clusters. You can have both, cluster-specific and +global affinity, defined and they will get merged on the pods. If +`node_readiness_label_merge` is configured to `"AND"` the node readiness +affinity will end up under the same `matchExpressions` section(s) from the +manifest affinity. + +```yaml + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: environment + operator: In + values: + - pci + - key: status1 + operator: In + values: + - ready + - key: status2 + ... +``` + +If `node_readiness_label_merge` is set to `"OR"` (default) the readiness label +affinty will be appended with its own expressions block: + +```yaml + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: environment + ... + - matchExpressions: + - key: storage + ... + - matchExpressions: + - key: status1 + ... + - key: status2 + ... +``` + ## Enable pod anti affinity To ensure Postgres pods are running on different topologies, you can use @@ -451,15 +601,39 @@ spec: ## Custom Pod Environment Variables -It is possible to configure a ConfigMap as well as a Secret which are used by -the Postgres pods as an additional provider for environment variables. One use -case is a customized Spilo image configured by extra environment variables. -Another case could be to provide custom cloud provider or backup settings. +The operator will assign a set of environment variables to the database pods +that cannot be overridden to guarantee core functionality. Only variables with +'WAL_' and 'LOG_' prefixes can be customized to allow for backup and log +shipping to be specified differently. There are three ways to specify extra +environment variables (or override existing ones) for database pods: -In general the Operator will give preference to the globally configured -variables, to not have the custom ones interfere with core functionality. -Variables with the 'WAL_' and 'LOG_' prefix can be overwritten though, to -allow backup and log shipping to be specified differently. +* [Via ConfigMap](#via-configmap) +* [Via Secret](#via-secret) +* [Via Postgres Cluster Manifest](#via-postgres-cluster-manifest) + +The first two options must be referenced from the operator configuration +making them global settings for all Postgres cluster the operator watches. +One use case is a customized Spilo image that must be configured by extra +environment variables. Another case could be to provide custom cloud +provider or backup settings. + +The last options allows for specifying environment variables individual to +every cluster via the `env` section in the manifest. For example, if you use +individual backup locations for each of your clusters. Or you want to disable +WAL archiving for a certain cluster by setting `WAL_S3_BUCKET`, `WAL_GS_BUCKET` +or `AZURE_STORAGE_ACCOUNT` to an empty string. + +The operator will give precedence to environment variables in the following +order (e.g. a variable defined in 4. overrides a variable with the same name +in 5.): + +1. Assigned by the operator +2. Clone section (with WAL settings from operator config when `s3_wal_path` is empty) +3. Standby section +4. `env` section in cluster manifest +5. Pod environment secret via operator config +6. Pod environment config map via operator config +7. WAL and logical backup settings from operator config ### Via ConfigMap @@ -556,6 +730,29 @@ data: The key-value pairs of the Secret are all accessible as environment variables to the Postgres StatefulSet/pods. +### Via Postgres Cluster Manifest + +It is possible to define environment variables directly in the Postgres cluster +manifest to configure it individually. The variables must be listed under the +`env` section in the same way you would do for [containers](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/). +Global parameters served from a custom config map or secret will be overridden. + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-test-cluster +spec: + env: + - name: wal_s3_bucket + value: my-custom-bucket + - name: minio_secret_key + valueFrom: + secretKeyRef: + name: my-custom-secret + key: minio_secret_key +``` + ## Limiting the number of min and max instances in clusters As a preventive measure, one can restrict the minimum and the maximum number of @@ -600,6 +797,11 @@ lead to K8s removing this field from the manifest due to its Then the resultant manifest will not contain the necessary change, and the operator will respectively do nothing with the existing source ranges. +Load balancer services can also be enabled for the [connection pooler](user.md#connection-pooler) +pods with manifest flags `enableMasterPoolerLoadBalancer` and/or +`enableReplicaPoolerLoadBalancer` or in the operator configuration with +`enable_master_pooler_load_balancer` and/or `enable_replica_pooler_load_balancer`. + ## Running periodic 'autorepair' scans of K8s objects The Postgres Operator periodically scans all K8s objects belonging to each @@ -762,7 +964,7 @@ WALE_S3_PREFIX=$WAL_S3_BUCKET/spilo/{WAL_BUCKET_SCOPE_PREFIX}{SCOPE}{WAL_BUCKET_ ``` The operator sets the prefix to an empty string so that spilo will generate it -from the configured `WAL_S3_BUCKET`. +from the configured `WAL_S3_BUCKET`. :warning: When you overwrite the configuration by defining `WAL_S3_BUCKET` in the [pod_environment_configmap](#custom-pod-environment-variables) you have @@ -773,6 +975,10 @@ When the `AWS_REGION` is set, `AWS_ENDPOINT` and `WALE_S3_ENDPOINT` are generated automatically. `WALG_S3_PREFIX` is identical to `WALE_S3_PREFIX`. `SCOPE` is the Postgres cluster name. +:warning: If both `AWS_REGION` and `AWS_ENDPOINT` or `WALE_S3_ENDPOINT` are +defined backups with WAL-E will fail. You can fix it by switching to WAL-G +with `USE_WALG_BACKUP: "true"`. + ### Google Cloud Platform setup To configure the operator on GCP these prerequisites that are needed: @@ -885,6 +1091,7 @@ data: USE_WALG_BACKUP: "true" USE_WALG_RESTORE: "true" CLONE_USE_WALG_RESTORE: "true" + WALG_AZ_PREFIX: "azure://container-name/$(SCOPE)/$(PGVERSION)" # Enables Azure Backups (SCOPE = Cluster name) (PGVERSION = Postgres version) ``` 3. Setup your operator configuration values. With the `psql-backup-creds` @@ -931,12 +1138,16 @@ data: ### Standby clusters -The setup for [standby clusters](user.md#setting-up-a-standby-cluster) is very -similar to cloning. At the moment, the operator only allows for streaming from -the S3 WAL archive of the master specified in the manifest. Like with cloning, -if you are using [additional environment variables](#custom-pod-environment-variables) -to access your backup location you have to copy those variables and prepend the -`STANDBY_` prefix for Spilo to find the backups and WAL files to stream. +The setup for [standby clusters](user.md#setting-up-a-standby-cluster) is +similar to cloning when they stream changes from a WAL archive (S3 or GCS). +If you are using [additional environment variables](#custom-pod-environment-variables) +to access your backup location you have to copy those variables and prepend +the `STANDBY_` prefix for Spilo to find the backups and WAL files to stream. + +Alternatively, standby clusters can also stream from a remote primary cluster. +You have to specify the host address. Port is optional and defaults to 5432. +Note, that only one of the options (`s3_wal_path`, `gs_wal_path`, +`standby_host`) can be present under the `standby` top-level key. ## Logical backups @@ -1075,7 +1286,7 @@ make docker # build in image in minikube docker env eval $(minikube docker-env) -docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.7.1 . +docker build -t registry.opensource.zalan.do/acid/postgres-operator-ui:v1.8.0 . # apply UI manifests next to a running Postgres Operator kubectl apply -f manifests/ diff --git a/docs/developer.md b/docs/developer.md index 09fa5f527..31f48d92d 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -7,7 +7,7 @@ features and tests. Postgres Operator is written in Go. Use the [installation instructions](https://golang.org/doc/install#install) if you don't have Go on your system. You won't be able to compile the operator -with Go older than 1.16. We recommend installing [the latest one](https://golang.org/dl/). +with Go older than 1.17. We recommend installing [the latest one](https://golang.org/dl/). Go projects expect their source code and all the dependencies to be located under the [GOPATH](https://github.com/golang/go/wiki/GOPATH). Normally, one @@ -72,22 +72,32 @@ make docker # kind make docker -kind load docker-image --name +kind load docker-image registry.opensource.zalan.do/acid/postgres-operator:${TAG} --name ``` -Then create a new Postgres Operator deployment. You can reuse the provided -manifest but replace the version and tag. Don't forget to also apply +Then create a new Postgres Operator deployment. + +### Deploying manually with manifests and kubectl + +You can reuse the provided manifest but replace the version and tag. Don't forget to also apply configuration and RBAC manifests first, e.g.: ```bash kubectl create -f manifests/configmap.yaml kubectl create -f manifests/operator-service-account-rbac.yaml -sed -e "s/\(image\:.*\:\).*$/\1$TAG/" manifests/postgres-operator.yaml | kubectl create -f - +sed -e "s/\(image\:.*\:\).*$/\1$TAG/" -e "s/\(imagePullPolicy\:\).*$/\1 Never/" manifests/postgres-operator.yaml | kubectl create -f - # check if the operator is coming up kubectl get pod -l name=postgres-operator ``` +### Deploying with Helm chart + +Yoy can reuse the provided Helm chart to deploy local operator build with the following command: +```bash +helm install postgres-operator ./charts/postgres-operator --namespace zalando-operator --set image.tag=${TAG} --set image.pullPolicy=Never +``` + ## Code generation The operator employs K8s-provided code generation to obtain deep copy methods @@ -176,7 +186,7 @@ go get -u github.com/derekparker/delve/cmd/dlv ``` RUN apk --no-cache add go git musl-dev -RUN go get github.com/derekparker/delve/cmd/dlv +RUN go get -d github.com/derekparker/delve/cmd/dlv ``` * Update the `Makefile` to build the project with debugging symbols. For that @@ -208,6 +218,13 @@ dlv connect 127.0.0.1:DLV_PORT ## Unit tests +Prerequisites: + +```bash +make deps +make mocks +``` + To run all unit tests, you can simply do: ```bash diff --git a/docs/quickstart.md b/docs/quickstart.md index ed01367b7..00ba2f3aa 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -37,7 +37,7 @@ The Postgres Operator can be deployed in the following ways: * Kustomization * Helm chart -### Manual deployment setup +### Manual deployment setup on Kubernetes The Postgres Operator can be installed simply by applying yaml manifests. Note, we provide the `/manifests` directory as an example only; you should consider @@ -71,21 +71,38 @@ manifest. ./run_operator_locally.sh ``` -### Helm chart +### Manual deployment setup on OpenShift -Alternatively, the operator can be installed by using the provided [Helm](https://helm.sh/) -chart which saves you the manual steps. Clone this repo and change directory to -the repo root. With Helm v3 installed you should be able to run: +To install the Postgres Operator in OpenShift you have to change the config +parameter `kubernetes_use_configmaps` to `"true"`. Otherwise, the operator +and Patroni will store leader and config keys in `Endpoints` that are not +supported in OpenShift. This requires also a slightly different set of rules +for the `postgres-operator` and `postgres-pod` cluster roles. ```bash -helm install postgres-operator ./charts/postgres-operator +oc create -f manifests/operator-service-account-rbac-openshift.yaml ``` -The chart works with both Helm 2 and Helm 3. The `crd-install` hook from v2 will -be skipped with warning when using v3. Documentation for installing applications -with Helm 2 can be found in the [v2 docs](https://v2.helm.sh/docs/). +### Helm chart -The chart is also hosted at: https://opensource.zalando.com/postgres-operator/charts/postgres-operator/ +Alternatively, the operator can be installed by using the provided +[Helm](https://helm.sh/) chart which saves you the manual steps. The charts +for both the Postgres Operator and its UI are hosted via the `gh-pages` branch. +They only work only with Helm v3. Helm v2 support was dropped with v1.8.0. + +```bash +# add repo for postgres-operator +helm repo add postgres-operator-charts https://opensource.zalando.com/postgres-operator/charts/postgres-operator + +# install the postgres-operator +helm install postgres-operator postgres-operator-charts/postgres-operator + +# add repo for postgres-operator-ui +helm repo add postgres-operator-ui-charts https://opensource.zalando.com/postgres-operator/charts/postgres-operator-ui + +# install the postgres-operator-ui +helm install postgres-operator-ui postgres-operator-ui-charts/postgres-operator-ui +``` ## Check if Postgres Operator is running diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index bf01b2a5c..b41550e22 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -91,11 +91,23 @@ These parameters are grouped directly under the `spec` key in the manifest. `enable_master_load_balancer` parameter) to define whether to enable the load balancer pointing to the Postgres primary. Optional. +* **enableMasterPoolerLoadBalancer** + boolean flag to override the operator defaults (set by the + `enable_master_pooler_load_balancer` parameter) to define whether to enable + the load balancer for master pooler pods pointing to the Postgres primary. + Optional. + * **enableReplicaLoadBalancer** boolean flag to override the operator defaults (set by the `enable_replica_load_balancer` parameter) to define whether to enable the load balancer pointing to the Postgres standby instances. Optional. +* **enableReplicaPoolerLoadBalancer** + boolean flag to override the operator defaults (set by the + `enable_replica_pooler_load_balancer` parameter) to define whether to enable + the load balancer for replica pooler pods pointing to the Postgres standby + instances. Optional. + * **allowedSourceRanges** when one or more load balancers are enabled for the cluster, this parameter defines the comma-separated range of IP networks (in CIDR-notation). The @@ -115,6 +127,22 @@ These parameters are grouped directly under the `spec` key in the manifest. create the K8s secret in that namespace. The part after the first `.` is considered to be the user name. Optional. +* **usersWithSecretRotation** + list of users to enable credential rotation in K8s secrets. The rotation + interval can only be configured globally. On each rotation a new user will + be added in the database replacing the `username` value in the secret of + the listed user. Although, rotation users inherit all rights from the + original role, keep in mind that ownership is not transferred. See more + details in the [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#password-rotation-in-k8s-secrets). + +* **usersWithInPlaceSecretRotation** + list of users to enable in-place password rotation in K8s secrets. The + rotation interval can only be configured globally. On each rotation the + password value will be replaced in the secrets which the operator reflects + in the database, too. List only users here that rarely connect to the + database, like a flyway user running a migration on Pod start. See more + details in the [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#password-replacement-without-extra-users). + * **databases** a map of database names to database owners for the databases that should be created by the operator. The owner users should already exist on the cluster @@ -287,13 +315,14 @@ explanation of `ttl` and `loop_wait` parameters. * **synchronous_mode_strict** Patroni `synchronous_mode_strict` parameter value. Can be used in addition to `synchronous_mode`. The default is set to `false`. Optional. +* **synchronous_node_count** + Patroni `synchronous_node_count` parameter value. Note, this option is only available for Spilo images with Patroni 2.0+. The default is set to `1`. Optional. + ## Postgres container resources Those parameters define [CPU and memory requests and limits](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the Postgres container. They are grouped under the `resources` top-level -key with subgroups `requests` and `limits`. The whole section is optional, -however if you specify a request or limit you have to define everything -(unless you are not modifying the default CRD schema validation). +key with subgroups `requests` and `limits`. ### Requests @@ -366,12 +395,22 @@ under the `clone` top-level key and do not affect the already running cluster. ## Standby cluster On startup, an existing `standby` top-level key creates a standby Postgres -cluster streaming from a remote location. So far only streaming from a S3 WAL -archive is supported. +cluster streaming from a remote location - either from a S3 or GCS WAL +archive or a remote primary. Only one of options is allowed and required +if the `standby` key is present. * **s3_wal_path** the url to S3 bucket containing the WAL archive of the remote primary. - Required when the `standby` section is present. + +* **gs_wal_path** + the url to GS bucket containing the WAL archive of the remote primary. + +* **standby_host** + hostname or IP address of the primary to stream from. + +* **standby_port** + TCP port on which the primary is listening for connections. Patroni will + use `"5432"` if not set. ## Volume properties @@ -513,3 +552,54 @@ Those parameters are grouped under the `tls` top-level key. relative to the "/tls/", which is mount path of the tls secret. If `caSecretName` is defined, the ca.crt path is relative to "/tlsca/", otherwise to the same "/tls/". + +## Change data capture streams + +This sections enables change data capture (CDC) streams via Postgres' +[logical decoding](https://www.postgresql.org/docs/14/logicaldecoding.html) +feature and `pgoutput` plugin. While the Postgres operator takes responsibility +for providing the setup to publish change events, it relies on external tools +to consume them. At Zalando, we are using a workflow based on +[Debezium Connector](https://debezium.io/documentation/reference/stable/connectors/postgresql.html) +which can feed streams into Zalando’s distributed event broker [Nakadi](https://nakadi.io/) +among others. + +The Postgres Operator creates custom resources for Zalando's internal CDC +operator which will be used to set up the consumer part. Each stream object +can have the following properties: + +* **applicationId** + The application name to which the database and CDC belongs to. For each + set of streams with a distinct `applicationId` a separate stream CR as well + as a separate logical replication slot will be created. This means there can + be different streams in the same database and streams with the same + `applicationId` are bundled in one stream CR. The stream CR will be called + like the Postgres cluster plus "-" suffix. Required. + +* **database** + Name of the database from where events will be published via Postgres' + logical decoding feature. The operator will take care of updating the + database configuration (setting `wal_level: logical`, creating logical + replication slots, using output plugin `pgoutput` and creating a dedicated + replication user). Required. + +* **tables** + Defines a map of table names and their properties (`eventType`, `idColumn` + and `payloadColumn`). The CDC operator is following the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/). + The application is responsible for putting events into a (JSON/B or VARCHAR) + payload column of the outbox table in the structure of the specified target + event type. The operator will create a [PUBLICATION](https://www.postgresql.org/docs/14/logical-replication-publication.html) + in Postgres for all tables specified for one `database` and `applicationId`. + The CDC operator will consume from it shortly after transactions are + committed to the outbox table. The `idColumn` will be used in telemetry for + the CDC operator. The names for `idColumn` and `payloadColumn` can be + configured. Defaults are `id` and `payload`. The target `eventType` has to + be defined. Required. + +* **filter** + Streamed events can be filtered by a jsonpath expression for each table. + Optional. + +* **batchSize** + Defines the size of batches in which events are consumed. Optional. + Defaults to 1. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index bba917d88..e4e64cc32 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -70,11 +70,18 @@ Variable names are underscore-separated words. Those are top-level keys, containing both leaf keys and groups. -* **enable_crd_validation** - toggles if the operator will create or update CRDs with - [OpenAPI v3 schema validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation) +* **enable_crd_registration** + Instruct the operator to create/update the CRDs. If disabled the operator will rely on the CRDs being managed separately. The default is `true`. +* **enable_crd_validation** + *deprecated*: toggles if the operator will create or update CRDs with + [OpenAPI v3 schema validation](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#validation) + The default is `true`. `false` will be ignored, since `apiextensions.io/v1` requires a structural schema definition. + +* **crd_categories** + The operator will register CRDs in the `all` category by default so that they will be returned by a `kubectl get all` call. You are free to change categories or leave them empty. + * **enable_lazy_spilo_upgrade** Instruct operator to update only the statefulsets with new images (Spilo and InitContainers) without immediately doing the rolling update. The assumption is pods will be re-started later with new images, for example due to the node rotation. The default is `false`. @@ -152,9 +159,9 @@ Those are top-level keys, containing both leaf keys and groups. at the cost of overprovisioning memory and potential scheduling problems for containers with high memory limits due to the lack of memory on Kubernetes cluster nodes. This affects all containers created by the operator (Postgres, - Scalyr sidecar, and other sidecars except **sidecars** defined in the operator - configuration); to set resources for the operator's own container, change the - [operator deployment manually](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L20). + connection pooler, logical backup, scalyr sidecar, and other sidecars except + **sidecars** defined in the operator configuration); to set resources for the + operator's own container, change the [operator deployment manually](https://github.com/zalando/postgres-operator/blob/master/manifests/postgres-operator.yaml#L20). The default is `false`. ## Postgres users @@ -170,6 +177,37 @@ under the `users` key. Postgres username used for replication between instances. The default is `standby`. +* **additional_owner_roles** + Specifies database roles that will become members of all database owners. + Then owners can use `SET ROLE` to obtain privileges of these roles to e.g. + create/update functionality from extensions as part of a migration script. + Note, that roles listed here should be preconfigured in the docker image + and already exist in the database cluster on startup. One such role can be + `cron_admin` which is provided by the Spilo docker image to set up cron + jobs inside the `postgres` database. Default is `empty`. + +* **enable_password_rotation** + For all `LOGIN` roles that are not database owners the operator can rotate + credentials in the corresponding K8s secrets by replacing the username and + password. This means, new users will be added on each rotation inheriting + all priviliges from the original roles. The rotation date (in YYMMDD format) + is appended to the names of the new user. The timestamp of the next rotation + is written to the secret. The default is `false`. + +* **password_rotation_interval** + If password rotation is enabled (either from config or cluster manifest) the + interval can be configured with this parameter. The measure is in days which + means daily rotation (`1`) is the most frequent interval possible. + Default is `90`. + +* **password_rotation_user_retention** + To avoid an ever growing amount of new users due to password rotation the + operator will remove the created users again after a certain amount of days + has passed. The number can be configured with this parameter. However, the + operator will check that the retention policy is at least twice as long as + the rotation interval and update to this minimum in case it is not. + Default is `180`. + ## Major version upgrades Parameters configuring automatic major version upgrades. In a @@ -184,6 +222,10 @@ CRD-configuration, they are grouped under the `major_version_upgrade` key. Note, that with all three modes increasing the version in the manifest will trigger a rolling update of the pods. The default is `"off"`. +* **major_version_upgrade_team_allow_list** + Upgrades will only be carried out for clusters of listed teams when mode is + set to "off". The default is empty. + * **minimal_major_version** The minimal Postgres major version that will not automatically be upgraded when `major_version_upgrade_mode` is set to `"full"`. The default is `"9.6"`. @@ -245,6 +287,12 @@ configuration they are grouped under the `kubernetes` key. Regular expressions like `downscaler/*` etc. are also accepted. Can be used with [kube-downscaler](https://github.com/hjacobs/kube-downscaler). +* **ignored_annotations** + Some K8s tools inject and update annotations out of the Postgres Operator + control. This can cause rolling updates on each cluster sync cycle. With + this option you can specify an array of annotation keys that should be + ignored when comparing K8s resources on sync. The default is empty. + * **watched_namespace** The operator watches for Postgres objects in the given namespace. If not specified, the value is taken from the operator namespace. A special `*` @@ -336,11 +384,16 @@ configuration they are grouped under the `kubernetes` key. * **node_readiness_label** a set of labels that a running and active node should possess to be - considered `ready`. The operator uses values of those labels to detect the - start of the Kubernetes cluster upgrade procedure and move master pods off - the nodes to be decommissioned. When the set is not empty, the operator also - assigns the `Affinity` clause to the Postgres pods to be scheduled only on - `ready` nodes. The default is empty. + considered `ready`. When the set is not empty, the operator assigns the + `nodeAffinity` clause to the Postgres pods to be scheduled only on `ready` + nodes. The default is empty. + +* **node_readiness_label_merge** + If a `nodeAffinity` is also specified in the postgres cluster manifest + it will get merged with the `node_readiness_label` affinity on the pods. + The merge strategy can be configured - it can either be "AND" or "OR". + See [user docs](../user.md#use-taints-tolerations-and-node-affinity-for-dedicated-postgresql-nodes) + for more details. Default is "OR". * **toleration** a dictionary that should contain `key`, `operator`, `value` and @@ -460,6 +513,13 @@ configuration `resource_check_interval` and `resource_check_timeout` have no effect, and the parameters are grouped under the `timeouts` key in the CRD-based configuration. +* **PatroniAPICheckInterval** + the interval between consecutive attempts waiting for the return of + Patroni Api. The default is `1s`. + +* **PatroniAPICheckTimeout** + the timeout for a response from Patroni Api. The default is `5s`. + * **resource_check_interval** interval to wait between consecutive attempts to check for the presence of some Kubernetes resource (i.e. `StatefulSet` or `PodDisruptionBudget`). The @@ -508,11 +568,21 @@ In the CRD-based configuration they are grouped under the `load_balancer` key. toggles service type load balancer pointing to the master pod of the cluster. Can be overridden by individual cluster settings. The default is `true`. -* **enable_replica_load_balancer** - toggles service type load balancer pointing to the replica pod of the - cluster. Can be overridden by individual cluster settings. The default is +* **enable_master_pooler_load_balancer** + toggles service type load balancer pointing to the master pooler pod of the + cluster. Can be overridden by individual cluster settings. The default is `false`. +* **enable_replica_load_balancer** + toggles service type load balancer pointing to the replica pod(s) of the + cluster. Can be overridden by individual cluster settings. The default is + `false`. + +* **enable_replica_pooler_load_balancer** + toggles service type load balancer pointing to the replica pooler pod(s) of + the cluster. Can be overridden by individual cluster settings. The default + is `false`. + * **external_traffic_policy** defines external traffic policy for load balancers. Allowed values are `Cluster` (default) and `Local`. @@ -575,7 +645,10 @@ yet officially supported. empty. * **aws_region** - AWS region used to store EBS volumes. The default is `eu-central-1`. + AWS region used to store EBS volumes. The default is `eu-central-1`. Note, + this option is not meant for specifying the AWS region for backups and + restore, since it can be separate from the EBS region. You have to define + AWS_REGION as a [custom environment variable](../administrator.md#custom-pod-environment-variables). * **additional_secret_mount** Additional Secret (aws or gcp credentials) to mount in the pod. @@ -606,7 +679,7 @@ grouped under the `logical_backup` key. runs `pg_dumpall` on a replica if possible and uploads compressed results to an S3 bucket under the key `/spilo/pg_cluster_name/cluster_k8s_uuid/logical_backups`. The default image is the same image built with the Zalando-internal CI - pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1" + pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0" * **logical_backup_google_application_credentials** Specifies the path of the google cloud service account json file. Default is empty. @@ -638,6 +711,11 @@ grouped under the `logical_backup` key. Specify server side encryption that S3 storage is using. If empty string is specified, no argument will be passed to `aws s3` command. Default: "AES256". +* **logical_backup_s3_retention_time** + Specify a retention time for logical backups stored in S3. Backups older than the specified retention + time will be deleted after a new backup was uploaded. If empty, all backups will be kept. Example values are + "3 days", "2 weeks", or "1 month". The default is empty. + * **logical_backup_schedule** Backup schedule in the cron format. Please take the [reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) @@ -710,7 +788,7 @@ key. * **protected_role_names** List of roles that cannot be overwritten by an application, team or - infrastructure role. The default is `admin`. + infrastructure role. The default list is `admin` and `cron_admin`. * **postgres_superuser_teams** List of teams which members need the superuser role in each PG database diff --git a/docs/user.md b/docs/user.md index d8783c42f..0fc7c35f2 100644 --- a/docs/user.md +++ b/docs/user.md @@ -83,9 +83,9 @@ kubectl port-forward $PGMASTER 6432:5432 -n default ``` Open another CLI and connect to the database using e.g. the psql client. -When connecting with the `postgres` user read its password from the K8s secret -which was generated when creating the `acid-minimal-cluster`. As non-encrypted -connections are rejected by default set the SSL mode to `require`: +When connecting with a manifest role like `foo_user` user, read its password +from the K8s secret which was generated when creating `acid-minimal-cluster`. +As non-encrypted connections are rejected by default set SSL mode to `require`: ```bash export PGPASSWORD=$(kubectl get secret postgres.acid-minimal-cluster.credentials.postgresql.acid.zalan.do -o 'jsonpath={.data.password}' | base64 -d) @@ -93,6 +93,26 @@ export PGSSLMODE=require psql -U postgres -h localhost -p 6432 ``` +## Password encryption + +Passwords are encrypted with `md5` hash generation by default. However, it is +possible to use the more recent `scram-sha-256` method by changing the +`password_encryption` parameter in the Postgres config. You can define it +directly from the cluster manifest: + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-minimal-cluster +spec: + [...] + postgresql: + version: "14" + parameters: + password_encryption: scram-sha-256 +``` + ## Defining database roles in the operator Postgres Operator allows defining roles to be created in the resulting database @@ -141,14 +161,18 @@ other roles. To define the secrets for the users in a different namespace than that of the cluster, one can set `enable_cross_namespace_secret` and declare the namespace -for the secrets in the manifest in the following manner, +for the secrets in the manifest in the following manner (note, that it has to +be reflected in the `database` section, too), ```yaml spec: users: - #users with secret in dfferent namespace - appspace.db_user: + # users with secret in different namespace + appspace.db_user: - createdb + databases: + # namespace notation is part of user name + app_db: appspace.db_user ``` Here, anything before the first dot is considered the namespace and the text after @@ -554,7 +578,8 @@ schema creation. This means they are currently not set when `defaultUsers` For all LOGIN roles the operator will create K8s secrets in the namespace specified in `secretNamespace`, if `enable_cross_namespace_secret` is set to `true` in the config. Otherwise, they are created in the same namespace like -the Postgres cluster. +the Postgres cluster. Unlike roles specified with `namespace.username` under +`users`, the namespace will not be part of the role name here. ```yaml spec: @@ -598,10 +623,9 @@ spec: ``` Some extensions require SUPERUSER rights on creation unless they are not -whitelisted by the [pgextwlist](https://github.com/dimitri/pgextwlist) -extension, that is shipped with the Spilo image. To see which extensions are -on the list check the `extwlist.extension` parameter in the postgresql.conf -file. +allowed by the [pgextwlist](https://github.com/dimitri/pgextwlist) extension, +that is shipped with the Spilo image. To see which extensions are on the list +check the `extwlist.extension` parameter in the postgresql.conf file. ```bash SHOW extwlist.extensions; @@ -667,7 +691,9 @@ configured [default requests](reference/operator_parameters.md#kubernetes-resour To ensure Postgres pods are running on nodes without any other application pods, you can use [taints and tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) -and configure the required toleration in the manifest. +and configure the required toleration in the manifest. Tolerations can also be +defined in the [operator config](administrator.md#use-taints-and-tolerations-for-dedicated-postgresql-nodes) +to apply for all Postgres clusters. ```yaml spec: @@ -699,6 +725,9 @@ spec: - pci ``` +If you need to define a `nodeAffinity` for all your Postgres clusters use the +`node_readiness_label` [configuration](administrator.md#node-readiness-labels). + ## In-place major version upgrade Starting with Spilo 13, operator supports in-place major version upgrade to a @@ -728,29 +757,41 @@ source cluster. If you create it in the same Kubernetes environment, use a different name. ```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-minimal-cluster-clone spec: clone: uid: "efd12e58-5786-11e8-b5a7-06148230260c" - cluster: "acid-batman" + cluster: "acid-minimal-cluster" timestamp: "2017-12-19T12:40:33+01:00" - s3_wal_path: "s3:///spilo///wal/" ``` Here `cluster` is a name of a source cluster that is going to be cloned. A new cluster will be cloned from S3, using the latest backup before the `timestamp`. -Note, that a time zone is required for `timestamp` in the format of +00:00 which -is UTC. You can specify the `s3_wal_path` of the source cluster or let the -operator try to find it based on the configured `wal_[s3|gs]_bucket` and the -specified `uid`. You can find the UID of the source cluster in its metadata: +Note, a time zone is required for `timestamp` in the format of `+00:00` (UTC). + +The operator will try to find the WAL location based on the configured +`wal_[s3|gs]_bucket` or `wal_az_storage_account` and the specified `uid`. +You can find the UID of the source cluster in its metadata: ```yaml apiVersion: acid.zalan.do/v1 kind: postgresql metadata: - name: acid-batman + name: acid-minimal-cluster uid: efd12e58-5786-11e8-b5a7-06148230260c ``` +If your source cluster uses a WAL location different from the global +configuration you can specify the full path under `s3_wal_path`. For +[Google Cloud Platform](administrator.md#google-cloud-platform-setup) +or [Azure](administrator.md#azure-setup) +it can only be set globally with [custom Pod environment variables](administrator.md#custom-pod-environment-variables) +or locally in the Postgres manifest's [`env`](administrator.md#via-postgres-cluster-manifest) section. + + For non AWS S3 following settings can be set to support cloning from other S3 implementations: @@ -758,8 +799,9 @@ implementations: spec: clone: uid: "efd12e58-5786-11e8-b5a7-06148230260c" - cluster: "acid-batman" + cluster: "acid-minimal-cluster" timestamp: "2017-12-19T12:40:33+01:00" + s3_wal_path: "s3://custom/path/to/bucket" s3_endpoint: https://s3.acme.org s3_access_key_id: 0123456789abcdef0123456789abcdef s3_secret_access_key: 0123456789abcdef0123456789abcdef @@ -779,23 +821,60 @@ namespace. ```yaml spec: clone: - cluster: "acid-batman" + cluster: "acid-minimal-cluster" ``` Be aware that on a busy source database this can result in an elevated load! +## Restore in place + +There is also a possibility to restore a database without cloning it. The +advantage to this is that there is no need to change anything on the +application side. However, as it involves deleting the database first, this +process is of course riskier than cloning (which involves adjusting the +connection parameters of the app). + +First, make sure there is no writing activity on your DB, and save the UID. +Then delete the `postgresql` K8S resource: + +```bash +zkubectl delete postgresql acid-test-restore +``` + +Then deploy a new manifest with the same name, referring to itself +(both name and UID) in the `clone` section: + +```yaml +metadata: + name: acid-minimal-cluster + # [...] +spec: + # [...] + clone: + cluster: "acid-minimal-cluster" # the same as metadata.name above! + uid: "" + timestamp: "2022-04-01T10:11:12.000+00:00" +``` + +This will create a new database cluster with the same name but different UID, +whereas the database will be in the state it was at the specified time. + +:warning: The backups and WAL files for the original DB are retained under the +original UID, making it possible retry restoring. However, it is probably +better to create a temporary clone for experimenting or finding out to which +point you should restore. + ## Setting up a standby cluster Standby cluster is a [Patroni feature](https://github.com/zalando/patroni/blob/master/docs/replica_bootstrap.rst#standby-cluster) -that first clones a database, and keeps replicating changes afterwards. As the -replication is happening by the means of archived WAL files (stored on S3 or -the equivalent of other cloud providers), the standby cluster can exist in a -different location than its source database. Unlike cloning, the PostgreSQL -version between source and target cluster has to be the same. +that first clones a database, and keeps replicating changes afterwards. It can +exist in a different location than its source database, but unlike cloning, +the PostgreSQL version between source and target cluster has to be the same. To start a cluster as standby, add the following `standby` section in the YAML -file and specify the S3 bucket path. An empty path will result in an error and -no statefulset will be created. +file. You can stream changes from archived WAL files (AWS S3 or Google Cloud +Storage) or from a remote primary. Only one option can be specfied in the +manifest: ```yaml spec: @@ -803,32 +882,55 @@ spec: s3_wal_path: "s3:///spilo///wal/" ``` -At the moment, the operator only allows to stream from the WAL archive of the -master. Thus, it is recommended to deploy standby clusters with only [one pod](https://github.com/zalando/postgres-operator/blob/master/manifests/standby-manifest.yaml#L10). -You can raise the instance count when detaching. Note, that the same pod role -labels like for normal clusters are used: The standby leader is labeled as -`master`. +For GCS, you have to define STANDBY_GOOGLE_APPLICATION_CREDENTIALS as a +[custom pod environment variable](administrator.md#custom-pod-environment-variables). +It is not set from the config to allow for overridding. + +```yaml +spec: + standby: + gs_wal_path: "gs:///spilo///wal/" +``` + +For a remote primary you specify the host address and optionally the port. +If you leave out the port Patroni will use `"5432"`. + +```yaml +spec: + standby: + standby_host: "acid-minimal-cluster.default" + standby_port: "5433" +``` + +Note, that the pods and services use the same role labels like for normal clusters: +The standby leader is labeled as `master`. When using the `standby_host` option +you have to copy the credentials from the source cluster's secrets to successfully +bootstrap a standby cluster (see next chapter). ### Providing credentials of source cluster A standby cluster is replicating the data (including users and passwords) from the source database and is read-only. The system and application users (like standby, postgres etc.) all have a password that does not match the credentials -stored in secrets which are created by the operator. One solution is to create -secrets beforehand and paste in the credentials of the source cluster. +stored in secrets which are created by the operator. You have two options: + +a. Create secrets manually beforehand and paste the credentials of the source + cluster +b. Let the operator create the secrets when it bootstraps the standby cluster. + Patch the secrets with the credentials of the source cluster. Replace the + spilo pods. + Otherwise, you will see errors in the Postgres logs saying users cannot log in and the operator logs will complain about not being able to sync resources. +If you stream changes from a remote primary you have to align the secrets or +the standby cluster will not start up. -When you only run a standby leader, you can safely ignore this, as it will be -sorted out once the cluster is detached from the source. It is also harmless if -you don’t plan it. But, when you created a standby replica, too, fix the -credentials right away. WAL files will pile up on the standby leader if no -connection can be established between standby replica(s). You can also edit the -secrets after their creation. Find them by: - -```bash -kubectl get secrets --all-namespaces | grep -``` +If you stream changes from WAL files and you only run a standby leader, you +can safely ignore the secret mismatch, as it will be sorted out once the +cluster is detached from the source. It is also harmless if you do not plan it. +But, when you create a standby replica, too, fix the credentials right away. +WAL files will pile up on the standby leader if no connection can be +established between standby replica(s). ### Promote the standby diff --git a/e2e/Makefile b/e2e/Makefile index 6e46d7238..b2da9c86e 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -45,9 +45,9 @@ push: docker tools: # install pinned version of 'kind' - # go get must run outside of a dir with a (module-based) Go project ! - # otherwise go get updates project's dependencies and/or behaves differently - cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.11.1 + # go install must run outside of a dir with a (module-based) Go project ! + # otherwise go install updates project's dependencies and/or behaves differently + cd "/tmp" && GO111MODULE=on go install sigs.k8s.io/kind@v0.11.1 e2etest: tools copy clean ./run.sh main diff --git a/e2e/tests/k8s_api.py b/e2e/tests/k8s_api.py index c3ad1c999..cf8f47be4 100644 --- a/e2e/tests/k8s_api.py +++ b/e2e/tests/k8s_api.py @@ -53,7 +53,7 @@ class K8s: return master_pod_node, replica_pod_nodes - def get_cluster_nodes(self, cluster_labels='cluster-name=acid-minimal-cluster', namespace='default'): + def get_cluster_nodes(self, cluster_labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): m = [] r = [] podsList = self.api.core_v1.list_namespaced_pod(namespace, label_selector=cluster_labels) @@ -321,6 +321,15 @@ class K8s: def get_cluster_replica_pod(self, labels='application=spilo,cluster-name=acid-minimal-cluster', namespace='default'): return self.get_cluster_pod('replica', labels, namespace) + def get_secret(self, username, clustername='acid-minimal-cluster', namespace='default'): + secret = self.api.core_v1.read_namespaced_secret( + "{}.{}.credentials.postgresql.acid.zalan.do".format(username.replace("_","-"), clustername), namespace) + secret.metadata.resource_version = None + secret.metadata.uid = None + return secret + + def create_secret(self, secret, namespace='default'): + return self.api.core_v1.create_namespaced_secret(namespace, secret) class K8sBase: ''' diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index e28e3e087..9b1cf50b4 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -4,8 +4,9 @@ import time import timeout_decorator import os import yaml +import base64 -from datetime import datetime +from datetime import datetime, date, timedelta from kubernetes import client from tests.k8s_api import K8s @@ -157,6 +158,37 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_additional_owner_roles(self): + ''' + Test adding additional member roles to existing database owner roles + ''' + k8s = self.k8s + + # enable PostgresTeam CRD and lower resync + owner_roles = { + "data": { + "additional_owner_roles": "cron_admin", + }, + } + k8s.update_config(owner_roles) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + + leader = k8s.get_cluster_leader_pod() + owner_query = """ + SELECT a2.rolname + FROM pg_catalog.pg_authid a + JOIN pg_catalog.pg_auth_members am + ON a.oid = am.member + AND a.rolname = 'cron_admin' + JOIN pg_catalog.pg_authid a2 + ON a2.oid = am.roleid + WHERE a2.rolname IN ('zalando', 'bar_owner', 'bar_data_owner'); + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", owner_query)), 3, + "Not all additional users found in database", 10, 5) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_additional_pod_capabilities(self): ''' @@ -176,13 +208,12 @@ class EndToEndTestCase(unittest.TestCase): try: k8s.update_config(patch_capabilities) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") # changed security context of postgres container should trigger a rolling update k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") self.eventuallyEqual(lambda: k8s.count_pods_with_container_capabilities(capabilities, cluster_label), 2, "Container capabilities not updated") @@ -203,12 +234,11 @@ class EndToEndTestCase(unittest.TestCase): "enable_postgres_team_crd": "true", "enable_team_member_deprecation": "true", "role_deletion_suffix": "_delete_me", - "resync_period": "15s" + "resync_period": "15s", + "repair_period": "15s", }, } k8s.update_config(enable_postgres_team_crd) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") k8s.api.custom_objects_api.patch_namespaced_custom_object( 'acid.zalan.do', 'v1', 'default', @@ -286,13 +316,141 @@ class EndToEndTestCase(unittest.TestCase): # revert config change revert_resync = { "data": { - "resync_period": "30m", + "resync_period": "4m", + "repair_period": "1m", }, } k8s.update_config(revert_resync) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_config_update(self): + ''' + Change Postgres config under Spec.Postgresql.Parameters and Spec.Patroni + and query Patroni config endpoint to check if manifest changes got applied + via restarting cluster through Patroni's rest api + ''' + k8s = self.k8s + leader = k8s.get_cluster_leader_pod() + replica = k8s.get_cluster_replica_pod() + masterCreationTimestamp = leader.metadata.creation_timestamp + replicaCreationTimestamp = replica.metadata.creation_timestamp + new_max_connections_value = "50" + + # adjust Postgres config + pg_patch_config = { + "spec": { + "postgresql": { + "parameters": { + "max_connections": new_max_connections_value + } + }, + "patroni": { + "slots": { + "test_slot": { + "type": "physical" + } + }, + "ttl": 29, + "loop_wait": 9, + "retry_timeout": 9, + "synchronous_mode": True + } + } + } + + try: + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config) + + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + def compare_config(): + effective_config = k8s.patroni_rest(leader.metadata.name, "config") + desired_config = pg_patch_config["spec"]["patroni"] + desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"] + effective_parameters = effective_config["postgresql"]["parameters"] + self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"], + "max_connections not updated") + self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added") + self.assertEqual(desired_config["ttl"], effective_config["ttl"], + "ttl not updated") + self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"], + "loop_wait not updated") + self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"], + "retry_timeout not updated") + self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"], + "synchronous_mode not updated") + return True + + # check if Patroni config has been updated + self.eventuallyTrue(compare_config, "Postgres config not applied") + + # make sure that pods were not recreated + leader = k8s.get_cluster_leader_pod() + replica = k8s.get_cluster_replica_pod() + self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp, + "Master pod creation timestamp is updated") + self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp, + "Master pod creation timestamp is updated") + + # query max_connections setting + setting_query = """ + SELECT setting + FROM pg_settings + WHERE name = 'max_connections'; + """ + self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value, + "New max_connections setting not applied on master", 10, 5) + self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value, + "Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5) + + # the next sync should restart the replica because it has pending_restart flag set + # force next sync by deleting the operator pod + k8s.delete_operator_pod() + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value, + "New max_connections setting not applied on replica", 10, 5) + + # decrease max_connections again + # this time restart will be correct and new value should appear on both instances + lower_max_connections_value = "30" + pg_patch_max_connections = { + "spec": { + "postgresql": { + "parameters": { + "max_connections": lower_max_connections_value + } + } + } + } + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections) + + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # check Patroni config again + pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value + self.eventuallyTrue(compare_config, "Postgres config not applied") + + # and query max_connections setting again + self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value, + "Previous max_connections setting not applied on master", 10, 5) + self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value, + "Previous max_connections setting not applied on replica", 10, 5) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + # make sure cluster is in a good state for further tests + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, + "No 2 pods running") + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_cross_namespace_secrets(self): ''' @@ -322,7 +480,7 @@ class EndToEndTestCase(unittest.TestCase): } } }) - + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") self.eventuallyEqual(lambda: k8s.count_secrets_with_label("cluster-name=acid-minimal-cluster,application=spilo", self.test_namespace), @@ -337,6 +495,9 @@ class EndToEndTestCase(unittest.TestCase): the end turn connection pooler off to not interfere with other tests. ''' k8s = self.k8s + pooler_label = 'application=db-connection-pooler,cluster-name=acid-minimal-cluster' + master_pooler_label = 'connection-pooler=acid-minimal-cluster-pooler' + replica_pooler_label = master_pooler_label + '-repl' self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -348,20 +509,30 @@ class EndToEndTestCase(unittest.TestCase): 'enableReplicaConnectionPooler': True, } }) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, - "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods( - "connection-pooler=acid-minimal-cluster-pooler"), - 2, "No pooler pods found") - self.eventuallyEqual(lambda: k8s.count_running_pods( - "connection-pooler=acid-minimal-cluster-pooler-repl"), - 2, "No pooler replica pods found") - self.eventuallyEqual(lambda: k8s.count_services_with_label( - 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'), - 2, "No pooler service found") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), - 1, "Pooler secret not created") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, "Deployment replicas is 2 default") + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 2, "No pooler pods found") + self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 2, "No pooler replica pods found") + self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 2, "No pooler service found") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), 1, "Pooler secret not created") + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableMasterPoolerLoadBalancer': True, + 'enableReplicaPoolerLoadBalancer': True, + } + }) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label), + 'LoadBalancer', + "Expected LoadBalancer service type for master pooler pod, found {}") + self.eventuallyEqual(lambda: k8s.get_service_type(replica_pooler_label+","+pooler_label), + 'LoadBalancer', + "Expected LoadBalancer service type for replica pooler pod, found {}") # Turn off only master connection pooler k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -374,20 +545,17 @@ class EndToEndTestCase(unittest.TestCase): } }) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler-repl"), 2, "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods( - "connection-pooler=acid-minimal-cluster-pooler"), + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 0, "Master pooler pods not deleted") - self.eventuallyEqual(lambda: k8s.count_running_pods( - "connection-pooler=acid-minimal-cluster-pooler-repl"), + self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 2, "Pooler replica pods not found") - self.eventuallyEqual(lambda: k8s.count_services_with_label( - 'application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 1, "No pooler service found") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), 1, "Secret not created") # Turn off only replica connection pooler @@ -398,20 +566,24 @@ class EndToEndTestCase(unittest.TestCase): 'spec': { 'enableConnectionPooler': True, 'enableReplicaConnectionPooler': False, + 'enableMasterPoolerLoadBalancer': False, } }) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 2, "Deployment replicas is 2 default") - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 2, "Master pooler pods not found") - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler-repl"), + self.eventuallyEqual(lambda: k8s.count_running_pods(replica_pooler_label), 0, "Pooler replica pods not deleted") - self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 1, "No pooler service found") - self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + self.eventuallyEqual(lambda: k8s.get_service_type(master_pooler_label+","+pooler_label), + 'ClusterIP', + "Expected LoadBalancer service type for master, found {}") + self.eventuallyEqual(lambda: k8s.count_secrets_with_label(pooler_label), 1, "Secret not created") # scale up connection pooler deployment @@ -428,7 +600,7 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(), 3, "Deployment replicas is scaled to 3") - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 3, "Scale up of pooler pods does not work") # turn it off, keeping config should be overwritten by false @@ -439,12 +611,13 @@ class EndToEndTestCase(unittest.TestCase): 'spec': { 'enableConnectionPooler': False, 'enableReplicaConnectionPooler': False, + 'enableReplicaPoolerLoadBalancer': False, } }) - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + self.eventuallyEqual(lambda: k8s.count_running_pods(master_pooler_label), 0, "Pooler pods not scaled down") - self.eventuallyEqual(lambda: k8s.count_services_with_label('application=db-connection-pooler,cluster-name=acid-minimal-cluster'), + self.eventuallyEqual(lambda: k8s.count_services_with_label(pooler_label), 0, "Pooler service not removed") self.eventuallyEqual(lambda: k8s.count_secrets_with_label('application=spilo,cluster-name=acid-minimal-cluster'), 4, "Secrets not deleted") @@ -531,6 +704,49 @@ class EndToEndTestCase(unittest.TestCase): print('Operator log: {}'.format(k8s.get_operator_log())) raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_ignored_annotations(self): + ''' + Test if injected annotation does not cause replacement of resources when listed under ignored_annotations + ''' + k8s = self.k8s + + annotation_patch = { + "metadata": { + "annotations": { + "k8s-status": "healthy" + }, + } + } + + try: + sts = k8s.api.apps_v1.read_namespaced_stateful_set('acid-minimal-cluster', 'default') + old_sts_creation_timestamp = sts.metadata.creation_timestamp + k8s.api.apps_v1.patch_namespaced_stateful_set(sts.metadata.name, sts.metadata.namespace, annotation_patch) + svc = k8s.api.core_v1.read_namespaced_service('acid-minimal-cluster', 'default') + old_svc_creation_timestamp = svc.metadata.creation_timestamp + k8s.api.core_v1.patch_namespaced_service(svc.metadata.name, svc.metadata.namespace, annotation_patch) + + patch_config_ignored_annotations = { + "data": { + "ignored_annotations": "k8s-status", + } + } + k8s.update_config(patch_config_ignored_annotations) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + sts = k8s.api.apps_v1.read_namespaced_stateful_set('acid-minimal-cluster', 'default') + new_sts_creation_timestamp = sts.metadata.creation_timestamp + svc = k8s.api.core_v1.read_namespaced_service('acid-minimal-cluster', 'default') + new_svc_creation_timestamp = svc.metadata.creation_timestamp + + self.assertEqual(old_sts_creation_timestamp, new_sts_creation_timestamp, "unexpected replacement of statefulset on sync") + self.assertEqual(old_svc_creation_timestamp, new_svc_creation_timestamp, "unexpected replacement of master service on sync") + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_infrastructure_roles(self): ''' @@ -579,6 +795,7 @@ class EndToEndTestCase(unittest.TestCase): "Parameters": None, "AdminRole": "", "Origin": 2, + "IsDbOwner": False, "Deleted": False }) return True @@ -600,7 +817,6 @@ class EndToEndTestCase(unittest.TestCase): but lets pods run with the old image until they are recreated for reasons other than operator's activity. That works because the operator configures stateful sets to use "onDelete" pod update policy. - The test covers: 1) enabling lazy upgrade in existing operator deployment 2) forcing the normal rolling upgrade by changing the operator @@ -695,7 +911,6 @@ class EndToEndTestCase(unittest.TestCase): Ensure we can (a) create the cron job at user request for a specific PG cluster (b) update the cluster-wide image for the logical backup pod (c) delete the job at user request - Limitations: (a) Does not run the actual batch job because there is no S3 mock to upload backups to (b) Assumes 'acid-minimal-cluster' exists as defined in setUp @@ -792,7 +1007,11 @@ class EndToEndTestCase(unittest.TestCase): Lower resource limits below configured minimum and let operator fix it ''' k8s = self.k8s - # self.eventuallyEqual(lambda: k8s.pg_get_status(), "Running", "Cluster not healthy at start") + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # get nodes of master and replica(s) (expected target of new master) + _, replica_nodes = k8s.get_pg_nodes(cluster_label) + self.assertNotEqual(replica_nodes, []) # configure minimum boundaries for CPU and memory limits minCPULimit = '503m' @@ -823,10 +1042,12 @@ class EndToEndTestCase(unittest.TestCase): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No two pods running after lazy rolling upgrade") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members()), 2, "Postgres status did not enter running") + # wait for switched over + k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) def verify_pod_limits(): pods = k8s.api.core_v1.list_namespaced_pod('default', label_selector="cluster-name=acid-minimal-cluster,application=spilo").items @@ -880,12 +1101,10 @@ class EndToEndTestCase(unittest.TestCase): # verify we are in good state from potential previous tests self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") # get nodes of master and replica(s) - master_node, replica_nodes = k8s.get_pg_nodes(cluster_label) - - self.assertNotEqual(master_node, []) + master_nodes, replica_nodes = k8s.get_cluster_nodes() + self.assertNotEqual(master_nodes, []) self.assertNotEqual(replica_nodes, []) # label node with environment=postgres @@ -898,9 +1117,8 @@ class EndToEndTestCase(unittest.TestCase): } try: - # patch current master node with the label - print('patching master node: {}'.format(master_node)) - k8s.api.core_v1.patch_node(master_node, node_label_body) + # patch master node with the label + k8s.api.core_v1.patch_node(master_nodes[0], node_label_body) # add node affinity to cluster patch_node_affinity_config = { @@ -924,7 +1142,6 @@ class EndToEndTestCase(unittest.TestCase): } } } - k8s.api.custom_objects_api.patch_namespaced_custom_object( group="acid.zalan.do", version="v1", @@ -932,17 +1149,21 @@ class EndToEndTestCase(unittest.TestCase): plural="postgresqls", name="acid-minimal-cluster", body=patch_node_affinity_config) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") # node affinity change should cause replica to relocate from replica node to master node due to node affinity requirement - k8s.wait_for_pod_failover(master_node, 'spilo-role=replica,' + cluster_label) + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + # next master will be switched over and pod needs to be replaced as well to finish the rolling update + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) podsList = k8s.api.core_v1.list_namespaced_pod('default', label_selector=cluster_label) for pod in podsList.items: if pod.metadata.labels.get('spilo-role') == 'replica': - self.assertEqual(master_node, pod.spec.node_name, - "Sanity check: expected replica to relocate to master node {}, but found on {}".format(master_node, pod.spec.node_name)) + self.assertEqual(master_nodes[0], pod.spec.node_name, + "Sanity check: expected replica to relocate to master node {}, but found on {}".format(master_nodes[0], pod.spec.node_name)) # check that pod has correct node affinity key = pod.spec.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms[0].match_expressions[0].key @@ -967,15 +1188,17 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") # node affinity change should cause another rolling update and relocation of replica - k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise + # toggle pod anti affinity to make sure replica and master run on separate nodes + self.assert_distributed_pods(master_nodes) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - @unittest.skip("Skipping this test until fixed") def test_node_readiness_label(self): ''' Remove node readiness label from master node. This must cause a failover. @@ -985,12 +1208,15 @@ class EndToEndTestCase(unittest.TestCase): readiness_label = 'lifecycle-status' readiness_value = 'ready' - try: - # get nodes of master and replica(s) (expected target of new master) - current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - num_replicas = len(current_replica_nodes) - failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + # verify we are in good state from potential previous tests + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + # get nodes of master and replica(s) (expected target of new master) + master_nodes, replica_nodes = k8s.get_cluster_nodes() + self.assertNotEqual(master_nodes, []) + self.assertNotEqual(replica_nodes, []) + + try: # add node_readiness_label to potential failover nodes patch_readiness_label = { "metadata": { @@ -999,36 +1225,50 @@ class EndToEndTestCase(unittest.TestCase): } } } - self.assertTrue(len(failover_targets) > 0, "No failover targets available") - for failover_target in failover_targets: - k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) + for replica_node in replica_nodes: + k8s.api.core_v1.patch_node(replica_node, patch_readiness_label) - # define node_readiness_label in config map which should trigger a failover of the master + # define node_readiness_label in config map which should trigger a rolling update patch_readiness_label_config = { "data": { "node_readiness_label": readiness_label + ':' + readiness_value, + "node_readiness_label_merge": "AND", } } k8s.update_config(patch_readiness_label_config, "setting readiness label") - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # first replica will be replaced and get the new affinity + # however, it might not start due to a volume node affinity conflict + # in this case only if the pvc and pod are deleted it can be scheduled + replica = k8s.get_cluster_replica_pod() + if replica.status.phase == 'Pending': + k8s.api.core_v1.delete_namespaced_persistent_volume_claim('pgdata-' + replica.metadata.name, 'default') + k8s.api.core_v1.delete_namespaced_pod(replica.metadata.name, 'default') + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + # next master will be switched over and pod needs to be replaced as well to finish the rolling update + k8s.wait_for_pod_failover(replica_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) # patch also node where master ran before - k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) - - # toggle pod anti affinity to move replica away from master node - self.eventuallyTrue(lambda: self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label), "Pods are redistributed") + k8s.api.core_v1.patch_node(master_nodes[0], patch_readiness_label) except timeout_decorator.TimeoutError: print('Operator log: {}'.format(k8s.get_operator_log())) raise + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(master_nodes) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_overwrite_pooler_deployment(self): + pooler_name = 'acid-minimal-cluster-pooler' k8s = self.k8s k8s.create_with_kubectl("manifests/minimal-fake-pooler-deployment.yaml") self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler"), 1, + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name=pooler_name), 1, "Initial broken deployment not rolled out") k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -1041,7 +1281,7 @@ class EndToEndTestCase(unittest.TestCase): }) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name="acid-minimal-cluster-pooler"), 2, + self.eventuallyEqual(lambda: k8s.get_deployment_replica_count(name=pooler_name), 2, "Operator did not succeed in overwriting labels") k8s.api.custom_objects_api.patch_namespaced_custom_object( @@ -1054,135 +1294,124 @@ class EndToEndTestCase(unittest.TestCase): }) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler=acid-minimal-cluster-pooler"), + self.eventuallyEqual(lambda: k8s.count_running_pods("connection-pooler="+pooler_name), 0, "Pooler pods not scaled down") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_patroni_config_update(self): + def test_password_rotation(self): ''' - Change Postgres config under Spec.Postgresql.Parameters and Spec.Patroni - and query Patroni config endpoint to check if manifest changes got applied - via restarting cluster through Patroni's rest api + Test password rotation and removal of users due to retention policy ''' k8s = self.k8s leader = k8s.get_cluster_leader_pod() - replica = k8s.get_cluster_replica_pod() - masterCreationTimestamp = leader.metadata.creation_timestamp - replicaCreationTimestamp = replica.metadata.creation_timestamp - new_max_connections_value = "50" + today = date.today() - # adjust Postgres config - pg_patch_config = { + # enable password rotation for owner of foo database + pg_patch_inplace_rotation_for_owner = { "spec": { - "postgresql": { - "parameters": { - "max_connections": new_max_connections_value - } - }, - "patroni": { - "slots": { - "test_slot": { - "type": "physical" - } - }, - "ttl": 29, - "loop_wait": 9, - "retry_timeout": 9, - "synchronous_mode": True - } + "usersWithInPlaceSecretRotation": [ + "zalando" + ] } } - - try: - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_config) - - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - def compare_config(): - effective_config = k8s.patroni_rest(leader.metadata.name, "config") - desired_config = pg_patch_config["spec"]["patroni"] - desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"] - effective_parameters = effective_config["postgresql"]["parameters"] - self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"], - "max_connections not updated") - self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added") - self.assertEqual(desired_config["ttl"], effective_config["ttl"], - "ttl not updated") - self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"], - "loop_wait not updated") - self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"], - "retry_timeout not updated") - self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"], - "synchronous_mode not updated") - return True - - # check if Patroni config has been updated - self.eventuallyTrue(compare_config, "Postgres config not applied") - - # make sure that pods were not recreated - leader = k8s.get_cluster_leader_pod() - replica = k8s.get_cluster_replica_pod() - self.assertEqual(masterCreationTimestamp, leader.metadata.creation_timestamp, - "Master pod creation timestamp is updated") - self.assertEqual(replicaCreationTimestamp, replica.metadata.creation_timestamp, - "Master pod creation timestamp is updated") - - # query max_connections setting - setting_query = """ - SELECT setting - FROM pg_settings - WHERE name = 'max_connections'; - """ - self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], new_max_connections_value, - "New max_connections setting not applied on master", 10, 5) - self.eventuallyNotEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value, - "Expected max_connections not to be updated on replica since Postgres was restarted there first", 10, 5) - - # the next sync should restart the replica because it has pending_restart flag set - # force next sync by deleting the operator pod - k8s.delete_operator_pod() - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], new_max_connections_value, - "New max_connections setting not applied on replica", 10, 5) - - # decrease max_connections again - # this time restart will be correct and new value should appear on both instances - lower_max_connections_value = "30" - pg_patch_max_connections = { - "spec": { - "postgresql": { - "parameters": { - "max_connections": lower_max_connections_value - } - } - } - } - - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_max_connections) - - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - - # check Patroni config again - pg_patch_config["spec"]["postgresql"]["parameters"]["max_connections"] = lower_max_connections_value - self.eventuallyTrue(compare_config, "Postgres config not applied") - - # and query max_connections setting again - self.eventuallyEqual(lambda: self.query_database(leader.metadata.name, "postgres", setting_query)[0], lower_max_connections_value, - "Previous max_connections setting not applied on master", 10, 5) - self.eventuallyEqual(lambda: self.query_database(replica.metadata.name, "postgres", setting_query)[0], lower_max_connections_value, - "Previous max_connections setting not applied on replica", 10, 5) - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - - # make sure cluster is in a good state for further tests + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_inplace_rotation_for_owner) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, - "No 2 pods running") + + # check if next rotation date was set in secret + zalando_secret = k8s.get_secret("zalando") + next_rotation_timestamp = datetime.strptime(str(base64.b64decode(zalando_secret.data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ") + today90days = today+timedelta(days=90) + self.assertEqual(today90days, next_rotation_timestamp.date(), + "Unexpected rotation date in secret of zalando user: expected {}, got {}".format(today90days, next_rotation_timestamp.date())) + + # create fake rotation users that should be removed by operator + # but have one that would still fit into the retention period + create_fake_rotation_user = """ + CREATE ROLE foo_user201031 IN ROLE foo_user; + CREATE ROLE foo_user211031 IN ROLE foo_user; + CREATE ROLE foo_user"""+(today-timedelta(days=40)).strftime("%y%m%d")+""" IN ROLE foo_user; + """ + self.query_database(leader.metadata.name, "postgres", create_fake_rotation_user) + + # patch foo_user secret with outdated rotation date + fake_rotation_date = today.isoformat() + 'T00:00:00Z' + fake_rotation_date_encoded = base64.b64encode(fake_rotation_date.encode('utf-8')) + secret_fake_rotation = { + "data": { + "nextRotation": str(fake_rotation_date_encoded, 'utf-8'), + }, + } + k8s.api.core_v1.patch_namespaced_secret( + name="foo-user.acid-minimal-cluster.credentials.postgresql.acid.zalan.do", + namespace="default", + body=secret_fake_rotation) + + # enable password rotation for all other users (foo_user) + # this will force a sync of secrets for further assertions + enable_password_rotation = { + "data": { + "enable_password_rotation": "true", + "password_rotation_interval": "30", + "password_rotation_user_retention": "30", # should be set to 60 + }, + } + k8s.update_config(enable_password_rotation) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + + # check if next rotation date and username have been replaced + foo_user_secret = k8s.get_secret("foo_user") + secret_username = str(base64.b64decode(foo_user_secret.data["username"]), 'utf-8') + next_rotation_timestamp = datetime.strptime(str(base64.b64decode(foo_user_secret.data["nextRotation"]), 'utf-8'), "%Y-%m-%dT%H:%M:%SZ") + rotation_user = "foo_user"+today.strftime("%y%m%d") + today30days = today+timedelta(days=30) + + self.assertEqual(rotation_user, secret_username, + "Unexpected username in secret of foo_user: expected {}, got {}".format(rotation_user, secret_username)) + self.assertEqual(today30days, next_rotation_timestamp.date(), + "Unexpected rotation date in secret of foo_user: expected {}, got {}".format(today30days, next_rotation_timestamp.date())) + + # check if oldest fake rotation users were deleted + # there should only be foo_user, foo_user+today and foo_user+today-40days + user_query = """ + SELECT rolname + FROM pg_catalog.pg_roles + WHERE rolname LIKE 'foo_user%'; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 3, + "Found incorrect number of rotation users", 10, 5) + + # disable password rotation for all other users (foo_user) + # and pick smaller intervals to see if the third fake rotation user is dropped + enable_password_rotation = { + "data": { + "enable_password_rotation": "false", + "password_rotation_interval": "15", + "password_rotation_user_retention": "30", # 2 * rotation interval + }, + } + k8s.update_config(enable_password_rotation) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") + + # check if username in foo_user secret is reset + foo_user_secret = k8s.get_secret("foo_user") + secret_username = str(base64.b64decode(foo_user_secret.data["username"]), 'utf-8') + next_rotation_timestamp = str(base64.b64decode(foo_user_secret.data["nextRotation"]), 'utf-8') + self.assertEqual("foo_user", secret_username, + "Unexpected username in secret of foo_user: expected {}, got {}".format("foo_user", secret_username)) + self.assertEqual('', next_rotation_timestamp, + "Unexpected rotation date in secret of foo_user: expected empty string, got {}".format(next_rotation_timestamp)) + + # check roles again, there should only be foo_user and foo_user+today + user_query = """ + SELECT rolname + FROM pg_catalog.pg_roles + WHERE rolname LIKE 'foo_user%'; + """ + self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "postgres", user_query)), 2, + "Found incorrect number of rotation users", 10, 5) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_rolling_update_flag(self): @@ -1270,6 +1499,7 @@ class EndToEndTestCase(unittest.TestCase): "data": { "pod_label_wait_timeout": "2s", "resync_period": "30s", + "repair_period": "30s", } } @@ -1310,7 +1540,8 @@ class EndToEndTestCase(unittest.TestCase): patch_resync_config = { "data": { "pod_label_wait_timeout": "10m", - "resync_period": "30m", + "resync_period": "4m", + "repair_period": "2m", } } k8s.update_config(patch_resync_config, "revert resync interval and pod_label_wait_timeout") @@ -1414,7 +1645,42 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyTrue(lambda: k8s.check_statefulset_annotations(cluster_label, annotations), "Annotations missing") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - @unittest.skip("Skipping this test until fixed") + def test_standby_cluster(self): + ''' + Create standby cluster streaming from remote primary + ''' + k8s = self.k8s + standby_cluster_name = 'acid-standby-cluster' + cluster_name_label = 'cluster-name' + cluster_label = 'application=spilo,{}={}'.format(cluster_name_label, standby_cluster_name) + superuser_name = 'postgres' + replication_user = 'standby' + secret_suffix = 'credentials.postgresql.acid.zalan.do' + + # copy secrets from remote cluster before operator creates them when bootstrapping the standby cluster + postgres_secret = k8s.get_secret(superuser_name) + postgres_secret.metadata.name = '{}.{}.{}'.format(superuser_name, standby_cluster_name, secret_suffix) + postgres_secret.metadata.labels[cluster_name_label] = standby_cluster_name + k8s.create_secret(postgres_secret) + standby_secret = k8s.get_secret(replication_user) + standby_secret.metadata.name = '{}.{}.{}'.format(replication_user, standby_cluster_name, secret_suffix) + standby_secret.metadata.labels[cluster_name_label] = standby_cluster_name + k8s.create_secret(standby_secret) + + try: + k8s.create_with_kubectl("manifests/standby-manifest.yaml") + k8s.wait_for_pod_start("spilo-role=master," + cluster_label) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + finally: + # delete the standby cluster so that the k8s_api.get_operator_state works correctly in subsequent tests + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-standby-cluster") + time.sleep(5) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): ''' Add taint "postgres=:NoExecute" to node with master. This must cause a failover. @@ -1428,7 +1694,6 @@ class EndToEndTestCase(unittest.TestCase): # get nodes of master and replica(s) (expected target of new master) master_nodes, replica_nodes = k8s.get_cluster_nodes() - self.assertNotEqual(master_nodes, []) self.assertNotEqual(replica_nodes, []) @@ -1443,10 +1708,7 @@ class EndToEndTestCase(unittest.TestCase): ] } } - k8s.api.core_v1.patch_node(master_nodes[0], body) - self.eventuallyTrue(lambda: k8s.get_cluster_nodes()[0], replica_nodes) - self.assertNotEqual(lambda: k8s.get_cluster_nodes()[0], master_nodes) # add toleration to pods patch_toleration_config = { @@ -1455,15 +1717,20 @@ class EndToEndTestCase(unittest.TestCase): } } - k8s.update_config(patch_toleration_config, step="allow tainted nodes") + try: + k8s.update_config(patch_toleration_config, step="allow tainted nodes") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, + "Operator does not get in sync") - self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") - self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") + self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") + self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise # toggle pod anti affinity to move replica away from master node - nm, new_replica_nodes = k8s.get_cluster_nodes() - new_master_node = nm[0] - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + self.assert_distributed_pods(master_nodes) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_zz_cluster_deletion(self): @@ -1528,6 +1795,8 @@ class EndToEndTestCase(unittest.TestCase): self.eventuallyEqual(lambda: len(k8s.api.custom_objects_api.list_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", label_selector="cluster-name=acid-minimal-cluster")["items"]), 0, "Manifest not deleted") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + # check if everything has been deleted self.eventuallyEqual(lambda: k8s.count_pods_with_label(cluster_label), 0, "Pods not deleted") self.eventuallyEqual(lambda: k8s.count_services_with_label(cluster_label), 0, "Service not deleted") @@ -1550,39 +1819,6 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_delete_annotations) - def get_failover_targets(self, master_node, replica_nodes): - ''' - If all pods live on the same node, failover will happen to other worker(s) - ''' - k8s = self.k8s - k8s_master_exclusion = 'kubernetes.io/hostname!=postgres-operator-e2e-tests-control-plane' - - failover_targets = [x for x in replica_nodes if x != master_node] - if len(failover_targets) == 0: - nodes = k8s.api.core_v1.list_node(label_selector=k8s_master_exclusion) - for n in nodes.items: - if n.metadata.name != master_node: - failover_targets.append(n.metadata.name) - - return failover_targets - - def assert_failover(self, current_master_node, num_replicas, failover_targets, cluster_label): - ''' - Check if master is failing over. The replica should move first to be the switchover target - ''' - k8s = self.k8s - k8s.wait_for_pod_failover(failover_targets, 'spilo-role=master,' + cluster_label) - k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - - new_master_node, new_replica_nodes = k8s.get_pg_nodes(cluster_label) - self.assertNotEqual(current_master_node, new_master_node, - "Master on {} did not fail over to one of {}".format(current_master_node, failover_targets)) - self.assertEqual(num_replicas, len(new_replica_nodes), - "Expected {} replicas, found {}".format(num_replicas, len(new_replica_nodes))) - self.assert_master_is_unique() - - return new_master_node, new_replica_nodes - def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"): ''' Check that there is a single pod in the k8s cluster with the label "spilo-role=master" @@ -1594,14 +1830,23 @@ class EndToEndTestCase(unittest.TestCase): num_of_master_pods = k8s.count_pods_with_label(labels, namespace) self.assertEqual(num_of_master_pods, 1, "Expected 1 master pod, found {}".format(num_of_master_pods)) - def assert_distributed_pods(self, master_node, replica_nodes, cluster_label): + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def assert_distributed_pods(self, target_nodes, cluster_labels='cluster-name=acid-minimal-cluster'): ''' Other tests can lead to the situation that master and replica are on the same node. Toggle pod anti affinty to distribute pods accross nodes (replica in particular). ''' k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - failover_targets = self.get_failover_targets(master_node, replica_nodes) + cluster_labels = 'application=spilo,cluster-name=acid-minimal-cluster' + + # get nodes of master and replica(s) + master_nodes, replica_nodes = k8s.get_cluster_nodes() + self.assertNotEqual(master_nodes, []) + self.assertNotEqual(replica_nodes, []) + + # if nodes are different we can quit here + if master_nodes[0] not in replica_nodes: + return True # enable pod anti affintiy in config map which should trigger movement of replica patch_enable_antiaffinity = { @@ -1609,18 +1854,40 @@ class EndToEndTestCase(unittest.TestCase): "enable_pod_antiaffinity": "true" } } - k8s.update_config(patch_enable_antiaffinity, "enable antiaffinity") - self.assert_failover(master_node, len(replica_nodes), failover_targets, cluster_label) - # now disable pod anti affintiy again which will cause yet another failover - patch_disable_antiaffinity = { - "data": { - "enable_pod_antiaffinity": "false" + try: + k8s.update_config(patch_enable_antiaffinity, "enable antiaffinity") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_labels) + k8s.wait_for_running_pods(cluster_labels, 2) + + # now disable pod anti affintiy again which will cause yet another failover + patch_disable_antiaffinity = { + "data": { + "enable_pod_antiaffinity": "false" + } } - } - k8s.update_config(patch_disable_antiaffinity, "disable antiaffinity") - k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) - k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + k8s.update_config(patch_disable_antiaffinity, "disable antiaffinity") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_labels) + k8s.wait_for_running_pods(cluster_labels, 2) + + master_nodes, replica_nodes = k8s.get_cluster_nodes() + self.assertNotEqual(master_nodes, []) + self.assertNotEqual(replica_nodes, []) + + # if nodes are different we can quit here + for target_node in target_nodes: + if (target_node not in master_nodes or target_node not in replica_nodes) and master_nodes[0] in replica_nodes: + print('Pods run on the same node') + return False + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + return True def list_databases(self, pod_name): @@ -1636,12 +1903,8 @@ class EndToEndTestCase(unittest.TestCase): try: q = exec_query.format(db_list_query, "postgres") q = "su postgres -c \"{}\"".format(q) - print('Get databases: {}'.format(q)) result = k8s.exec_with_kubectl(pod_name, q) db_list = clean_list(result.stdout.split(b'\n')) - print('db_list: {}, stdout: {}, stderr {}'.format( - db_list, result.stdout, result.stderr - )) except Exception as ex: print('Could not get databases: {}'.format(ex)) print('Stdout: {}'.format(result.stdout)) @@ -1665,12 +1928,8 @@ class EndToEndTestCase(unittest.TestCase): try: q = exec_query.format(query, db_name) q = "su postgres -c \"{}\"".format(q) - print('Send query: {}'.format(q)) result = k8s.exec_with_kubectl(pod_name, q) result_set = clean_list(result.stdout.split(b'\n')) - print('result: {}, stdout: {}, stderr {}'.format( - result_set, result.stdout, result.stderr - )) except Exception as ex: print('Error on query execution: {}'.format(ex)) print('Stdout: {}'.format(result.stdout)) diff --git a/go.mod b/go.mod index ac550f810..26e629beb 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,69 @@ module github.com/zalando/postgres-operator -go 1.16 +go 1.17 require ( - github.com/aws/aws-sdk-go v1.41.16 + github.com/aws/aws-sdk-go v1.42.18 github.com/golang/mock v1.6.0 - github.com/lib/pq v1.10.3 + github.com/lib/pq v1.10.4 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d + github.com/pkg/errors v0.9.1 github.com/r3labs/diff v1.1.0 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/crypto v0.0.0-20211202192323-5770296d904e gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.22.3 - k8s.io/apiextensions-apiserver v0.22.3 - k8s.io/apimachinery v0.22.3 - k8s.io/client-go v0.22.3 - k8s.io/code-generator v0.22.3 + k8s.io/api v0.22.4 + k8s.io/apiextensions-apiserver v0.22.4 + k8s.io/apimachinery v0.22.4 + k8s.io/client-go v0.22.4 + k8s.io/code-generator v0.22.4 +) + +require ( + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful v2.9.5+incompatible // indirect + github.com/evanphx/json-patch v4.11.0+incompatible // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.5 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/googleapis/gnostic v0.5.5 // indirect + github.com/imdario/mergo v0.3.5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/moby/spdystream v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/mod v0.5.1 // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect + golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 // indirect + golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect + golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + golang.org/x/tools v0.1.7 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 // indirect + k8s.io/klog/v2 v2.9.0 // indirect + k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c // indirect + k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index 5666613bb..fda15408b 100644 --- a/go.sum +++ b/go.sum @@ -9,18 +9,28 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= @@ -50,8 +60,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.41.16 h1:rNbPreEO4K2b8LwhHOe5+iNRH3Ih35L+H9U5tLdcevo= -github.com/aws/aws-sdk-go v1.41.16/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.42.18 h1:2f/cDNwQ3e+yHxtPn1si0to3GalbNHwkRm461IjwRiM= +github.com/aws/aws-sdk-go v1.42.18/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -153,6 +163,8 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -160,6 +172,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -179,7 +192,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -187,11 +202,14 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= @@ -266,8 +284,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= -github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -392,9 +410,12 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -408,6 +429,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= @@ -436,8 +458,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= +golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -469,8 +491,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -487,6 +510,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -496,28 +520,37 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -552,9 +585,15 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -566,8 +605,10 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -621,13 +662,24 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -642,13 +694,21 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -666,10 +726,20 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -680,7 +750,10 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= @@ -736,18 +809,19 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.22.3 h1:wOoES2GoSkUsdped2RB4zYypPqWtvprGoKCENTOOjP4= -k8s.io/api v0.22.3/go.mod h1:azgiXFiXqiWyLCfI62/eYBOu19rj2LKmIhFPP4+33fs= -k8s.io/apiextensions-apiserver v0.22.3 h1:bKku7MqawIbtTZc084BZoMV4fz0WZuvCnB5E+yrQXGM= -k8s.io/apiextensions-apiserver v0.22.3/go.mod h1:f4plF+CXeqI89jAXL0Ml4LI/kSAZ54JS94+XOX1sae8= -k8s.io/apimachinery v0.22.3 h1:mrvBG5CZnEfwgpVqWcrRKvdsYECTrhAR6cApAgdsflk= -k8s.io/apimachinery v0.22.3/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apiserver v0.22.3/go.mod h1:oam7lH/F1Kto/WTamyQYrD68fS0mGUBORAFf6x/9Mxs= -k8s.io/client-go v0.22.3 h1:6onkOSc+YNdwq5zXE0wFXicq64rrym+mXwHu/CPVGO4= -k8s.io/client-go v0.22.3/go.mod h1:ElDjYf8gvZsKDYexmsmnMQ0DYO8W9RwBjfQ1PI53yow= -k8s.io/code-generator v0.22.3 h1:24xLuKySzFl1XupMarNBkpt10q0N+73R9dF7wzJO/hE= -k8s.io/code-generator v0.22.3/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= -k8s.io/component-base v0.22.3/go.mod h1:kuybv1miLCMoOk3ebrqF93GbQHQx6W2287FC0YEQY6s= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.22.4 h1:UvyHW0ezB2oIgHAxlYoo6UJQObYXU7awuNarwoHEOjw= +k8s.io/api v0.22.4/go.mod h1:Rgs+9gIGYC5laXQSZZ9JqT5NevNgoGiOdVWi1BAB3qk= +k8s.io/apiextensions-apiserver v0.22.4 h1:2iGpcVyw4MnAyyXVJU2Xg6ZsbIxAOfRHo0LF5A5J0RA= +k8s.io/apiextensions-apiserver v0.22.4/go.mod h1:kH9lxD8dbJ+k0ZizGET55lFgdGjO8t45fgZnCVdZEpw= +k8s.io/apimachinery v0.22.4 h1:9uwcvPpukBw/Ri0EUmWz+49cnFtaoiyEhQTK+xOe7Ck= +k8s.io/apimachinery v0.22.4/go.mod h1:yU6oA6Gnax9RrxGzVvPFFJ+mpnW6PBSqp0sx0I0HHW0= +k8s.io/apiserver v0.22.4/go.mod h1:38WmcUZiiy41A7Aty8/VorWRa8vDGqoUzDf2XYlku0E= +k8s.io/client-go v0.22.4 h1:aAQ1Wk+I3bjCNk35YWUqbaueqrIonkfDPJSPDDe8Kfg= +k8s.io/client-go v0.22.4/go.mod h1:Yzw4e5e7h1LNHA4uqnMVrpEpUs1hJOiuBsJKIlRCHDA= +k8s.io/code-generator v0.22.4 h1:h7lBa5IuEUC4OQ45q/gIip/a0iQcML2iwrRmXksau30= +k8s.io/code-generator v0.22.4/go.mod h1:qjYl54pQ/emhkT0UxbufbREYJMWsHNNV/jSVwhYZQGw= +k8s.io/component-base v0.22.4/go.mod h1:MrSaQy4a3tFVViff8TZL6JHYSewNCLshZCwHYM58v5A= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -755,8 +829,8 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 280da9385..d34db9c45 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -4,10 +4,22 @@ set -o errexit set -o nounset set -o pipefail +GENERATED_PACKAGE_ROOT="github.com" +OPERATOR_PACKAGE_ROOT="${GENERATED_PACKAGE_ROOT}/zalando/postgres-operator" SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. -CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)} +TARGET_CODE_DIR=${1-${SCRIPT_ROOT}/pkg} +CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo "${GOPATH}"/src/k8s.io/code-generator)} + +cleanup() { + rm -rf "${GENERATED_PACKAGE_ROOT}" +} +trap "cleanup" EXIT SIGINT bash "${CODEGEN_PKG}/generate-groups.sh" all \ - github.com/zalando/postgres-operator/pkg/generated github.com/zalando/postgres-operator/pkg/apis \ - "acid.zalan.do:v1" \ + "${OPERATOR_PACKAGE_ROOT}/pkg/generated" "${OPERATOR_PACKAGE_ROOT}/pkg/apis" \ + "acid.zalan.do:v1 zalando.org:v1" \ --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt + +cp -r "${OPERATOR_PACKAGE_ROOT}"/pkg/* "${TARGET_CODE_DIR}" + +cleanup diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh index 68710015e..451ac76b7 100755 --- a/hack/verify-codegen.sh +++ b/hack/verify-codegen.sh @@ -19,15 +19,14 @@ cleanup mkdir -p "${TMP_DIFFROOT}" cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}" -"${SCRIPT_ROOT}/hack/update-codegen.sh" +"${SCRIPT_ROOT}/hack/update-codegen.sh" "${TMP_DIFFROOT}" echo "diffing ${DIFFROOT} against freshly generated codegen" ret=0 diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$? -cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}" if [[ $ret -eq 0 ]] then echo "${DIFFROOT} up to date." else - echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh" + echo "${DIFFROOT} is out of date. Please run 'make codegen'" exit 1 fi diff --git a/kubectl-pg/cmd/root.go b/kubectl-pg/cmd/root.go index 916852998..163d6f6ea 100644 --- a/kubectl-pg/cmd/root.go +++ b/kubectl-pg/cmd/root.go @@ -24,9 +24,10 @@ package cmd import ( "fmt" + "os" + "github.com/spf13/cobra" "github.com/spf13/viper" - "os" ) var rootCmd = &cobra.Command{ diff --git a/kubectl-pg/cmd/version.go b/kubectl-pg/cmd/version.go index 91976fbc2..e9a1e8056 100644 --- a/kubectl-pg/cmd/version.go +++ b/kubectl-pg/cmd/version.go @@ -24,10 +24,11 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" - "k8s.io/client-go/kubernetes" "log" "strings" + + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" ) var KubectlPgVersion string = "1.0" diff --git a/kubectl-pg/go.mod b/kubectl-pg/go.mod index 8c54afc99..d82b2392f 100644 --- a/kubectl-pg/go.mod +++ b/kubectl-pg/go.mod @@ -1,13 +1,59 @@ module github.com/zalando/postgres-operator/kubectl-pg -go 1.16 +go 1.17 require ( - github.com/spf13/cobra v1.1.3 - github.com/spf13/viper v1.7.1 - github.com/zalando/postgres-operator v1.7.1 - k8s.io/api v0.22.2 - k8s.io/apiextensions-apiserver v0.22.2 - k8s.io/apimachinery v0.22.2 - k8s.io/client-go v0.22.2 + github.com/spf13/cobra v1.2.1 + github.com/spf13/viper v1.9.0 + github.com/zalando/postgres-operator v1.8.0 + k8s.io/api v0.22.4 + k8s.io/apiextensions-apiserver v0.22.4 + k8s.io/apimachinery v0.22.4 + k8s.io/client-go v0.22.4 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/googleapis/gnostic v0.5.5 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.5 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/moby/spdystream v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect + golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect + golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect + golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.63.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/klog/v2 v2.9.0 // indirect + k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/kubectl-pg/go.sum b/kubectl-pg/go.sum index f1f097b98..aaa4b2e36 100644 --- a/kubectl-pg/go.sum +++ b/kubectl-pg/go.sum @@ -9,35 +9,49 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -54,14 +68,16 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.36.29/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.41.16/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= @@ -73,23 +89,20 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -97,9 +110,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -108,20 +119,22 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -136,14 +149,10 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -154,7 +163,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -165,12 +173,16 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -183,6 +195,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -190,50 +203,66 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -245,10 +274,14 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -260,13 +293,11 @@ github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUB github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -276,41 +307,48 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -329,27 +367,28 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -369,7 +408,6 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/r3labs/diff v1.1.0/go.mod h1:7WjXasNzi0vJetRcB/RqNl5dlIsmXcTTLmF5IoH6Xig= @@ -378,6 +416,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -386,36 +425,36 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk= +github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -425,21 +464,18 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/zalando/postgres-operator v1.7.0 h1:IgTXIVLJM37bFbvbdCks+HCIq7eGSistuLoFCUVr/tY= -github.com/zalando/postgres-operator v1.7.0/go.mod h1:zPFdBewKFBf6726fnGc4GLXYxzbtwyVaU/7XFknyTl4= +github.com/zalando/postgres-operator v1.7.1 h1:tDh7utqbrNoCNoQjy3seZiEoO+vT6SgP2+VlnSJ5mpg= +github.com/zalando/postgres-operator v1.7.1/go.mod h1:hZTzOQBITJvv5nDHZpGIwyKqOghomxpZQRyWOfkPzKs= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -451,6 +487,9 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= @@ -463,7 +502,6 @@ go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= @@ -476,13 +514,15 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -505,6 +545,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -515,6 +556,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -532,9 +574,10 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -542,27 +585,52 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -574,47 +642,70 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -628,11 +719,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -649,11 +738,11 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -672,11 +761,31 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -691,13 +800,33 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -715,13 +844,43 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -730,11 +889,24 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -746,21 +918,23 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -779,7 +953,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -788,46 +961,41 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= -k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= -k8s.io/apiextensions-apiserver v0.20.6/go.mod h1:qO8YMqeMmZH+lV21LUNzV41vfpoE9QVAJRA+MNqj0mo= -k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= -k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= -k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= -k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= -k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= -k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= -k8s.io/code-generator v0.20.6/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU= -k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= -k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.22.3/go.mod h1:azgiXFiXqiWyLCfI62/eYBOu19rj2LKmIhFPP4+33fs= +k8s.io/api v0.22.4 h1:UvyHW0ezB2oIgHAxlYoo6UJQObYXU7awuNarwoHEOjw= +k8s.io/api v0.22.4/go.mod h1:Rgs+9gIGYC5laXQSZZ9JqT5NevNgoGiOdVWi1BAB3qk= +k8s.io/apiextensions-apiserver v0.22.3/go.mod h1:f4plF+CXeqI89jAXL0Ml4LI/kSAZ54JS94+XOX1sae8= +k8s.io/apiextensions-apiserver v0.22.4 h1:2iGpcVyw4MnAyyXVJU2Xg6ZsbIxAOfRHo0LF5A5J0RA= +k8s.io/apiextensions-apiserver v0.22.4/go.mod h1:kH9lxD8dbJ+k0ZizGET55lFgdGjO8t45fgZnCVdZEpw= +k8s.io/apimachinery v0.22.3/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.4 h1:9uwcvPpukBw/Ri0EUmWz+49cnFtaoiyEhQTK+xOe7Ck= +k8s.io/apimachinery v0.22.4/go.mod h1:yU6oA6Gnax9RrxGzVvPFFJ+mpnW6PBSqp0sx0I0HHW0= +k8s.io/apiserver v0.22.3/go.mod h1:oam7lH/F1Kto/WTamyQYrD68fS0mGUBORAFf6x/9Mxs= +k8s.io/apiserver v0.22.4/go.mod h1:38WmcUZiiy41A7Aty8/VorWRa8vDGqoUzDf2XYlku0E= +k8s.io/client-go v0.22.3/go.mod h1:ElDjYf8gvZsKDYexmsmnMQ0DYO8W9RwBjfQ1PI53yow= +k8s.io/client-go v0.22.4 h1:aAQ1Wk+I3bjCNk35YWUqbaueqrIonkfDPJSPDDe8Kfg= +k8s.io/client-go v0.22.4/go.mod h1:Yzw4e5e7h1LNHA4uqnMVrpEpUs1hJOiuBsJKIlRCHDA= +k8s.io/code-generator v0.22.3/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= +k8s.io/code-generator v0.22.4/go.mod h1:qjYl54pQ/emhkT0UxbufbREYJMWsHNNV/jSVwhYZQGw= +k8s.io/component-base v0.22.3/go.mod h1:kuybv1miLCMoOk3ebrqF93GbQHQx6W2287FC0YEQY6s= +k8s.io/component-base v0.22.4/go.mod h1:MrSaQy4a3tFVViff8TZL6JHYSewNCLshZCwHYM58v5A= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 46e79ba21..e2d798a94 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -3,19 +3,27 @@ kind: postgresql metadata: name: acid-test-cluster # labels: +# application: test-app # environment: demo # annotations: # "acid.zalan.do/controller": "second-operator" # "delete-date": "2020-08-31" # can only be deleted on that day if "delete-date "key is configured # "delete-clustername": "acid-test-cluster" # can only be deleted when name matches if "delete-clustername" key is configured spec: - dockerImage: registry.opensource.zalan.do/acid/spilo-14:2.1-p3 + dockerImage: registry.opensource.zalan.do/acid/spilo-14:2.1-p5 teamId: "acid" numberOfInstances: 2 users: # Application/Robot users zalando: - superuser - createdb + foo_user: [] +# flyway: [] +# usersWithSecretRotation: +# - foo_user +# usersWithInPlaceSecretRotation: +# - flyway +# - bar_owner_user enableMasterLoadBalancer: false enableReplicaLoadBalancer: false enableConnectionPooler: false # enable/disable connection pooler deployment @@ -41,6 +49,10 @@ spec: shared_buffers: "32MB" max_connections: "10" log_statement: "all" +# env: +# - name: wal_s3_bucket +# value: my-custom-bucket + volume: size: 1Gi # storageClass: my-sc @@ -112,10 +124,11 @@ spec: # database: foo # plugin: pgoutput ttl: 30 - loop_wait: &loop_wait 10 + loop_wait: 10 retry_timeout: 10 synchronous_mode: false synchronous_mode_strict: false + synchronous_node_count: 1 maximum_lag_on_failover: 33554432 # restore a Postgres DB with point-in-time-recovery @@ -123,7 +136,7 @@ spec: # with an empty/absent timestamp, clone from an existing alive cluster using pg_basebackup # clone: # uid: "efd12e58-5786-11e8-b5a7-06148230260c" -# cluster: "acid-batman" +# cluster: "acid-minimal-cluster" # timestamp: "2017-12-19T12:40:33+01:00" # timezone required (offset relative to UTC, see RFC 3339 section 5.6) # s3_wal_path: "s3://custom/path/to/bucket" @@ -195,3 +208,21 @@ spec: # operator: In # values: # - enabled + +# Enables change data capture streams for defined database tables +# streams: +# - applicationId: test-app +# database: foo +# tables: +# data.state_pending_outbox: +# eventType: test-app.status-pending +# data.state_approved_outbox: +# eventType: test-app.status-approved +# data.orders_outbox: +# eventType: test-app.order-completed +# idColumn: o_id +# payloadColumn: o_payload +# # Optional. Filter ignores events before a certain txnId and lsn. Can be used to skip bad events +# filter: +# data.orders_outbox: "[?(@.source.txId > 500 && @.source.lsn > 123456)]" +# batchSize: 1000 diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index e91e2f19e..64e01fb50 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -3,6 +3,7 @@ kind: ConfigMap metadata: name: postgres-operator data: + # additional_owner_roles: "cron_admin" # additional_pod_capabilities: "SYS_NICE" # additional_secret_mount: "some-secret-name" # additional_secret_mount_path: "/some/dir" @@ -16,12 +17,13 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-19" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-22" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 # connection_pooler_schema: "pooler" # connection_pooler_user: "pooler" + crd_categories: "all" # custom_service_annotations: "keyx:valuez,keya:valuea" # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com @@ -32,10 +34,10 @@ data: # default_memory_request: 100Mi # delete_annotation_date_key: delete-date # delete_annotation_name_key: delete-clustername - docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p3 + docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p5 # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" - # enable_crd_validation: "true" + # enable_crd_registration: "true" # enable_cross_namespace_secret: "false" # enable_database_access: "true" enable_ebs_gp3_migration: "false" @@ -43,12 +45,15 @@ data: # enable_init_containers: "true" # enable_lazy_spilo_upgrade: "false" enable_master_load_balancer: "false" + enable_master_pooler_load_balancer: "false" + enable_password_rotation: "false" enable_pgversion_env_var: "true" # enable_pod_antiaffinity: "false" # enable_pod_disruption_budget: "true" # enable_postgres_team_crd: "false" # enable_postgres_team_crd_superusers: "false" enable_replica_load_balancer: "false" + enable_replica_pooler_load_balancer: "false" # enable_shm_volume: "true" # enable_sidecars: "true" enable_spilo_wal_path_compat: "true" @@ -58,14 +63,15 @@ data: # etcd_host: "" external_traffic_policy: "Cluster" # gcp_credentials: "" - # kubernetes_use_configmaps: "false" + # ignored_annotations: "" # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole" # inherited_annotations: owned-by # inherited_labels: application,environment # kube_iam_role: "" + # kubernetes_use_configmaps: "false" # log_s3_bucket: "" - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0" # logical_backup_google_application_credentials: "" logical_backup_job_prefix: "logical-backup-" logical_backup_provider: "s3" @@ -75,8 +81,10 @@ data: # logical_backup_s3_endpoint: "" # logical_backup_s3_secret_access_key: "" logical_backup_s3_sse: "AES256" + # logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" major_version_upgrade_mode: "manual" + # major_version_upgrade_team_allow_list: "" master_dns_name_format: "{cluster}.{team}.{hostedzone}" # master_pod_move_timeout: 20m # max_instances: "-1" @@ -84,11 +92,16 @@ data: # min_cpu_limit: 250m # min_memory_limit: 250Mi # minimal_major_version: "9.6" - # node_readiness_label: "" + # node_readiness_label: "status:ready" + # node_readiness_label_merge: "OR" # oauth_token_secret_name: postgresql-operator # pam_configuration: | # https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees # pam_role_name: zalandos + patroni_api_check_interval: "1s" + patroni_api_check_timeout: "5s" + # password_rotation_interval: "90" + # password_rotation_user_retention: "180" pdb_name_format: "postgres-{cluster}-pdb" # pod_antiaffinity_topology_key: "kubernetes.io/hostname" pod_deletion_wait_timeout: 10m @@ -103,7 +116,7 @@ data: # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m # postgres_superuser_teams: "postgres_superusers" - # protected_role_names: "admin" + # protected_role_names: "admin,cron_admin" ready_wait_interval: 3s ready_wait_timeout: 30s repair_period: 5m diff --git a/manifests/minimal-fake-pooler-deployment.yaml b/manifests/minimal-fake-pooler-deployment.yaml index c432b4613..7bd661a87 100644 --- a/manifests/minimal-fake-pooler-deployment.yaml +++ b/manifests/minimal-fake-pooler-deployment.yaml @@ -23,7 +23,7 @@ spec: serviceAccountName: postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/pgbouncer:master-19 + image: registry.opensource.zalan.do/acid/pgbouncer:master-22 imagePullPolicy: IfNotPresent resources: requests: diff --git a/manifests/minimal-master-replica-svcmonitor.yaml b/manifests/minimal-master-replica-svcmonitor.yaml new file mode 100644 index 000000000..67ed28c81 --- /dev/null +++ b/manifests/minimal-master-replica-svcmonitor.yaml @@ -0,0 +1,131 @@ +# Here we use https://github.com/prometheus-community/helm-charts/charts/kube-prometheus-stack +# Please keep the ServiceMonitor's label same as the Helm release name of kube-prometheus-stack + +apiVersion: v1 +kind: Namespace +metadata: + name: test-pg +--- +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-minimal-cluster + namespace: test-pg + labels: + app: test-pg +spec: + teamId: "acid" + volume: + size: 1Gi + numberOfInstances: 2 + users: + zalando: # database owner + - superuser + - createdb + foo_user: [] # role for application foo + databases: + foo: zalando # dbname: owner + preparedDatabases: + bar: {} + postgresql: + version: "13" + sidecars: + - name: "exporter" + image: "wrouesnel/postgres_exporter" + ports: + - name: exporter + containerPort: 9187 + protocol: TCP + resources: + limits: + cpu: 500m + memory: 256M + requests: + cpu: 100m + memory: 200M +--- +apiVersion: v1 +kind: Service +metadata: + name: acid-minimal-cluster-svc-metrics-master + namespace: test-pg + labels: + app: test-pg + spilo-role: master + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9187" +spec: + type: ClusterIP + ports: + - name: exporter + port: 9187 + targetPort: exporter + selector: + application: spilo + cluster-name: acid-minimal-cluster + spilo-role: master +--- +apiVersion: v1 +kind: Service +metadata: + name: acid-minimal-cluster-svc-metrics-replica + namespace: test-pg + labels: + app: test-pg + spilo-role: replica + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9187" +spec: + type: ClusterIP + ports: + - name: exporter + port: 9187 + targetPort: exporter + selector: + application: spilo + cluster-name: acid-minimal-cluster + spilo-role: replica +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: acid-minimal-cluster-svcm-master + namespace: test-pg + labels: + app: test-pg + spilo-role: master +spec: + endpoints: + - port: exporter + interval: 15s + scrapeTimeout: 10s + namespaceSelector: + matchNames: + - test-pg + selector: + matchLabels: + app: test-pg + spilo-role: master +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: acid-minimal-cluster-svcm-replica + namespace: test-pg + labels: + app: test-pg + spilo-role: replica +spec: + endpoints: + - port: exporter + interval: 15s + scrapeTimeout: 10s + namespaceSelector: + matchNames: + - test-pg + selector: + matchLabels: + app: test-pg + spilo-role: replica diff --git a/manifests/operator-service-account-rbac-openshift.yaml b/manifests/operator-service-account-rbac-openshift.yaml new file mode 100644 index 000000000..e0e45cc54 --- /dev/null +++ b/manifests/operator-service-account-rbac-openshift.yaml @@ -0,0 +1,283 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: postgres-operator + namespace: default + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: postgres-operator +rules: +# all verbs allowed for custom operator resources +- apiGroups: + - acid.zalan.do + resources: + - postgresqls + - postgresqls/status + - operatorconfigurations + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# operator only reads PostgresTeams +- apiGroups: + - acid.zalan.do + resources: + - postgresteams + verbs: + - get + - list + - watch +# all verbs allowed for event streams (Zalando-internal feature) +# - apiGroups: +# - zalando.org +# resources: +# - fabriceventstreams +# verbs: +# - create +# - delete +# - deletecollection +# - get +# - list +# - patch +# - update +# - watch +# to create or get/update CRDs when starting up +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - patch + - update +# to read configuration and manage ConfigMaps used by Patroni +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# to send events to the CRs +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - update + - watch +# to CRUD secrets for database access +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - update +# to check nodes for node readiness label +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +# to read or delete existing PVCs. Creation via StatefulSet +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - delete + - get + - list + - patch + - update + # to read existing PVs. Creation should be done via dynamic provisioning +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update # only for resizing AWS volumes +# to watch Spilo pods and do rolling updates. Creation via StatefulSet +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +# to resize the filesystem in Spilo pods when increasing volume size +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +# to CRUD services to point to Postgres cluster instances +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - patch + - update +# to CRUD the StatefulSet which controls the Postgres cluster instances +- apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - create + - delete + - get + - list + - patch +# to CRUD cron jobs for logical backups +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update +# to get namespaces operator resources can run in +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +# to define PDBs. Update happens via delete/create +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - get +# to create ServiceAccounts in each namespace the operator watches +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - create +# to create role bindings to the postgres-pod service account +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - get + - create +# to grant privilege to run privileged pods (not needed by default) +#- apiGroups: +# - extensions +# resources: +# - podsecuritypolicies +# resourceNames: +# - privileged +# verbs: +# - use + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: postgres-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: postgres-operator +subjects: +- kind: ServiceAccount + name: postgres-operator + namespace: default + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: postgres-pod +rules: +# Patroni needs to watch and manage config maps +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +# Patroni needs to watch pods +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch +# to let Patroni create a headless service +- apiGroups: + - "" + resources: + - services + verbs: + - create +# to grant privilege to run privileged pods (not needed by default) +#- apiGroups: +# - extensions +# resources: +# - podsecuritypolicies +# resourceNames: +# - privileged +# verbs: +# - use diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index f0307f6a0..c10dc5fd7 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -35,6 +35,20 @@ rules: - get - list - watch +# all verbs allowed for event streams (Zalando-internal feature) +# - apiGroups: +# - zalando.org +# resources: +# - fabriceventstreams +# verbs: +# - create +# - delete +# - deletecollection +# - get +# - list +# - patch +# - update +# - watch # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index d202125f3..d5594f6d4 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -59,11 +59,20 @@ spec: configuration: type: object properties: + crd_categories: + type: array + nullable: true + items: + type: string docker_image: type: string - default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p3" + default: "registry.opensource.zalan.do/acid/spilo-14:2.1-p5" + enable_crd_registration: + type: boolean + default: true enable_crd_validation: type: boolean + description: deprecated default: true enable_lazy_spilo_upgrade: type: boolean @@ -85,11 +94,13 @@ spec: default: false max_instances: type: integer - minimum: -1 # -1 = disabled + description: "-1 = disabled" + minimum: -1 default: -1 min_instances: type: integer - minimum: -1 # -1 = disabled + description: "-1 = disabled" + minimum: -1 default: -1 resync_period: type: string @@ -117,6 +128,20 @@ spec: users: type: object properties: + additional_owner_roles: + type: array + nullable: true + items: + type: string + enable_password_rotation: + type: boolean + default: false + password_rotation_interval: + type: integer + default: 90 + password_rotation_user_retention: + type: integer + default: 180 replication_username: type: string default: standby @@ -129,6 +154,10 @@ spec: major_version_upgrade_mode: type: string default: "off" + major_version_upgrade_team_allow_list: + type: array + items: + type: string minimal_major_version: type: string default: "9.6" @@ -166,6 +195,9 @@ spec: type: array items: type: string + enable_cross_namespace_secret: + type: boolean + default: false enable_init_containers: type: boolean default: true @@ -178,6 +210,10 @@ spec: enable_sidecars: type: boolean default: true + ignored_annotations: + type: array + items: + type: string infrastructure_roles_secret_name: type: string infrastructure_roles_secrets: @@ -221,6 +257,11 @@ spec: type: object additionalProperties: type: string + node_readiness_label_merge: + type: string + enum: + - "AND" + - "OR" oauth_token_secret_name: type: string default: "postgresql-operator" @@ -315,6 +356,12 @@ spec: timeouts: type: object properties: + patroni_api_check_interval: + type: string + default: "1s" + patroni_api_check_timeout: + type: string + default: "5s" pod_label_wait_timeout: type: string default: "10m" @@ -346,9 +393,15 @@ spec: enable_master_load_balancer: type: boolean default: true + enable_master_pooler_load_balancer: + type: boolean + default: false enable_replica_load_balancer: type: boolean default: false + enable_replica_pooler_load_balancer: + type: boolean + default: false external_traffic_policy: type: string enum: @@ -395,7 +448,7 @@ spec: properties: logical_backup_docker_image: type: string - default: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1" + default: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0" logical_backup_google_application_credentials: type: string logical_backup_job_prefix: @@ -416,6 +469,8 @@ spec: type: string logical_backup_s3_sse: type: string + logical_backup_s3_retention_time: + type: string logical_backup_schedule: type: string pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' @@ -466,6 +521,7 @@ spec: type: string default: - admin + - cron_admin role_deletion_suffix: type: string default: "_deleted" @@ -530,7 +586,7 @@ spec: default: "pooler" connection_pooler_image: type: string - default: "registry.opensource.zalan.do/acid/pgbouncer:master-19" + default: "registry.opensource.zalan.do/acid/pgbouncer:master-22" connection_pooler_max_db_connections: type: integer default: 60 diff --git a/manifests/postgres-operator.yaml b/manifests/postgres-operator.yaml index eddd44650..f06ba50e6 100644 --- a/manifests/postgres-operator.yaml +++ b/manifests/postgres-operator.yaml @@ -19,7 +19,7 @@ spec: serviceAccountName: postgres-operator containers: - name: postgres-operator - image: registry.opensource.zalan.do/acid/postgres-operator:v1.7.1 + image: registry.opensource.zalan.do/acid/postgres-operator:v1.8.0 imagePullPolicy: IfNotPresent resources: requests: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 24d496b36..d2e42e4f8 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -3,8 +3,10 @@ kind: OperatorConfiguration metadata: name: postgresql-operator-default-configuration configuration: - docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p3 - # enable_crd_validation: true + docker_image: registry.opensource.zalan.do/acid/spilo-14:2.1-p5 + # enable_crd_registration: true + # crd_categories: + # - all # enable_lazy_spilo_upgrade: false enable_pgversion_env_var: true # enable_shm_volume: true @@ -24,10 +26,17 @@ configuration: # protocol: TCP workers: 8 users: + # additional_owner_roles: + # - cron_admin + enable_password_rotation: false + password_rotation_interval: 90 + password_rotation_user_retention: 180 replication_username: standby super_username: postgres major_version_upgrade: major_version_upgrade_mode: "off" + # major_version_upgrade_team_allow_list: + # - acid minimal_major_version: "9.6" target_major_version: "14" kubernetes: @@ -50,6 +59,8 @@ configuration: enable_pod_antiaffinity: false enable_pod_disruption_budget: true enable_sidecars: true + # ignored_annotations: + # - k8s.v1.cni.cncf.io/network-status # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" # infrastructure_roles_secrets: # - secretname: "monitoring-roles" @@ -67,6 +78,7 @@ configuration: master_pod_move_timeout: 20m # node_readiness_label: # status: ready + # node_readiness_label_merge: "OR" oauth_token_secret_name: postgresql-operator pdb_name_format: "postgres-{cluster}-pdb" pod_antiaffinity_topology_key: "kubernetes.io/hostname" @@ -99,6 +111,8 @@ configuration: # min_cpu_limit: 250m # min_memory_limit: 250Mi timeouts: + patroni_api_check_interval: 1s + patroni_api_check_timeout: 5s pod_label_wait_timeout: 10m pod_deletion_wait_timeout: 10m ready_wait_interval: 4s @@ -111,7 +125,9 @@ configuration: # keyy: valuey # db_hosted_zone: "" enable_master_load_balancer: false + enable_master_pooler_load_balancer: false enable_replica_load_balancer: false + enable_replica_pooler_load_balancer: false external_traffic_policy: "Cluster" master_dns_name_format: "{cluster}.{team}.{hostedzone}" replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}" @@ -128,7 +144,7 @@ configuration: # wal_gs_bucket: "" # wal_s3_bucket: "" logical_backup: - logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.7.1" + logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.8.0" # logical_backup_google_application_credentials: "" logical_backup_job_prefix: "logical-backup-" logical_backup_provider: "s3" @@ -138,6 +154,7 @@ configuration: # logical_backup_s3_region: "" # logical_backup_s3_secret_access_key: "" logical_backup_s3_sse: "AES256" + # logical_backup_s3_retention_time: "" logical_backup_schedule: "30 00 * * *" debug: debug_logging: true @@ -155,6 +172,7 @@ configuration: # - postgres_superusers protected_role_names: - admin + - cron_admin role_deletion_suffix: "_deleted" team_admin_role: admin team_api_role_configuration: @@ -169,7 +187,7 @@ configuration: connection_pooler_default_cpu_request: "500m" connection_pooler_default_memory_limit: 100Mi connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-19" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-22" # connection_pooler_max_db_connections: 60 connection_pooler_mode: "transaction" connection_pooler_number_of_instances: 2 diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 0f7524bcc..34a4298dd 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -145,18 +145,12 @@ spec: - "transaction" numberOfInstances: type: integer - minimum: 2 + minimum: 1 resources: type: object - required: - - requests - - limits properties: limits: type: object - required: - - cpu - - memory properties: cpu: type: string @@ -166,9 +160,6 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' requests: type: object - required: - - cpu - - memory properties: cpu: type: string @@ -195,16 +186,27 @@ spec: type: boolean enableMasterLoadBalancer: type: boolean + enableMasterPoolerLoadBalancer: + type: boolean enableReplicaLoadBalancer: type: boolean + enableReplicaPoolerLoadBalancer: + type: boolean enableShmVolume: type: boolean - init_containers: # deprecated + env: type: array nullable: true items: type: object x-kubernetes-preserve-unknown-fields: true + init_containers: + type: array + description: deprecated + nullable: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true initContainers: type: array nullable: true @@ -227,8 +229,8 @@ spec: items: type: object required: - - weight - preference + - weight properties: preference: type: object @@ -340,14 +342,17 @@ spec: type: boolean synchronous_mode_strict: type: boolean + synchronous_node_count: + type: integer ttl: type: integer podAnnotations: type: object additionalProperties: type: string - pod_priority_class_name: # deprecated + pod_priority_class_name: type: string + description: deprecated podPriorityClassName: type: string postgresql: @@ -391,19 +396,14 @@ spec: type: boolean secretNamespace: type: string - replicaLoadBalancer: # deprecated + replicaLoadBalancer: type: boolean + description: deprecated resources: type: object - required: - - requests - - limits properties: limits: type: object - required: - - cpu - - memory properties: cpu: type: string @@ -432,9 +432,6 @@ spec: # than the corresponding limit. requests: type: object - required: - - cpu - - memory properties: cpu: type: string @@ -462,11 +459,55 @@ spec: type: integer standby: type: object - required: - - s3_wal_path properties: s3_wal_path: type: string + gs_wal_path: + type: string + standby_host: + type: string + standby_port: + type: string + oneOf: + - required: + - s3_wal_path + - required: + - gs_wal_path + - required: + - standby_host + streams: + type: array + nullable: true + items: + type: object + required: + - applicationId + - database + - tables + properties: + applicationId: + type: string + batchSize: + type: integer + database: + type: string + filter: + type: object + additionalProperties: + type: string + tables: + type: object + additionalProperties: + type: object + required: + - eventType + properties: + eventType: + type: string + idColumn: + type: string + payloadColumn: + type: string teamId: type: string tls: @@ -488,10 +529,6 @@ spec: type: array items: type: object - required: - - key - - operator - - effect properties: key: type: string @@ -510,14 +547,14 @@ spec: - PreferNoSchedule tolerationSeconds: type: integer - useLoadBalancer: # deprecated + useLoadBalancer: type: boolean + description: deprecated users: type: object additionalProperties: type: array nullable: true - description: "Role flags specified here must not contradict each other" items: type: string enum: @@ -549,6 +586,16 @@ spec: - SUPERUSER - nosuperuser - NOSUPERUSER + usersWithPasswordRotation: + type: array + nullable: true + items: + type: string + usersWithInPlacePasswordRotation: + type: array + nullable: true + items: + type: string volume: type: object required: diff --git a/manifests/standby-manifest.yaml b/manifests/standby-manifest.yaml index 3ba8d6b9d..66f515518 100644 --- a/manifests/standby-manifest.yaml +++ b/manifests/standby-manifest.yaml @@ -10,6 +10,8 @@ spec: numberOfInstances: 1 postgresql: version: "14" -# Make this a standby cluster and provide the s3 bucket path of source cluster for continuous streaming. + # Make this a standby cluster and provide either the s3 bucket path of source cluster or the remote primary host for continuous streaming. standby: - s3_wal_path: "s3://path/to/bucket/containing/wal/of/source/cluster/" + # s3_wal_path: "s3://mybucket/spilo/acid-minimal-cluster/abcd1234-2a4b-4b2a-8c9c-c1234defg567/wal/14/" + standby_host: "acid-minimal-cluster.default" + # standby_port: "5432" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 582b1379e..2851d31ac 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1,6 +1,8 @@ package v1 import ( + "fmt" + acidzalando "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do" "github.com/zalando/postgres-operator/pkg/util" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -11,11 +13,13 @@ import ( const ( PostgresCRDResourceKind = "postgresql" PostgresCRDResourcePlural = "postgresqls" + PostgresCRDResourceList = PostgresCRDResourceKind + "List" PostgresCRDResouceName = PostgresCRDResourcePlural + "." + acidzalando.GroupName PostgresCRDResourceShort = "pg" OperatorConfigCRDResouceKind = "OperatorConfiguration" OperatorConfigCRDResourcePlural = "operatorconfigurations" + OperatorConfigCRDResourceList = OperatorConfigCRDResouceKind + "List" OperatorConfigCRDResourceName = OperatorConfigCRDResourcePlural + "." + acidzalando.GroupName OperatorConfigCRDResourceShort = "opconfig" ) @@ -148,7 +152,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "string", }, "targetContainers": { - Type: "array", + Type: "array", + Nullable: true, Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ Type: "string", @@ -199,9 +204,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "string", }, "timestamp": { - Type: "string", - Description: "Date-time format that specifies a timezone as an offset relative to UTC e.g. 1996-12-19T16:39:57-08:00", - Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", + Type: "string", + Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", }, "uid": { Type: "string", @@ -234,38 +238,31 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Minimum: &min1, }, "resources": { - Type: "object", - Required: []string{"requests", "limits"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "limits": { - Type: "object", - Required: []string{"cpu", "memory"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "cpu": { - Type: "string", - Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, "memory": { - Type: "string", - Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, }, }, "requests": { - Type: "object", - Required: []string{"cpu", "memory"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "cpu": { - Type: "string", - Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, "memory": { - Type: "string", - Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, }, }, @@ -283,8 +280,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "object", AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ Schema: &apiextv1.JSONSchemaProps{ - Type: "string", - Description: "User names specified here as database owners must be declared in the users key of the spec key", + Type: "string", }, }, }, @@ -303,15 +299,32 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "enableMasterLoadBalancer": { Type: "boolean", }, + "enableMasterPoolerLoadBalancer": { + Type: "boolean", + }, "enableReplicaLoadBalancer": { Type: "boolean", }, + "enableReplicaPoolerLoadBalancer": { + Type: "boolean", + }, "enableShmVolume": { Type: "boolean", }, + "env": { + Type: "array", + Nullable: true, + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + XPreserveUnknownFields: util.True(), + }, + }, + }, "init_containers": { Type: "array", - Description: "Deprecated", + Description: "deprecated", + Nullable: true, Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ Type: "object", @@ -320,7 +333,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, "initContainers": { - Type: "array", + Type: "array", + Nullable: true, Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ Type: "object", @@ -358,9 +372,23 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, + Type: "object", + Required: []string{"key", "operator"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "key": { + Type: "string", + }, + "operator": { + Type: "string", + }, + "values": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, }, }, }, @@ -369,9 +397,23 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, + Type: "object", + Required: []string{"key", "operator"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "key": { + Type: "string", + }, + "operator": { + Type: "string", + }, + "values": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, }, }, }, @@ -400,9 +442,23 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, + Type: "object", + Required: []string{"key", "operator"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "key": { + Type: "string", + }, + "operator": { + Type: "string", + }, + "values": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, }, }, }, @@ -411,9 +467,23 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ - Allows: true, + Type: "object", + Required: []string{"key", "operator"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "key": { + Type: "string", + }, + "operator": { + Type: "string", + }, + "values": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, }, }, }, @@ -477,6 +547,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "synchronous_mode_strict": { Type: "boolean", }, + "synchronous_node_count": { + Type: "integer", + }, "ttl": { Type: "integer", }, @@ -492,7 +565,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, "pod_priority_class_name": { Type: "string", - Description: "Deprecated", + Description: "deprecated", }, "podPriorityClassName": { Type: "string", @@ -579,41 +652,34 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, "replicaLoadBalancer": { Type: "boolean", - Description: "Deprecated", + Description: "deprecated", }, "resources": { - Type: "object", - Required: []string{"requests", "limits"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "limits": { - Type: "object", - Required: []string{"cpu", "memory"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "cpu": { - Type: "string", - Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, "memory": { - Type: "string", - Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, }, }, "requests": { - Type: "object", - Required: []string{"cpu", "memory"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "cpu": { - Type: "string", - Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", - Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", }, "memory": { - Type: "string", - Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, }, }, @@ -631,7 +697,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, "sidecars": { - Type: "array", + Type: "array", + Nullable: true, Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ Type: "object", @@ -649,12 +716,73 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "integer", }, "standby": { - Type: "object", - Required: []string{"s3_wal_path"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "s3_wal_path": { Type: "string", }, + "gs_wal_path": { + Type: "string", + }, + "standby_host": { + Type: "string", + }, + "standby_port": { + Type: "string", + }, + }, + OneOf: []apiextv1.JSONSchemaProps{ + apiextv1.JSONSchemaProps{Required: []string{"s3_wal_path"}}, + apiextv1.JSONSchemaProps{Required: []string{"gs_wal_path"}}, + apiextv1.JSONSchemaProps{Required: []string{"standby_host"}}, + }, + }, + "streams": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + Required: []string{"applicationId", "database", "tables"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "applicationId": { + Type: "string", + }, + "batchSize": { + Type: "integer", + }, + "database": { + Type: "string", + }, + "filter": { + Type: "object", + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "tables": { + Type: "object", + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + Required: []string{"eventType"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "eventType": { + Type: "string", + }, + "idColumn": { + Type: "string", + }, + "payloadColumn": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, }, }, "teamId": { @@ -685,8 +813,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"key", "operator", "effect"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "key": { Type: "string", @@ -728,15 +855,14 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, "useLoadBalancer": { Type: "boolean", - Description: "Deprecated", + Description: "deprecated", }, "users": { Type: "object", AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ Schema: &apiextv1.JSONSchemaProps{ - Type: "array", - Description: "Role flags specified here must not contradict each other", - Nullable: true, + Type: "array", + Nullable: true, Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ Type: "string", @@ -831,6 +957,24 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, + "usersWithSecretRotation": { + Type: "array", + Nullable: true, + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "usersWithInPlaceSecretRotation": { + Type: "array", + Nullable: true, + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "volume": { Type: "object", Required: []string{"size"}, @@ -887,9 +1031,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, "size": { - Type: "string", - Description: "Value must not be zero", - Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", }, "storageClass": { Type: "string", @@ -941,12 +1084,25 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "configuration": { Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ + "crd_categories": { + Type: "array", + Nullable: true, + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "docker_image": { Type: "string", }, - "enable_crd_validation": { + "enable_crd_registration": { Type: "boolean", }, + "enable_crd_validation": { + Type: "boolean", + Description: "deprecated", + }, "enable_lazy_spilo_upgrade": { Type: "boolean", }, @@ -990,7 +1146,8 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, "sidecars": { - Type: "array", + Type: "array", + Nullable: true, Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ Type: "object", @@ -1005,6 +1162,24 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "users": { Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ + "additional_owner_roles": { + Type: "array", + Nullable: true, + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "enable_password_rotation": { + Type: "boolean", + }, + "password_rotation_interval": { + Type: "integer", + }, + "password_rotation_user_retention": { + Type: "integer", + }, "replication_username": { Type: "string", }, @@ -1019,6 +1194,14 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "major_version_upgrade_mode": { Type: "string", }, + "major_version_upgrade_team_allow_list": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "minimal_major_version": { Type: "string", }, @@ -1089,11 +1272,20 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "enable_sidecars": { Type: "boolean", }, + "ignored_annotations": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, "infrastructure_roles_secret_name": { Type: "string", }, "infrastructure_roles_secrets": { - Type: "array", + Type: "array", + Nullable: true, Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ Type: "object", @@ -1154,6 +1346,17 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, + "node_readiness_label_merge": { + Type: "string", + Enum: []apiextv1.JSON{ + { + Raw: []byte(`"AND"`), + }, + { + Raw: []byte(`"OR"`), + }, + }, + }, "oauth_token_secret_name": { Type: "string", }, @@ -1275,6 +1478,12 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "timeouts": { Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ + "patroni_api_check_interval": { + Type: "string", + }, + "patroni_api_check_timeout": { + Type: "string", + }, "pod_label_wait_timeout": { Type: "string", }, @@ -1312,9 +1521,15 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "enable_master_load_balancer": { Type: "boolean", }, + "enable_master_pooler_load_balancer": { + Type: "boolean", + }, "enable_replica_load_balancer": { Type: "boolean", }, + "enable_replica_pooler_load_balancer": { + Type: "boolean", + }, "external_traffic_policy": { Type: "string", Enum: []apiextv1.JSON{ @@ -1399,6 +1614,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "logical_backup_s3_sse": { Type: "string", }, + "logical_backup_s3_retention_time": { + Type: "string", + }, "logical_backup_schedule": { Type: "string", Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$", @@ -1584,18 +1802,27 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ }, } -func buildCRD(name, kind, plural, short string, columns []apiextv1.CustomResourceColumnDefinition, validation apiextv1.CustomResourceValidation) *apiextv1.CustomResourceDefinition { +func buildCRD(name, kind, plural, list, short string, + categories []string, + columns []apiextv1.CustomResourceColumnDefinition, + validation apiextv1.CustomResourceValidation) *apiextv1.CustomResourceDefinition { return &apiextv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: fmt.Sprintf("%s/%s", apiextv1.GroupName, apiextv1.SchemeGroupVersion.Version), + Kind: "CustomResourceDefinition", + }, ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: apiextv1.CustomResourceDefinitionSpec{ Group: SchemeGroupVersion.Group, Names: apiextv1.CustomResourceDefinitionNames{ - Plural: plural, - ShortNames: []string{short}, Kind: kind, - Categories: []string{"all"}, + ListKind: list, + Plural: plural, + Singular: kind, + ShortNames: []string{short}, + Categories: categories, }, Scope: apiextv1.NamespaceScoped, Versions: []apiextv1.CustomResourceDefinitionVersion{ @@ -1615,33 +1842,25 @@ func buildCRD(name, kind, plural, short string, columns []apiextv1.CustomResourc } // PostgresCRD returns CustomResourceDefinition built from PostgresCRDResource -func PostgresCRD(enableValidation *bool) *apiextv1.CustomResourceDefinition { - postgresCRDvalidation := apiextv1.CustomResourceValidation{} - - if enableValidation != nil && *enableValidation { - postgresCRDvalidation = PostgresCRDResourceValidation - } - +func PostgresCRD(crdCategories []string) *apiextv1.CustomResourceDefinition { return buildCRD(PostgresCRDResouceName, PostgresCRDResourceKind, PostgresCRDResourcePlural, + PostgresCRDResourceList, PostgresCRDResourceShort, + crdCategories, PostgresCRDResourceColumns, - postgresCRDvalidation) + PostgresCRDResourceValidation) } // ConfigurationCRD returns CustomResourceDefinition built from OperatorConfigCRDResource -func ConfigurationCRD(enableValidation *bool) *apiextv1.CustomResourceDefinition { - opconfigCRDvalidation := apiextv1.CustomResourceValidation{} - - if enableValidation != nil && *enableValidation { - opconfigCRDvalidation = OperatorConfigCRDResourceValidation - } - +func ConfigurationCRD(crdCategories []string) *apiextv1.CustomResourceDefinition { return buildCRD(OperatorConfigCRDResourceName, OperatorConfigCRDResouceKind, OperatorConfigCRDResourcePlural, + OperatorConfigCRDResourceList, OperatorConfigCRDResourceShort, + crdCategories, OperatorConfigCRDResourceColumns, - opconfigCRDvalidation) + OperatorConfigCRDResourceValidation) } diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 6d0dd136a..0be275553 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -37,15 +37,20 @@ type OperatorConfigurationList struct { // PostgresUsersConfiguration defines the system users of Postgres. type PostgresUsersConfiguration struct { - SuperUsername string `json:"super_username,omitempty"` - ReplicationUsername string `json:"replication_username,omitempty"` + SuperUsername string `json:"super_username,omitempty"` + ReplicationUsername string `json:"replication_username,omitempty"` + AdditionalOwnerRoles []string `json:"additional_owner_roles,omitempty"` + EnablePasswordRotation bool `json:"enable_password_rotation,omitempty"` + PasswordRotationInterval uint32 `json:"password_rotation_interval,omitempty"` + PasswordRotationUserRetention uint32 `json:"password_rotation_user_retention,omitempty"` } // MajorVersionUpgradeConfiguration defines how to execute major version upgrades of Postgres. type MajorVersionUpgradeConfiguration struct { - MajorVersionUpgradeMode string `json:"major_version_upgrade_mode" default:"off"` // off - no actions, manual - manifest triggers action, full - manifest and minimal version violation trigger upgrade - MinimalMajorVersion string `json:"minimal_major_version" default:"9.6"` - TargetMajorVersion string `json:"target_major_version" default:"14"` + MajorVersionUpgradeMode string `json:"major_version_upgrade_mode" default:"off"` // off - no actions, manual - manifest triggers action, full - manifest and minimal version violation trigger upgrade + MajorVersionUpgradeTeamAllowList []string `json:"major_version_upgrade_team_allow_list,omitempty"` + MinimalMajorVersion string `json:"minimal_major_version" default:"9.6"` + TargetMajorVersion string `json:"target_major_version" default:"14"` } // KubernetesMetaConfiguration defines k8s conf required for all Postgres clusters and the operator itself @@ -77,10 +82,12 @@ type KubernetesMetaConfiguration struct { InheritedLabels []string `json:"inherited_labels,omitempty"` InheritedAnnotations []string `json:"inherited_annotations,omitempty"` DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` + IgnoredAnnotations []string `json:"ignored_annotations,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` DeleteAnnotationDateKey string `json:"delete_annotation_date_key,omitempty"` DeleteAnnotationNameKey string `json:"delete_annotation_name_key,omitempty"` NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` + NodeReadinessLabelMerge string `json:"node_readiness_label_merge,omitempty"` CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` @@ -106,23 +113,27 @@ type PostgresPodResourcesDefaults struct { // OperatorTimeouts defines the timeout of ResourceCheck, PodWait, ReadyWait type OperatorTimeouts struct { - ResourceCheckInterval Duration `json:"resource_check_interval,omitempty"` - ResourceCheckTimeout Duration `json:"resource_check_timeout,omitempty"` - PodLabelWaitTimeout Duration `json:"pod_label_wait_timeout,omitempty"` - PodDeletionWaitTimeout Duration `json:"pod_deletion_wait_timeout,omitempty"` - ReadyWaitInterval Duration `json:"ready_wait_interval,omitempty"` - ReadyWaitTimeout Duration `json:"ready_wait_timeout,omitempty"` + ResourceCheckInterval Duration `json:"resource_check_interval,omitempty"` + ResourceCheckTimeout Duration `json:"resource_check_timeout,omitempty"` + PodLabelWaitTimeout Duration `json:"pod_label_wait_timeout,omitempty"` + PodDeletionWaitTimeout Duration `json:"pod_deletion_wait_timeout,omitempty"` + ReadyWaitInterval Duration `json:"ready_wait_interval,omitempty"` + ReadyWaitTimeout Duration `json:"ready_wait_timeout,omitempty"` + PatroniAPICheckInterval Duration `json:"patroni_api_check_interval,omitempty"` + PatroniAPICheckTimeout Duration `json:"patroni_api_check_timeout,omitempty"` } // LoadBalancerConfiguration defines the LB configuration type LoadBalancerConfiguration struct { - DbHostedZone string `json:"db_hosted_zone,omitempty"` - EnableMasterLoadBalancer bool `json:"enable_master_load_balancer,omitempty"` - EnableReplicaLoadBalancer bool `json:"enable_replica_load_balancer,omitempty"` - CustomServiceAnnotations map[string]string `json:"custom_service_annotations,omitempty"` - MasterDNSNameFormat config.StringTemplate `json:"master_dns_name_format,omitempty"` - ReplicaDNSNameFormat config.StringTemplate `json:"replica_dns_name_format,omitempty"` - ExternalTrafficPolicy string `json:"external_traffic_policy" default:"Cluster"` + DbHostedZone string `json:"db_hosted_zone,omitempty"` + EnableMasterLoadBalancer bool `json:"enable_master_load_balancer,omitempty"` + EnableMasterPoolerLoadBalancer bool `json:"enable_master_pooler_load_balancer,omitempty"` + EnableReplicaLoadBalancer bool `json:"enable_replica_load_balancer,omitempty"` + EnableReplicaPoolerLoadBalancer bool `json:"enable_replica_pooler_load_balancer,omitempty"` + CustomServiceAnnotations map[string]string `json:"custom_service_annotations,omitempty"` + MasterDNSNameFormat config.StringTemplate `json:"master_dns_name_format,omitempty"` + ReplicaDNSNameFormat config.StringTemplate `json:"replica_dns_name_format,omitempty"` + ExternalTrafficPolicy string `json:"external_traffic_policy" default:"Cluster"` } // AWSGCPConfiguration defines the configuration for AWS @@ -208,13 +219,16 @@ type OperatorLogicalBackupConfiguration struct { S3AccessKeyID string `json:"logical_backup_s3_access_key_id,omitempty"` S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"` S3SSE string `json:"logical_backup_s3_sse,omitempty"` + RetentionTime string `json:"logical_backup_s3_retention_time,omitempty"` GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"` JobPrefix string `json:"logical_backup_job_prefix,omitempty"` } // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { + EnableCRDRegistration *bool `json:"enable_crd_registration,omitempty"` EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` + CRDCategories []string `json:"crd_categories,omitempty"` EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` EnablePgVersionEnvVar bool `json:"enable_pgversion_env_var,omitempty"` EnableSpiloWalPathCompat bool `json:"enable_spilo_wal_path_compat,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 079cb8b98..3d37dacfb 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -27,7 +27,7 @@ type PostgresSpec struct { PostgresqlParam `json:"postgresql"` Volume `json:"volume,omitempty"` Patroni `json:"patroni,omitempty"` - Resources `json:"resources,omitempty"` + *Resources `json:"resources,omitempty"` EnableConnectionPooler *bool `json:"enableConnectionPooler,omitempty"` EnableReplicaConnectionPooler *bool `json:"enableReplicaConnectionPooler,omitempty"` @@ -42,8 +42,10 @@ type PostgresSpec struct { // vars that enable load balancers are pointers because it is important to know if any of them is omitted from the Postgres manifest // in that case the var evaluates to nil and the value is taken from the operator config - EnableMasterLoadBalancer *bool `json:"enableMasterLoadBalancer,omitempty"` - EnableReplicaLoadBalancer *bool `json:"enableReplicaLoadBalancer,omitempty"` + EnableMasterLoadBalancer *bool `json:"enableMasterLoadBalancer,omitempty"` + EnableMasterPoolerLoadBalancer *bool `json:"enableMasterPoolerLoadBalancer,omitempty"` + EnableReplicaLoadBalancer *bool `json:"enableReplicaLoadBalancer,omitempty"` + EnableReplicaPoolerLoadBalancer *bool `json:"enableReplicaPoolerLoadBalancer,omitempty"` // deprecated load balancer settings maintained for backward compatibility // see "Load balancers" operator docs @@ -53,8 +55,11 @@ type PostgresSpec struct { // load balancers' source ranges are the same for master and replica services AllowedSourceRanges []string `json:"allowedSourceRanges"` + Users map[string]UserFlags `json:"users,omitempty"` + UsersWithSecretRotation []string `json:"usersWithSecretRotation,omitempty"` + UsersWithInPlaceSecretRotation []string `json:"usersWithInPlaceSecretRotation,omitempty"` + NumberOfInstances int32 `json:"numberOfInstances"` - Users map[string]UserFlags `json:"users,omitempty"` MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` Clone *CloneDescription `json:"clone,omitempty"` ClusterName string `json:"-"` @@ -74,6 +79,8 @@ type PostgresSpec struct { ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` TLS *TLSDescription `json:"tls,omitempty"` AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` + Streams []Stream `json:"streams,omitempty"` + Env []v1.EnvVar `json:"env,omitempty"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` @@ -161,11 +168,15 @@ type Patroni struct { Slots map[string]map[string]string `json:"slots,omitempty"` SynchronousMode bool `json:"synchronous_mode,omitempty"` SynchronousModeStrict bool `json:"synchronous_mode_strict,omitempty"` + SynchronousNodeCount uint32 `json:"synchronous_node_count,omitempty" defaults:"1"` } -// StandbyDescription contains s3 wal path +// StandbyDescription contains remote primary config or s3/gs wal path type StandbyDescription struct { - S3WalPath string `json:"s3_wal_path,omitempty"` + S3WalPath string `json:"s3_wal_path,omitempty"` + GSWalPath string `json:"gs_wal_path,omitempty"` + StandbyHost string `json:"standby_host,omitempty"` + StandbyPort string `json:"standby_port,omitempty"` } // TLSDescription specs TLS properties @@ -191,7 +202,7 @@ type CloneDescription struct { // Sidecar defines a container to be run in the same pod as the Postgres container. type Sidecar struct { - Resources `json:"resources,omitempty"` + *Resources `json:"resources,omitempty"` Name string `json:"name,omitempty"` DockerImage string `json:"image,omitempty"` Ports []v1.ContainerPort `json:"ports,omitempty"` @@ -224,5 +235,19 @@ type ConnectionPooler struct { DockerImage string `json:"dockerImage,omitempty"` MaxDBConnections *int32 `json:"maxDBConnections,omitempty"` - Resources `json:"resources,omitempty"` + *Resources `json:"resources,omitempty"` +} + +type Stream struct { + ApplicationId string `json:"applicationId"` + Database string `json:"database"` + Tables map[string]StreamTable `json:"tables"` + Filter map[string]*string `json:"filter,omitempty"` + BatchSize *uint32 `json:"batchSize,omitempty"` +} + +type StreamTable struct { + EventType string `json:"eventType"` + IdColumn *string `json:"idColumn,omitempty"` + PayloadColumn *string `json:"payloadColumn,omitempty"` } diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index bf6875a82..2ff40d347 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -163,7 +163,7 @@ var unmarshalCluster = []struct { "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":"Invalid"}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":"Invalid"}`), err: nil}, { about: "example with /status subresource", @@ -184,7 +184,7 @@ var unmarshalCluster = []struct { "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), err: nil}, { about: "example with detailed input manifest and deprecated pod_priority_class_name -> podPriorityClassName", @@ -300,7 +300,7 @@ var unmarshalCluster = []struct { MaximumLagOnFailover: 33554432, Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, }, - Resources: Resources{ + Resources: &Resources{ ResourceRequests: ResourceDescription{CPU: "10m", Memory: "50Mi"}, ResourceLimits: ResourceDescription{CPU: "300m", Memory: "3000Mi"}, }, @@ -351,7 +351,7 @@ var unmarshalCluster = []struct { Status: PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid}, Error: errors.New("name must match {TEAM}-{NAME} format").Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null} ,"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), err: nil}, { about: "example with clone", @@ -373,7 +373,7 @@ var unmarshalCluster = []struct { }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{"cluster":"team-batman"}},"status":{"PostgresClusterStatus":""}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{"cluster":"team-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, { about: "standby example", @@ -395,7 +395,7 @@ var unmarshalCluster = []struct { }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"standby":{"s3_wal_path":"s3://custom/path/to/bucket/"}},"status":{"PostgresClusterStatus":""}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"standby":{"s3_wal_path":"s3://custom/path/to/bucket/"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, { about: "expect error on malformatted JSON", diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index c0be8fdf9..b338421a2 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -1,7 +1,8 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -105,7 +106,11 @@ func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) { *out = new(int32) **out = **in } - out.Resources = in.Resources + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + **out = **in + } return } @@ -223,6 +228,11 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura *out = make([]string, len(*in)) copy(*out, *in) } + if in.IgnoredAnnotations != nil { + in, out := &in.IgnoredAnnotations, &out.IgnoredAnnotations + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.NodeReadinessLabel != nil { in, out := &in.NodeReadinessLabel, &out.NodeReadinessLabel *out = make(map[string]string, len(*in)) @@ -318,6 +328,11 @@ func (in *MaintenanceWindow) DeepCopy() *MaintenanceWindow { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MajorVersionUpgradeConfiguration) DeepCopyInto(out *MajorVersionUpgradeConfiguration) { *out = *in + if in.MajorVersionUpgradeTeamAllowList != nil { + in, out := &in.MajorVersionUpgradeTeamAllowList, &out.MajorVersionUpgradeTeamAllowList + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -361,11 +376,21 @@ func (in *OperatorConfiguration) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData) { *out = *in + if in.EnableCRDRegistration != nil { + in, out := &in.EnableCRDRegistration, &out.EnableCRDRegistration + *out = new(bool) + **out = **in + } if in.EnableCRDValidation != nil { in, out := &in.EnableCRDValidation, &out.EnableCRDValidation *out = new(bool) **out = **in } + if in.CRDCategories != nil { + in, out := &in.CRDCategories, &out.CRDCategories + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.ShmVolume != nil { in, out := &in.ShmVolume, &out.ShmVolume *out = new(bool) @@ -385,8 +410,8 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData (*in)[i].DeepCopyInto(&(*out)[i]) } } - out.PostgresUsersConfiguration = in.PostgresUsersConfiguration - out.MajorVersionUpgrade = in.MajorVersionUpgrade + in.PostgresUsersConfiguration.DeepCopyInto(&out.PostgresUsersConfiguration) + in.MajorVersionUpgrade.DeepCopyInto(&out.MajorVersionUpgrade) in.Kubernetes.DeepCopyInto(&out.Kubernetes) out.PostgresPodResources = in.PostgresPodResources out.Timeouts = in.Timeouts @@ -559,7 +584,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { in.PostgresqlParam.DeepCopyInto(&out.PostgresqlParam) in.Volume.DeepCopyInto(&out.Volume) in.Patroni.DeepCopyInto(&out.Patroni) - out.Resources = in.Resources + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + **out = **in + } if in.EnableConnectionPooler != nil { in, out := &in.EnableConnectionPooler, &out.EnableConnectionPooler *out = new(bool) @@ -595,11 +624,21 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(bool) **out = **in } + if in.EnableMasterPoolerLoadBalancer != nil { + in, out := &in.EnableMasterPoolerLoadBalancer, &out.EnableMasterPoolerLoadBalancer + *out = new(bool) + **out = **in + } if in.EnableReplicaLoadBalancer != nil { in, out := &in.EnableReplicaLoadBalancer, &out.EnableReplicaLoadBalancer *out = new(bool) **out = **in } + if in.EnableReplicaPoolerLoadBalancer != nil { + in, out := &in.EnableReplicaPoolerLoadBalancer, &out.EnableReplicaPoolerLoadBalancer + *out = new(bool) + **out = **in + } if in.UseLoadBalancer != nil { in, out := &in.UseLoadBalancer, &out.UseLoadBalancer *out = new(bool) @@ -630,6 +669,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*out)[key] = outVal } } + if in.UsersWithSecretRotation != nil { + in, out := &in.UsersWithSecretRotation, &out.UsersWithSecretRotation + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.UsersWithInPlaceSecretRotation != nil { + in, out := &in.UsersWithInPlaceSecretRotation, &out.UsersWithInPlaceSecretRotation + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.MaintenanceWindows != nil { in, out := &in.MaintenanceWindows, &out.MaintenanceWindows *out = make([]MaintenanceWindow, len(*in)) @@ -723,6 +772,20 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Streams != nil { + in, out := &in.Streams, &out.Streams + *out = make([]Stream, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) @@ -883,6 +946,11 @@ func (in *PostgresTeamSpec) DeepCopy() *PostgresTeamSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PostgresUsersConfiguration) DeepCopyInto(out *PostgresUsersConfiguration) { *out = *in + if in.AdditionalOwnerRoles != nil { + in, out := &in.AdditionalOwnerRoles, &out.AdditionalOwnerRoles + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -1084,7 +1152,11 @@ func (in *ScalyrConfiguration) DeepCopy() *ScalyrConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Sidecar) DeepCopyInto(out *Sidecar) { *out = *in - out.Resources = in.Resources + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + **out = **in + } if in.Ports != nil { in, out := &in.Ports, &out.Ports *out = make([]corev1.ContainerPort, len(*in)) @@ -1126,6 +1198,75 @@ func (in *StandbyDescription) DeepCopy() *StandbyDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Stream) DeepCopyInto(out *Stream) { + *out = *in + if in.Tables != nil { + in, out := &in.Tables, &out.Tables + *out = make(map[string]StreamTable, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.Filter != nil { + in, out := &in.Filter, &out.Filter + *out = make(map[string]*string, len(*in)) + for key, val := range *in { + var outVal *string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(string) + **out = **in + } + (*out)[key] = outVal + } + } + if in.BatchSize != nil { + in, out := &in.BatchSize, &out.BatchSize + *out = new(uint32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Stream. +func (in *Stream) DeepCopy() *Stream { + if in == nil { + return nil + } + out := new(Stream) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StreamTable) DeepCopyInto(out *StreamTable) { + *out = *in + if in.IdColumn != nil { + in, out := &in.IdColumn, &out.IdColumn + *out = new(string) + **out = **in + } + if in.PayloadColumn != nil { + in, out := &in.PayloadColumn, &out.PayloadColumn + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StreamTable. +func (in *StreamTable) DeepCopy() *StreamTable { + if in == nil { + return nil + } + out := new(StreamTable) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSDescription) DeepCopyInto(out *TLSDescription) { *out = *in diff --git a/pkg/apis/zalando.org/register.go b/pkg/apis/zalando.org/register.go new file mode 100644 index 000000000..3dbd3f089 --- /dev/null +++ b/pkg/apis/zalando.org/register.go @@ -0,0 +1,6 @@ +package zalando + +const ( + // GroupName is the group name for the operator CRDs + GroupName = "zalando.org" +) diff --git a/pkg/apis/zalando.org/v1/fabriceventstream.go b/pkg/apis/zalando.org/v1/fabriceventstream.go new file mode 100644 index 000000000..559c52d3e --- /dev/null +++ b/pkg/apis/zalando.org/v1/fabriceventstream.go @@ -0,0 +1,84 @@ +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// FabricEventStream defines FabricEventStream Custom Resource Definition Object. +type FabricEventStream struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FabricEventStreamSpec `json:"spec"` +} + +// FabricEventStreamSpec defines the specification for the FabricEventStream TPR. +type FabricEventStreamSpec struct { + ApplicationId string `json:"applicationId"` + EventStreams []EventStream `json:"eventStreams"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// FabricEventStreamList defines a list of FabricEventStreams . +type FabricEventStreamList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []FabricEventStream `json:"items"` +} + +// EventStream defines the source, flow and sink of the event stream +type EventStream struct { + EventStreamFlow EventStreamFlow `json:"flow"` + EventStreamSink EventStreamSink `json:"sink"` + EventStreamSource EventStreamSource `json:"source"` +} + +// EventStreamFlow defines the flow characteristics of the event stream +type EventStreamFlow struct { + Type string `json:"type"` + PayloadColumn *string `json:"payloadColumn,omitempty"` +} + +// EventStreamSink defines the target of the event stream +type EventStreamSink struct { + Type string `json:"type"` + EventType string `json:"eventType,omitempty"` + MaxBatchSize *uint32 `json:"maxBatchSize,omitempty"` +} + +// EventStreamSource defines the source of the event stream and connection for FES operator +type EventStreamSource struct { + Type string `json:"type"` + Schema string `json:"schema,omitempty" defaults:"public"` + EventStreamTable EventStreamTable `json:"table"` + Filter *string `json:"filter,omitempty"` + Connection Connection `json:"jdbcConnection"` +} + +// EventStreamTable defines the name and ID column to be used for streaming +type EventStreamTable struct { + Name string `json:"name"` + IDColumn *string `json:"idColumn,omitempty"` +} + +// Connection to be used for allowing the FES operator to connect to a database +type Connection struct { + Url string `json:"jdbcUrl"` + SlotName string `json:"slotName"` + PluginType string `json:"pluginType,omitempty"` + PublicationName *string `json:"publicationName,omitempty"` + DBAuth DBAuth `json:"databaseAuthentication"` +} + +// DBAuth specifies the credentials to be used for connecting with the database +type DBAuth struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + UserKey string `json:"userKey,omitempty"` + PasswordKey string `json:"passwordKey,omitempty"` +} diff --git a/pkg/apis/zalando.org/v1/register.go b/pkg/apis/zalando.org/v1/register.go new file mode 100644 index 000000000..33a2c718b --- /dev/null +++ b/pkg/apis/zalando.org/v1/register.go @@ -0,0 +1,44 @@ +package v1 + +import ( + "github.com/zalando/postgres-operator/pkg/apis/zalando.org" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" +) + +// APIVersion of the `fabriceventstream` CRD +const ( + APIVersion = "v1" +) + +var ( + schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = schemeBuilder.AddToScheme +) + +func init() { + err := AddToScheme(scheme.Scheme) + if err != nil { + panic(err) + } +} + +// SchemeGroupVersion is the group version used to register these objects. +var SchemeGroupVersion = schema.GroupVersion{Group: zalando.GroupName, Version: APIVersion} + +// Resource takes an unqualified resource and returns a Group-qualified GroupResource. +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +// addKnownTypes adds the set of types defined in this package to the supplied scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &FabricEventStream{}, + &FabricEventStreamList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go new file mode 100644 index 000000000..a44439a94 --- /dev/null +++ b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go @@ -0,0 +1,227 @@ +// +build !ignore_autogenerated + +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connection) DeepCopyInto(out *Connection) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connection) DeepCopy() *Connection { + if in == nil { + return nil + } + out := new(Connection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DBAuth) DeepCopyInto(out *DBAuth) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DBAuth) DeepCopy() *DBAuth { + if in == nil { + return nil + } + out := new(DBAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStream) DeepCopyInto(out *EventStream) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStream) DeepCopy() *EventStream { + if in == nil { + return nil + } + out := new(EventStream) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamFlow) DeepCopyInto(out *EventStreamFlow) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamFlow) DeepCopy() *EventStreamFlow { + if in == nil { + return nil + } + out := new(EventStreamFlow) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSink) DeepCopyInto(out *EventStreamSink) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSink) DeepCopy() *EventStreamSink { + if in == nil { + return nil + } + out := new(EventStreamSink) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSource) DeepCopyInto(out *EventStreamSource) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSource) DeepCopy() *EventStreamSource { + if in == nil { + return nil + } + out := new(EventStreamSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamTable) DeepCopyInto(out *EventStreamTable) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamTable) DeepCopy() *EventStreamTable { + if in == nil { + return nil + } + out := new(EventStreamTable) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FabricEventStreamSpec) DeepCopyInto(out *FabricEventStreamSpec) { + *out = *in + if in.EventStreams != nil { + in, out := &in.EventStreams, &out.EventStreams + *out = make([]EventStream, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamSpec. +func (in *FabricEventStreamSpec) DeepCopy() *FabricEventStreamSpec { + if in == nil { + return nil + } + out := new(FabricEventStreamSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FabricEventStream) DeepCopyInto(out *FabricEventStream) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStream. +func (in *FabricEventStream) DeepCopy() *FabricEventStream { + if in == nil { + return nil + } + out := new(FabricEventStream) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FabricEventStream) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FabricEventStreamList) DeepCopyInto(out *FabricEventStreamList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FabricEventStream, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamList. +func (in *FabricEventStreamList) DeepCopy() *FabricEventStreamList { + if in == nil { + return nil + } + out := new(FabricEventStreamList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FabricEventStreamList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 8e1dcb22e..a92b09094 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -43,7 +43,7 @@ var ( alphaNumericRegexp = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9]*$") databaseNameRegexp = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") userRegexp = regexp.MustCompile(`^[a-z0-9]([-_a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-_a-z0-9]*[a-z0-9])?)*$`) - patroniObjectSuffixes = []string{"config", "failover", "sync"} + patroniObjectSuffixes = []string{"leader", "config", "sync", "failover"} ) // Config contains operator-wide clients and configuration used from a cluster. TODO: remove struct duplication. @@ -91,6 +91,7 @@ type Cluster struct { currentProcess Process processMu sync.RWMutex // protects the current operation for reporting, no need to hold the master mutex specMu sync.RWMutex // protects the spec for reporting, no need to hold the master mutex + streamApplications []string ConnectionPooler map[PostgresRole]*ConnectionPoolerObjects EBSVolumes map[string]volumes.VolumeProperties VolumeResizer volumes.VolumeResizer @@ -148,7 +149,6 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres cluster.EBSVolumes = make(map[string]volumes.VolumeProperties) if cfg.OpConfig.StorageResizeMode != "pvc" || cfg.OpConfig.EnableEBSGp3Migration { cluster.VolumeResizer = &volumes.EBSVolumeResizer{AWSRegion: cfg.OpConfig.AWSRegion} - } return cluster @@ -228,6 +228,8 @@ func (c *Cluster) initUsers() error { return fmt.Errorf("could not init human users: %v", err) } + c.initAdditionalOwnerRoles() + return nil } @@ -254,24 +256,24 @@ func (c *Cluster) Create() error { c.KubeClient.SetPostgresCRDStatus(c.clusterName(), acidv1.ClusterStatusCreating) c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Create", "Started creation of new cluster resources") - if err = c.enforceMinResourceLimits(&c.Spec); err != nil { - return fmt.Errorf("could not enforce minimum resource limits: %v", err) - } - for _, role := range []PostgresRole{Master, Replica} { - if c.Endpoints[role] != nil { - return fmt.Errorf("%s endpoint already exists in the cluster", role) - } - if role == Master { - // replica endpoint will be created by the replica service. Master endpoint needs to be created by us, - // since the corresponding master service does not define any selectors. - ep, err = c.createEndpoint(role) - if err != nil { - return fmt.Errorf("could not create %s endpoint: %v", role, err) + // if kubernetes_use_configmaps is set Patroni will create configmaps + // otherwise it will use endpoints + if !c.patroniKubernetesUseConfigMaps() { + if c.Endpoints[role] != nil { + return fmt.Errorf("%s endpoint already exists in the cluster", role) + } + if role == Master { + // replica endpoint will be created by the replica service. Master endpoint needs to be created by us, + // since the corresponding master service does not define any selectors. + ep, err = c.createEndpoint(role) + if err != nil { + return fmt.Errorf("could not create %s endpoint: %v", role, err) + } + c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) } - c.logger.Infof("endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Endpoints", "Endpoint %q has been successfully created", util.NameFromMeta(ep.ObjectMeta)) } if c.Services[role] != nil { @@ -361,6 +363,12 @@ func (c *Cluster) Create() error { // something fails, report warning c.createConnectionPooler(c.installLookupFunction) + if len(c.Spec.Streams) > 0 { + if err = c.syncStreams(); err != nil { + c.logger.Errorf("could not create streams: %v", err) + } + } + return nil } @@ -374,10 +382,10 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa match = false reasons = append(reasons, "new statefulset's number of replicas does not match the current one") } - if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) { + if changed, reason := c.compareAnnotations(c.Statefulset.Annotations, statefulSet.Annotations); changed { match = false needsReplace = true - reasons = append(reasons, "new statefulset's annotations do not match the current one") + reasons = append(reasons, "new statefulset's annotations do not match: "+reason) } needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons) @@ -406,6 +414,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa needsRollUpdate = true reasons = append(reasons, "new statefulset's pod affinity does not match the current one") } + if len(c.Statefulset.Spec.Template.Spec.Tolerations) != len(statefulSet.Spec.Template.Spec.Tolerations) { + needsReplace = true + needsRollUpdate = true + reasons = append(reasons, "new statefulset's pod tolerations does not match the current one") + } // Some generated fields like creationTimestamp make it not possible to use DeepCompare on Spec.Template.ObjectMeta if !reflect.DeepEqual(c.Statefulset.Spec.Template.Labels, statefulSet.Spec.Template.Labels) { @@ -426,14 +439,13 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa } } - if !reflect.DeepEqual(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations) { + if changed, reason := c.compareAnnotations(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations); changed { match = false needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one") + reasons = append(reasons, "new statefulset's pod template metadata annotations does not match "+reason) } if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) { - match = false needsReplace = true needsRollUpdate = true reasons = append(reasons, "new statefulset's pod template security context in spec does not match the current one") @@ -469,7 +481,6 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa // we assume any change in priority happens by rolling out a new priority class // changing the priority value in an existing class is not supproted if c.Statefulset.Spec.Template.Spec.PriorityClassName != statefulSet.Spec.Template.Spec.PriorityClassName { - match = false needsReplace = true needsRollUpdate = true reasons = append(reasons, "new statefulset's pod priority class in spec does not match the current one") @@ -516,7 +527,7 @@ func (c *Cluster) compareContainers(description string, setA, setB []v1.Containe newCheck("new statefulset %s's %s (index %d) name does not match the current one", func(a, b v1.Container) bool { return a.Name != b.Name }), newCheck("new statefulset %s's %s (index %d) ports do not match the current one", - func(a, b v1.Container) bool { return !reflect.DeepEqual(a.Ports, b.Ports) }), + func(a, b v1.Container) bool { return !comparePorts(a.Ports, b.Ports) }), newCheck("new statefulset %s's %s (index %d) resources do not match the current ones", func(a, b v1.Container) bool { return !compareResources(&a.Resources, &b.Resources) }), newCheck("new statefulset %s's %s (index %d) environment does not match the current one", @@ -627,44 +638,99 @@ func compareSpiloConfiguration(configa, configb string) bool { return reflect.DeepEqual(oa, ob) } -func (c *Cluster) enforceMinResourceLimits(spec *acidv1.PostgresSpec) error { +func areProtocolsEqual(a, b v1.Protocol) bool { + return a == b || + (a == "" && b == v1.ProtocolTCP) || + (a == v1.ProtocolTCP && b == "") +} - var ( - isSmaller bool - err error - ) +func comparePorts(a, b []v1.ContainerPort) bool { + if len(a) != len(b) { + return false + } - // setting limits too low can cause unnecessary evictions / OOM kills - minCPULimit := c.OpConfig.MinCPULimit - minMemoryLimit := c.OpConfig.MinMemoryLimit + areContainerPortsEqual := func(a, b v1.ContainerPort) bool { + return a.Name == b.Name && + a.HostPort == b.HostPort && + areProtocolsEqual(a.Protocol, b.Protocol) && + a.HostIP == b.HostIP + } - cpuLimit := spec.Resources.ResourceLimits.CPU - if cpuLimit != "" { - isSmaller, err = util.IsSmallerQuantity(cpuLimit, minCPULimit) - if err != nil { - return fmt.Errorf("could not compare defined CPU limit %s with configured minimum value %s: %v", cpuLimit, minCPULimit, err) + findByPortValue := func(portSpecs []v1.ContainerPort, port int32) (v1.ContainerPort, bool) { + for _, portSpec := range portSpecs { + if portSpec.ContainerPort == port { + return portSpec, true + } } - if isSmaller { - c.logger.Warningf("defined CPU limit %s is below required minimum %s and will be increased", cpuLimit, minCPULimit) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined CPU limit %s is below required minimum %s and will be set to it", cpuLimit, minCPULimit) - spec.Resources.ResourceLimits.CPU = minCPULimit + return v1.ContainerPort{}, false + } + + for _, portA := range a { + portB, found := findByPortValue(b, portA.ContainerPort) + if !found { + return false + } + if !areContainerPortsEqual(portA, portB) { + return false } } - memoryLimit := spec.Resources.ResourceLimits.Memory - if memoryLimit != "" { - isSmaller, err = util.IsSmallerQuantity(memoryLimit, minMemoryLimit) - if err != nil { - return fmt.Errorf("could not compare defined memory limit %s with configured minimum value %s: %v", memoryLimit, minMemoryLimit, err) + return true +} + +func (c *Cluster) compareAnnotations(old, new map[string]string) (bool, string) { + reason := "" + ignoredAnnotations := make(map[string]bool) + for _, ignore := range c.OpConfig.IgnoredAnnotations { + ignoredAnnotations[ignore] = true + } + + for key := range old { + if _, ok := ignoredAnnotations[key]; ok { + continue } - if isSmaller { - c.logger.Warningf("defined memory limit %s is below required minimum %s and will be increased", memoryLimit, minMemoryLimit) - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", "defined memory limit %s is below required minimum %s and will be set to it", memoryLimit, minMemoryLimit) - spec.Resources.ResourceLimits.Memory = minMemoryLimit + if _, ok := new[key]; !ok { + reason += fmt.Sprintf(" Removed %q.", key) } } - return nil + for key := range new { + if _, ok := ignoredAnnotations[key]; ok { + continue + } + v, ok := old[key] + if !ok { + reason += fmt.Sprintf(" Added %q with value %q.", key, new[key]) + } else if v != new[key] { + reason += fmt.Sprintf(" %q changed from %q to %q.", key, v, new[key]) + } + } + + return reason != "", reason + +} + +func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) { + if old.Spec.Type != new.Spec.Type { + return false, fmt.Sprintf("new service's type %q does not match the current one %q", + new.Spec.Type, old.Spec.Type) + } + + oldSourceRanges := old.Spec.LoadBalancerSourceRanges + newSourceRanges := new.Spec.LoadBalancerSourceRanges + + /* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */ + if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) { + if !util.IsEqualIgnoreOrder(oldSourceRanges, newSourceRanges) { + return false, "new service's LoadBalancerSourceRange does not match the current one" + } + } + + if changed, reason := c.compareAnnotations(old.Annotations, new.Annotations); changed { + return !changed, "new service's annotations does not match the current one:" + reason + } + + return true, "" } // Update changes Kubernetes objects according to the new specification. Unlike the sync case, the missing object @@ -710,13 +776,18 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - // connection pooler needs one system user created, which is done in - // initUsers. Check if it needs to be called. + // check if users need to be synced sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) && reflect.DeepEqual(oldSpec.Spec.PreparedDatabases, newSpec.Spec.PreparedDatabases) + sameRotatedUsers := reflect.DeepEqual(oldSpec.Spec.UsersWithSecretRotation, newSpec.Spec.UsersWithSecretRotation) && + reflect.DeepEqual(oldSpec.Spec.UsersWithInPlaceSecretRotation, newSpec.Spec.UsersWithInPlaceSecretRotation) + + // connection pooler needs one system user created, which is done in + // initUsers. Check if it needs to be called. needConnectionPooler := needMasterConnectionPoolerWorker(&newSpec.Spec) || needReplicaConnectionPoolerWorker(&newSpec.Spec) - if !sameUsers || needConnectionPooler { + + if !sameUsers || !sameRotatedUsers || needConnectionPooler { c.logger.Debugf("initialize users") if err := c.initUsers(); err != nil { c.logger.Errorf("could not init users: %v", err) @@ -741,12 +812,6 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { // Statefulset func() { - if err := c.enforceMinResourceLimits(&c.Spec); err != nil { - c.logger.Errorf("could not sync resources: %v", err) - updateFailed = true - return - } - oldSs, err := c.generateStatefulSet(&oldSpec.Spec) if err != nil { c.logger.Errorf("could not generate old statefulset spec: %v", err) @@ -754,16 +819,13 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { return } - // update newSpec to for latter comparison with oldSpec - c.enforceMinResourceLimits(&newSpec.Spec) - newSs, err := c.generateStatefulSet(&newSpec.Spec) if err != nil { c.logger.Errorf("could not generate new statefulset spec: %v", err) updateFailed = true return } - if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) { + if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) { c.logger.Debugf("syncing statefulsets") syncStatefulSet = false // TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet @@ -855,6 +917,13 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed = true } + if len(c.Spec.Streams) > 0 { + if err := c.syncStreams(); err != nil { + c.logger.Errorf("could not sync streams: %v", err) + updateFailed = true + } + } + if !updateFailed { // Major version upgrade must only fire after success of earlier operations and should stay last if err := c.majorVersionUpgrade(); err != nil { @@ -890,6 +959,10 @@ func (c *Cluster) Delete() { defer c.mu.Unlock() c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Delete", "Started deletion of new cluster resources") + if err := c.deleteStreams(); err != nil { + c.logger.Warningf("could not delete event streams: %v", err) + } + // delete the backup job before the stateful set of the cluster to prevent connections to non-existing pods // deleting the cron job also removes pods and batch jobs it created if err := c.deleteLogicalBackupJob(); err != nil { @@ -1000,6 +1073,7 @@ func (c *Cluster) initSystemUsers() { Origin: spec.RoleOriginSystem, Name: c.OpConfig.ReplicationUsername, Namespace: c.Namespace, + Flags: []string{constants.RoleFlagLogin}, Password: util.RandomPassword(constants.PasswordLength), } @@ -1043,6 +1117,23 @@ func (c *Cluster) initSystemUsers() { c.systemUsers[constants.ConnectionPoolerUserKeyName] = connectionPoolerUser } } + + // replication users for event streams are another exception + // the operator will create one replication user for all streams + if len(c.Spec.Streams) > 0 { + username := constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix + streamUser := spec.PgUser{ + Origin: spec.RoleConnectionPooler, + Name: username, + Namespace: c.Namespace, + Flags: []string{constants.RoleFlagLogin, constants.RoleFlagReplication}, + Password: util.RandomPassword(constants.PasswordLength), + } + + if _, exists := c.pgUsers[username]; !exists { + c.pgUsers[username] = streamUser + } + } } func (c *Cluster) initPreparedDatabaseRoles() error { @@ -1112,7 +1203,6 @@ func (c *Cluster) initPreparedDatabaseRoles() error { func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix, searchPath, secretNamespace string) error { for defaultRole, inherits := range defaultRoles { - namespace := c.Namespace //if namespaced secrets are allowed if secretNamespace != "" { @@ -1135,8 +1225,10 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix } adminRole := "" + isOwner := false if strings.Contains(defaultRole, constants.OwnerRoleNameSuffix) { adminRole = admin + isOwner = true } else { adminRole = prefix + constants.OwnerRoleNameSuffix } @@ -1150,6 +1242,7 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix MemberOf: memberOf, Parameters: map[string]string{"search_path": searchPath}, AdminRole: adminRole, + IsDbOwner: isOwner, } if currentRole, present := c.pgUsers[roleName]; present { c.pgUsers[roleName] = c.resolveNameConflict(¤tRole, &newRole) @@ -1171,11 +1264,20 @@ func (c *Cluster) initRobotUsers() error { } namespace := c.Namespace + // check if role is specified as database owner + isOwner := false + for _, owner := range c.Spec.Databases { + if username == owner { + isOwner = true + } + } + //if namespaced secrets are allowed if c.Config.OpConfig.EnableCrossNamespaceSecret { if strings.Contains(username, ".") { splits := strings.Split(username, ".") namespace = splits[0] + c.logger.Warningf("enable_cross_namespace_secret is set. Database role name contains the respective namespace i.e. %s is the created user", username) } } @@ -1194,6 +1296,7 @@ func (c *Cluster) initRobotUsers() error { Password: util.RandomPassword(constants.PasswordLength), Flags: flags, AdminRole: adminRole, + IsDbOwner: isOwner, } if currentRole, present := c.pgUsers[username]; present { c.pgUsers[username] = c.resolveNameConflict(¤tRole, &newRole) @@ -1204,6 +1307,33 @@ func (c *Cluster) initRobotUsers() error { return nil } +func (c *Cluster) initAdditionalOwnerRoles() { + for _, additionalOwner := range c.OpConfig.AdditionalOwnerRoles { + // fetch all database owners the additional should become a member of + memberOf := make([]string, 0) + for username, pgUser := range c.pgUsers { + if pgUser.IsDbOwner { + memberOf = append(memberOf, username) + } + } + + if len(memberOf) > 0 { + namespace := c.Namespace + additionalOwnerPgUser := spec.PgUser{ + Origin: spec.RoleOriginSpilo, + MemberOf: memberOf, + Name: additionalOwner, + Namespace: namespace, + } + if currentRole, present := c.pgUsers[additionalOwner]; present { + c.pgUsers[additionalOwner] = c.resolveNameConflict(¤tRole, &additionalOwnerPgUser) + } else { + c.pgUsers[additionalOwner] = additionalOwnerPgUser + } + } + } +} + func (c *Cluster) initTeamMembers(teamID string, isPostgresSuperuserTeam bool) error { teamMembers, err := c.getTeamMembers(teamID) @@ -1354,22 +1484,26 @@ func (c *Cluster) GetCurrentProcess() Process { // GetStatus provides status of the cluster func (c *Cluster) GetStatus() *ClusterStatus { - return &ClusterStatus{ - Cluster: c.Spec.ClusterName, - Team: c.Spec.TeamID, - Status: c.Status, - Spec: c.Spec, - + status := &ClusterStatus{ + Cluster: c.Spec.ClusterName, + Team: c.Spec.TeamID, + Status: c.Status, + Spec: c.Spec, MasterService: c.GetServiceMaster(), ReplicaService: c.GetServiceReplica(), - MasterEndpoint: c.GetEndpointMaster(), - ReplicaEndpoint: c.GetEndpointReplica(), StatefulSet: c.GetStatefulSet(), PodDisruptionBudget: c.GetPodDisruptionBudget(), CurrentProcess: c.GetCurrentProcess(), Error: fmt.Errorf("error: %s", c.Error), } + + if !c.patroniKubernetesUseConfigMaps() { + status.MasterEndpoint = c.GetEndpointMaster() + status.ReplicaEndpoint = c.GetEndpointReplica() + } + + return status } // Switchover does a switchover (via Patroni) to a candidate pod @@ -1434,7 +1568,7 @@ func (c *Cluster) Unlock() { c.mu.Unlock() } -type simpleActionWithResult func() error +type simpleActionWithResult func() type clusterObjectGet func(name string) (spec.NamespacedName, error) @@ -1448,45 +1582,47 @@ func (c *Cluster) deletePatroniClusterObjects() error { c.logger.Infof("not cleaning up Etcd Patroni objects on cluster delete") } - if !c.patroniKubernetesUseConfigMaps() { + actionsList = append(actionsList, c.deletePatroniClusterServices) + if c.patroniKubernetesUseConfigMaps() { + actionsList = append(actionsList, c.deletePatroniClusterConfigMaps) + } else { actionsList = append(actionsList, c.deletePatroniClusterEndpoints) } - actionsList = append(actionsList, c.deletePatroniClusterServices, c.deletePatroniClusterConfigMaps) c.logger.Debugf("removing leftover Patroni objects (endpoints / services and configmaps)") for _, deleter := range actionsList { - if err := deleter(); err != nil { - return err - } + deleter() } return nil } -func (c *Cluster) deleteClusterObject( +func deleteClusterObject( get clusterObjectGet, del clusterObjectDelete, - objType string) error { + objType string, + clusterName string, + logger *logrus.Entry) { for _, suffix := range patroniObjectSuffixes { - name := fmt.Sprintf("%s-%s", c.Name, suffix) + name := fmt.Sprintf("%s-%s", clusterName, suffix) - if namespacedName, err := get(name); err == nil { - c.logger.Debugf("deleting Patroni cluster object %q with name %q", + namespacedName, err := get(name) + if err == nil { + logger.Debugf("deleting %s %q", objType, namespacedName) if err = del(name); err != nil { - return fmt.Errorf("could not delete Patroni cluster object %q with name %q: %v", + logger.Warningf("could not delete %s %q: %v", objType, namespacedName, err) } } else if !k8sutil.ResourceNotFound(err) { - return fmt.Errorf("could not fetch Patroni Endpoint %q: %v", - namespacedName, err) + logger.Warningf("could not fetch %s %q: %v", + objType, namespacedName, err) } } - return nil } -func (c *Cluster) deletePatroniClusterServices() error { +func (c *Cluster) deletePatroniClusterServices() { get := func(name string) (spec.NamespacedName, error) { svc, err := c.KubeClient.Services(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) return util.NameFromMeta(svc.ObjectMeta), err @@ -1496,10 +1632,10 @@ func (c *Cluster) deletePatroniClusterServices() error { return c.KubeClient.Services(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) } - return c.deleteClusterObject(get, deleteServiceFn, "service") + deleteClusterObject(get, deleteServiceFn, "service", c.Name, c.logger) } -func (c *Cluster) deletePatroniClusterEndpoints() error { +func (c *Cluster) deletePatroniClusterEndpoints() { get := func(name string) (spec.NamespacedName, error) { ep, err := c.KubeClient.Endpoints(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) return util.NameFromMeta(ep.ObjectMeta), err @@ -1509,10 +1645,10 @@ func (c *Cluster) deletePatroniClusterEndpoints() error { return c.KubeClient.Endpoints(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) } - return c.deleteClusterObject(get, deleteEndpointFn, "endpoint") + deleteClusterObject(get, deleteEndpointFn, "endpoint", c.Name, c.logger) } -func (c *Cluster) deletePatroniClusterConfigMaps() error { +func (c *Cluster) deletePatroniClusterConfigMaps() { get := func(name string) (spec.NamespacedName, error) { cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) return util.NameFromMeta(cm.ObjectMeta), err @@ -1522,5 +1658,5 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error { return c.KubeClient.ConfigMaps(c.Namespace).Delete(context.TODO(), name, c.deleteOptions) } - return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") + deleteClusterObject(get, deleteConfigMapFn, "configmap", c.Name, c.logger) } diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index dc1f5ff03..7581d1473 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -3,12 +3,16 @@ package cluster import ( "fmt" "reflect" + "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/sirupsen/logrus" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" @@ -22,6 +26,8 @@ import ( const ( superUserName = "postgres" replicationUserName = "standby" + exampleSpiloConfig = `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}` + spiloConfigDiff = `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}` ) var logger = logrus.New().WithField("test", "cluster") @@ -31,10 +37,11 @@ var cl = New( Config{ OpConfig: config.Config{ PodManagementPolicy: "ordered_ready", - ProtectedRoles: []string{"admin"}, + ProtectedRoles: []string{"admin", "cron_admin", "part_man"}, Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + AdditionalOwnerRoles: []string{"cron_admin", "part_man"}, }, Resources: config.Resources{ DownscalerAnnotations: []string{"downscaler/*"}, @@ -42,7 +49,13 @@ var cl = New( }, }, k8sutil.NewMockKubernetesClient(), - acidv1.Postgresql{ObjectMeta: metav1.ObjectMeta{Name: "acid-test", Namespace: "test", Annotations: map[string]string{"downscaler/downtime_replicas": "0"}}}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: "acid-test", + Namespace: "test", + Annotations: map[string]string{"downscaler/downtime_replicas": "0"}, + }, + }, logger, eventRecorder, ) @@ -51,7 +64,7 @@ func TestStatefulSetAnnotations(t *testing.T) { testName := "CheckStatefulsetAnnotations" spec := acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ + Resources: &acidv1.Resources{ ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, }, @@ -130,6 +143,48 @@ func TestInitRobotUsers(t *testing.T) { } } +func TestInitAdditionalOwnerRoles(t *testing.T) { + testName := "TestInitAdditionalOwnerRoles" + + manifestUsers := map[string]acidv1.UserFlags{"foo_owner": {}, "bar_owner": {}, "app_user": {}} + expectedUsers := map[string]spec.PgUser{ + "foo_owner": {Origin: spec.RoleOriginManifest, Name: "foo_owner", Namespace: cl.Namespace, Password: "f123", Flags: []string{"LOGIN"}, IsDbOwner: true}, + "bar_owner": {Origin: spec.RoleOriginManifest, Name: "bar_owner", Namespace: cl.Namespace, Password: "b123", Flags: []string{"LOGIN"}, IsDbOwner: true}, + "app_user": {Origin: spec.RoleOriginManifest, Name: "app_user", Namespace: cl.Namespace, Password: "a123", Flags: []string{"LOGIN"}, IsDbOwner: false}, + "cron_admin": {Origin: spec.RoleOriginSpilo, Name: "cron_admin", Namespace: cl.Namespace, MemberOf: []string{"foo_owner", "bar_owner"}}, + "part_man": {Origin: spec.RoleOriginSpilo, Name: "part_man", Namespace: cl.Namespace, MemberOf: []string{"foo_owner", "bar_owner"}}, + } + + cl.Spec.Databases = map[string]string{"foo_db": "foo_owner", "bar_db": "bar_owner"} + cl.Spec.Users = manifestUsers + + // this should set IsDbOwner field for manifest users + if err := cl.initRobotUsers(); err != nil { + t.Errorf("%s could not init manifest users", testName) + } + + // update passwords to compare with result + for manifestUser := range manifestUsers { + pgUser := cl.pgUsers[manifestUser] + pgUser.Password = manifestUser[0:1] + "123" + cl.pgUsers[manifestUser] = pgUser + } + + cl.initAdditionalOwnerRoles() + + for _, additionalOwnerRole := range cl.Config.OpConfig.AdditionalOwnerRoles { + expectedPgUser := expectedUsers[additionalOwnerRole] + existingPgUser, exists := cl.pgUsers[additionalOwnerRole] + if !exists { + t.Errorf("%s additional owner role %q not initilaized", testName, additionalOwnerRole) + } + if !util.IsEqualIgnoreOrder(expectedPgUser.MemberOf, existingPgUser.MemberOf) { + t.Errorf("%s unexpected membership of additional owner role %q: expected member of %#v, got member of %#v", + testName, additionalOwnerRole, expectedPgUser.MemberOf, existingPgUser.MemberOf) + } + } +} + type mockOAuthTokenGetter struct { } @@ -141,8 +196,8 @@ type mockTeamsAPIClient struct { members []string } -func (m *mockTeamsAPIClient) TeamInfo(teamID, token string) (tm *teams.Team, err error) { - return &teams.Team{Members: m.members}, nil +func (m *mockTeamsAPIClient) TeamInfo(teamID, token string) (tm *teams.Team, statusCode int, err error) { + return &teams.Team{Members: m.members}, statusCode, nil } func (m *mockTeamsAPIClient) setMembers(members []string) { @@ -207,15 +262,15 @@ type mockTeamsAPIClientMultipleTeams struct { teams []mockTeam } -func (m *mockTeamsAPIClientMultipleTeams) TeamInfo(teamID, token string) (tm *teams.Team, err error) { +func (m *mockTeamsAPIClientMultipleTeams) TeamInfo(teamID, token string) (tm *teams.Team, statusCode int, err error) { for _, team := range m.teams { if team.teamID == teamID { - return &teams.Team{Members: team.members}, nil + return &teams.Team{Members: team.members}, statusCode, nil } } // should not be reached if a slice with teams is populated correctly - return nil, nil + return nil, statusCode, nil } // Test adding members of maintenance teams that get superuser rights for all PG databases @@ -904,7 +959,7 @@ func TestCompareEnv(t *testing.T) { }, { Name: "SPILO_CONFIGURATION", - Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`, + Value: exampleSpiloConfig, }, }, ExpectedResult: true, @@ -925,7 +980,7 @@ func TestCompareEnv(t *testing.T) { }, { Name: "SPILO_CONFIGURATION", - Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`, + Value: spiloConfigDiff, }, }, ExpectedResult: true, @@ -946,7 +1001,7 @@ func TestCompareEnv(t *testing.T) { }, { Name: "SPILO_CONFIGURATION", - Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`, + Value: exampleSpiloConfig, }, }, ExpectedResult: false, @@ -971,7 +1026,7 @@ func TestCompareEnv(t *testing.T) { }, { Name: "SPILO_CONFIGURATION", - Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`, + Value: exampleSpiloConfig, }, }, ExpectedResult: false, @@ -988,7 +1043,7 @@ func TestCompareEnv(t *testing.T) { }, { Name: "SPILO_CONFIGURATION", - Value: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/12/bin","parameters":{"autovacuum_analyze_scale_factor":"0.1"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"test":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"postgresql":{"parameters":{"max_connections":"100","max_locks_per_transaction":"64","max_worker_processes":"4"}}}}}`, + Value: exampleSpiloConfig, }, }, ExpectedResult: false, @@ -1002,6 +1057,336 @@ func TestCompareEnv(t *testing.T) { } } +func newService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service { + svc := &v1.Service{ + Spec: v1.ServiceSpec{ + Type: svcT, + LoadBalancerSourceRanges: lbSr, + }, + } + svc.Annotations = ann + return svc +} + +func TestCompareServices(t *testing.T) { + testName := "TestCompareServices" + cluster := Cluster{ + Config: Config{ + OpConfig: config.Config{ + Resources: config.Resources{ + IgnoredAnnotations: []string{ + "k8s.v1.cni.cncf.io/network-status", + }, + }, + }, + }, + } + + tests := []struct { + about string + current *v1.Service + new *v1.Service + reason string + match bool + }{ + { + about: "two equal services", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: true, + }, + { + about: "services differ on service type", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeClusterIP, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`, + }, + { + about: "services differ on lb source ranges", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"185.249.56.0/22"}), + match: false, + reason: `new service's LoadBalancerSourceRange does not match the current one`, + }, + { + about: "new service doesn't have lb source ranges", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{}), + match: false, + reason: `new service's LoadBalancerSourceRange does not match the current one`, + }, + { + about: "services differ on DNS annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: "external-dns.alpha.kubernetes.io/hostname" changed from "clstr.acid.zalan.do" to "new_clstr.acid.zalan.do".`, + }, + { + about: "services differ on AWS ELB annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: "1800", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout" changed from "3600" to "1800".`, + }, + { + about: "service changes existing annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "baz", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: "foo" changed from "bar" to "baz".`, + }, + { + about: "service changes multiple existing annotations", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + "bar": "foo", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "baz", + "bar": "fooz", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + // Test just the prefix to avoid flakiness and map sorting + reason: `new service's annotations does not match the current one:`, + }, + { + about: "service adds a new custom annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: Added "foo" with value "bar".`, + }, + { + about: "service removes a custom annotation", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: Removed "foo".`, + }, + { + about: "service removes a custom annotation and adds a new one", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "bar": "foo", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + reason: `new service's annotations does not match the current one: Removed "foo". Added "bar" with value "foo".`, + }, + { + about: "service removes a custom annotation, adds a new one and change another", + current: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "foo": "bar", + "zalan": "do", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + "bar": "foo", + "zalan": "do.com", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + // Test just the prefix to avoid flakiness and map sorting + reason: `new service's annotations does not match the current one: Removed "foo".`, + }, + { + about: "service add annotations", + current: newService( + map[string]string{}, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", + constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: false, + // Test just the prefix to avoid flakiness and map sorting + reason: `new service's annotations does not match the current one: Added `, + }, + { + about: "ignored annotations", + current: newService( + map[string]string{}, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newService( + map[string]string{ + "k8s.v1.cni.cncf.io/network-status": "up", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: true, + }, + } + + for _, tt := range tests { + t.Run(tt.about, func(t *testing.T) { + match, reason := cluster.compareServices(tt.current, tt.new) + if match && !tt.match { + t.Logf("match=%v current=%v, old=%v reason=%s", match, tt.current.Annotations, tt.new.Annotations, reason) + t.Errorf("%s - expected services to do not match: %q and %q", testName, tt.current, tt.new) + return + } + if !match && tt.match { + t.Errorf("%s - expected services to be the same: %q and %q", testName, tt.current, tt.new) + return + } + if !match && !tt.match { + if !strings.HasPrefix(reason, tt.reason) { + t.Errorf("%s - expected reason prefix %s, found %s", testName, tt.reason, reason) + return + } + } + }) + } +} + func TestCrossNamespacedSecrets(t *testing.T) { testName := "test secrets in different namespace" clientSet := fake.NewSimpleClientset() @@ -1039,7 +1424,7 @@ func TestCrossNamespacedSecrets(t *testing.T) { ConnectionPoolerDefaultCPULimit: "100m", ConnectionPoolerDefaultMemoryRequest: "100Mi", ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + NumberOfInstances: k8sutil.Int32ToPointer(1), }, PodManagementPolicy: "ordered_ready", Resources: config.Resources{ @@ -1088,3 +1473,102 @@ func TestValidUsernames(t *testing.T) { } } } + +func TestComparePorts(t *testing.T) { + testCases := []struct { + name string + setA []v1.ContainerPort + setB []v1.ContainerPort + expected bool + }{ + { + name: "different ports", + setA: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + + setB: []v1.ContainerPort{ + { + Name: "http", + ContainerPort: 80, + Protocol: v1.ProtocolTCP, + }, + }, + expected: false, + }, + { + name: "no difference", + setA: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + setB: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + expected: true, + }, + { + name: "same ports, different order", + setA: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + { + Name: "http", + ContainerPort: 80, + Protocol: v1.ProtocolTCP, + }, + }, + setB: []v1.ContainerPort{ + { + Name: "http", + ContainerPort: 80, + Protocol: v1.ProtocolTCP, + }, + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + expected: true, + }, + { + name: "same ports, but one with default protocol", + setA: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + Protocol: v1.ProtocolTCP, + }, + }, + setB: []v1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 9187, + }, + }, + expected: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + got := comparePorts(testCase.setA, testCase.setB) + assert.Equal(t, testCase.expected, got) + }) + } +} diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 5bde71458..1fba2eed7 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -3,7 +3,6 @@ package cluster import ( "context" "fmt" - "reflect" "strings" "github.com/r3labs/diff" @@ -211,9 +210,10 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( connectionPoolerSpec = &acidv1.ConnectionPooler{} } gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) - resources, err := generateResourceRequirements( + resources, err := c.generateResourceRequirements( connectionPoolerSpec.Resources, - makeDefaultConnectionPoolerResources(&c.OpConfig)) + makeDefaultConnectionPoolerResources(&c.OpConfig), + connectionPoolerContainer) effectiveDockerImage := util.Coalesce( connectionPoolerSpec.DockerImage, @@ -247,7 +247,7 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( }, { Name: "PGPORT", - Value: c.servicePort(role), + Value: fmt.Sprint(c.servicePort(role)), }, { Name: "PGUSER", @@ -309,7 +309,7 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( }, } - nodeAffinity := nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity) + nodeAffinity := c.nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity) if c.OpConfig.EnablePodAntiAffinity { labelsSet := labels.Set(c.connectionPoolerLabels(role, false).MatchLabels) podTemplate.Spec.Affinity = generatePodAffinity(labelsSet, c.OpConfig.PodAntiAffinityTopologyKey, nodeAffinity) @@ -344,7 +344,7 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio } if *numberOfInstances < constants.ConnectionPoolerMinInstances { - msg := "Adjusted number of connection pooler instances from %d to %d" + msg := "adjusted number of connection pooler instances from %d to %d" c.logger.Warningf(msg, *numberOfInstances, constants.ConnectionPoolerMinInstances) *numberOfInstances = constants.ConnectionPoolerMinInstances @@ -386,7 +386,7 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo { Name: connectionPooler.Name, Port: pgPort, - TargetPort: intstr.IntOrString{StrVal: c.servicePort(connectionPooler.Role)}, + TargetPort: intstr.IntOrString{IntVal: c.servicePort(connectionPooler.Role)}, }, }, Type: v1.ServiceTypeClusterIP, @@ -395,6 +395,10 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo }, } + if c.shouldCreateLoadBalancerForPoolerService(connectionPooler.Role, spec) { + c.configureLoadBalanceService(&serviceSpec, spec.AllowedSourceRanges) + } + service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: connectionPooler.Name, @@ -415,6 +419,29 @@ func (c *Cluster) generateConnectionPoolerService(connectionPooler *ConnectionPo return service } +func (c *Cluster) shouldCreateLoadBalancerForPoolerService(role PostgresRole, spec *acidv1.PostgresSpec) bool { + + switch role { + + case Replica: + // if the value is explicitly set in a Postgresql manifest, follow this setting + if spec.EnableReplicaPoolerLoadBalancer != nil { + return *spec.EnableReplicaPoolerLoadBalancer + } + // otherwise, follow the operator configuration + return c.OpConfig.EnableReplicaPoolerLoadBalancer + + case Master: + if spec.EnableMasterPoolerLoadBalancer != nil { + return *spec.EnableMasterPoolerLoadBalancer + } + return c.OpConfig.EnableMasterPoolerLoadBalancer + + default: + panic(fmt.Sprintf("Unknown role %v", role)) + } +} + //delete connection pooler func (c *Cluster) deleteConnectionPooler(role PostgresRole) (err error) { c.logger.Infof("deleting connection pooler spilo-role=%s", role) @@ -587,7 +614,7 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1. *deployment.Spec.Replicas != *config.NumberOfInstances { sync = true - msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)", + msg := fmt.Sprintf("numberOfInstances is different (having %d, required %d)", *deployment.Spec.Replicas, *config.NumberOfInstances) reasons = append(reasons, msg) } @@ -596,13 +623,14 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1. poolerContainer.Image != config.Image { sync = true - msg := fmt.Sprintf("DockerImage is different (having %s, required %s)", + msg := fmt.Sprintf("dockerImage is different (having %s, required %s)", poolerContainer.Image, config.Image) reasons = append(reasons, msg) } - expectedResources, err := generateResourceRequirements(spec.Resources, - makeDefaultConnectionPoolerResources(&Config.OpConfig)) + expectedResources, err := c.generateResourceRequirements(spec.Resources, + makeDefaultConnectionPoolerResources(&Config.OpConfig), + connectionPoolerContainer) // An error to generate expected resources means something is not quite // right, but for the purpose of robustness do not panic here, just report @@ -610,7 +638,7 @@ func (c *Cluster) needSyncConnectionPoolerDefaults(Config *Config, spec *acidv1. // updates for new resource values). if err == nil && syncResources(&poolerContainer.Resources, expectedResources) { sync = true - msg := fmt.Sprintf("Resources are different (having %+v, required %+v)", + msg := fmt.Sprintf("resources are different (having %+v, required %+v)", poolerContainer.Resources, expectedResources) reasons = append(reasons, msg) } @@ -695,28 +723,6 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look var err error var connectionPoolerNeeded bool - needSync := !reflect.DeepEqual(oldSpec.Spec.ConnectionPooler, newSpec.Spec.ConnectionPooler) - masterChanges, err := diff.Diff(oldSpec.Spec.EnableConnectionPooler, newSpec.Spec.EnableConnectionPooler) - if err != nil { - c.logger.Error("Error in getting diff of master connection pooler changes") - } - replicaChanges, err := diff.Diff(oldSpec.Spec.EnableReplicaConnectionPooler, newSpec.Spec.EnableReplicaConnectionPooler) - if err != nil { - c.logger.Error("Error in getting diff of replica connection pooler changes") - } - - // skip pooler sync when theres no diff or it's deactivated - // but, handling the case when connectionPooler is not there but it is required - // as per spec, hence do not skip syncing in that case, even though there - // is no diff in specs - if (!needSync && len(masterChanges) <= 0 && len(replicaChanges) <= 0) && - ((!needConnectionPooler(&newSpec.Spec) && (c.ConnectionPooler == nil || !needConnectionPooler(&oldSpec.Spec))) || - (c.ConnectionPooler != nil && needConnectionPooler(&newSpec.Spec) && - (c.ConnectionPooler[Master].LookupFunction || c.ConnectionPooler[Replica].LookupFunction))) { - c.logger.Debugln("syncing pooler is not required") - return nil, nil - } - logPoolerEssentials(c.logger, oldSpec, newSpec) // Check and perform the sync requirements for each of the roles. @@ -753,7 +759,8 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look // in between // in this case also do not forget to install lookup function - if !c.ConnectionPooler[role].LookupFunction { + // skip installation in standby clusters, since they are read-only + if !c.ConnectionPooler[role].LookupFunction && c.Spec.StandbyCluster == nil { connectionPooler := c.Spec.ConnectionPooler specSchema := "" specUser := "" @@ -810,31 +817,37 @@ func (c *Cluster) syncConnectionPooler(oldSpec, newSpec *acidv1.Postgresql, Look func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql, role PostgresRole) ( SyncReason, error) { - deployment, err := c.KubeClient. + var ( + deployment *appsv1.Deployment + newDeployment *appsv1.Deployment + service *v1.Service + newService *v1.Service + err error + ) + + syncReason := make([]string, 0) + deployment, err = c.KubeClient. Deployments(c.Namespace). Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}) if err != nil && k8sutil.ResourceNotFound(err) { - msg := "deployment %s for connection pooler synchronization is not found, create it" - c.logger.Warningf(msg, c.connectionPoolerName(role)) + c.logger.Warningf("deployment %s for connection pooler synchronization is not found, create it", c.connectionPoolerName(role)) - deploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) + newDeployment, err = c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) if err != nil { - msg = "could not generate deployment for connection pooler: %v" - return NoSync, fmt.Errorf(msg, err) + return NoSync, fmt.Errorf("could not generate deployment for connection pooler: %v", err) } - deployment, err := c.KubeClient. - Deployments(deploymentSpec.Namespace). - Create(context.TODO(), deploymentSpec, metav1.CreateOptions{}) + deployment, err = c.KubeClient. + Deployments(newDeployment.Namespace). + Create(context.TODO(), newDeployment, metav1.CreateOptions{}) if err != nil { return NoSync, err } c.ConnectionPooler[role].Deployment = deployment } else if err != nil { - msg := "could not get connection pooler deployment to sync: %v" - return NoSync, fmt.Errorf(msg, err) + return NoSync, fmt.Errorf("could not get connection pooler deployment to sync: %v", err) } else { c.ConnectionPooler[role].Deployment = deployment // actual synchronization @@ -864,25 +877,24 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql if oldSpec != nil { specSync, specReason = needSyncConnectionPoolerSpecs(oldConnectionPooler, newConnectionPooler, c.logger) + syncReason = append(syncReason, specReason...) } defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(&c.Config, newConnectionPooler, deployment) - reason := append(specReason, defaultsReason...) + syncReason = append(syncReason, defaultsReason...) if specSync || defaultsSync { - c.logger.Infof("Update connection pooler deployment %s, reason: %+v", - c.connectionPoolerName(role), reason) - newDeploymentSpec, err := c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) + c.logger.Infof("update connection pooler deployment %s, reason: %+v", + c.connectionPoolerName(role), syncReason) + newDeployment, err = c.generateConnectionPoolerDeployment(c.ConnectionPooler[role]) if err != nil { - msg := "could not generate deployment for connection pooler: %v" - return reason, fmt.Errorf(msg, err) + return syncReason, fmt.Errorf("could not generate deployment for connection pooler: %v", err) } - deployment, err := updateConnectionPoolerDeployment(c.KubeClient, - newDeploymentSpec) + deployment, err = updateConnectionPoolerDeployment(c.KubeClient, newDeployment) if err != nil { - return reason, err + return syncReason, err } c.ConnectionPooler[role].Deployment = deployment } @@ -897,31 +909,38 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql c.ConnectionPooler[role].Deployment = deployment } - service, err := c.KubeClient. - Services(c.Namespace). - Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}) - - if err != nil && k8sutil.ResourceNotFound(err) { - msg := "Service %s for connection pooler synchronization is not found, create it" - c.logger.Warningf(msg, c.connectionPoolerName(role)) - - serviceSpec := c.generateConnectionPoolerService(c.ConnectionPooler[role]) - service, err := c.KubeClient. - Services(serviceSpec.Namespace). - Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) - - if err != nil { - return NoSync, err + if service, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.connectionPoolerName(role), metav1.GetOptions{}); err == nil { + c.ConnectionPooler[role].Service = service + desiredSvc := c.generateConnectionPoolerService(c.ConnectionPooler[role]) + if match, reason := c.compareServices(service, desiredSvc); !match { + syncReason = append(syncReason, reason) + c.logServiceChanges(role, service, desiredSvc, false, reason) + newService, err = c.updateService(role, service, desiredSvc) + if err != nil { + return syncReason, fmt.Errorf("could not update %s service to match desired state: %v", role, err) + } + c.ConnectionPooler[role].Service = newService + c.logger.Infof("%s service %q is in the desired state now", role, util.NameFromMeta(desiredSvc.ObjectMeta)) } - c.ConnectionPooler[role].Service = service - - } else if err != nil { - msg := "could not get connection pooler service to sync: %v" - return NoSync, fmt.Errorf(msg, err) - } else { - // Service updates are not supported and probably not that useful anyway - c.ConnectionPooler[role].Service = service + return NoSync, nil } + if !k8sutil.ResourceNotFound(err) { + return NoSync, fmt.Errorf("could not get connection pooler service to sync: %v", err) + } + + c.ConnectionPooler[role].Service = nil + c.logger.Warningf("service %s for connection pooler synchronization is not found, create it", c.connectionPoolerName(role)) + + serviceSpec := c.generateConnectionPoolerService(c.ConnectionPooler[role]) + newService, err = c.KubeClient. + Services(serviceSpec.Namespace). + Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) + + if err != nil { + return NoSync, err + } + c.ConnectionPooler[role].Service = newService + return NoSync, nil } diff --git a/pkg/cluster/connection_pooler_test.go b/pkg/cluster/connection_pooler_test.go index 9b983c7b0..da45899b4 100644 --- a/pkg/cluster/connection_pooler_test.go +++ b/pkg/cluster/connection_pooler_test.go @@ -27,10 +27,6 @@ func boolToPointer(value bool) *bool { return &value } -func int32ToPointer(value int32) *int32 { - return &value -} - func deploymentUpdated(cluster *Cluster, err error, reason SyncReason) error { for _, role := range [2]PostgresRole{Master, Replica} { @@ -294,7 +290,7 @@ func TestConnectionPoolerCreateDeletion(t *testing.T) { ConnectionPoolerDefaultCPULimit: "100m", ConnectionPoolerDefaultMemoryRequest: "100Mi", ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + NumberOfInstances: k8sutil.Int32ToPointer(1), }, PodManagementPolicy: "ordered_ready", Resources: config.Resources{ @@ -401,7 +397,7 @@ func TestConnectionPoolerSync(t *testing.T) { ConnectionPoolerDefaultCPULimit: "100m", ConnectionPoolerDefaultMemoryRequest: "100Mi", ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + NumberOfInstances: k8sutil.Int32ToPointer(1), }, PodManagementPolicy: "ordered_ready", Resources: config.Resources{ @@ -639,7 +635,7 @@ func TestConnectionPoolerSync(t *testing.T) { for _, tt := range tests { tt.cluster.OpConfig.ConnectionPooler.Image = tt.defaultImage tt.cluster.OpConfig.ConnectionPooler.NumberOfInstances = - int32ToPointer(tt.defaultInstances) + k8sutil.Int32ToPointer(tt.defaultInstances) t.Logf("running test for %s [%s]", testName, tt.subTest) @@ -664,7 +660,7 @@ func TestConnectionPoolerPodSpec(t *testing.T) { ReplicationUsername: replicationUserName, }, ConnectionPooler: config.ConnectionPooler{ - MaxDBConnections: int32ToPointer(60), + MaxDBConnections: k8sutil.Int32ToPointer(60), ConnectionPoolerDefaultCPURequest: "100m", ConnectionPoolerDefaultCPULimit: "100m", ConnectionPoolerDefaultMemoryRequest: "100Mi", diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index aa3a5e3be..652f0d0ae 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -14,18 +14,25 @@ import ( "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/retryutil" + "github.com/zalando/postgres-operator/pkg/util/users" ) const ( getUserSQL = `SELECT a.rolname, COALESCE(a.rolpassword, ''), a.rolsuper, a.rolinherit, - a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig, - ARRAY(SELECT b.rolname - FROM pg_catalog.pg_auth_members m - JOIN pg_catalog.pg_authid b ON (m.roleid = b.oid) - WHERE m.member = a.oid) as memberof - FROM pg_catalog.pg_authid a LEFT JOIN pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid) - WHERE a.rolname = ANY($1) - ORDER BY 1;` + a.rolcreaterole, a.rolcreatedb, a.rolcanlogin, s.setconfig, + ARRAY(SELECT b.rolname + FROM pg_catalog.pg_auth_members m + JOIN pg_catalog.pg_authid b ON (m.roleid = b.oid) + WHERE m.member = a.oid) as memberof + FROM pg_catalog.pg_authid a LEFT JOIN pg_db_role_setting s ON (a.oid = s.setrole AND s.setdatabase = 0::oid) + WHERE a.rolname = ANY($1) + ORDER BY 1;` + + getUsersForRetention = `SELECT r.rolname, right(r.rolname, 6) AS roldatesuffix + FROM pg_roles r + JOIN unnest($1::text[]) AS u(name) ON r.rolname LIKE u.name || '%' + AND right(r.rolname, 6) ~ '^[0-9\.]+$' + ORDER BY 1;` getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` getSchemasSQL = `SELECT n.nspname AS dbschema FROM pg_catalog.pg_namespace n @@ -39,6 +46,13 @@ const ( createExtensionSQL = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"` alterExtensionSQL = `ALTER EXTENSION "%s" SET SCHEMA "%s"` + getPublicationsSQL = `SELECT p.pubname, string_agg(pt.schemaname || '.' || pt.tablename, ', ' ORDER BY pt.schemaname, pt.tablename) + FROM pg_publication p + JOIN pg_publication_tables pt ON pt.pubname = p.pubname + GROUP BY p.pubname;` + createPublicationSQL = `CREATE PUBLICATION "%s" FOR TABLE %s WITH (publish = 'insert, update');` + alterPublicationSQL = `ALTER PUBLICATION "%s" SET TABLE %s;` + globalDefaultPrivilegesSQL = `SET ROLE TO "%s"; ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO "%s","%s"; ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO "%s"; @@ -227,6 +241,65 @@ func (c *Cluster) readPgUsersFromDatabase(userNames []string) (users spec.PgUser return users, nil } +func findUsersFromRotation(rotatedUsers []string, db *sql.DB) (map[string]string, error) { + extraUsers := make(map[string]string, 0) + rows, err := db.Query(getUsersForRetention, pq.Array(rotatedUsers)) + if err != nil { + return nil, fmt.Errorf("query failed: %v", err) + } + defer func() { + if err2 := rows.Close(); err2 != nil { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + }() + + for rows.Next() { + var ( + rolname, roldatesuffix string + ) + err := rows.Scan(&rolname, &roldatesuffix) + if err != nil { + return nil, fmt.Errorf("error when processing rows of deprecated users: %v", err) + } + extraUsers[rolname] = roldatesuffix + } + + return extraUsers, nil +} + +func (c *Cluster) cleanupRotatedUsers(rotatedUsers []string, db *sql.DB) error { + c.setProcessName("checking for rotated users to remove from the database due to configured retention") + extraUsers, err := findUsersFromRotation(rotatedUsers, db) + if err != nil { + return fmt.Errorf("error when querying for deprecated users from password rotation: %v", err) + } + + // make sure user retention policy aligns with rotation interval + retenionDays := c.OpConfig.PasswordRotationUserRetention + if retenionDays < 2*c.OpConfig.PasswordRotationInterval { + retenionDays = 2 * c.OpConfig.PasswordRotationInterval + c.logger.Warnf("user retention days too few compared to rotation interval %d - setting it to %d", c.OpConfig.PasswordRotationInterval, retenionDays) + } + retentionDate := time.Now().AddDate(0, 0, int(retenionDays)*-1) + + for rotatedUser, dateSuffix := range extraUsers { + userCreationDate, err := time.Parse("060102", dateSuffix) + if err != nil { + c.logger.Errorf("could not parse creation date suffix of user %q: %v", rotatedUser, err) + continue + } + if retentionDate.After(userCreationDate) { + c.logger.Infof("dropping user %q due to configured days in password_rotation_user_retention", rotatedUser) + if err = users.DropPgUser(rotatedUser, db); err != nil { + c.logger.Errorf("could not drop role %q: %v", rotatedUser, err) + continue + } + } + } + + return nil +} + // getDatabases returns the map of current databases with owners // The caller is responsible for opening and closing the database connection func (c *Cluster) getDatabases() (dbs map[string]string, err error) { @@ -506,6 +579,68 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi return nil } +// getPublications returns the list of current database publications with tables +// The caller is responsible for opening and closing the database connection +func (c *Cluster) getPublications() (publications map[string]string, err error) { + var ( + rows *sql.Rows + ) + + if rows, err = c.pgDb.Query(getPublicationsSQL); err != nil { + return nil, fmt.Errorf("could not query database publications: %v", err) + } + + defer func() { + if err2 := rows.Close(); err2 != nil { + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + } + }() + + dbPublications := make(map[string]string) + + for rows.Next() { + var ( + dbPublication string + dbPublicationTables string + ) + + if err = rows.Scan(&dbPublication, &dbPublicationTables); err != nil { + return nil, fmt.Errorf("error when processing row: %v", err) + } + dbPublications[dbPublication] = dbPublicationTables + } + + return dbPublications, err +} + +// executeCreatePublication creates new publication for given tables +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeCreatePublication(pubName, tableList string) error { + return c.execCreateOrAlterPublication(pubName, tableList, createPublicationSQL, + "creating publication", "create publication") +} + +// executeAlterExtension changes the table list of the given publication. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeAlterPublication(pubName, tableList string) error { + return c.execCreateOrAlterPublication(pubName, tableList, alterPublicationSQL, + "changing publication", "alter publication tables") +} + +func (c *Cluster) execCreateOrAlterPublication(pubName, tableList, statement, doing, operation string) error { + + c.logger.Debugf("%s %q with table list %q", doing, pubName, tableList) + if _, err := c.pgDb.Exec(fmt.Sprintf(statement, pubName, tableList)); err != nil { + return fmt.Errorf("could not execute %s: %v", operation, err) + } + + return nil +} + // Creates a connection pool credentials lookup function in every database to // perform remote authentication. func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 366570c3c..6cdf379bd 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -8,11 +8,13 @@ import ( "sort" "strings" + "github.com/pkg/errors" "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -24,6 +26,8 @@ import ( "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "github.com/zalando/postgres-operator/pkg/util/patroni" + "github.com/zalando/postgres-operator/pkg/util/retryutil" batchv1 "k8s.io/api/batch/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" "k8s.io/apimachinery/pkg/labels" @@ -34,6 +38,8 @@ const ( patroniPGBinariesParameterName = "bin_dir" patroniPGHBAConfParameterName = "pg_hba" localHost = "127.0.0.1/32" + scalyrSidecarName = "scalyr-sidecar" + logicalBackupContainerName = "logical-backup" connectionPoolerContainer = "connection-pooler" pgPort = 5432 ) @@ -50,6 +56,7 @@ type patroniDCS struct { MaximumLagOnFailover float32 `json:"maximum_lag_on_failover,omitempty"` SynchronousMode bool `json:"synchronous_mode,omitempty"` SynchronousModeStrict bool `json:"synchronous_mode_strict,omitempty"` + SynchronousNodeCount uint32 `json:"synchronous_node_count,omitempty"` PGBootstrapConfiguration map[string]interface{} `json:"postgresql,omitempty"` Slots map[string]map[string]string `json:"slots,omitempty"` } @@ -98,24 +105,22 @@ func (c *Cluster) serviceAddress(role PostgresRole) string { return "" } -func (c *Cluster) servicePort(role PostgresRole) string { +func (c *Cluster) servicePort(role PostgresRole) int32 { service, exist := c.Services[role] if exist { - return fmt.Sprint(service.Spec.Ports[0].Port) + return service.Spec.Ports[0].Port } - c.logger.Warningf("No service for role %s", role) - return "" + c.logger.Warningf("No service for role %s - defaulting to port %d", role, pgPort) + return pgPort } func (c *Cluster) podDisruptionBudgetName() string { return c.OpConfig.PDBNameFormat.Format("cluster", c.Name) } -func (c *Cluster) makeDefaultResources() acidv1.Resources { - - config := c.OpConfig +func makeDefaultResources(config *config.Config) acidv1.Resources { defaultRequests := acidv1.ResourceDescription{ CPU: config.Resources.DefaultCPURequest, @@ -132,25 +137,61 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources { } } -func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) { - var err error +func (c *Cluster) enforceMinResourceLimits(resources *v1.ResourceRequirements) error { + var ( + isSmaller bool + err error + msg string + ) - specRequests := resources.ResourceRequests - specLimits := resources.ResourceLimits - - result := v1.ResourceRequirements{} - - result.Requests, err = fillResourceList(specRequests, defaultResources.ResourceRequests) - if err != nil { - return nil, fmt.Errorf("could not fill resource requests: %v", err) + // setting limits too low can cause unnecessary evictions / OOM kills + cpuLimit := resources.Limits[v1.ResourceCPU] + minCPULimit := c.OpConfig.MinCPULimit + if minCPULimit != "" { + isSmaller, err = util.IsSmallerQuantity(cpuLimit.String(), minCPULimit) + if err != nil { + return fmt.Errorf("could not compare defined CPU limit %s for %q container with configured minimum value %s: %v", + cpuLimit.String(), constants.PostgresContainerName, minCPULimit, err) + } + if isSmaller { + msg = fmt.Sprintf("defined CPU limit %s for %q container is below required minimum %s and will be increased", + cpuLimit.String(), constants.PostgresContainerName, minCPULimit) + c.logger.Warningf(msg) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", msg) + resources.Limits[v1.ResourceCPU], _ = resource.ParseQuantity(minCPULimit) + } } - result.Limits, err = fillResourceList(specLimits, defaultResources.ResourceLimits) - if err != nil { - return nil, fmt.Errorf("could not fill resource limits: %v", err) + memoryLimit := resources.Limits[v1.ResourceMemory] + minMemoryLimit := c.OpConfig.MinMemoryLimit + if minMemoryLimit != "" { + isSmaller, err = util.IsSmallerQuantity(memoryLimit.String(), minMemoryLimit) + if err != nil { + return fmt.Errorf("could not compare defined memory limit %s for %q container with configured minimum value %s: %v", + memoryLimit.String(), constants.PostgresContainerName, minMemoryLimit, err) + } + if isSmaller { + msg = fmt.Sprintf("defined memory limit %s for %q container is below required minimum %s and will be increased", + memoryLimit.String(), constants.PostgresContainerName, minMemoryLimit) + c.logger.Warningf(msg) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "ResourceLimits", msg) + resources.Limits[v1.ResourceMemory], _ = resource.ParseQuantity(minMemoryLimit) + } } - return &result, nil + return nil +} + +func setMemoryRequestToLimit(resources *v1.ResourceRequirements, containerName string, logger *logrus.Entry) { + + requests := resources.Requests[v1.ResourceMemory] + limits := resources.Limits[v1.ResourceMemory] + isSmaller := requests.Cmp(limits) == -1 + if isSmaller { + logger.Warningf("memory request of %s for %q container is increased to match memory limit of %s", + requests.String(), containerName, limits.String()) + resources.Requests[v1.ResourceMemory] = limits + } } func fillResourceList(spec acidv1.ResourceDescription, defaults acidv1.ResourceDescription) (v1.ResourceList, error) { @@ -183,6 +224,44 @@ func fillResourceList(spec acidv1.ResourceDescription, defaults acidv1.ResourceD return requests, nil } +func (c *Cluster) generateResourceRequirements( + resources *acidv1.Resources, + defaultResources acidv1.Resources, + containerName string) (*v1.ResourceRequirements, error) { + var err error + specRequests := acidv1.ResourceDescription{} + specLimits := acidv1.ResourceDescription{} + result := v1.ResourceRequirements{} + + if resources != nil { + specRequests = resources.ResourceRequests + specLimits = resources.ResourceLimits + } + + result.Requests, err = fillResourceList(specRequests, defaultResources.ResourceRequests) + if err != nil { + return nil, fmt.Errorf("could not fill resource requests: %v", err) + } + + result.Limits, err = fillResourceList(specLimits, defaultResources.ResourceLimits) + if err != nil { + return nil, fmt.Errorf("could not fill resource limits: %v", err) + } + + // enforce minimum cpu and memory limits for Postgres containers only + if containerName == constants.PostgresContainerName { + if err = c.enforceMinResourceLimits(&result); err != nil { + return nil, fmt.Errorf("could not enforce minimum resource limits: %v", err) + } + } + + if c.OpConfig.SetMemoryRequestToLimit { + setMemoryRequestToLimit(&result, containerName, c.logger) + } + + return &result, nil +} + func generateSpiloJSONConfiguration(pg *acidv1.PostgresqlParam, patroni *acidv1.Patroni, pamRoleName string, EnablePgVersionEnvVar bool, logger *logrus.Entry) (string, error) { config := spiloConfiguration{} @@ -262,6 +341,9 @@ PatroniInitDBParams: if patroni.SynchronousModeStrict { config.Bootstrap.DCS.SynchronousModeStrict = patroni.SynchronousModeStrict } + if patroni.SynchronousNodeCount >= 1 { + config.Bootstrap.DCS.SynchronousNodeCount = patroni.SynchronousNodeCount + } config.PgLocalConfiguration = make(map[string]interface{}) @@ -327,7 +409,7 @@ func generateCapabilities(capabilities []string) *v1.Capabilities { return nil } -func nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAffinity) *v1.Affinity { +func (c *Cluster) nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAffinity) *v1.Affinity { if len(nodeReadinessLabel) == 0 && nodeAffinity == nil { return nil } @@ -352,8 +434,18 @@ func nodeAffinity(nodeReadinessLabel map[string]string, nodeAffinity *v1.NodeAff }, } } else { - nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{ - NodeSelectorTerms: append(nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, nodeReadinessSelectorTerm), + if c.OpConfig.NodeReadinessLabelMerge == "OR" { + manifestTerms := nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + manifestTerms = append(manifestTerms, nodeReadinessSelectorTerm) + nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{ + NodeSelectorTerms: manifestTerms, + } + } else if c.OpConfig.NodeReadinessLabelMerge == "AND" { + for i, nodeSelectorTerm := range nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + manifestExpressions := nodeSelectorTerm.MatchExpressions + manifestExpressions = append(manifestExpressions, matchExpressions...) + nodeAffinityCopy.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i] = v1.NodeSelectorTerm{MatchExpressions: manifestExpressions} + } } } } @@ -467,15 +559,15 @@ func generateContainer( Resources: *resourceRequirements, Ports: []v1.ContainerPort{ { - ContainerPort: 8008, + ContainerPort: patroni.ApiPort, Protocol: v1.ProtocolTCP, }, { - ContainerPort: 5432, + ContainerPort: pgPort, Protocol: v1.ProtocolTCP, }, { - ContainerPort: 8080, + ContainerPort: patroni.ApiPort, Protocol: v1.ProtocolTCP, }, }, @@ -490,22 +582,20 @@ func generateContainer( } } -func generateSidecarContainers(sidecars []acidv1.Sidecar, - defaultResources acidv1.Resources, startIndex int, logger *logrus.Entry) ([]v1.Container, error) { +func (c *Cluster) generateSidecarContainers(sidecars []acidv1.Sidecar, + defaultResources acidv1.Resources, startIndex int) ([]v1.Container, error) { if len(sidecars) > 0 { result := make([]v1.Container, 0) for index, sidecar := range sidecars { + var resourcesSpec acidv1.Resources + if sidecar.Resources == nil { + resourcesSpec = acidv1.Resources{} + } else { + sidecar.Resources.DeepCopyInto(&resourcesSpec) + } - resources, err := generateResourceRequirements( - makeResources( - sidecar.Resources.ResourceRequests.CPU, - sidecar.Resources.ResourceRequests.Memory, - sidecar.Resources.ResourceLimits.CPU, - sidecar.Resources.ResourceLimits.Memory, - ), - defaultResources, - ) + resources, err := c.generateResourceRequirements(&resourcesSpec, defaultResources, sidecar.Name) if err != nil { return nil, err } @@ -559,8 +649,7 @@ func patchSidecarContainers(in []v1.Container, volumeMounts []v1.VolumeMount, su }, }, } - mergedEnv := append(env, container.Env...) - container.Env = deduplicateEnvVars(mergedEnv, container.Name, logger) + container.Env = appendEnvVars(env, container.Env...) result = append(result, container) } @@ -673,7 +762,14 @@ func (c *Cluster) generatePodTemplate( } // generatePodEnvVars generates environment variables for the Spilo Pod -func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration string, cloneDescription *acidv1.CloneDescription, standbyDescription *acidv1.StandbyDescription, customPodEnvVarsList []v1.EnvVar) []v1.EnvVar { +func (c *Cluster) generateSpiloPodEnvVars( + uid types.UID, + spiloConfiguration string, + cloneDescription *acidv1.CloneDescription, + standbyDescription *acidv1.StandbyDescription) []v1.EnvVar { + + // hard-coded set of environment variables we need + // to guarantee core functionality of the operator envVars := []v1.EnvVar{ { Name: "SCOPE", @@ -748,6 +844,11 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri Value: c.OpConfig.PamRoleName, }, } + + if c.OpConfig.EnableSpiloWalPathCompat { + envVars = append(envVars, v1.EnvVar{Name: "ENABLE_WAL_PATH_COMPAT", Value: "true"}) + } + if c.OpConfig.EnablePgVersionEnvVar { envVars = append(envVars, v1.EnvVar{Name: "PGVERSION", Value: c.GetDesiredMajorVersion()}) } @@ -775,77 +876,87 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...) } - if c.Spec.StandbyCluster != nil { + if standbyDescription != nil { envVars = append(envVars, c.generateStandbyEnvironment(standbyDescription)...) } - // add vars taken from pod_environment_configmap and pod_environment_secret first - // (to allow them to override the globals set in the operator config) - if len(customPodEnvVarsList) > 0 { - envVars = append(envVars, customPodEnvVarsList...) + // fetch cluster-specific variables that will override all subsequent global variables + if len(c.Spec.Env) > 0 { + envVars = appendEnvVars(envVars, c.Spec.Env...) } + // fetch variables from custom environment Secret + // that will override all subsequent global variables + secretEnvVarsList, err := c.getPodEnvironmentSecretVariables() + if err != nil { + c.logger.Warningf("%v", err) + } + envVars = appendEnvVars(envVars, secretEnvVarsList...) + + // fetch variables from custom environment ConfigMap + // that will override all subsequent global variables + configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables() + if err != nil { + c.logger.Warningf("%v", err) + } + envVars = appendEnvVars(envVars, configMapEnvVarsList...) + + // global variables derived from operator configuration + opConfigEnvVars := make([]v1.EnvVar, 0) if c.OpConfig.WALES3Bucket != "" { - envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } if c.OpConfig.WALGSBucket != "" { - envVars = append(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } if c.OpConfig.WALAZStorageAccount != "" { - envVars = append(envVars, v1.EnvVar{Name: "AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } if c.OpConfig.GCPCredentials != "" { - envVars = append(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) } if c.OpConfig.LogS3Bucket != "" { - envVars = append(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket}) - envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + opConfigEnvVars = append(opConfigEnvVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""}) } + envVars = appendEnvVars(envVars, opConfigEnvVars...) + return envVars } -// deduplicateEnvVars makes sure there are no duplicate in the target envVar array. While Kubernetes already -// deduplicates variables defined in a container, it leaves the last definition in the list and this behavior is not -// well-documented, which means that the behavior can be reversed at some point (it may also start producing an error). -// Therefore, the merge is done by the operator, the entries that are ahead in the passed list take priority over those -// that are behind, and only the name is considered in order to eliminate duplicates. -func deduplicateEnvVars(input []v1.EnvVar, containerName string, logger *logrus.Entry) []v1.EnvVar { - result := make([]v1.EnvVar, 0) - names := make(map[string]int) - - for i, va := range input { - if names[va.Name] == 0 { - names[va.Name]++ - result = append(result, input[i]) - } else if names[va.Name] == 1 { - names[va.Name]++ - - // Some variables (those to configure the WAL_ and LOG_ shipping) may be overwritten, only log as info - if strings.HasPrefix(va.Name, "WAL_") || strings.HasPrefix(va.Name, "LOG_") { - logger.Infof("global variable %q has been overwritten by configmap/secret for container %q", - va.Name, containerName) - } else { - logger.Warningf("variable %q is defined in %q more than once, the subsequent definitions are ignored", - va.Name, containerName) - } +func appendEnvVars(envs []v1.EnvVar, appEnv ...v1.EnvVar) []v1.EnvVar { + collectedEnvs := envs + for _, env := range appEnv { + env.Name = strings.ToUpper(env.Name) + if !isEnvVarPresent(collectedEnvs, env.Name) { + collectedEnvs = append(collectedEnvs, env) } } - return result + return collectedEnvs } -// Return list of variables the pod recieved from the configured ConfigMap +func isEnvVarPresent(envs []v1.EnvVar, key string) bool { + for _, env := range envs { + if env.Name == key { + return true + } + } + return false +} + +// Return list of variables the pod received from the configured ConfigMap func (c *Cluster) getPodEnvironmentConfigMapVariables() ([]v1.EnvVar, error) { configMapPodEnvVarsList := make([]v1.EnvVar, 0) @@ -869,9 +980,11 @@ func (c *Cluster) getPodEnvironmentConfigMapVariables() ([]v1.EnvVar, error) { return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) } } + for k, v := range cm.Data { configMapPodEnvVarsList = append(configMapPodEnvVarsList, v1.EnvVar{Name: k, Value: v}) } + sort.Slice(configMapPodEnvVarsList, func(i, j int) bool { return configMapPodEnvVarsList[i].Name < configMapPodEnvVarsList[j].Name }) return configMapPodEnvVarsList, nil } @@ -883,12 +996,30 @@ func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) { return secretPodEnvVarsList, nil } - secret, err := c.KubeClient.Secrets(c.Namespace).Get( - context.TODO(), - c.OpConfig.PodEnvironmentSecret, - metav1.GetOptions{}) + secret := &v1.Secret{} + var notFoundErr error + err := retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, + func() (bool, error) { + var err error + secret, err = c.KubeClient.Secrets(c.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentSecret, + metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + notFoundErr = err + return false, nil + } + return false, err + } + return true, nil + }, + ) + if notFoundErr != nil && err != nil { + err = errors.Wrap(notFoundErr, err.Error()) + } if err != nil { - return nil, fmt.Errorf("could not read Secret PodEnvironmentSecretName: %v", err) + return nil, errors.Wrap(err, "could not read Secret PodEnvironmentSecretName") } for k := range secret.Data { @@ -903,6 +1034,7 @@ func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) { }}) } + sort.Slice(secretPodEnvVarsList, func(i, j int) bool { return secretPodEnvVarsList[i].Name < secretPodEnvVarsList[j].Name }) return secretPodEnvVarsList, nil } @@ -951,6 +1083,22 @@ func extractPgVersionFromBinPath(binPath string, template string) (string, error return fmt.Sprintf("%v", pgVersion), nil } +func generateSpiloReadinessProbe() *v1.Probe { + return &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/readiness", + Port: intstr.IntOrString{IntVal: patroni.ApiPort}, + }, + }, + InitialDelaySeconds: 6, + PeriodSeconds: 10, + TimeoutSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 3, + } +} + func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.StatefulSet, error) { var ( @@ -962,63 +1110,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef additionalVolumes = spec.AdditionalVolumes ) - // Improve me. Please. - if c.OpConfig.SetMemoryRequestToLimit { - - // controller adjusts the default memory request at operator startup - - request := spec.Resources.ResourceRequests.Memory - if request == "" { - request = c.OpConfig.Resources.DefaultMemoryRequest - } - - limit := spec.Resources.ResourceLimits.Memory - if limit == "" { - limit = c.OpConfig.Resources.DefaultMemoryLimit - } - - isSmaller, err := util.IsSmallerQuantity(request, limit) - if err != nil { - return nil, err - } - if isSmaller { - c.logger.Warningf("The memory request of %v for the Postgres container is increased to match the memory limit of %v.", request, limit) - spec.Resources.ResourceRequests.Memory = limit - - } - - // controller adjusts the Scalyr sidecar request at operator startup - // as this sidecar is managed separately - - // adjust sidecar containers defined for that particular cluster - for _, sidecar := range spec.Sidecars { - - // TODO #413 - sidecarRequest := sidecar.Resources.ResourceRequests.Memory - if request == "" { - request = c.OpConfig.Resources.DefaultMemoryRequest - } - - sidecarLimit := sidecar.Resources.ResourceLimits.Memory - if limit == "" { - limit = c.OpConfig.Resources.DefaultMemoryLimit - } - - isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit) - if err != nil { - return nil, err - } - if isSmaller { - c.logger.Warningf("The memory request of %v for the %v sidecar container is increased to match the memory limit of %v.", sidecar.Resources.ResourceRequests.Memory, sidecar.Name, sidecar.Resources.ResourceLimits.Memory) - sidecar.Resources.ResourceRequests.Memory = sidecar.Resources.ResourceLimits.Memory - } - } - - } - - defaultResources := c.makeDefaultResources() - - resourceRequirements, err := generateResourceRequirements(spec.Resources, defaultResources) + defaultResources := makeDefaultResources(&c.OpConfig) + resourceRequirements, err := c.generateResourceRequirements( + spec.Resources, defaultResources, constants.PostgresContainerName) if err != nil { return nil, fmt.Errorf("could not generate resource requirements: %v", err) } @@ -1030,41 +1124,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef initContainers = spec.InitContainers } - spiloCompathWalPathList := make([]v1.EnvVar, 0) - if c.OpConfig.EnableSpiloWalPathCompat { - spiloCompathWalPathList = append(spiloCompathWalPathList, - v1.EnvVar{ - Name: "ENABLE_WAL_PATH_COMPAT", - Value: "true", - }, - ) - } - - // fetch env vars from custom ConfigMap - configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables() - if err != nil { - return nil, err - } - - // fetch env vars from custom ConfigMap - secretEnvVarsList, err := c.getPodEnvironmentSecretVariables() - if err != nil { - return nil, err - } - - // concat all custom pod env vars and sort them - customPodEnvVarsList := append(spiloCompathWalPathList, configMapEnvVarsList...) - customPodEnvVarsList = append(customPodEnvVarsList, secretEnvVarsList...) - sort.Slice(customPodEnvVarsList, - func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name }) - - if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" { - return nil, fmt.Errorf("s3_wal_path is empty for standby cluster") - } - // backward compatible check for InitContainers if spec.InitContainersOld != nil { - msg := "Manifest parameter init_containers is deprecated." + msg := "manifest parameter init_containers is deprecated." if spec.InitContainers == nil { c.logger.Warningf("%s Consider using initContainers instead.", msg) spec.InitContainers = spec.InitContainersOld @@ -1075,7 +1137,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // backward compatible check for PodPriorityClassName if spec.PodPriorityClassNameOld != "" { - msg := "Manifest parameter pod_priority_class_name is deprecated." + msg := "manifest parameter pod_priority_class_name is deprecated." if spec.PodPriorityClassName == "" { c.logger.Warningf("%s Consider using podPriorityClassName instead.", msg) spec.PodPriorityClassName = spec.PodPriorityClassNameOld @@ -1094,9 +1156,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.Postgresql.GetUID(), spiloConfiguration, spec.Clone, - spec.StandbyCluster, - customPodEnvVarsList, - ) + spec.StandbyCluster) // pickup the docker image for the spilo container effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage) @@ -1139,7 +1199,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // use the same filenames as Secret resources by default certFile := ensurePath(spec.TLS.CertificateFile, mountPath, "tls.crt") privateKeyFile := ensurePath(spec.TLS.PrivateKeyFile, mountPath, "tls.key") - spiloEnvVars = append( + spiloEnvVars = appendEnvVars( spiloEnvVars, v1.EnvVar{Name: "SSL_CERTIFICATE_FILE", Value: certFile}, v1.EnvVar{Name: "SSL_PRIVATE_KEY_FILE", Value: privateKeyFile}, @@ -1153,7 +1213,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef } caFile := ensurePath(spec.TLS.CAFile, mountPathCA, "") - spiloEnvVars = append( + spiloEnvVars = appendEnvVars( spiloEnvVars, v1.EnvVar{Name: "SSL_CA_FILE", Value: caFile}, ) @@ -1178,13 +1238,16 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef spiloContainer := generateContainer(constants.PostgresContainerName, &effectiveDockerImage, resourceRequirements, - deduplicateEnvVars(spiloEnvVars, constants.PostgresContainerName, c.logger), + spiloEnvVars, volumeMounts, c.OpConfig.Resources.SpiloPrivileged, c.OpConfig.Resources.SpiloAllowPrivilegeEscalation, generateCapabilities(c.OpConfig.AdditionalPodCapabilities), ) + // Patroni responds 200 to probe only if it either owns the leader lock or postgres is running and DCS is accessible + spiloContainer.ReadinessProbe = generateSpiloReadinessProbe() + // generate container specs for sidecars specified in the cluster manifest clusterSpecificSidecars := []v1.Container{} if spec.Sidecars != nil && len(spec.Sidecars) > 0 { @@ -1193,7 +1256,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.logger.Warningf("sidecars specified but disabled in configuration - next statefulset creation would fail") } - if clusterSpecificSidecars, err = generateSidecarContainers(spec.Sidecars, defaultResources, 0, c.logger); err != nil { + if clusterSpecificSidecars, err = c.generateSidecarContainers(spec.Sidecars, defaultResources, 0); err != nil { return nil, fmt.Errorf("could not generate sidecar containers: %v", err) } } @@ -1204,7 +1267,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef for name, dockerImage := range c.OpConfig.SidecarImages { globalSidecarsByDockerImage = append(globalSidecarsByDockerImage, acidv1.Sidecar{Name: name, DockerImage: dockerImage}) } - if globalSidecarContainersByDockerImage, err = generateSidecarContainers(globalSidecarsByDockerImage, defaultResources, len(clusterSpecificSidecars), c.logger); err != nil { + if globalSidecarContainersByDockerImage, err = c.generateSidecarContainers(globalSidecarsByDockerImage, defaultResources, len(clusterSpecificSidecars)); err != nil { return nil, fmt.Errorf("could not generate sidecar containers: %v", err) } // make the resulting list reproducible @@ -1217,7 +1280,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // generate scalyr sidecar container var scalyrSidecars []v1.Container if scalyrSidecar, err := - generateScalyrSidecarSpec(c.Name, + c.generateScalyrSidecarSpec(c.Name, c.OpConfig.ScalyrAPIKey, c.OpConfig.ScalyrServerURL, c.OpConfig.ScalyrImage, @@ -1225,8 +1288,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef c.OpConfig.ScalyrMemoryRequest, c.OpConfig.ScalyrCPULimit, c.OpConfig.ScalyrMemoryLimit, - defaultResources, - c.logger); err != nil { + defaultResources); err != nil { return nil, fmt.Errorf("could not generate Scalyr sidecar: %v", err) } else { if scalyrSidecar != nil { @@ -1236,7 +1298,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef sidecarContainers, conflicts := mergeContainers(clusterSpecificSidecars, c.Config.OpConfig.SidecarContainers, globalSidecarContainersByDockerImage, scalyrSidecars) for containerName := range conflicts { - c.logger.Warningf("a sidecar is specified twice. Ignoring sidecar %q in favor of %q with high a precendence", + c.logger.Warningf("a sidecar is specified twice. Ignoring sidecar %q in favor of %q with high a precedence", containerName, containerName) } @@ -1259,7 +1321,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef effectiveRunAsUser, effectiveRunAsGroup, effectiveFSGroup, - nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity), + c.nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity), spec.SchedulerName, int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, @@ -1276,7 +1338,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef return nil, fmt.Errorf("could not generate pod template: %v", err) } - if volumeClaimTemplate, err = generatePersistentVolumeClaimTemplate(spec.Volume.Size, + if volumeClaimTemplate, err = c.generatePersistentVolumeClaimTemplate(spec.Volume.Size, spec.Volume.StorageClass, spec.Volume.Selector); err != nil { return nil, fmt.Errorf("could not generate volume claim template: %v", err) } @@ -1336,12 +1398,12 @@ func (c *Cluster) generatePodAnnotations(spec *acidv1.PostgresSpec) map[string]s return annotations } -func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string, +func (c *Cluster) generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage string, scalyrCPURequest string, scalyrMemoryRequest string, scalyrCPULimit string, scalyrMemoryLimit string, - defaultResources acidv1.Resources, logger *logrus.Entry) (*v1.Container, error) { + defaultResources acidv1.Resources) (*v1.Container, error) { if APIKey == "" || dockerImage == "" { if APIKey == "" && dockerImage != "" { - logger.Warning("Not running Scalyr sidecar: SCALYR_API_KEY must be defined") + c.logger.Warning("Not running Scalyr sidecar: SCALYR_API_KEY must be defined") } return nil, nil } @@ -1351,7 +1413,8 @@ func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage strin scalyrCPULimit, scalyrMemoryLimit, ) - resourceRequirementsScalyrSidecar, err := generateResourceRequirements(resourcesScalyrSidecar, defaultResources) + resourceRequirementsScalyrSidecar, err := c.generateResourceRequirements( + &resourcesScalyrSidecar, defaultResources, scalyrSidecarName) if err != nil { return nil, fmt.Errorf("invalid resources for Scalyr sidecar: %v", err) } @@ -1369,7 +1432,7 @@ func generateScalyrSidecarSpec(clusterName, APIKey, serverURL, dockerImage strin env = append(env, v1.EnvVar{Name: "SCALYR_SERVER_URL", Value: serverURL}) } return &v1.Container{ - Name: "scalyr-sidecar", + Name: scalyrSidecarName, Image: dockerImage, Env: env, ImagePullPolicy: v1.PullIfNotPresent, @@ -1468,13 +1531,13 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec, mountPaths := map[string]acidv1.AdditionalVolume{} for i, additionalVolume := range additionalVolumes { if previousVolume, exist := mountPaths[additionalVolume.MountPath]; exist { - msg := "Volume %+v cannot be mounted to the same path as %+v" + msg := "volume %+v cannot be mounted to the same path as %+v" c.logger.Warningf(msg, additionalVolume, previousVolume) continue } if additionalVolume.MountPath == constants.PostgresDataMount { - msg := "Cannot mount volume on postgresql data directory, %+v" + msg := "cannot mount volume on postgresql data directory, %+v" c.logger.Warningf(msg, additionalVolume) continue } @@ -1487,7 +1550,7 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec, for _, target := range additionalVolume.TargetContainers { if target == "all" && len(additionalVolume.TargetContainers) != 1 { - msg := `Target containers could be either "all" or a list + msg := `target containers could be either "all" or a list of containers, mixing those is not allowed, %+v` c.logger.Warningf(msg, additionalVolume) continue @@ -1525,21 +1588,12 @@ func (c *Cluster) addAdditionalVolumes(podSpec *v1.PodSpec, podSpec.Volumes = volumes } -func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string, +func (c *Cluster) generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string, volumeSelector *metav1.LabelSelector) (*v1.PersistentVolumeClaim, error) { var storageClassName *string - - metadata := metav1.ObjectMeta{ - Name: constants.DataVolumeName, - } if volumeStorageClass != "" { - // TODO: remove the old annotation, switching completely to the StorageClassName field. - metadata.Annotations = map[string]string{"volume.beta.kubernetes.io/storage-class": volumeStorageClass} storageClassName = &volumeStorageClass - } else { - metadata.Annotations = map[string]string{"volume.alpha.kubernetes.io/storage-class": "default"} - storageClassName = nil } quantity, err := resource.ParseQuantity(volumeSize) @@ -1549,7 +1603,11 @@ func generatePersistentVolumeClaimTemplate(volumeSize, volumeStorageClass string volumeMode := v1.PersistentVolumeFilesystem volumeClaim := &v1.PersistentVolumeClaim{ - ObjectMeta: metadata, + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DataVolumeName, + Annotations: c.annotationsSet(nil), + Labels: c.labelsSet(true), + }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, Resources: v1.ResourceRequirements{ @@ -1591,7 +1649,7 @@ func (c *Cluster) generateUserSecrets() map[string]*v1.Secret { func (c *Cluster) generateSingleUserSecret(namespace string, pgUser spec.PgUser) *v1.Secret { //Skip users with no password i.e. human users (they'll be authenticated using pam) if pgUser.Password == "" { - if pgUser.Origin != spec.RoleOriginTeamsAPI { + if pgUser.Origin != spec.RoleOriginTeamsAPI && pgUser.Origin != spec.RoleOriginSpilo { c.logger.Warningf("could not generate secret for a non-teamsAPI role %q: role has no password", pgUser.Name) } @@ -1659,32 +1717,18 @@ func (c *Cluster) shouldCreateLoadBalancerForService(role PostgresRole, spec *ac func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) *v1.Service { serviceSpec := v1.ServiceSpec{ - Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}}, + Ports: []v1.ServicePort{{Name: "postgresql", Port: pgPort, TargetPort: intstr.IntOrString{IntVal: pgPort}}}, Type: v1.ServiceTypeClusterIP, } + // no selector for master, see https://github.com/zalando/postgres-operator/issues/340 + // if kubernetes_use_configmaps is set master service needs a selector if role == Replica || c.patroniKubernetesUseConfigMaps() { serviceSpec.Selector = c.roleLabelsSet(false, role) } if c.shouldCreateLoadBalancerForService(role, spec) { - - // spec.AllowedSourceRanges evaluates to the empty slice of zero length - // when omitted or set to 'null'/empty sequence in the PG manifest - if len(spec.AllowedSourceRanges) > 0 { - serviceSpec.LoadBalancerSourceRanges = spec.AllowedSourceRanges - } else { - // safe default value: lock a load balancer only to the local address unless overridden explicitly - serviceSpec.LoadBalancerSourceRanges = []string{localHost} - } - - c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges) - serviceSpec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(c.OpConfig.ExternalTrafficPolicy) - serviceSpec.Type = v1.ServiceTypeLoadBalancer - } else if role == Replica { - // before PR #258, the replica service was only created if allocated a LB - // now we always create the service but warn if the LB is absent - c.logger.Debugf("No load balancer created for the replica service") + c.configureLoadBalanceService(&serviceSpec, spec.AllowedSourceRanges) } service := &v1.Service{ @@ -1700,6 +1744,21 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) return service } +func (c *Cluster) configureLoadBalanceService(serviceSpec *v1.ServiceSpec, sourceRanges []string) { + // spec.AllowedSourceRanges evaluates to the empty slice of zero length + // when omitted or set to 'null'/empty sequence in the PG manifest + if len(sourceRanges) > 0 { + serviceSpec.LoadBalancerSourceRanges = sourceRanges + } else { + // safe default value: lock a load balancer only to the local address unless overridden explicitly + serviceSpec.LoadBalancerSourceRanges = []string{localHost} + } + + c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges) + serviceSpec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(c.OpConfig.ExternalTrafficPolicy) + serviceSpec.Type = v1.ServiceTypeLoadBalancer +} + func (c *Cluster) generateServiceAnnotations(role PostgresRole, spec *acidv1.PostgresSpec) map[string]string { annotations := make(map[string]string) @@ -1761,6 +1820,7 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) cluster := description.ClusterName result = append(result, v1.EnvVar{Name: "CLONE_SCOPE", Value: cluster}) if description.EndTimestamp == "" { + c.logger.Infof("cloning with basebackup from %s", cluster) // cloning with basebackup, make a connection string to the cluster to clone from host, port := c.getClusterServiceConnectionParameters(cluster) // TODO: make some/all of those constants @@ -1782,67 +1842,47 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) }, }) } else { - // cloning with S3, find out the bucket to clone - msg := "Clone from S3 bucket" - c.logger.Info(msg, description.S3WalPath) - + c.logger.Info("cloning from WAL location") if description.S3WalPath == "" { - msg := "Figure out which S3 bucket to use from env" - c.logger.Info(msg, description.S3WalPath) + c.logger.Info("no S3 WAL path defined - taking value from global config", description.S3WalPath) if c.OpConfig.WALES3Bucket != "" { - envs := []v1.EnvVar{ - { - Name: "CLONE_WAL_S3_BUCKET", - Value: c.OpConfig.WALES3Bucket, - }, - } - result = append(result, envs...) + c.logger.Debugf("found WALES3Bucket %s - will set CLONE_WAL_S3_BUCKET", c.OpConfig.WALES3Bucket) + result = append(result, v1.EnvVar{Name: "CLONE_WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) } else if c.OpConfig.WALGSBucket != "" { - envs := []v1.EnvVar{ - { - Name: "CLONE_WAL_GS_BUCKET", - Value: c.OpConfig.WALGSBucket, - }, - { - Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", - Value: c.OpConfig.GCPCredentials, - }, + c.logger.Debugf("found WALGSBucket %s - will set CLONE_WAL_GS_BUCKET", c.OpConfig.WALGSBucket) + result = append(result, v1.EnvVar{Name: "CLONE_WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) + if c.OpConfig.GCPCredentials != "" { + result = append(result, v1.EnvVar{Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) } - result = append(result, envs...) } else if c.OpConfig.WALAZStorageAccount != "" { - envs := []v1.EnvVar{ - { - Name: "CLONE_AZURE_STORAGE_ACCOUNT", - Value: c.OpConfig.WALAZStorageAccount, - }, - } - result = append(result, envs...) + c.logger.Debugf("found WALAZStorageAccount %s - will set CLONE_AZURE_STORAGE_ACCOUNT", c.OpConfig.WALAZStorageAccount) + result = append(result, v1.EnvVar{Name: "CLONE_AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount}) } else { - c.logger.Error("Cannot figure out S3 or GS bucket. Both are empty.") + c.logger.Error("cannot figure out S3 or GS bucket or AZ storage account. All options are empty in the config.") } + // append suffix because WAL location name is not the whole path + result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID)}) + } else { + c.logger.Debugf("use S3WalPath %s from the manifest", description.S3WalPath) + envs := []v1.EnvVar{ + { + Name: "CLONE_WALE_S3_PREFIX", + Value: description.S3WalPath, + }, { Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", - Value: getBucketScopeSuffix(description.UID), + Value: "", }, } result = append(result, envs...) - } else { - msg := "Use custom parsed S3WalPath %s from the manifest" - c.logger.Warningf(msg, description.S3WalPath) - - result = append(result, v1.EnvVar{ - Name: "CLONE_WALE_S3_PREFIX", - Value: description.S3WalPath, - }) } result = append(result, v1.EnvVar{Name: "CLONE_METHOD", Value: "CLONE_WITH_WALE"}) result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIME", Value: description.EndTimestamp}) - result = append(result, v1.EnvVar{Name: "CLONE_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) if description.S3Endpoint != "" { result = append(result, v1.EnvVar{Name: "CLONE_AWS_ENDPOINT", Value: description.S3Endpoint}) @@ -1874,20 +1914,38 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescription) []v1.EnvVar { result := make([]v1.EnvVar, 0) - if description.S3WalPath == "" { - return nil + if description.StandbyHost != "" { + c.logger.Info("standby cluster streaming from remote primary") + result = append(result, v1.EnvVar{ + Name: "STANDBY_HOST", + Value: description.StandbyHost, + }) + if description.StandbyPort != "" { + result = append(result, v1.EnvVar{ + Name: "STANDBY_PORT", + Value: description.StandbyPort, + }) + } + } else { + c.logger.Info("standby cluster streaming from WAL location") + if description.S3WalPath != "" { + result = append(result, v1.EnvVar{ + Name: "STANDBY_WALE_S3_PREFIX", + Value: description.S3WalPath, + }) + } else if description.GSWalPath != "" { + result = append(result, v1.EnvVar{ + Name: "STANDBY_WALE_GS_PREFIX", + Value: description.GSWalPath, + }) + } else { + c.logger.Error("no WAL path specified in standby section") + return result + } + + result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) + result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } - // standby with S3, find out the bucket to setup standby - msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s " - c.logger.Infof(msg, description.S3WalPath) - - result = append(result, v1.EnvVar{ - Name: "STANDBY_WALE_S3_PREFIX", - Value: description.S3WalPath, - }) - - result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) - result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""}) return result } @@ -1922,7 +1980,7 @@ func (c *Cluster) generatePodDisruptionBudget() *policybeta1.PodDisruptionBudget // TODO: handle clusters in different namespaces func (c *Cluster) getClusterServiceConnectionParameters(clusterName string) (host string, port string) { host = clusterName - port = "5432" + port = fmt.Sprintf("%d", pgPort) return } @@ -1939,15 +1997,15 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { c.logger.Debug("Generating logical backup pod template") // allocate for the backup pod the same amount of resources as for normal DB pods - defaultResources := c.makeDefaultResources() - resourceRequirements, err = generateResourceRequirements(c.Spec.Resources, defaultResources) + resourceRequirements, err = c.generateResourceRequirements( + c.Spec.Resources, makeDefaultResources(&c.OpConfig), logicalBackupContainerName) if err != nil { return nil, fmt.Errorf("could not generate resource requirements for logical backup pods: %v", err) } envVars := c.generateLogicalBackupPodEnvVars() logicalBackupContainer := generateContainer( - "logical-backup", + logicalBackupContainerName, &c.OpConfig.LogicalBackup.LogicalBackupDockerImage, resourceRequirements, envVars, @@ -1990,7 +2048,7 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { nil, nil, nil, - nodeAffinity(c.OpConfig.NodeReadinessLabel, nil), + c.nodeAffinity(c.OpConfig.NodeReadinessLabel, nil), nil, int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, @@ -2084,6 +2142,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { Name: "LOGICAL_BACKUP_S3_SSE", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE, }, + { + Name: "LOGICAL_BACKUP_S3_RETENTION_TIME", + Value: c.OpConfig.LogicalBackup.LogicalBackupS3RetentionTime, + }, { Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())), @@ -2099,7 +2161,7 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { }, { Name: "PGPORT", - Value: "5432", + Value: fmt.Sprintf("%d", pgPort), }, { Name: "PGUSER", diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 0f99d5f31..7a1bdeaf9 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "sort" + "time" "testing" @@ -21,12 +22,15 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/fake" v1core "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" ) func newFakeK8sTestClient() (k8sutil.KubernetesClient, *fake.Clientset) { @@ -45,11 +49,7 @@ type ExpectedValue struct { envIndex int envVarConstant string envVarValue string -} - -func toIntStr(val int) *intstr.IntOrString { - b := intstr.FromInt(val) - return &b + envVarValueRef *v1.EnvVarSource } func TestGenerateSpiloJSONConfiguration(t *testing.T) { @@ -97,11 +97,12 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { MaximumLagOnFailover: 33554432, SynchronousMode: true, SynchronousModeStrict: true, + SynchronousNodeCount: 1, Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, }, role: "zalandos", opConfig: config.Config{}, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"synchronous_node_count":1,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, }, } for _, tt := range tests { @@ -117,57 +118,496 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { } } -func TestGenerateSpiloPodEnvVars(t *testing.T) { - var cluster = New( - Config{ - OpConfig: config.Config{ - WALGSBucket: "wale-gs-bucket", - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, +func TestExtractPgVersionFromBinPath(t *testing.T) { + testName := "TestExtractPgVersionFromBinPath" + tests := []struct { + subTest string + binPath string + template string + expected string + }{ + { + subTest: "test current bin path with decimal against hard coded template", + binPath: "/usr/lib/postgresql/9.6/bin", + template: pgBinariesLocationTemplate, + expected: "9.6", + }, + { + subTest: "test current bin path against hard coded template", + binPath: "/usr/lib/postgresql/12/bin", + template: pgBinariesLocationTemplate, + expected: "12", + }, + { + subTest: "test alternative bin path against a matching template", + binPath: "/usr/pgsql-12/bin", + template: "/usr/pgsql-%v/bin", + expected: "12", + }, + } + + for _, tt := range tests { + pgVersion, err := extractPgVersionFromBinPath(tt.binPath, tt.template) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if pgVersion != tt.expected { + t.Errorf("%s %s: Expected version %s, have %s instead", + testName, tt.subTest, tt.expected, pgVersion) + } + } +} + +const ( + testPodEnvironmentConfigMapName = "pod_env_cm" + testPodEnvironmentSecretName = "pod_env_sc" + testPodEnvironmentObjectNotExists = "idonotexist" + testPodEnvironmentSecretNameAPIError = "pod_env_sc_apierror" + testResourceCheckInterval = 3 + testResourceCheckTimeout = 10 +) + +type mockSecret struct { + v1core.SecretInterface +} + +type mockConfigMap struct { + v1core.ConfigMapInterface +} + +func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { + if name == testPodEnvironmentSecretNameAPIError { + return nil, fmt.Errorf("Secret PodEnvironmentSecret API error") + } + if name != testPodEnvironmentSecretName { + return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "core", Resource: "secret"}, name) + } + secret := &v1.Secret{} + secret.Name = testPodEnvironmentSecretName + secret.Data = map[string][]byte{ + "clone_aws_access_key_id": []byte("0123456789abcdef0123456789abcdef"), + "custom_variable": []byte("secret-test"), + "standby_google_application_credentials": []byte("0123456789abcdef0123456789abcdef"), + } + return secret, nil +} + +func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { + if name != testPodEnvironmentConfigMapName { + return nil, fmt.Errorf("NotFound") + } + configmap := &v1.ConfigMap{} + configmap.Name = testPodEnvironmentConfigMapName + configmap.Data = map[string]string{ + // hard-coded clone env variable, can set when not specified in manifest + "clone_aws_endpoint": "s3.eu-west-1.amazonaws.com", + // custom variable, can be overridden by c.Spec.Env + "custom_variable": "configmap-test", + // hard-coded env variable, can not be overridden + "kubernetes_scope_label": "pgaas", + // hard-coded env variable, can be overridden + "wal_s3_bucket": "global-s3-bucket-configmap", + } + return configmap, nil +} + +type MockSecretGetter struct { +} + +type MockConfigMapsGetter struct { +} + +func (c *MockSecretGetter) Secrets(namespace string) v1core.SecretInterface { + return &mockSecret{} +} + +func (c *MockConfigMapsGetter) ConfigMaps(namespace string) v1core.ConfigMapInterface { + return &mockConfigMap{} +} + +func newMockKubernetesClient() k8sutil.KubernetesClient { + return k8sutil.KubernetesClient{ + SecretsGetter: &MockSecretGetter{}, + ConfigMapsGetter: &MockConfigMapsGetter{}, + } +} +func newMockCluster(opConfig config.Config) *Cluster { + cluster := &Cluster{ + Config: Config{OpConfig: opConfig}, + KubeClient: newMockKubernetesClient(), + logger: logger, + } + return cluster +} + +func TestPodEnvironmentConfigMapVariables(t *testing.T) { + testName := "TestPodEnvironmentConfigMapVariables" + tests := []struct { + subTest string + opConfig config.Config + envVars []v1.EnvVar + err error + }{ + { + subTest: "no PodEnvironmentConfigMap", + envVars: []v1.EnvVar{}, + }, + { + subTest: "missing PodEnvironmentConfigMap", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentObjectNotExists, + }, }, }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - - expectedValuesGSBucket := []ExpectedValue{ - ExpectedValue{ - envIndex: 15, - envVarConstant: "WAL_GS_BUCKET", - envVarValue: "wale-gs-bucket", + err: fmt.Errorf("could not read PodEnvironmentConfigMap: NotFound"), }, - ExpectedValue{ + { + subTest: "Pod environment vars configured by PodEnvironmentConfigMap", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + }, + envVars: []v1.EnvVar{ + { + Name: "clone_aws_endpoint", + Value: "s3.eu-west-1.amazonaws.com", + }, + { + Name: "custom_variable", + Value: "configmap-test", + }, + { + Name: "kubernetes_scope_label", + Value: "pgaas", + }, + { + Name: "wal_s3_bucket", + Value: "global-s3-bucket-configmap", + }, + }, + }, + } + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + vars, err := c.getPodEnvironmentConfigMapVariables() + if !reflect.DeepEqual(vars, tt.envVars) { + t.Errorf("%s %s: expected `%v` but got `%v`", + testName, tt.subTest, tt.envVars, vars) + } + if tt.err != nil { + if err.Error() != tt.err.Error() { + t.Errorf("%s %s: expected error `%v` but got `%v`", + testName, tt.subTest, tt.err, err) + } + } else { + if err != nil { + t.Errorf("%s %s: expected no error but got error: `%v`", + testName, tt.subTest, err) + } + } + } +} + +// Test if the keys of an existing secret are properly referenced +func TestPodEnvironmentSecretVariables(t *testing.T) { + maxRetries := int(testResourceCheckTimeout / testResourceCheckInterval) + testName := "TestPodEnvironmentSecretVariables" + tests := []struct { + subTest string + opConfig config.Config + envVars []v1.EnvVar + err error + }{ + { + subTest: "No PodEnvironmentSecret configured", + envVars: []v1.EnvVar{}, + }, + { + subTest: "Secret referenced by PodEnvironmentSecret does not exist", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentObjectNotExists, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: still failing after %d retries: secret.core %q not found", maxRetries, testPodEnvironmentObjectNotExists), + }, + { + subTest: "API error during PodEnvironmentSecret retrieval", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretNameAPIError, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret API error"), + }, + { + subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + envVars: []v1.EnvVar{ + { + Name: "clone_aws_access_key_id", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "clone_aws_access_key_id", + }, + }, + }, + { + Name: "custom_variable", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "custom_variable", + }, + }, + }, + { + Name: "standby_google_application_credentials", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "standby_google_application_credentials", + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + vars, err := c.getPodEnvironmentSecretVariables() + sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) + if !reflect.DeepEqual(vars, tt.envVars) { + t.Errorf("%s %s: expected `%v` but got `%v`", + testName, tt.subTest, tt.envVars, vars) + } + if tt.err != nil { + if err.Error() != tt.err.Error() { + t.Errorf("%s %s: expected error `%v` but got `%v`", + testName, tt.subTest, tt.err, err) + } + } else { + if err != nil { + t.Errorf("%s %s: expected no error but got error: `%v`", + testName, tt.subTest, err) + } + } + } + +} + +func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { + required := map[string]bool{ + "PGHOST": false, + "PGPORT": false, + "PGUSER": false, + "PGSCHEMA": false, + "PGPASSWORD": false, + "CONNECTION_POOLER_MODE": false, + "CONNECTION_POOLER_PORT": false, + } + + container := getPostgresContainer(&podSpec.Spec) + envs := container.Env + for _, env := range envs { + required[env.Name] = true + } + + for env, value := range required { + if !value { + return fmt.Errorf("Environment variable %s is not present", env) + } + } + + return nil +} + +func TestGenerateSpiloPodEnvVars(t *testing.T) { + var dummyUUID = "efd12e58-5786-11e8-b5a7-06148230260c" + + expectedClusterNameLabel := []ExpectedValue{ + { + envIndex: 5, + envVarConstant: "KUBERNETES_SCOPE_LABEL", + envVarValue: "cluster-name", + }, + } + expectedSpiloWalPathCompat := []ExpectedValue{ + { + envIndex: 12, + envVarConstant: "ENABLE_WAL_PATH_COMPAT", + envVarValue: "true", + }, + } + expectedValuesS3Bucket := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "WAL_S3_BUCKET", + envVarValue: "global-s3-bucket", + }, + { envIndex: 16, envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", - envVarValue: "/SomeUUID", + envVarValue: fmt.Sprintf("/%s", dummyUUID), }, - ExpectedValue{ + { envIndex: 17, envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", envVarValue: "", }, } - expectedValuesGCPCreds := []ExpectedValue{ - ExpectedValue{ + { envIndex: 15, envVarConstant: "WAL_GS_BUCKET", - envVarValue: "wale-gs-bucket", + envVarValue: "global-gs-bucket", }, - ExpectedValue{ + { envIndex: 16, envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", - envVarValue: "/SomeUUID", + envVarValue: fmt.Sprintf("/%s", dummyUUID), }, - ExpectedValue{ + { envIndex: 17, envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", envVarValue: "", }, - ExpectedValue{ + { envIndex: 18, envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS", - envVarValue: "some_path_to_credentials", + envVarValue: "some-path-to-credentials", + }, + } + expectedS3BucketConfigMap := []ExpectedValue{ + { + envIndex: 17, + envVarConstant: "WAL_S3_BUCKET", + envVarValue: "global-s3-bucket-configmap", + }, + } + expectedCustomS3BucketSpec := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "WAL_S3_BUCKET", + envVarValue: "custom-s3-bucket", + }, + } + expectedCustomVariableSecret := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "CUSTOM_VARIABLE", + envVarValueRef: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "custom_variable", + }, + }, + }, + } + expectedCustomVariableConfigMap := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "CUSTOM_VARIABLE", + envVarValue: "configmap-test", + }, + } + expectedCustomVariableSpec := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "CUSTOM_VARIABLE", + envVarValue: "spec-env-test", + }, + } + expectedCloneEnvSpec := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "CLONE_WALE_S3_PREFIX", + envVarValue: "s3://another-bucket", + }, + { + envIndex: 17, + envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: "", + }, + { + envIndex: 20, + envVarConstant: "CLONE_AWS_ENDPOINT", + envVarValue: "s3.eu-central-1.amazonaws.com", + }, + } + expectedCloneEnvConfigMap := []ExpectedValue{ + { + envIndex: 16, + envVarConstant: "CLONE_WAL_S3_BUCKET", + envVarValue: "global-s3-bucket", + }, + { + envIndex: 17, + envVarConstant: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", + envVarValue: fmt.Sprintf("/%s", dummyUUID), + }, + { + envIndex: 20, + envVarConstant: "CLONE_AWS_ENDPOINT", + envVarValue: "s3.eu-west-1.amazonaws.com", + }, + } + expectedCloneEnvSecret := []ExpectedValue{ + { + envIndex: 20, + envVarConstant: "CLONE_AWS_ACCESS_KEY_ID", + envVarValueRef: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "clone_aws_access_key_id", + }, + }, + }, + } + expectedStandbyEnvSecret := []ExpectedValue{ + { + envIndex: 15, + envVarConstant: "STANDBY_WALE_GS_PREFIX", + envVarValue: "gs://some/path/", + }, + { + envIndex: 20, + envVarConstant: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS", + envVarValueRef: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "standby_google_application_credentials", + }, + }, }, } @@ -175,284 +615,255 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { tests := []struct { subTest string opConfig config.Config - uid types.UID - spiloConfig string cloneDescription *acidv1.CloneDescription standbyDescription *acidv1.StandbyDescription - customEnvList []v1.EnvVar expectedValues []ExpectedValue + pgsql acidv1.Postgresql }{ { - subTest: "Will set WAL_GS_BUCKET env", + subTest: "will set ENABLE_WAL_PATH_COMPAT env", opConfig: config.Config{ - WALGSBucket: "wale-gs-bucket", + EnableSpiloWalPathCompat: true, }, - uid: "SomeUUID", - spiloConfig: "someConfig", cloneDescription: &acidv1.CloneDescription{}, standbyDescription: &acidv1.StandbyDescription{}, - customEnvList: []v1.EnvVar{}, - expectedValues: expectedValuesGSBucket, + expectedValues: expectedSpiloWalPathCompat, }, { - subTest: "Will set GOOGLE_APPLICATION_CREDENTIALS env", + subTest: "will set WAL_S3_BUCKET env", opConfig: config.Config{ - WALGSBucket: "wale-gs-bucket", - GCPCredentials: "some_path_to_credentials", + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedValuesS3Bucket, + }, + { + subTest: "will set GOOGLE_APPLICATION_CREDENTIALS env", + opConfig: config.Config{ + WALGSBucket: "global-gs-bucket", + GCPCredentials: "some-path-to-credentials", }, - uid: "SomeUUID", - spiloConfig: "someConfig", cloneDescription: &acidv1.CloneDescription{}, standbyDescription: &acidv1.StandbyDescription{}, - customEnvList: []v1.EnvVar{}, expectedValues: expectedValuesGCPCreds, }, + { + subTest: "will not override global config KUBERNETES_SCOPE_LABEL parameter", + opConfig: config.Config{ + Resources: config.Resources{ + ClusterNameLabel: "cluster-name", + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, // contains kubernetes_scope_label, too + }, + }, + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedClusterNameLabel, + pgsql: acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + Env: []v1.EnvVar{ + { + Name: "KUBERNETES_SCOPE_LABEL", + Value: "my-scope-label", + }, + }, + }, + }, + }, + { + subTest: "will override global WAL_S3_BUCKET parameter from pod environment config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedS3BucketConfigMap, + }, + { + subTest: "will override global WAL_S3_BUCKET parameter from manifest `env` section", + opConfig: config.Config{ + WALGSBucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCustomS3BucketSpec, + pgsql: acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + Env: []v1.EnvVar{ + { + Name: "WAL_S3_BUCKET", + Value: "custom-s3-bucket", + }, + }, + }, + }, + }, + { + subTest: "will set CUSTOM_VARIABLE from pod environment secret and not config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCustomVariableSecret, + }, + { + subTest: "will set CUSTOM_VARIABLE from pod environment config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCustomVariableConfigMap, + }, + { + subTest: "will override CUSTOM_VARIABLE of pod environment secret/configmap from manifest `env` section", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCustomVariableSpec, + pgsql: acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + Env: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "spec-env-test", + }, + }, + }, + }, + }, + { + subTest: "will set CLONE_ parameters from spec and not global config or pod environment config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: dummyUUID, + S3WalPath: "s3://another-bucket", + S3Endpoint: "s3.eu-central-1.amazonaws.com", + }, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCloneEnvSpec, + }, + { + subTest: "will set CLONE_AWS_ENDPOINT parameter from pod environment config map", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: dummyUUID, + }, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCloneEnvConfigMap, + }, + { + subTest: "will set CLONE_AWS_ACCESS_KEY_ID parameter from pod environment secret", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{ + ClusterName: "test-cluster", + EndTimestamp: "somewhen", + UID: dummyUUID, + }, + standbyDescription: &acidv1.StandbyDescription{}, + expectedValues: expectedCloneEnvSecret, + }, + { + subTest: "will set STANDBY_GOOGLE_APPLICATION_CREDENTIALS parameter from pod environment secret", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretName, + ResourceCheckInterval: time.Duration(testResourceCheckInterval), + ResourceCheckTimeout: time.Duration(testResourceCheckTimeout), + }, + WALES3Bucket: "global-s3-bucket", + }, + cloneDescription: &acidv1.CloneDescription{}, + standbyDescription: &acidv1.StandbyDescription{ + GSWalPath: "gs://some/path/", + }, + expectedValues: expectedStandbyEnvSecret, + }, } for _, tt := range tests { - cluster.OpConfig = tt.opConfig - - actualEnvs := cluster.generateSpiloPodEnvVars(tt.uid, tt.spiloConfig, tt.cloneDescription, tt.standbyDescription, tt.customEnvList) + c := newMockCluster(tt.opConfig) + c.Postgresql = tt.pgsql + actualEnvs := c.generateSpiloPodEnvVars( + types.UID(dummyUUID), exampleSpiloConfig, tt.cloneDescription, tt.standbyDescription) for _, ev := range tt.expectedValues { env := actualEnvs[ev.envIndex] if env.Name != ev.envVarConstant { - t.Errorf("%s %s: Expected env name %s, have %s instead", + t.Errorf("%s %s: expected env name %s, have %s instead", testName, tt.subTest, ev.envVarConstant, env.Name) } + if ev.envVarValueRef != nil { + if !reflect.DeepEqual(env.ValueFrom, ev.envVarValueRef) { + t.Errorf("%s %s: expected env value reference %#v, have %#v instead", + testName, tt.subTest, ev.envVarValueRef, env.ValueFrom) + } + continue + } + if env.Value != ev.envVarValue { - t.Errorf("%s %s: Expected env value %s, have %s instead", + t.Errorf("%s %s: expected env value %s, have %s instead", testName, tt.subTest, ev.envVarValue, env.Value) } } } } -func TestCreateLoadBalancerLogic(t *testing.T) { - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - - testName := "TestCreateLoadBalancerLogic" - tests := []struct { - subtest string - role PostgresRole - spec *acidv1.PostgresSpec - opConfig config.Config - result bool - }{ - { - subtest: "new format, load balancer is enabled for replica", - role: Replica, - spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.True()}, - opConfig: config.Config{}, - result: true, - }, - { - subtest: "new format, load balancer is disabled for replica", - role: Replica, - spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.False()}, - opConfig: config.Config{}, - result: false, - }, - { - subtest: "new format, load balancer isn't specified for replica", - role: Replica, - spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil}, - opConfig: config.Config{EnableReplicaLoadBalancer: true}, - result: true, - }, - { - subtest: "new format, load balancer isn't specified for replica", - role: Replica, - spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil}, - opConfig: config.Config{EnableReplicaLoadBalancer: false}, - result: false, - }, - } - for _, tt := range tests { - cluster.OpConfig = tt.opConfig - result := cluster.shouldCreateLoadBalancerForService(tt.role, tt.spec) - if tt.result != result { - t.Errorf("%s %s: Load balancer is %t, expect %t for role %#v and spec %#v", - testName, tt.subtest, result, tt.result, tt.role, tt.spec) - } - } -} - -func TestGeneratePodDisruptionBudget(t *testing.T) { - tests := []struct { - c *Cluster - out policyv1beta1.PodDisruptionBudget - }{ - // With multiple instances. - { - New( - Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, - k8sutil.KubernetesClient{}, - acidv1.Postgresql{ - ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, - Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger, - eventRecorder), - policyv1beta1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres-myapp-database-pdb", - Namespace: "myapp", - Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, - }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ - MinAvailable: toIntStr(1), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, - }, - }, - }, - }, - // With zero instances. - { - New( - Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, - k8sutil.KubernetesClient{}, - acidv1.Postgresql{ - ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, - Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}}, - logger, - eventRecorder), - policyv1beta1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres-myapp-database-pdb", - Namespace: "myapp", - Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, - }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ - MinAvailable: toIntStr(0), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, - }, - }, - }, - }, - // With PodDisruptionBudget disabled. - { - New( - Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.False()}}, - k8sutil.KubernetesClient{}, - acidv1.Postgresql{ - ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, - Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger, - eventRecorder), - policyv1beta1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres-myapp-database-pdb", - Namespace: "myapp", - Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, - }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ - MinAvailable: toIntStr(0), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, - }, - }, - }, - }, - // With non-default PDBNameFormat and PodDisruptionBudget explicitly enabled. - { - New( - Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-databass-budget", EnablePodDisruptionBudget: util.True()}}, - k8sutil.KubernetesClient{}, - acidv1.Postgresql{ - ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, - Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, - logger, - eventRecorder), - policyv1beta1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres-myapp-database-databass-budget", - Namespace: "myapp", - Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, - }, - Spec: policyv1beta1.PodDisruptionBudgetSpec{ - MinAvailable: toIntStr(1), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, - }, - }, - }, - }, - } - - for _, tt := range tests { - result := tt.c.generatePodDisruptionBudget() - if !reflect.DeepEqual(*result, tt.out) { - t.Errorf("Expected PodDisruptionBudget: %#v, got %#v", tt.out, *result) - } - } -} - -func TestShmVolume(t *testing.T) { - testName := "TestShmVolume" - tests := []struct { - subTest string - podSpec *v1.PodSpec - shmPos int - }{ - { - subTest: "empty PodSpec", - podSpec: &v1.PodSpec{ - Volumes: []v1.Volume{}, - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{}, - }, - }, - }, - shmPos: 0, - }, - { - subTest: "non empty PodSpec", - podSpec: &v1.PodSpec{ - Volumes: []v1.Volume{{}}, - Containers: []v1.Container{ - { - Name: "postgres", - VolumeMounts: []v1.VolumeMount{ - {}, - }, - }, - }, - }, - shmPos: 1, - }, - } - for _, tt := range tests { - addShmVolume(tt.podSpec) - postgresContainer := getPostgresContainer(tt.podSpec) - - volumeName := tt.podSpec.Volumes[tt.shmPos].Name - volumeMountName := postgresContainer.VolumeMounts[tt.shmPos].Name - - if volumeName != constants.ShmVolumeName { - t.Errorf("%s %s: Expected volume %s was not created, have %s instead", - testName, tt.subTest, constants.ShmVolumeName, volumeName) - } - if volumeMountName != constants.ShmVolumeName { - t.Errorf("%s %s: Expected mount %s was not created, have %s instead", - testName, tt.subTest, constants.ShmVolumeName, volumeMountName) - } - } -} - func TestCloneEnv(t *testing.T) { testName := "TestCloneEnv" tests := []struct { @@ -531,355 +942,176 @@ func TestCloneEnv(t *testing.T) { } } -func TestExtractPgVersionFromBinPath(t *testing.T) { - testName := "TestExtractPgVersionFromBinPath" +func TestAppendEnvVar(t *testing.T) { + testName := "TestAppendEnvVar" tests := []struct { - subTest string - binPath string - template string - expected string + subTest string + envs []v1.EnvVar + envsToAppend []v1.EnvVar + expectedSize int }{ { - subTest: "test current bin path with decimal against hard coded template", - binPath: "/usr/lib/postgresql/9.6/bin", - template: pgBinariesLocationTemplate, - expected: "9.6", + subTest: "append two variables - one with same key that should get rejected", + envs: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "test", + }, + }, + envsToAppend: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "new-test", + }, + { + Name: "ANOTHER_CUSTOM_VARIABLE", + Value: "another-test", + }, + }, + expectedSize: 2, }, { - subTest: "test current bin path against hard coded template", - binPath: "/usr/lib/postgresql/12/bin", - template: pgBinariesLocationTemplate, - expected: "12", + subTest: "append empty slice", + envs: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "test", + }, + }, + envsToAppend: []v1.EnvVar{}, + expectedSize: 1, }, { - subTest: "test alternative bin path against a matching template", - binPath: "/usr/pgsql-12/bin", - template: "/usr/pgsql-%v/bin", - expected: "12", + subTest: "append nil", + envs: []v1.EnvVar{ + { + Name: "CUSTOM_VARIABLE", + Value: "test", + }, + }, + envsToAppend: nil, + expectedSize: 1, }, } for _, tt := range tests { - pgVersion, err := extractPgVersionFromBinPath(tt.binPath, tt.template) - if err != nil { - t.Errorf("Unexpected error: %v", err) + finalEnvs := appendEnvVars(tt.envs, tt.envsToAppend...) + + if len(finalEnvs) != tt.expectedSize { + t.Errorf("%s %s: expected %d env variables, got %d", + testName, tt.subTest, tt.expectedSize, len(finalEnvs)) } - if pgVersion != tt.expected { - t.Errorf("%s %s: Expected version %s, have %s instead", - testName, tt.subTest, tt.expected, pgVersion) + + for _, env := range tt.envs { + for _, finalEnv := range finalEnvs { + if env.Name == finalEnv.Name { + if env.Value != finalEnv.Value { + t.Errorf("%s %s: expected env value %s of variable %s, got %s instead", + testName, tt.subTest, env.Value, env.Name, finalEnv.Value) + } + } + } } } } -func TestSecretVolume(t *testing.T) { - testName := "TestSecretVolume" +func TestStandbyEnv(t *testing.T) { + testName := "TestStandbyEnv" tests := []struct { - subTest string - podSpec *v1.PodSpec - secretPos int + subTest string + standbyOpts *acidv1.StandbyDescription + env v1.EnvVar + envPos int + envLen int }{ { - subTest: "empty PodSpec", - podSpec: &v1.PodSpec{ - Volumes: []v1.Volume{}, - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{}, - }, - }, + subTest: "from custom s3 path", + standbyOpts: &acidv1.StandbyDescription{ + S3WalPath: "s3://some/path/", }, - secretPos: 0, + env: v1.EnvVar{ + Name: "STANDBY_WALE_S3_PREFIX", + Value: "s3://some/path/", + }, + envPos: 0, + envLen: 3, }, { - subTest: "non empty PodSpec", - podSpec: &v1.PodSpec{ - Volumes: []v1.Volume{{}}, - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "data", - ReadOnly: false, - MountPath: "/data", - }, - }, - }, - }, + subTest: "ignore gs path if s3 is set", + standbyOpts: &acidv1.StandbyDescription{ + S3WalPath: "s3://some/path/", + GSWalPath: "gs://some/path/", }, - secretPos: 1, - }, - } - for _, tt := range tests { - additionalSecretMount := "aws-iam-s3-role" - additionalSecretMountPath := "/meta/credentials" - postgresContainer := getPostgresContainer(tt.podSpec) - - numMounts := len(postgresContainer.VolumeMounts) - - addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath) - - volumeName := tt.podSpec.Volumes[tt.secretPos].Name - - if volumeName != additionalSecretMount { - t.Errorf("%s %s: Expected volume %s was not created, have %s instead", - testName, tt.subTest, additionalSecretMount, volumeName) - } - - for i := range tt.podSpec.Containers { - volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name - - if volumeMountName != additionalSecretMount { - t.Errorf("%s %s: Expected mount %s was not created, have %s instead", - testName, tt.subTest, additionalSecretMount, volumeMountName) - } - } - - postgresContainer = getPostgresContainer(tt.podSpec) - numMountsCheck := len(postgresContainer.VolumeMounts) - - if numMountsCheck != numMounts+1 { - t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", - numMountsCheck, numMounts+1) - } - } -} - -const ( - testPodEnvironmentConfigMapName = "pod_env_cm" - testPodEnvironmentSecretName = "pod_env_sc" -) - -type mockSecret struct { - v1core.SecretInterface -} - -type mockConfigMap struct { - v1core.ConfigMapInterface -} - -func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { - if name != testPodEnvironmentSecretName { - return nil, fmt.Errorf("Secret PodEnvironmentSecret not found") - } - secret := &v1.Secret{} - secret.Name = testPodEnvironmentSecretName - secret.Data = map[string][]byte{ - "minio_access_key": []byte("alpha"), - "minio_secret_key": []byte("beta"), - } - return secret, nil -} - -func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { - if name != testPodEnvironmentConfigMapName { - return nil, fmt.Errorf("NotFound") - } - configmap := &v1.ConfigMap{} - configmap.Name = testPodEnvironmentConfigMapName - configmap.Data = map[string]string{ - "foo1": "bar1", - "foo2": "bar2", - } - return configmap, nil -} - -type MockSecretGetter struct { -} - -type MockConfigMapsGetter struct { -} - -func (c *MockSecretGetter) Secrets(namespace string) v1core.SecretInterface { - return &mockSecret{} -} - -func (c *MockConfigMapsGetter) ConfigMaps(namespace string) v1core.ConfigMapInterface { - return &mockConfigMap{} -} - -func newMockKubernetesClient() k8sutil.KubernetesClient { - return k8sutil.KubernetesClient{ - SecretsGetter: &MockSecretGetter{}, - ConfigMapsGetter: &MockConfigMapsGetter{}, - } -} -func newMockCluster(opConfig config.Config) *Cluster { - cluster := &Cluster{ - Config: Config{OpConfig: opConfig}, - KubeClient: newMockKubernetesClient(), - } - return cluster -} - -func TestPodEnvironmentConfigMapVariables(t *testing.T) { - testName := "TestPodEnvironmentConfigMapVariables" - tests := []struct { - subTest string - opConfig config.Config - envVars []v1.EnvVar - err error - }{ - { - subTest: "no PodEnvironmentConfigMap", - envVars: []v1.EnvVar{}, + env: v1.EnvVar{ + Name: "STANDBY_METHOD", + Value: "STANDBY_WITH_WALE", + }, + envPos: 1, + envLen: 3, }, { - subTest: "missing PodEnvironmentConfigMap", - opConfig: config.Config{ - Resources: config.Resources{ - PodEnvironmentConfigMap: spec.NamespacedName{ - Name: "idonotexist", - }, - }, + subTest: "from remote primary", + standbyOpts: &acidv1.StandbyDescription{ + StandbyHost: "remote-primary", }, - err: fmt.Errorf("could not read PodEnvironmentConfigMap: NotFound"), + env: v1.EnvVar{ + Name: "STANDBY_HOST", + Value: "remote-primary", + }, + envPos: 0, + envLen: 1, }, { - subTest: "simple PodEnvironmentConfigMap", - opConfig: config.Config{ - Resources: config.Resources{ - PodEnvironmentConfigMap: spec.NamespacedName{ - Name: testPodEnvironmentConfigMapName, - }, - }, + subTest: "from remote primary with port", + standbyOpts: &acidv1.StandbyDescription{ + StandbyHost: "remote-primary", + StandbyPort: "9876", }, - envVars: []v1.EnvVar{ - { - Name: "foo1", - Value: "bar1", - }, - { - Name: "foo2", - Value: "bar2", - }, + env: v1.EnvVar{ + Name: "STANDBY_PORT", + Value: "9876", }, + envPos: 1, + envLen: 2, + }, + { + subTest: "from remote primary - ignore WAL path", + standbyOpts: &acidv1.StandbyDescription{ + GSWalPath: "gs://some/path/", + StandbyHost: "remote-primary", + }, + env: v1.EnvVar{ + Name: "STANDBY_HOST", + Value: "remote-primary", + }, + envPos: 0, + envLen: 1, }, } - for _, tt := range tests { - c := newMockCluster(tt.opConfig) - vars, err := c.getPodEnvironmentConfigMapVariables() - sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) - if !reflect.DeepEqual(vars, tt.envVars) { - t.Errorf("%s %s: expected `%v` but got `%v`", - testName, tt.subTest, tt.envVars, vars) - } - if tt.err != nil { - if err.Error() != tt.err.Error() { - t.Errorf("%s %s: expected error `%v` but got `%v`", - testName, tt.subTest, tt.err, err) - } - } else { - if err != nil { - t.Errorf("%s %s: expected no error but got error: `%v`", - testName, tt.subTest, err) - } - } - } -} -// Test if the keys of an existing secret are properly referenced -func TestPodEnvironmentSecretVariables(t *testing.T) { - testName := "TestPodEnvironmentSecretVariables" - tests := []struct { - subTest string - opConfig config.Config - envVars []v1.EnvVar - err error - }{ - { - subTest: "No PodEnvironmentSecret configured", - envVars: []v1.EnvVar{}, - }, - { - subTest: "Secret referenced by PodEnvironmentSecret does not exist", - opConfig: config.Config{ - Resources: config.Resources{ - PodEnvironmentSecret: "idonotexist", - }, - }, - err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret not found"), - }, - { - subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret", - opConfig: config.Config{ - Resources: config.Resources{ - PodEnvironmentSecret: testPodEnvironmentSecretName, - }, - }, - envVars: []v1.EnvVar{ - { - Name: "minio_access_key", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: testPodEnvironmentSecretName, - }, - Key: "minio_access_key", - }, - }, - }, - { - Name: "minio_secret_key", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: testPodEnvironmentSecretName, - }, - Key: "minio_secret_key", - }, - }, - }, - }, - }, - } + var cluster = New( + Config{}, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) for _, tt := range tests { - c := newMockCluster(tt.opConfig) - vars, err := c.getPodEnvironmentSecretVariables() - sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) - if !reflect.DeepEqual(vars, tt.envVars) { - t.Errorf("%s %s: expected `%v` but got `%v`", - testName, tt.subTest, tt.envVars, vars) + envs := cluster.generateStandbyEnvironment(tt.standbyOpts) + + env := envs[tt.envPos] + + if env.Name != tt.env.Name { + t.Errorf("%s %s: Expected env name %s, have %s instead", + testName, tt.subTest, tt.env.Name, env.Name) } - if tt.err != nil { - if err.Error() != tt.err.Error() { - t.Errorf("%s %s: expected error `%v` but got `%v`", - testName, tt.subTest, tt.err, err) - } - } else { - if err != nil { - t.Errorf("%s %s: expected no error but got error: `%v`", - testName, tt.subTest, err) - } + + if env.Value != tt.env.Value { + t.Errorf("%s %s: Expected env value %s, have %s instead", + testName, tt.subTest, tt.env.Value, env.Value) + } + + if len(envs) != tt.envLen { + t.Errorf("%s %s: Expected number of env variables %d, have %d instead", + testName, tt.subTest, tt.envLen, len(envs)) } } - -} - -func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec, role PostgresRole) error { - required := map[string]bool{ - "PGHOST": false, - "PGPORT": false, - "PGUSER": false, - "PGSCHEMA": false, - "PGPASSWORD": false, - "CONNECTION_POOLER_MODE": false, - "CONNECTION_POOLER_PORT": false, - } - - container := getPostgresContainer(&podSpec.Spec) - envs := container.Env - for _, env := range envs { - required[env.Name] = true - } - - for env, value := range required { - if !value { - return fmt.Errorf("Environment variable %s is not present", env) - } - } - - return nil } func TestNodeAffinity(t *testing.T) { @@ -893,7 +1125,7 @@ func TestNodeAffinity(t *testing.T) { makeSpec := func(nodeAffinity *v1.NodeAffinity) acidv1.PostgresSpec { return acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ + Resources: &acidv1.Resources{ ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, }, @@ -989,7 +1221,7 @@ func TestTLS(t *testing.T) { }, Spec: acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ + Resources: &acidv1.Resources{ ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, }, @@ -1059,6 +1291,132 @@ func TestTLS(t *testing.T) { assert.Contains(t, postgresContainer.Env, v1.EnvVar{Name: "SSL_CA_FILE", Value: "/tls/ca.crt"}) } +func TestShmVolume(t *testing.T) { + testName := "TestShmVolume" + tests := []struct { + subTest string + podSpec *v1.PodSpec + shmPos int + }{ + { + subTest: "empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + shmPos: 0, + }, + { + subTest: "non empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + Name: "postgres", + VolumeMounts: []v1.VolumeMount{ + {}, + }, + }, + }, + }, + shmPos: 1, + }, + } + for _, tt := range tests { + addShmVolume(tt.podSpec) + postgresContainer := getPostgresContainer(tt.podSpec) + + volumeName := tt.podSpec.Volumes[tt.shmPos].Name + volumeMountName := postgresContainer.VolumeMounts[tt.shmPos].Name + + if volumeName != constants.ShmVolumeName { + t.Errorf("%s %s: Expected volume %s was not created, have %s instead", + testName, tt.subTest, constants.ShmVolumeName, volumeName) + } + if volumeMountName != constants.ShmVolumeName { + t.Errorf("%s %s: Expected mount %s was not created, have %s instead", + testName, tt.subTest, constants.ShmVolumeName, volumeMountName) + } + } +} + +func TestSecretVolume(t *testing.T) { + testName := "TestSecretVolume" + tests := []struct { + subTest string + podSpec *v1.PodSpec + secretPos int + }{ + { + subTest: "empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{}, + }, + }, + }, + secretPos: 0, + }, + { + subTest: "non empty PodSpec", + podSpec: &v1.PodSpec{ + Volumes: []v1.Volume{{}}, + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + ReadOnly: false, + MountPath: "/data", + }, + }, + }, + }, + }, + secretPos: 1, + }, + } + for _, tt := range tests { + additionalSecretMount := "aws-iam-s3-role" + additionalSecretMountPath := "/meta/credentials" + postgresContainer := getPostgresContainer(tt.podSpec) + + numMounts := len(postgresContainer.VolumeMounts) + + addSecretVolume(tt.podSpec, additionalSecretMount, additionalSecretMountPath) + + volumeName := tt.podSpec.Volumes[tt.secretPos].Name + + if volumeName != additionalSecretMount { + t.Errorf("%s %s: Expected volume %s was not created, have %s instead", + testName, tt.subTest, additionalSecretMount, volumeName) + } + + for i := range tt.podSpec.Containers { + volumeMountName := tt.podSpec.Containers[i].VolumeMounts[tt.secretPos].Name + + if volumeMountName != additionalSecretMount { + t.Errorf("%s %s: Expected mount %s was not created, have %s instead", + testName, tt.subTest, additionalSecretMount, volumeMountName) + } + } + + postgresContainer = getPostgresContainer(tt.podSpec) + numMountsCheck := len(postgresContainer.VolumeMounts) + + if numMountsCheck != numMounts+1 { + t.Errorf("Unexpected number of VolumeMounts: got %v instead of %v", + numMountsCheck, numMounts+1) + } + } +} + func TestAdditionalVolume(t *testing.T) { testName := "TestAdditionalVolume" @@ -1108,7 +1466,7 @@ func TestAdditionalVolume(t *testing.T) { }, Spec: acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ + Resources: &acidv1.Resources{ ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, }, @@ -1179,6 +1537,109 @@ func TestAdditionalVolume(t *testing.T) { } } +func TestVolumeSelector(t *testing.T) { + testName := "TestVolumeSelector" + makeSpec := func(volume acidv1.Volume) acidv1.PostgresSpec { + return acidv1.PostgresSpec{ + TeamID: "myapp", + NumberOfInstances: 0, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: volume, + } + } + + tests := []struct { + subTest string + volume acidv1.Volume + wantSelector *metav1.LabelSelector + }{ + { + subTest: "PVC template has no selector", + volume: acidv1.Volume{ + Size: "1G", + }, + wantSelector: nil, + }, + { + subTest: "PVC template has simple label selector", + volume: acidv1.Volume{ + Size: "1G", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "unittest"}, + }, + }, + wantSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "unittest"}, + }, + }, + { + subTest: "PVC template has full selector", + volume: acidv1.Volume{ + Size: "1G", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "unittest"}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "flavour", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"banana", "chocolate"}, + }, + }, + }, + }, + wantSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "unittest"}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "flavour", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"banana", "chocolate"}, + }, + }, + }, + }, + } + + cluster := New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + for _, tt := range tests { + pgSpec := makeSpec(tt.volume) + sts, err := cluster.generateStatefulSet(&pgSpec) + if err != nil { + t.Fatalf("%s %s: no statefulset created %v", testName, tt.subTest, err) + } + + volIdx := len(sts.Spec.VolumeClaimTemplates) + for i, ct := range sts.Spec.VolumeClaimTemplates { + if ct.ObjectMeta.Name == constants.DataVolumeName { + volIdx = i + break + } + } + if volIdx == len(sts.Spec.VolumeClaimTemplates) { + t.Errorf("%s %s: no datavolume found in sts", testName, tt.subTest) + } + + selector := sts.Spec.VolumeClaimTemplates[volIdx].Spec.Selector + if !reflect.DeepEqual(selector, tt.wantSelector) { + t.Errorf("%s %s: expected: %#v but got: %#v", testName, tt.subTest, tt.wantSelector, selector) + } + } +} + // inject sidecars through all available mechanisms and check the resulting container specs func TestSidecars(t *testing.T) { var err error @@ -1214,7 +1675,7 @@ func TestSidecars(t *testing.T) { }, }, TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ + Resources: &acidv1.Resources{ ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, }, @@ -1227,14 +1688,14 @@ func TestSidecars(t *testing.T) { }, acidv1.Sidecar{ Name: "cluster-specific-sidecar-with-resources", - Resources: acidv1.Resources{ + Resources: &acidv1.Resources{ ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"}, ResourceLimits: acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"}, }, }, acidv1.Sidecar{ Name: "replace-sidecar", - DockerImage: "overwrite-image", + DockerImage: "override-image", }, }, } @@ -1362,7 +1823,7 @@ func TestSidecars(t *testing.T) { // replaced sidecar assert.Contains(t, s.Spec.Template.Spec.Containers, v1.Container{ Name: "replace-sidecar", - Image: "overwrite-image", + Image: "override-image", Resources: generateKubernetesResources("200m", "500m", "0.7Gi", "1.3Gi"), ImagePullPolicy: v1.PullIfNotPresent, Env: env, @@ -1383,13 +1844,124 @@ func TestSidecars(t *testing.T) { } +func TestGeneratePodDisruptionBudget(t *testing.T) { + tests := []struct { + c *Cluster + out policyv1beta1.PodDisruptionBudget + }{ + // With multiple instances. + { + New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-myapp-database-pdb", + Namespace: "myapp", + Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + MinAvailable: util.ToIntStr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, + }, + }, + }, + }, + // With zero instances. + { + New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb"}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 0}}, + logger, + eventRecorder), + policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-myapp-database-pdb", + Namespace: "myapp", + Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + MinAvailable: util.ToIntStr(0), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, + }, + }, + }, + }, + // With PodDisruptionBudget disabled. + { + New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-pdb", EnablePodDisruptionBudget: util.False()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-myapp-database-pdb", + Namespace: "myapp", + Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + MinAvailable: util.ToIntStr(0), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, + }, + }, + }, + }, + // With non-default PDBNameFormat and PodDisruptionBudget explicitly enabled. + { + New( + Config{OpConfig: config.Config{Resources: config.Resources{ClusterNameLabel: "cluster-name", PodRoleLabel: "spilo-role"}, PDBNameFormat: "postgres-{cluster}-databass-budget", EnablePodDisruptionBudget: util.True()}}, + k8sutil.KubernetesClient{}, + acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{Name: "myapp-database", Namespace: "myapp"}, + Spec: acidv1.PostgresSpec{TeamID: "myapp", NumberOfInstances: 3}}, + logger, + eventRecorder), + policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-myapp-database-databass-budget", + Namespace: "myapp", + Labels: map[string]string{"team": "myapp", "cluster-name": "myapp-database"}, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + MinAvailable: util.ToIntStr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"spilo-role": "master", "cluster-name": "myapp-database"}, + }, + }, + }, + }, + } + + for _, tt := range tests { + result := tt.c.generatePodDisruptionBudget() + if !reflect.DeepEqual(*result, tt.out) { + t.Errorf("Expected PodDisruptionBudget: %#v, got %#v", tt.out, *result) + } + } +} + func TestGenerateService(t *testing.T) { var spec acidv1.PostgresSpec var cluster *Cluster var enableLB bool = true spec = acidv1.PostgresSpec{ TeamID: "myapp", NumberOfInstances: 1, - Resources: acidv1.Resources{ + Resources: &acidv1.Resources{ ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, }, @@ -1402,14 +1974,14 @@ func TestGenerateService(t *testing.T) { }, acidv1.Sidecar{ Name: "cluster-specific-sidecar-with-resources", - Resources: acidv1.Resources{ + Resources: &acidv1.Resources{ ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"}, ResourceLimits: acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"}, }, }, acidv1.Sidecar{ Name: "replace-sidecar", - DockerImage: "overwrite-image", + DockerImage: "override-image", }, }, EnableMasterLoadBalancer: &enableLB, @@ -1463,6 +2035,560 @@ func TestGenerateService(t *testing.T) { } +func TestCreateLoadBalancerLogic(t *testing.T) { + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + testName := "TestCreateLoadBalancerLogic" + tests := []struct { + subtest string + role PostgresRole + spec *acidv1.PostgresSpec + opConfig config.Config + result bool + }{ + { + subtest: "new format, load balancer is enabled for replica", + role: Replica, + spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.True()}, + opConfig: config.Config{}, + result: true, + }, + { + subtest: "new format, load balancer is disabled for replica", + role: Replica, + spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: util.False()}, + opConfig: config.Config{}, + result: false, + }, + { + subtest: "new format, load balancer isn't specified for replica", + role: Replica, + spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil}, + opConfig: config.Config{EnableReplicaLoadBalancer: true}, + result: true, + }, + { + subtest: "new format, load balancer isn't specified for replica", + role: Replica, + spec: &acidv1.PostgresSpec{EnableReplicaLoadBalancer: nil}, + opConfig: config.Config{EnableReplicaLoadBalancer: false}, + result: false, + }, + } + for _, tt := range tests { + cluster.OpConfig = tt.opConfig + result := cluster.shouldCreateLoadBalancerForService(tt.role, tt.spec) + if tt.result != result { + t.Errorf("%s %s: Load balancer is %t, expect %t for role %#v and spec %#v", + testName, tt.subtest, result, tt.result, tt.role, tt.spec) + } + } +} + +func newLBFakeClient() (k8sutil.KubernetesClient, *fake.Clientset) { + clientSet := fake.NewSimpleClientset() + + return k8sutil.KubernetesClient{ + DeploymentsGetter: clientSet.AppsV1(), + ServicesGetter: clientSet.CoreV1(), + }, clientSet +} + +func getServices(serviceType v1.ServiceType, sourceRanges []string, extTrafficPolicy, clusterName string) []v1.ServiceSpec { + return []v1.ServiceSpec{ + v1.ServiceSpec{ + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyType(extTrafficPolicy), + LoadBalancerSourceRanges: sourceRanges, + Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}}, + Type: serviceType, + }, + v1.ServiceSpec{ + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyType(extTrafficPolicy), + LoadBalancerSourceRanges: sourceRanges, + Ports: []v1.ServicePort{{Name: clusterName + "-pooler", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}}, + Selector: map[string]string{"connection-pooler": clusterName + "-pooler"}, + Type: serviceType, + }, + v1.ServiceSpec{ + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyType(extTrafficPolicy), + LoadBalancerSourceRanges: sourceRanges, + Ports: []v1.ServicePort{{Name: "postgresql", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}}, + Selector: map[string]string{"spilo-role": "replica", "application": "spilo", "cluster-name": clusterName}, + Type: serviceType, + }, + v1.ServiceSpec{ + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyType(extTrafficPolicy), + LoadBalancerSourceRanges: sourceRanges, + Ports: []v1.ServicePort{{Name: clusterName + "-pooler-repl", Port: 5432, TargetPort: intstr.IntOrString{IntVal: 5432}}}, + Selector: map[string]string{"connection-pooler": clusterName + "-pooler-repl"}, + Type: serviceType, + }, + } +} + +func TestEnableLoadBalancers(t *testing.T) { + testName := "Test enabling LoadBalancers" + client, _ := newLBFakeClient() + clusterName := "acid-test-cluster" + namespace := "default" + clusterNameLabel := "cluster-name" + roleLabel := "spilo-role" + roles := []PostgresRole{Master, Replica} + sourceRanges := []string{"192.186.1.2/22"} + extTrafficPolicy := "Cluster" + + tests := []struct { + subTest string + config config.Config + pgSpec acidv1.Postgresql + expectedServices []v1.ServiceSpec + }{ + { + subTest: "LBs enabled in config, disabled in manifest", + config: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(1), + }, + EnableMasterLoadBalancer: true, + EnableMasterPoolerLoadBalancer: true, + EnableReplicaLoadBalancer: true, + EnableReplicaPoolerLoadBalancer: true, + ExternalTrafficPolicy: extTrafficPolicy, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + PodRoleLabel: roleLabel, + }, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + AllowedSourceRanges: sourceRanges, + EnableConnectionPooler: util.True(), + EnableReplicaConnectionPooler: util.True(), + EnableMasterLoadBalancer: util.False(), + EnableMasterPoolerLoadBalancer: util.False(), + EnableReplicaLoadBalancer: util.False(), + EnableReplicaPoolerLoadBalancer: util.False(), + NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedServices: getServices(v1.ServiceTypeClusterIP, nil, "", clusterName), + }, + { + subTest: "LBs enabled in manifest, disabled in config", + config: config.Config{ + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(1), + }, + EnableMasterLoadBalancer: false, + EnableMasterPoolerLoadBalancer: false, + EnableReplicaLoadBalancer: false, + EnableReplicaPoolerLoadBalancer: false, + ExternalTrafficPolicy: extTrafficPolicy, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + PodRoleLabel: roleLabel, + }, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + AllowedSourceRanges: sourceRanges, + EnableConnectionPooler: util.True(), + EnableReplicaConnectionPooler: util.True(), + EnableMasterLoadBalancer: util.True(), + EnableMasterPoolerLoadBalancer: util.True(), + EnableReplicaLoadBalancer: util.True(), + EnableReplicaPoolerLoadBalancer: util.True(), + NumberOfInstances: 1, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedServices: getServices(v1.ServiceTypeLoadBalancer, sourceRanges, extTrafficPolicy, clusterName), + }, + } + + for _, tt := range tests { + var cluster = New( + Config{ + OpConfig: tt.config, + }, client, tt.pgSpec, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{} + generatedServices := make([]v1.ServiceSpec, 0) + for _, role := range roles { + cluster.syncService(role) + cluster.ConnectionPooler[role] = &ConnectionPoolerObjects{ + Name: cluster.connectionPoolerName(role), + ClusterName: cluster.ClusterName, + Namespace: cluster.Namespace, + Role: role, + } + cluster.syncConnectionPoolerWorker(&tt.pgSpec, &tt.pgSpec, role) + generatedServices = append(generatedServices, cluster.Services[role].Spec) + generatedServices = append(generatedServices, cluster.ConnectionPooler[role].Service.Spec) + } + if !reflect.DeepEqual(tt.expectedServices, generatedServices) { + t.Errorf("%s %s: expected %#v but got %#v", testName, tt.subTest, tt.expectedServices, generatedServices) + } + } +} + +func TestGenerateResourceRequirements(t *testing.T) { + testName := "TestGenerateResourceRequirements" + client, _ := newFakeK8sTestClient() + clusterName := "acid-test-cluster" + namespace := "default" + clusterNameLabel := "cluster-name" + roleLabel := "spilo-role" + sidecarName := "postgres-exporter" + + // two test cases will call enforceMinResourceLimits which emits 2 events per call + // hence bufferSize of 4 is required + newEventRecorder := record.NewFakeRecorder(4) + + configResources := config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: clusterNameLabel, + DefaultCPURequest: "100m", + DefaultCPULimit: "1", + DefaultMemoryRequest: "100Mi", + DefaultMemoryLimit: "500Mi", + MinCPULimit: "250m", + MinMemoryLimit: "250Mi", + PodRoleLabel: roleLabel, + } + + tests := []struct { + subTest string + config config.Config + pgSpec acidv1.Postgresql + expectedResources acidv1.Resources + }{ + { + subTest: "test generation of default resources when empty in manifest", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "500Mi"}, + }, + }, + { + subTest: "test generation of default resources for sidecar", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Sidecars: []acidv1.Sidecar{ + acidv1.Sidecar{ + Name: sidecarName, + }, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "500Mi"}, + }, + }, + { + subTest: "test generation of resources when only requests are defined in manifest", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "50m", Memory: "50Mi"}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "50m", Memory: "50Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "500Mi"}, + }, + }, + { + subTest: "test generation of resources when only memory is defined in manifest", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{Memory: "100Mi"}, + ResourceLimits: acidv1.ResourceDescription{Memory: "1Gi"}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "1Gi"}, + }, + }, + { + subTest: "test SetMemoryRequestToLimit flag", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: true, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "500Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "500Mi"}, + }, + }, + { + subTest: "test SetMemoryRequestToLimit flag for sidecar container, too", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: true, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Sidecars: []acidv1.Sidecar{ + acidv1.Sidecar{ + Name: sidecarName, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "10Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + }, + }, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "100Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + }, + }, + { + subTest: "test generating resources from manifest", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "250Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "400m", Memory: "800Mi"}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "250Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "400m", Memory: "800Mi"}, + }, + }, + { + subTest: "test enforcing min cpu and memory limit", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "200m", Memory: "200Mi"}, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "250m", Memory: "250Mi"}, + }, + }, + { + subTest: "test min cpu and memory limit are not enforced on sidecar", + config: config.Config{ + Resources: configResources, + PodManagementPolicy: "ordered_ready", + SetMemoryRequestToLimit: false, + }, + pgSpec: acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Sidecars: []acidv1.Sidecar{ + acidv1.Sidecar{ + Name: sidecarName, + Resources: &acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "10Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + }, + }, + }, + TeamID: "acid", + Volume: acidv1.Volume{ + Size: "1G", + }, + }, + }, + expectedResources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "10m", Memory: "10Mi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "100m", Memory: "100Mi"}, + }, + }, + } + + for _, tt := range tests { + var cluster = New( + Config{ + OpConfig: tt.config, + }, client, tt.pgSpec, logger, newEventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + _, err := cluster.createStatefulSet() + if k8sutil.ResourceAlreadyExists(err) { + err = cluster.syncStatefulSet() + } + assert.NoError(t, err) + + containers := cluster.Statefulset.Spec.Template.Spec.Containers + clusterResources, err := parseResourceRequirements(containers[0].Resources) + if len(containers) > 1 { + clusterResources, err = parseResourceRequirements(containers[1].Resources) + } + assert.NoError(t, err) + if !reflect.DeepEqual(tt.expectedResources, clusterResources) { + t.Errorf("%s - %s: expected %#v but got %#v", testName, tt.subTest, tt.expectedResources, clusterResources) + } + } +} + func TestGenerateCapabilities(t *testing.T) { testName := "TestGenerateCapabilities" @@ -1509,106 +2635,3 @@ func TestGenerateCapabilities(t *testing.T) { } } } - -func TestVolumeSelector(t *testing.T) { - testName := "TestVolumeSelector" - makeSpec := func(volume acidv1.Volume) acidv1.PostgresSpec { - return acidv1.PostgresSpec{ - TeamID: "myapp", - NumberOfInstances: 0, - Resources: acidv1.Resources{ - ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, - ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, - }, - Volume: volume, - } - } - - tests := []struct { - subTest string - volume acidv1.Volume - wantSelector *metav1.LabelSelector - }{ - { - subTest: "PVC template has no selector", - volume: acidv1.Volume{ - Size: "1G", - }, - wantSelector: nil, - }, - { - subTest: "PVC template has simple label selector", - volume: acidv1.Volume{ - Size: "1G", - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"environment": "unittest"}, - }, - }, - wantSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"environment": "unittest"}, - }, - }, - { - subTest: "PVC template has full selector", - volume: acidv1.Volume{ - Size: "1G", - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"environment": "unittest"}, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "flavour", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"banana", "chocolate"}, - }, - }, - }, - }, - wantSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"environment": "unittest"}, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "flavour", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"banana", "chocolate"}, - }, - }, - }, - }, - } - - cluster := New( - Config{ - OpConfig: config.Config{ - PodManagementPolicy: "ordered_ready", - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) - - for _, tt := range tests { - pgSpec := makeSpec(tt.volume) - sts, err := cluster.generateStatefulSet(&pgSpec) - if err != nil { - t.Fatalf("%s %s: no statefulset created %v", testName, tt.subTest, err) - } - - volIdx := len(sts.Spec.VolumeClaimTemplates) - for i, ct := range sts.Spec.VolumeClaimTemplates { - if ct.ObjectMeta.Name == constants.DataVolumeName { - volIdx = i - break - } - } - if volIdx == len(sts.Spec.VolumeClaimTemplates) { - t.Errorf("%s %s: no datavolume found in sts", testName, tt.subTest) - } - - selector := sts.Spec.VolumeClaimTemplates[volIdx].Spec.Selector - if !reflect.DeepEqual(selector, tt.wantSelector) { - t.Errorf("%s %s: expected: %#v but got: %#v", testName, tt.subTest, tt.wantSelector, selector) - } - } -} diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index edb55c882..d42f2c93d 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -2,8 +2,10 @@ package cluster import ( "fmt" + "strings" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util" v1 "k8s.io/api/core/v1" ) @@ -44,9 +46,25 @@ func (c *Cluster) GetDesiredMajorVersion() string { return c.Spec.PgVersion } +func (c *Cluster) isUpgradeAllowedForTeam(owningTeam string) bool { + allowedTeams := c.OpConfig.MajorVersionUpgradeTeamAllowList + + if len(allowedTeams) == 0 { + return false + } + + return util.SliceContains(allowedTeams, owningTeam) +} + +/* + Execute upgrade when mode is set to manual or full or when the owning team is allowed for upgrade (and mode is "off"). + + Manual upgrade means, it is triggered by the user via manifest version change + Full upgrade means, operator also determines the minimal version used accross all clusters and upgrades violators. +*/ func (c *Cluster) majorVersionUpgrade() error { - if c.OpConfig.MajorVersionUpgradeMode == "off" { + if c.OpConfig.MajorVersionUpgradeMode == "off" && !c.isUpgradeAllowedForTeam(c.Spec.TeamID) { return nil } @@ -66,7 +84,7 @@ func (c *Cluster) majorVersionUpgrade() error { var masterPod *v1.Pod - for _, pod := range pods { + for i, pod := range pods { ps, _ := c.patroni.GetMemberData(&pod) if ps.State != "running" { @@ -75,7 +93,7 @@ func (c *Cluster) majorVersionUpgrade() error { } if ps.Role == "master" { - masterPod = &pod + masterPod = &pods[i] c.currentMajorVersion = ps.ServerVersion } } @@ -88,14 +106,28 @@ func (c *Cluster) majorVersionUpgrade() error { c.logger.Infof("triggering major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Starting major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) upgradeCommand := fmt.Sprintf("/usr/bin/python3 /scripts/inplace_upgrade.py %d 2>&1 | tee last_upgrade.log", numberOfPods) - - result, err := c.ExecCommand(podName, "/bin/su", "postgres", "-c", upgradeCommand) - if err != nil { - c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, err) - return err + + c.logger.Debugf("checking if the spilo image runs with root or non-root (check for user id=0)") + resultIdCheck, errIdCheck := c.ExecCommand(podName, "/bin/bash", "-c", "/usr/bin/id -u") + if errIdCheck != nil { + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "Checking user id to run upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, errIdCheck) } - c.logger.Infof("upgrade action triggered and command completed: %s", result[:50]) + resultIdCheck = strings.TrimSuffix(resultIdCheck, "\n") + var result string + if resultIdCheck != "0" { + c.logger.Infof("User id was identified as: %s, hence default user is non-root already", resultIdCheck) + result, err = c.ExecCommand(podName, "/bin/bash", "-c", upgradeCommand) + } else { + c.logger.Infof("User id was identified as: %s, using su to reach the postgres user", resultIdCheck) + result, err = c.ExecCommand(podName, "/bin/su", "postgres", "-c", upgradeCommand) + } + if err != nil { + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "Upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, err) + return err + } + c.logger.Infof("upgrade action triggered and command completed: %s", result[:100]) + c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "Upgrade from %d to %d finished", c.currentMajorVersion, desiredVersion) } } diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 229648dd1..26c4c332d 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/rand" + "sort" "strconv" "time" @@ -12,6 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/patroni" @@ -65,7 +67,7 @@ func (c *Cluster) markRollingUpdateFlagForPod(pod *v1.Pod, msg string) error { return fmt.Errorf("could not form patch for pod's rolling update flag: %v", err) } - err = retryutil.Retry(1*time.Second, 5*time.Second, + err = retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, func() (bool, error) { _, err2 := c.KubeClient.Pods(pod.Namespace).Patch( context.TODO(), @@ -348,12 +350,60 @@ func (c *Cluster) MigrateReplicaPod(podName spec.NamespacedName, fromNodeName st return nil } +func (c *Cluster) getPatroniConfig(pod *v1.Pod) (acidv1.Patroni, map[string]string, error) { + var ( + patroniConfig acidv1.Patroni + pgParameters map[string]string + ) + podName := util.NameFromMeta(pod.ObjectMeta) + err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, + func() (bool, error) { + var err error + patroniConfig, pgParameters, err = c.patroni.GetConfig(pod) + + if err != nil { + return false, err + } + return true, nil + }, + ) + + if err != nil { + return acidv1.Patroni{}, nil, fmt.Errorf("could not get Postgres config from pod %s: %v", podName, err) + } + + return patroniConfig, pgParameters, nil +} + +func (c *Cluster) getPatroniMemberData(pod *v1.Pod) (patroni.MemberData, error) { + var memberData patroni.MemberData + err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, + func() (bool, error) { + var err error + memberData, err = c.patroni.GetMemberData(pod) + + if err != nil { + return false, err + } + return true, nil + }, + ) + if err != nil { + return patroni.MemberData{}, fmt.Errorf("could not get member data: %v", err) + } + if memberData.State == "creating replica" { + return patroni.MemberData{}, fmt.Errorf("replica currently being initialized") + } + + return memberData, nil +} + func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { ch := c.registerPodSubscriber(podName) defer c.unregisterPodSubscriber(podName) stopChan := make(chan struct{}) - err := retryutil.Retry(1*time.Second, 5*time.Second, + err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, func() (bool, error) { err2 := c.KubeClient.Pods(podName.Namespace).Delete( context.TODO(), @@ -379,54 +429,10 @@ func (c *Cluster) recreatePod(podName spec.NamespacedName) (*v1.Pod, error) { return pod, nil } -func (c *Cluster) isSafeToRecreatePods(pods []v1.Pod) bool { - - /* - Operator should not re-create pods if there is at least one replica being bootstrapped - because Patroni might use other replicas to take basebackup from (see Patroni's "clonefrom" tag). - - XXX operator cannot forbid replica re-init, so we might still fail if re-init is started - after this check succeeds but before a pod is re-created - */ - for _, pod := range pods { - c.logger.Debugf("name=%s phase=%s ip=%s", pod.Name, pod.Status.Phase, pod.Status.PodIP) - } - - for _, pod := range pods { - - var data patroni.MemberData - - err := retryutil.Retry(1*time.Second, 5*time.Second, - func() (bool, error) { - var err error - data, err = c.patroni.GetMemberData(&pod) - - if err != nil { - return false, err - } - return true, nil - }, - ) - - if err != nil { - c.logger.Errorf("failed to get Patroni state for pod: %s", err) - return false - } else if data.State == "creating replica" { - c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name) - return false - } - } - return true -} - func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.NamespacedName) error { c.setProcessName("starting to recreate pods") c.logger.Infof("there are %d pods in the cluster to recreate", len(pods)) - if !c.isSafeToRecreatePods(pods) { - return fmt.Errorf("postpone pod recreation until next Sync: recreation is unsafe because pods are being initialized") - } - var ( masterPod, newMasterPod *v1.Pod ) @@ -459,8 +465,13 @@ func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.Namesp // 1. we have not observed a new master pod when re-creating former replicas // 2. we know possible switchover targets even when no replicas were recreated if newMasterPod == nil && len(replicas) > 0 { - if err := c.Switchover(masterPod, masterCandidate(replicas)); err != nil { - c.logger.Warningf("could not perform switch over: %v", err) + masterCandidate, err := c.getSwitchoverCandidate(masterPod) + if err != nil { + // do not recreate master now so it will keep the update flag and switchover will be retried on next sync + return fmt.Errorf("skipping switchover: %v", err) + } + if err := c.Switchover(masterPod, masterCandidate); err != nil { + return fmt.Errorf("could not perform switch over: %v", err) } } else if newMasterPod == nil && len(replicas) == 0 { c.logger.Warningf("cannot perform switch over before re-creating the pod: no replicas") @@ -475,6 +486,54 @@ func (c *Cluster) recreatePods(pods []v1.Pod, switchoverCandidates []spec.Namesp return nil } +func (c *Cluster) getSwitchoverCandidate(master *v1.Pod) (spec.NamespacedName, error) { + + var members []patroni.ClusterMember + candidates := make([]patroni.ClusterMember, 0) + syncCandidates := make([]patroni.ClusterMember, 0) + + err := retryutil.Retry(c.OpConfig.PatroniAPICheckInterval, c.OpConfig.PatroniAPICheckTimeout, + func() (bool, error) { + var err error + members, err = c.patroni.GetClusterMembers(master) + + if err != nil { + return false, err + } + return true, nil + }, + ) + if err != nil { + return spec.NamespacedName{}, fmt.Errorf("failed to get Patroni cluster members: %s", err) + } + + for _, member := range members { + if PostgresRole(member.Role) != Leader && PostgresRole(member.Role) != StandbyLeader && member.State == "running" { + candidates = append(candidates, member) + if PostgresRole(member.Role) == SyncStandby { + syncCandidates = append(syncCandidates, member) + } + } + } + + // pick candidate with lowest lag + // if sync_standby replicas were found assume synchronous_mode is enabled and ignore other candidates list + if len(syncCandidates) > 0 { + sort.Slice(syncCandidates, func(i, j int) bool { + return util.IntFromIntStr(syncCandidates[i].Lag) < util.IntFromIntStr(syncCandidates[j].Lag) + }) + return spec.NamespacedName{Namespace: master.Namespace, Name: syncCandidates[0].Name}, nil + } + if len(candidates) > 0 { + sort.Slice(candidates, func(i, j int) bool { + return util.IntFromIntStr(candidates[i].Lag) < util.IntFromIntStr(candidates[j].Lag) + }) + return spec.NamespacedName{Namespace: master.Namespace, Name: candidates[0].Name}, nil + } + + return spec.NamespacedName{}, fmt.Errorf("no switchover candidate found") +} + func (c *Cluster) podIsEndOfLife(pod *v1.Pod) (bool, error) { node, err := c.KubeClient.Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{}) if err != nil { diff --git a/pkg/cluster/pod_test.go b/pkg/cluster/pod_test.go new file mode 100644 index 000000000..068145312 --- /dev/null +++ b/pkg/cluster/pod_test.go @@ -0,0 +1,94 @@ +package cluster + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/zalando/postgres-operator/mocks" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + "github.com/zalando/postgres-operator/pkg/util/patroni" +) + +func TestGetSwitchoverCandidate(t *testing.T) { + testName := "test getting right switchover candidate" + namespace := "default" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + var cluster = New( + Config{ + OpConfig: config.Config{ + PatroniAPICheckInterval: time.Duration(1), + PatroniAPICheckTimeout: time.Duration(5), + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + // simulate different member scenarios + tests := []struct { + subtest string + clusterJson string + expectedCandidate spec.NamespacedName + expectedError error + }{ + { + subtest: "choose sync_standby over replica", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 0}]}`, + expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-1"}, + expectedError: nil, + }, + { + subtest: "choose replica with lowest lag", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "replica", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 5}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 2}]}`, + expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-2"}, + expectedError: nil, + }, + { + subtest: "choose first replica when lag is equal evrywhere", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "replica", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 5}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": 5}]}`, + expectedCandidate: spec.NamespacedName{Namespace: namespace, Name: "acid-test-cluster-1"}, + expectedError: nil, + }, + { + subtest: "no running replica available", + clusterJson: `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 2}, {"name": "acid-test-cluster-1", "role": "replica", "state": "starting", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 2}]}`, + expectedCandidate: spec.NamespacedName{}, + expectedError: fmt.Errorf("no switchover candidate found"), + }, + } + + for _, tt := range tests { + // mocking cluster members + r := ioutil.NopCloser(bytes.NewReader([]byte(tt.clusterJson))) + + response := http.Response{ + StatusCode: 200, + Body: r, + } + + mockClient := mocks.NewMockHTTPClient(ctrl) + mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil).AnyTimes() + + p := patroni.New(patroniLogger, mockClient) + cluster.patroni = p + mockMasterPod := newMockPod("192.168.100.1") + mockMasterPod.Namespace = namespace + + candidate, err := cluster.getSwitchoverCandidate(mockMasterPod) + if err != nil && err.Error() != tt.expectedError.Error() { + t.Errorf("%s - %s: unexpected error, %v", testName, tt.subtest, err) + } + + if candidate != tt.expectedCandidate { + t.Errorf("%s - %s: unexpect switchover candidate, got %s, expected %s", testName, tt.subtest, candidate, tt.expectedCandidate) + } + } +} diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index f078c6434..9a57051df 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -35,8 +35,10 @@ func (c *Cluster) listResources() error { c.logger.Infof("found secret: %q (uid: %q) namesapce: %s", util.NameFromMeta(obj.ObjectMeta), obj.UID, obj.ObjectMeta.Namespace) } - for role, endpoint := range c.Endpoints { - c.logger.Infof("found %s endpoint: %q (uid: %q)", role, util.NameFromMeta(endpoint.ObjectMeta), endpoint.UID) + if !c.patroniKubernetesUseConfigMaps() { + for role, endpoint := range c.Endpoints { + c.logger.Infof("found %s endpoint: %q (uid: %q)", role, util.NameFromMeta(endpoint.ObjectMeta), endpoint.UID) + } } for role, service := range c.Services { @@ -275,7 +277,7 @@ func (c *Cluster) createService(role PostgresRole) (*v1.Service, error) { return service, nil } -func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error { +func (c *Cluster) updateService(role PostgresRole, oldService *v1.Service, newService *v1.Service) (*v1.Service, error) { var ( svc *v1.Service err error @@ -283,11 +285,7 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error c.setProcessName("updating %v service", role) - if c.Services[role] == nil { - return fmt.Errorf("there is no service in the cluster") - } - - serviceName := util.NameFromMeta(c.Services[role].ObjectMeta) + serviceName := util.NameFromMeta(oldService.ObjectMeta) // update the service annotation in order to propagate ELB notation. if len(newService.ObjectMeta.Annotations) > 0 { @@ -301,39 +299,38 @@ func (c *Cluster) updateService(role PostgresRole, newService *v1.Service) error "") if err != nil { - return fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err) + return nil, fmt.Errorf("could not replace annotations for the service %q: %v", serviceName, err) } } else { - return fmt.Errorf("could not form patch for the service metadata: %v", err) + return nil, fmt.Errorf("could not form patch for the service metadata: %v", err) } } // now, patch the service spec, but when disabling LoadBalancers do update instead // patch does not work because of LoadBalancerSourceRanges field (even if set to nil) - oldServiceType := c.Services[role].Spec.Type + oldServiceType := oldService.Spec.Type newServiceType := newService.Spec.Type if newServiceType == "ClusterIP" && newServiceType != oldServiceType { - newService.ResourceVersion = c.Services[role].ResourceVersion - newService.Spec.ClusterIP = c.Services[role].Spec.ClusterIP + newService.ResourceVersion = oldService.ResourceVersion + newService.Spec.ClusterIP = oldService.Spec.ClusterIP svc, err = c.KubeClient.Services(serviceName.Namespace).Update(context.TODO(), newService, metav1.UpdateOptions{}) if err != nil { - return fmt.Errorf("could not update service %q: %v", serviceName, err) + return nil, fmt.Errorf("could not update service %q: %v", serviceName, err) } } else { patchData, err := specPatch(newService.Spec) if err != nil { - return fmt.Errorf("could not form patch for the service %q: %v", serviceName, err) + return nil, fmt.Errorf("could not form patch for the service %q: %v", serviceName, err) } svc, err = c.KubeClient.Services(serviceName.Namespace).Patch( context.TODO(), serviceName.Name, types.MergePatchType, patchData, metav1.PatchOptions{}, "") if err != nil { - return fmt.Errorf("could not patch service %q: %v", serviceName, err) + return nil, fmt.Errorf("could not patch service %q: %v", serviceName, err) } } - c.Services[role] = svc - return nil + return svc, nil } func (c *Cluster) deleteService(role PostgresRole) error { @@ -496,18 +493,17 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error { func (c *Cluster) deleteSecrets() error { c.setProcessName("deleting secrets") - var errors []string - errorCount := 0 + errors := make([]string, 0) + for uid, secret := range c.Secrets { err := c.deleteSecret(uid, *secret) if err != nil { errors = append(errors, fmt.Sprintf("%v", err)) - errorCount++ } } - if errorCount > 0 { - return fmt.Errorf("could not delete all secrets: %v", errors) + if len(errors) > 0 { + return fmt.Errorf("could not delete all secrets: %v", strings.Join(errors, `', '`)) } return nil @@ -595,7 +591,7 @@ func (c *Cluster) GetEndpointMaster() *v1.Endpoints { return c.Endpoints[Master] } -// GetEndpointReplica returns cluster's kubernetes master Endpoint +// GetEndpointReplica returns cluster's kubernetes replica Endpoint func (c *Cluster) GetEndpointReplica() *v1.Endpoints { return c.Endpoints[Replica] } diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go new file mode 100644 index 000000000..0236925ca --- /dev/null +++ b/pkg/cluster/streams.go @@ -0,0 +1,398 @@ +package cluster + +import ( + "context" + "fmt" + "reflect" + "sort" + "strings" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + "github.com/zalando/postgres-operator/pkg/util" + "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (c *Cluster) createStreams(appId string) error { + c.setProcessName("creating streams") + + fes := c.generateFabricEventStream(appId) + if _, err := c.KubeClient.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}); err != nil { + return err + } + + return nil +} + +func (c *Cluster) updateStreams(newEventStreams *zalandov1.FabricEventStream) error { + c.setProcessName("updating event streams") + + if _, err := c.KubeClient.FabricEventStreams(newEventStreams.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}); err != nil { + return err + } + + return nil +} + +func (c *Cluster) deleteStreams() error { + c.setProcessName("deleting event streams") + + // check if stream CRD is installed before trying a delete + _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamCRDName, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + return nil + } + + errors := make([]string, 0) + for _, appId := range c.streamApplications { + fesName := fmt.Sprintf("%s-%s", c.Name, appId) + err = c.KubeClient.FabricEventStreams(c.Namespace).Delete(context.TODO(), fesName, metav1.DeleteOptions{}) + if err != nil { + errors = append(errors, fmt.Sprintf("could not delete event stream %q: %v", fesName, err)) + } + } + + if len(errors) > 0 { + return fmt.Errorf("could not delete all event stream custom resources: %v", strings.Join(errors, `', '`)) + } + + return nil +} + +func gatherApplicationIds(streams []acidv1.Stream) []string { + appIds := make([]string, 0) + for _, stream := range streams { + if !util.SliceContains(appIds, stream.ApplicationId) { + appIds = append(appIds, stream.ApplicationId) + } + } + + return appIds +} + +func (c *Cluster) syncPostgresConfig(requiredPatroniConfig acidv1.Patroni) error { + errorMsg := "no pods found to update config" + + // if streams are defined wal_level must be switched to logical + requiredPgParameters := map[string]string{"wal_level": "logical"} + + // apply config changes in pods + pods, err := c.listPods() + if err != nil { + errorMsg = fmt.Sprintf("could not list pods of the statefulset: %v", err) + } + for i, pod := range pods { + podName := util.NameFromMeta(pods[i].ObjectMeta) + effectivePatroniConfig, effectivePgParameters, err := c.patroni.GetConfig(&pod) + if err != nil { + errorMsg = fmt.Sprintf("could not get Postgres config from pod %s: %v", podName, err) + continue + } + + _, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, effectivePatroniConfig, requiredPatroniConfig, effectivePgParameters, requiredPgParameters) + if err != nil { + errorMsg = fmt.Sprintf("could not set PostgreSQL configuration options for pod %s: %v", podName, err) + continue + } + + // Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used + return nil + } + + return fmt.Errorf(errorMsg) +} + +func (c *Cluster) syncPublication(publication, dbName string, tables map[string]acidv1.StreamTable) error { + createPublications := make(map[string]string) + alterPublications := make(map[string]string) + + defer func() { + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + + // check for existing publications + if err := c.initDbConnWithName(dbName); err != nil { + return fmt.Errorf("could not init database connection") + } + + currentPublications, err := c.getPublications() + if err != nil { + return fmt.Errorf("could not get current publications: %v", err) + } + + tableNames := make([]string, len(tables)) + i := 0 + for t := range tables { + tableName, schemaName := getTableSchema(t) + tableNames[i] = fmt.Sprintf("%s.%s", schemaName, tableName) + i++ + } + sort.Strings(tableNames) + tableList := strings.Join(tableNames, ", ") + + currentTables, exists := currentPublications[publication] + if !exists { + createPublications[publication] = tableList + } else if currentTables != tableList { + alterPublications[publication] = tableList + } + + if len(createPublications)+len(alterPublications) == 0 { + return nil + } + + for publicationName, tables := range createPublications { + if err = c.executeCreatePublication(publicationName, tables); err != nil { + return fmt.Errorf("creation of publication %q failed: %v", publicationName, err) + } + } + for publicationName, tables := range alterPublications { + if err = c.executeAlterPublication(publicationName, tables); err != nil { + return fmt.Errorf("update of publication %q failed: %v", publicationName, err) + } + } + + return nil +} + +func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEventStream { + eventStreams := make([]zalandov1.EventStream, 0) + + for _, stream := range c.Spec.Streams { + if stream.ApplicationId != appId { + continue + } + for tableName, table := range stream.Tables { + streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn) + streamFlow := getEventStreamFlow(stream, table.PayloadColumn) + streamSink := getEventStreamSink(stream, table.EventType) + + eventStreams = append(eventStreams, zalandov1.EventStream{ + EventStreamFlow: streamFlow, + EventStreamSink: streamSink, + EventStreamSource: streamSource}) + } + } + + return &zalandov1.FabricEventStream{ + TypeMeta: metav1.TypeMeta{ + APIVersion: constants.EventStreamCRDApiVersion, + Kind: constants.EventStreamCRDKind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", c.Name, appId), + Namespace: c.Namespace, + Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)), + // make cluster StatefulSet the owner (like with connection pooler objects) + OwnerReferences: c.ownerReferences(), + }, + Spec: zalandov1.FabricEventStreamSpec{ + ApplicationId: appId, + EventStreams: eventStreams, + }, + } +} + +func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName string, idColumn *string) zalandov1.EventStreamSource { + table, schema := getTableSchema(tableName) + streamFilter := stream.Filter[tableName] + return zalandov1.EventStreamSource{ + Type: constants.EventStreamSourcePGType, + Schema: schema, + EventStreamTable: getOutboxTable(table, idColumn), + Filter: streamFilter, + Connection: c.getStreamConnection( + stream.Database, + constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix, + stream.ApplicationId), + } +} + +func getEventStreamFlow(stream acidv1.Stream, payloadColumn *string) zalandov1.EventStreamFlow { + return zalandov1.EventStreamFlow{ + Type: constants.EventStreamFlowPgGenericType, + PayloadColumn: payloadColumn, + } +} + +func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1.EventStreamSink { + return zalandov1.EventStreamSink{ + Type: constants.EventStreamSinkNakadiType, + EventType: eventType, + MaxBatchSize: stream.BatchSize, + } +} + +func getTableSchema(fullTableName string) (tableName, schemaName string) { + schemaName = "public" + tableName = fullTableName + if strings.Contains(fullTableName, ".") { + schemaName = strings.Split(fullTableName, ".")[0] + tableName = strings.Split(fullTableName, ".")[1] + } + + return tableName, schemaName +} + +func getOutboxTable(tableName string, idColumn *string) zalandov1.EventStreamTable { + return zalandov1.EventStreamTable{ + Name: tableName, + IDColumn: idColumn, + } +} + +func getSlotName(dbName, appId string) string { + return fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)) +} + +func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1.Connection { + return zalandov1.Connection{ + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), + SlotName: getSlotName(database, appId), + PluginType: constants.EventStreamSourcePluginType, + DBAuth: zalandov1.DBAuth{ + Type: constants.EventStreamSourceAuthType, + Name: c.credentialSecretNameForCluster(user, c.Name), + UserKey: "username", + PasswordKey: "password", + }, + } +} + +func (c *Cluster) syncStreams() error { + + c.setProcessName("syncing streams") + + _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamCRDName, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("event stream CRD not installed, skipping") + return nil + } + + // fetch different application IDs from streams section + // there will be a separate event stream resource for each ID + appIds := gatherApplicationIds(c.Spec.Streams) + c.streamApplications = appIds + + slots := make(map[string]map[string]string) + publications := make(map[string]map[string]acidv1.StreamTable) + + requiredPatroniConfig := c.Spec.Patroni + if len(requiredPatroniConfig.Slots) > 0 { + slots = requiredPatroniConfig.Slots + } + + // gather list of required slots and publications + for _, stream := range c.Spec.Streams { + slot := map[string]string{ + "database": stream.Database, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + } + slotName := getSlotName(stream.Database, stream.ApplicationId) + if _, exists := slots[slotName]; !exists { + slots[slotName] = slot + publications[slotName] = stream.Tables + } else { + streamTables := publications[slotName] + for tableName, table := range stream.Tables { + if _, exists := streamTables[tableName]; !exists { + streamTables[tableName] = table + } + } + publications[slotName] = streamTables + } + } + + // no slots = no streams defined + if len(slots) > 0 { + requiredPatroniConfig.Slots = slots + } else { + return nil + } + + // add extra logical slots to Patroni config + c.logger.Debug("syncing Postgres config for logical decoding") + err = c.syncPostgresConfig(requiredPatroniConfig) + if err != nil { + return fmt.Errorf("failed to snyc Postgres config for event streaming: %v", err) + } + + // next, create publications to each created slot + c.logger.Debug("syncing database publications") + for publication, tables := range publications { + // but first check for existing publications + dbName := slots[publication]["database"] + err = c.syncPublication(publication, dbName, tables) + if err != nil { + c.logger.Warningf("could not sync publication %q in database %q: %v", publication, dbName, err) + } + } + + err = c.createOrUpdateStreams() + if err != nil { + return err + } + + return nil +} + +func (c *Cluster) createOrUpdateStreams() error { + for _, appId := range c.streamApplications { + fesName := fmt.Sprintf("%s-%s", c.Name, appId) + effectiveStreams, err := c.KubeClient.FabricEventStreams(c.Namespace).Get(context.TODO(), fesName, metav1.GetOptions{}) + if err != nil { + if !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("failed reading event stream %s: %v", fesName, err) + } + + c.logger.Infof("event streams do not exist, create it") + err = c.createStreams(appId) + if err != nil { + return fmt.Errorf("failed creating event stream %s: %v", fesName, err) + } + c.logger.Infof("event stream %q has been successfully created", fesName) + } else { + desiredStreams := c.generateFabricEventStream(appId) + if match, reason := sameStreams(effectiveStreams.Spec.EventStreams, desiredStreams.Spec.EventStreams); !match { + c.logger.Debugf("updating event streams: %s", reason) + desiredStreams.ObjectMeta.ResourceVersion = effectiveStreams.ObjectMeta.ResourceVersion + err = c.updateStreams(desiredStreams) + if err != nil { + return fmt.Errorf("failed updating event stream %s: %v", fesName, err) + } + c.logger.Infof("event stream %q has been successfully updated", fesName) + } + } + } + + return nil +} + +func sameStreams(curEventStreams, newEventStreams []zalandov1.EventStream) (match bool, reason string) { + if len(newEventStreams) != len(curEventStreams) { + return false, "number of defined streams is different" + } + + for _, newStream := range newEventStreams { + match = false + reason = "event stream specs differ" + for _, curStream := range curEventStreams { + if reflect.DeepEqual(newStream.EventStreamSource, curStream.EventStreamSource) && + reflect.DeepEqual(newStream.EventStreamFlow, curStream.EventStreamFlow) && + reflect.DeepEqual(newStream.EventStreamSink, curStream.EventStreamSink) { + match = true + break + } + } + if !match { + return false, reason + } + } + + return true, "" +} diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go new file mode 100644 index 000000000..0094708a4 --- /dev/null +++ b/pkg/cluster/streams_test.go @@ -0,0 +1,375 @@ +package cluster + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + "context" + "testing" + + "github.com/stretchr/testify/assert" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + fakezalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" + "github.com/zalando/postgres-operator/pkg/util" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" +) + +func newFakeK8sStreamClient() (k8sutil.KubernetesClient, *fake.Clientset) { + zalandoClientSet := fakezalandov1.NewSimpleClientset() + clientSet := fake.NewSimpleClientset() + + return k8sutil.KubernetesClient{ + FabricEventStreamsGetter: zalandoClientSet.ZalandoV1(), + PostgresqlsGetter: zalandoClientSet.AcidV1(), + PodsGetter: clientSet.CoreV1(), + StatefulSetsGetter: clientSet.AppsV1(), + }, clientSet +} + +var ( + clusterName string = "acid-test-cluster" + namespace string = "default" + appId string = "test-app" + dbName string = "foo" + fesUser string = constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix + fesName string = fmt.Sprintf("%s-%s", clusterName, appId) + slotName string = fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)) + + pg = acidv1.Postgresql{ + TypeMeta: metav1.TypeMeta{ + Kind: "Postgresql", + APIVersion: "acid.zalan.do/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Databases: map[string]string{ + dbName: dbName + constants.UserRoleNameSuffix, + }, + Streams: []acidv1.Stream{ + { + ApplicationId: appId, + Database: "foo", + Tables: map[string]acidv1.StreamTable{ + "data.bar": acidv1.StreamTable{ + EventType: "stream-type-a", + IdColumn: k8sutil.StringToPointer("b_id"), + PayloadColumn: k8sutil.StringToPointer("b_payload"), + }, + "data.foobar": acidv1.StreamTable{ + EventType: "stream-type-b", + }, + }, + Filter: map[string]*string{ + "data.bar": k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"), + }, + BatchSize: k8sutil.UInt32ToPointer(uint32(100)), + }, + }, + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + fes = &zalandov1.FabricEventStream{ + TypeMeta: metav1.TypeMeta{ + APIVersion: constants.EventStreamCRDApiVersion, + Kind: constants.EventStreamCRDKind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fesName, + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + metav1.OwnerReference{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "acid-test-cluster", + Controller: util.True(), + }, + }, + }, + Spec: zalandov1.FabricEventStreamSpec{ + ApplicationId: appId, + EventStreams: []zalandov1.EventStream{ + zalandov1.EventStream{ + EventStreamFlow: zalandov1.EventStreamFlow{ + PayloadColumn: k8sutil.StringToPointer("b_payload"), + Type: constants.EventStreamFlowPgGenericType, + }, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-a", + MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)), + Type: constants.EventStreamSinkNakadiType, + }, + EventStreamSource: zalandov1.EventStreamSource{ + Filter: k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"), + Connection: zalandov1.Connection{ + DBAuth: zalandov1.DBAuth{ + Name: fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName), + PasswordKey: "password", + Type: constants.EventStreamSourceAuthType, + UserKey: "username", + }, + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), + SlotName: slotName, + PluginType: constants.EventStreamSourcePluginType, + }, + Schema: "data", + EventStreamTable: zalandov1.EventStreamTable{ + IDColumn: k8sutil.StringToPointer("b_id"), + Name: "bar", + }, + Type: constants.EventStreamSourcePGType, + }, + }, + zalandov1.EventStream{ + EventStreamFlow: zalandov1.EventStreamFlow{ + Type: constants.EventStreamFlowPgGenericType, + }, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-b", + MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)), + Type: constants.EventStreamSinkNakadiType, + }, + EventStreamSource: zalandov1.EventStreamSource{ + Connection: zalandov1.Connection{ + DBAuth: zalandov1.DBAuth{ + Name: fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName), + PasswordKey: "password", + Type: constants.EventStreamSourceAuthType, + UserKey: "username", + }, + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), + SlotName: slotName, + PluginType: constants.EventStreamSourcePluginType, + }, + Schema: "data", + EventStreamTable: zalandov1.EventStreamTable{ + Name: "foobar", + }, + Type: constants.EventStreamSourcePGType, + }, + }, + }, + }, + } +) + +func TestGenerateFabricEventStream(t *testing.T) { + client, _ := newFakeK8sStreamClient() + + var cluster = New( + Config{ + OpConfig: config.Config{ + Auth: config.Auth{ + SecretNameTemplate: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}", + }, + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + + // create statefulset to have ownerReference for streams + _, err := cluster.createStatefulSet() + assert.NoError(t, err) + + // createOrUpdateStreams will loop over existing apps + cluster.streamApplications = []string{appId} + + // create the streams + err = cluster.createOrUpdateStreams() + assert.NoError(t, err) + + // compare generated stream with expected stream + result := cluster.generateFabricEventStream(appId) + if match, _ := sameStreams(result.Spec.EventStreams, fes.Spec.EventStreams); !match { + t.Errorf("malformed FabricEventStream, expected %#v, got %#v", fes, result) + } + + // compare stream resturned from API with expected stream + streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), fesName, metav1.GetOptions{}) + assert.NoError(t, err) + if match, _ := sameStreams(streamCRD.Spec.EventStreams, fes.Spec.EventStreams); !match { + t.Errorf("malformed FabricEventStream returned from API, expected %#v, got %#v", fes, streamCRD) + } + + // sync streams once again + err = cluster.createOrUpdateStreams() + assert.NoError(t, err) + + // compare stream resturned from API with generated stream + streamCRD, err = cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), fesName, metav1.GetOptions{}) + assert.NoError(t, err) + if match, _ := sameStreams(streamCRD.Spec.EventStreams, result.Spec.EventStreams); !match { + t.Errorf("returned FabricEventStream differs from generated one, expected %#v, got %#v", result, streamCRD) + } +} + +func TestSameStreams(t *testing.T) { + testName := "TestSameStreams" + + stream1 := zalandov1.EventStream{ + EventStreamFlow: zalandov1.EventStreamFlow{}, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-a", + }, + EventStreamSource: zalandov1.EventStreamSource{ + EventStreamTable: zalandov1.EventStreamTable{ + Name: "foo", + }, + }, + } + + stream2 := zalandov1.EventStream{ + EventStreamFlow: zalandov1.EventStreamFlow{}, + EventStreamSink: zalandov1.EventStreamSink{ + EventType: "stream-type-b", + }, + EventStreamSource: zalandov1.EventStreamSource{ + EventStreamTable: zalandov1.EventStreamTable{ + Name: "bar", + }, + }, + } + + tests := []struct { + subTest string + streamsA []zalandov1.EventStream + streamsB []zalandov1.EventStream + match bool + reason string + }{ + { + subTest: "identical streams", + streamsA: []zalandov1.EventStream{stream1, stream2}, + streamsB: []zalandov1.EventStream{stream1, stream2}, + match: true, + reason: "", + }, + { + subTest: "same streams different order", + streamsA: []zalandov1.EventStream{stream1, stream2}, + streamsB: []zalandov1.EventStream{stream2, stream1}, + match: true, + reason: "", + }, + { + subTest: "same streams different order", + streamsA: []zalandov1.EventStream{stream1}, + streamsB: []zalandov1.EventStream{stream1, stream2}, + match: false, + reason: "number of defined streams is different", + }, + { + subTest: "different number of streams", + streamsA: []zalandov1.EventStream{stream1}, + streamsB: []zalandov1.EventStream{stream1, stream2}, + match: false, + reason: "number of defined streams is different", + }, + { + subTest: "event stream specs differ", + streamsA: []zalandov1.EventStream{stream1, stream2}, + streamsB: fes.Spec.EventStreams, + match: false, + reason: "number of defined streams is different", + }, + } + + for _, tt := range tests { + streamsMatch, matchReason := sameStreams(tt.streamsA, tt.streamsB) + if streamsMatch != tt.match { + t.Errorf("%s %s: unexpected match result when comparing streams: got %s, epxected %s", + testName, tt.subTest, matchReason, tt.reason) + } + } +} + +func TestUpdateFabricEventStream(t *testing.T) { + client, _ := newFakeK8sStreamClient() + + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + _, err := cluster.KubeClient.Postgresqls(namespace).Create( + context.TODO(), &pg, metav1.CreateOptions{}) + assert.NoError(t, err) + + // createOrUpdateStreams will loop over existing apps + cluster.streamApplications = []string{appId} + + err = cluster.createOrUpdateStreams() + assert.NoError(t, err) + + var pgSpec acidv1.PostgresSpec + pgSpec.Streams = []acidv1.Stream{ + { + ApplicationId: appId, + Database: dbName, + Tables: map[string]acidv1.StreamTable{ + "data.bar": acidv1.StreamTable{ + EventType: "stream-type-c", + IdColumn: k8sutil.StringToPointer("b_id"), + PayloadColumn: k8sutil.StringToPointer("b_payload"), + }, + }, + BatchSize: k8sutil.UInt32ToPointer(uint32(250)), + }, + } + patch, err := json.Marshal(struct { + PostgresSpec interface{} `json:"spec"` + }{&pgSpec}) + assert.NoError(t, err) + + pgPatched, err := cluster.KubeClient.Postgresqls(namespace).Patch( + context.TODO(), cluster.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "spec") + assert.NoError(t, err) + + cluster.Postgresql.Spec = pgPatched.Spec + err = cluster.createOrUpdateStreams() + assert.NoError(t, err) + + streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), fesName, metav1.GetOptions{}) + assert.NoError(t, err) + + result := cluster.generateFabricEventStream(appId) + if !reflect.DeepEqual(result, streamCRD) { + t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", streamCRD, result) + } +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 2df168a6e..a34279533 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -69,19 +69,14 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } - if c.OpConfig.EnableEBSGp3Migration { + if c.OpConfig.EnableEBSGp3Migration && len(c.EBSVolumes) > 0 { err = c.executeEBSMigration() if nil != err { return err } } - if err = c.enforceMinResourceLimits(&c.Spec); err != nil { - err = fmt.Errorf("could not enforce minimum resource limits: %v", err) - return err - } - - c.logger.Debugf("syncing statefulsets") + c.logger.Debug("syncing statefulsets") if err = c.syncStatefulSet(); err != nil { if !k8sutil.ResourceAlreadyExists(err) { err = fmt.Errorf("could not sync statefulsets: %v", err) @@ -107,17 +102,17 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { // create database objects unless we are running without pods or disabled that feature explicitly if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&newSpec.Spec) <= 0 || c.Spec.StandbyCluster != nil) { - c.logger.Debugf("syncing roles") + c.logger.Debug("syncing roles") if err = c.syncRoles(); err != nil { err = fmt.Errorf("could not sync roles: %v", err) return err } - c.logger.Debugf("syncing databases") + c.logger.Debug("syncing databases") if err = c.syncDatabases(); err != nil { err = fmt.Errorf("could not sync databases: %v", err) return err } - c.logger.Debugf("syncing prepared databases with schemas") + c.logger.Debug("syncing prepared databases with schemas") if err = c.syncPreparedDatabases(); err != nil { err = fmt.Errorf("could not sync prepared database: %v", err) return err @@ -129,6 +124,14 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return fmt.Errorf("could not sync connection pooler: %v", err) } + if len(c.Spec.Streams) > 0 { + c.logger.Debug("syncing streams") + if err = c.syncStreams(); err != nil { + err = fmt.Errorf("could not sync streams: %v", err) + return err + } + } + // Major version upgrade must only run after success of all earlier operations, must remain last item in sync if err := c.majorVersionUpgrade(); err != nil { c.logger.Errorf("major version upgrade failed: %v", err) @@ -164,11 +167,13 @@ func (c *Cluster) syncService(role PostgresRole) error { if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil { c.Services[role] = svc desiredSvc := c.generateService(role, &c.Spec) - if match, reason := k8sutil.SameService(svc, desiredSvc); !match { + if match, reason := c.compareServices(svc, desiredSvc); !match { c.logServiceChanges(role, svc, desiredSvc, false, reason) - if err = c.updateService(role, desiredSvc); err != nil { + updatedSvc, err := c.updateService(role, svc, desiredSvc) + if err != nil { return fmt.Errorf("could not update %s service to match desired state: %v", role, err) } + c.Services[role] = updatedSvc c.logger.Infof("%s service %q is in the desired state now", role, util.NameFromMeta(desiredSvc.ObjectMeta)) } return nil @@ -277,6 +282,7 @@ func (c *Cluster) syncStatefulSet() error { restartMasterFirst bool ) podsToRecreate := make([]v1.Pod, 0) + isSafeToRecreatePods := true switchoverCandidates := make([]spec.NamespacedName, 0) pods, err := c.listPods() @@ -400,25 +406,26 @@ func (c *Cluster) syncStatefulSet() error { } // get Postgres config, compare with manifest and update via Patroni PATCH endpoint if it differs - // Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used. + // Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used for i, pod := range pods { - emptyPatroniConfig := acidv1.Patroni{} - podName := util.NameFromMeta(pods[i].ObjectMeta) - patroniConfig, pgParameters, err := c.patroni.GetConfig(&pod) + patroniConfig, pgParameters, err := c.getPatroniConfig(&pod) if err != nil { - c.logger.Warningf("could not get Postgres config from pod %s: %v", podName, err) + c.logger.Warningf("%v", err) + isSafeToRecreatePods = false continue } restartWait = patroniConfig.LoopWait // empty config probably means cluster is not fully initialized yet, e.g. restoring from backup // do not attempt a restart - if !reflect.DeepEqual(patroniConfig, emptyPatroniConfig) || len(pgParameters) > 0 { - restartMasterFirst, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, patroniConfig, pgParameters) + if !reflect.DeepEqual(patroniConfig, acidv1.Patroni{}) || len(pgParameters) > 0 { + // compare config returned from Patroni with what is specified in the manifest + restartMasterFirst, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, patroniConfig, c.Spec.Patroni, pgParameters, c.Spec.Parameters) if err != nil { - c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", podName, err) + c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", pods[i].Name, err) continue } + // it could take up to LoopWait to apply the config time.Sleep(time.Duration(restartWait)*time.Second + time.Second*2) break @@ -437,50 +444,59 @@ func (c *Cluster) syncStatefulSet() error { remainingPods = append(remainingPods, &pods[i]) continue } - c.restartInstance(&pod, restartWait) + if err = c.restartInstance(&pod, restartWait); err != nil { + c.logger.Errorf("%v", err) + isSafeToRecreatePods = false + } } // in most cases only the master should be left to restart if len(remainingPods) > 0 { for _, remainingPod := range remainingPods { - c.restartInstance(remainingPod, restartWait) + if err = c.restartInstance(remainingPod, restartWait); err != nil { + c.logger.Errorf("%v", err) + isSafeToRecreatePods = false + } } } // if we get here we also need to re-create the pods (either leftovers from the old // statefulset or those that got their configuration from the outdated statefulset) if len(podsToRecreate) > 0 { - c.logger.Debugln("performing rolling update") - c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update") - if err := c.recreatePods(podsToRecreate, switchoverCandidates); err != nil { - return fmt.Errorf("could not recreate pods: %v", err) + if isSafeToRecreatePods { + c.logger.Debugln("performing rolling update") + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Performing rolling update") + if err := c.recreatePods(podsToRecreate, switchoverCandidates); err != nil { + return fmt.Errorf("could not recreate pods: %v", err) + } + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated") + } else { + c.logger.Warningf("postpone pod recreation until next sync") } - c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", "Rolling update done - pods have been recreated") } return nil } -func (c *Cluster) restartInstance(pod *v1.Pod, restartWait uint32) { +func (c *Cluster) restartInstance(pod *v1.Pod, restartWait uint32) error { + // if the config update requires a restart, call Patroni restart podName := util.NameFromMeta(pod.ObjectMeta) role := PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel]) - - // if the config update requires a restart, call Patroni restart - memberData, err := c.patroni.GetMemberData(pod) + memberData, err := c.getPatroniMemberData(pod) if err != nil { - c.logger.Debugf("could not get member data of %s pod %s - skipping possible restart attempt: %v", role, podName, err) - return + return fmt.Errorf("could not restart Postgres in %s pod %s: %v", role, podName, err) } // do restart only when it is pending if memberData.PendingRestart { - c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("restarting Postgres server within %s pod %s", role, pod.Name)) + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("restarting Postgres server within %s pod %s", role, podName)) if err := c.patroni.Restart(pod); err != nil { - c.logger.Warningf("could not restart Postgres server within %s pod %s: %v", role, podName, err) - return + return err } time.Sleep(time.Duration(restartWait) * time.Second) - c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("Postgres server restart done for %s pod %s", role, pod.Name)) + c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Update", fmt.Sprintf("Postgres server restart done for %s pod %s", role, podName)) } + + return nil } // AnnotationsToPropagate get the annotations to update if required @@ -517,40 +533,39 @@ func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[stri // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters // (like max_connections) have changed and if necessary sets it via the Patroni API -func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniConfig acidv1.Patroni, effectivePgParameters map[string]string) (bool, error) { +func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectivePatroniConfig, desiredPatroniConfig acidv1.Patroni, effectivePgParameters, desiredPgParameters map[string]string) (bool, error) { configToSet := make(map[string]interface{}) parametersToSet := make(map[string]string) restartMaster := make([]bool, 0) requiresMasterRestart := false - // compare options from config with c.Spec.Patroni from manifest - desiredPatroniConfig := c.Spec.Patroni - if desiredPatroniConfig.LoopWait > 0 && desiredPatroniConfig.LoopWait != patroniConfig.LoopWait { + // compare effective and desired Patroni config options + if desiredPatroniConfig.LoopWait > 0 && desiredPatroniConfig.LoopWait != effectivePatroniConfig.LoopWait { configToSet["loop_wait"] = desiredPatroniConfig.LoopWait } - if desiredPatroniConfig.MaximumLagOnFailover > 0 && desiredPatroniConfig.MaximumLagOnFailover != patroniConfig.MaximumLagOnFailover { + if desiredPatroniConfig.MaximumLagOnFailover > 0 && desiredPatroniConfig.MaximumLagOnFailover != effectivePatroniConfig.MaximumLagOnFailover { configToSet["maximum_lag_on_failover"] = desiredPatroniConfig.MaximumLagOnFailover } - if desiredPatroniConfig.PgHba != nil && !reflect.DeepEqual(desiredPatroniConfig.PgHba, patroniConfig.PgHba) { + if desiredPatroniConfig.PgHba != nil && !reflect.DeepEqual(desiredPatroniConfig.PgHba, effectivePatroniConfig.PgHba) { configToSet["pg_hba"] = desiredPatroniConfig.PgHba } - if desiredPatroniConfig.RetryTimeout > 0 && desiredPatroniConfig.RetryTimeout != patroniConfig.RetryTimeout { + if desiredPatroniConfig.RetryTimeout > 0 && desiredPatroniConfig.RetryTimeout != effectivePatroniConfig.RetryTimeout { configToSet["retry_timeout"] = desiredPatroniConfig.RetryTimeout } - if desiredPatroniConfig.SynchronousMode != patroniConfig.SynchronousMode { + if desiredPatroniConfig.SynchronousMode != effectivePatroniConfig.SynchronousMode { configToSet["synchronous_mode"] = desiredPatroniConfig.SynchronousMode } - if desiredPatroniConfig.SynchronousModeStrict != patroniConfig.SynchronousModeStrict { + if desiredPatroniConfig.SynchronousModeStrict != effectivePatroniConfig.SynchronousModeStrict { configToSet["synchronous_mode_strict"] = desiredPatroniConfig.SynchronousModeStrict } - if desiredPatroniConfig.TTL > 0 && desiredPatroniConfig.TTL != patroniConfig.TTL { + if desiredPatroniConfig.TTL > 0 && desiredPatroniConfig.TTL != effectivePatroniConfig.TTL { configToSet["ttl"] = desiredPatroniConfig.TTL } // check if specified slots exist in config and if they differ slotsToSet := make(map[string]map[string]string) for slotName, desiredSlot := range desiredPatroniConfig.Slots { - if effectiveSlot, exists := patroniConfig.Slots[slotName]; exists { + if effectiveSlot, exists := effectivePatroniConfig.Slots[slotName]; exists { if reflect.DeepEqual(desiredSlot, effectiveSlot) { continue } @@ -561,8 +576,7 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniC configToSet["slots"] = slotsToSet } - // compare parameters under postgresql section with c.Spec.Postgresql.Parameters from manifest - desiredPgParameters := c.Spec.Parameters + // compare effective and desired parameters under postgresql section in Patroni config for desiredOption, desiredValue := range desiredPgParameters { effectiveValue := effectivePgParameters[desiredOption] if isBootstrapOnlyParameter(desiredOption) && (effectiveValue != desiredValue) { @@ -605,66 +619,184 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniC c.logger.Debugf("patching Postgres config via Patroni API on pod %s with following options: %s", podName, configToSetJson) if err = c.patroni.SetConfig(pod, configToSet); err != nil { - return requiresMasterRestart, fmt.Errorf("could not patch postgres parameters with a pod %s: %v", podName, err) + return requiresMasterRestart, fmt.Errorf("could not patch postgres parameters within pod %s: %v", podName, err) } return requiresMasterRestart, nil } func (c *Cluster) syncSecrets() error { - var ( - err error - secret *v1.Secret - ) + c.logger.Info("syncing secrets") c.setProcessName("syncing secrets") - secrets := c.generateUserSecrets() + generatedSecrets := c.generateUserSecrets() + rotationUsers := make(spec.PgUserMap) + retentionUsers := make([]string, 0) + currentTime := time.Now() - for secretUsername, secretSpec := range secrets { - if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Create(context.TODO(), secretSpec, metav1.CreateOptions{}); err == nil { + for secretUsername, generatedSecret := range generatedSecrets { + secret, err := c.KubeClient.Secrets(generatedSecret.Namespace).Create(context.TODO(), generatedSecret, metav1.CreateOptions{}) + if err == nil { c.Secrets[secret.UID] = secret - c.logger.Debugf("created new secret %s, namespace: %s, uid: %s", util.NameFromMeta(secret.ObjectMeta), secretSpec.Namespace, secret.UID) + c.logger.Debugf("created new secret %s, namespace: %s, uid: %s", util.NameFromMeta(secret.ObjectMeta), generatedSecret.Namespace, secret.UID) continue } if k8sutil.ResourceAlreadyExists(err) { - var userMap map[string]spec.PgUser - if secret, err = c.KubeClient.Secrets(secretSpec.Namespace).Get(context.TODO(), secretSpec.Name, metav1.GetOptions{}); err != nil { - return fmt.Errorf("could not get current secret: %v", err) - } - if secretUsername != string(secret.Data["username"]) { - c.logger.Errorf("secret %s does not contain the role %s", secretSpec.Name, secretUsername) - continue - } - c.Secrets[secret.UID] = secret - c.logger.Debugf("secret %s already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta)) - if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name { - secretUsername = constants.SuperuserKeyName - userMap = c.systemUsers - } else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name { - secretUsername = constants.ReplicationUserKeyName - userMap = c.systemUsers - } else { - userMap = c.pgUsers - } - pwdUser := userMap[secretUsername] - // if this secret belongs to the infrastructure role and the password has changed - replace it in the secret - if pwdUser.Password != string(secret.Data["password"]) && - pwdUser.Origin == spec.RoleOriginInfrastructure { - - c.logger.Debugf("updating the secret %s from the infrastructure roles", secretSpec.Name) - if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(context.TODO(), secretSpec, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err) - } - } else { - // for non-infrastructure role - update the role with the password from the secret - pwdUser.Password = string(secret.Data["password"]) - userMap[secretUsername] = pwdUser + if err = c.updateSecret(secretUsername, generatedSecret, &rotationUsers, &retentionUsers, currentTime); err != nil { + c.logger.Warningf("syncing secret %s failed: %v", util.NameFromMeta(secret.ObjectMeta), err) } } else { - return fmt.Errorf("could not create secret for user %s: in namespace %s: %v", secretUsername, secretSpec.Namespace, err) + return fmt.Errorf("could not create secret for user %s: in namespace %s: %v", secretUsername, generatedSecret.Namespace, err) } } + // add new user with date suffix and use it in the secret of the original user + if len(rotationUsers) > 0 { + err := c.initDbConn() + if err != nil { + return fmt.Errorf("could not init db connection: %v", err) + } + pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(spec.PgUserMap{}, rotationUsers) + if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil { + return fmt.Errorf("error creating database roles for password rotation: %v", err) + } + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection after creating users for password rotation: %v", err) + } + } + + // remove rotation users that exceed the retention interval + if len(retentionUsers) > 0 { + err := c.initDbConn() + if err != nil { + return fmt.Errorf("could not init db connection: %v", err) + } + if err = c.cleanupRotatedUsers(retentionUsers, c.pgDb); err != nil { + return fmt.Errorf("error removing users exceeding configured retention interval: %v", err) + } + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection after removing users exceeding configured retention interval: %v", err) + } + } + + return nil +} + +func (c *Cluster) getNextRotationDate(currentDate time.Time) (time.Time, string) { + nextRotationDate := currentDate.AddDate(0, 0, int(c.OpConfig.PasswordRotationInterval)) + return nextRotationDate, nextRotationDate.Format(time.RFC3339) +} + +func (c *Cluster) updateSecret( + secretUsername string, + generatedSecret *v1.Secret, + rotationUsers *spec.PgUserMap, + retentionUsers *[]string, + currentTime time.Time) error { + var ( + secret *v1.Secret + err error + updateSecret bool + updateSecretMsg string + nextRotationDate time.Time + nextRotationDateStr string + ) + + // get the secret first + if secret, err = c.KubeClient.Secrets(generatedSecret.Namespace).Get(context.TODO(), generatedSecret.Name, metav1.GetOptions{}); err != nil { + return fmt.Errorf("could not get current secret: %v", err) + } + c.Secrets[secret.UID] = secret + + // fetch user map to update later + var userMap map[string]spec.PgUser + var userKey string + if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name { + userKey = constants.SuperuserKeyName + userMap = c.systemUsers + } else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name { + userKey = constants.ReplicationUserKeyName + userMap = c.systemUsers + } else { + userKey = secretUsername + userMap = c.pgUsers + } + pwdUser := userMap[userKey] + secretName := util.NameFromMeta(secret.ObjectMeta) + + // if password rotation is enabled update password and username if rotation interval has been passed + if (c.OpConfig.EnablePasswordRotation && !pwdUser.IsDbOwner && + pwdUser.Origin != spec.RoleOriginInfrastructure && pwdUser.Origin != spec.RoleOriginSystem) || + util.SliceContains(c.Spec.UsersWithSecretRotation, secretUsername) || + util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) { + + // initialize password rotation setting first rotation date + nextRotationDateStr = string(secret.Data["nextRotation"]) + if nextRotationDate, err = time.ParseInLocation(time.RFC3339, nextRotationDateStr, currentTime.UTC().Location()); err != nil { + nextRotationDate, nextRotationDateStr = c.getNextRotationDate(currentTime) + secret.Data["nextRotation"] = []byte(nextRotationDateStr) + updateSecret = true + updateSecretMsg = fmt.Sprintf("rotation date not found in secret %q. Setting it to %s", secretName, nextRotationDateStr) + } + + // check if next rotation can happen sooner + // if rotation interval has been decreased + currentRotationDate, nextRotationDateStr := c.getNextRotationDate(currentTime) + if nextRotationDate.After(currentRotationDate) { + nextRotationDate = currentRotationDate + } + + // update password and next rotation date if configured interval has passed + if currentTime.After(nextRotationDate) { + // create rotation user if role is not listed for in-place password update + if !util.SliceContains(c.Spec.UsersWithInPlaceSecretRotation, secretUsername) { + rotationUser := pwdUser + newRotationUsername := secretUsername + currentTime.Format("060102") + rotationUser.Name = newRotationUsername + rotationUser.MemberOf = []string{secretUsername} + (*rotationUsers)[newRotationUsername] = rotationUser + secret.Data["username"] = []byte(newRotationUsername) + + // whenever there is a rotation, check if old rotation users can be deleted + *retentionUsers = append(*retentionUsers, secretUsername) + } + secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength)) + secret.Data["nextRotation"] = []byte(nextRotationDateStr) + + updateSecret = true + updateSecretMsg = fmt.Sprintf("updating secret %q due to password rotation - next rotation date: %s", secretName, nextRotationDateStr) + } + } else { + // username might not match if password rotation has been disabled again + if secretUsername != string(secret.Data["username"]) { + *retentionUsers = append(*retentionUsers, secretUsername) + secret.Data["username"] = []byte(secretUsername) + secret.Data["password"] = []byte(util.RandomPassword(constants.PasswordLength)) + secret.Data["nextRotation"] = []byte{} + updateSecret = true + updateSecretMsg = fmt.Sprintf("secret %s does not contain the role %s - updating username and resetting password", secretName, secretUsername) + } + } + + // if this secret belongs to the infrastructure role and the password has changed - replace it in the secret + if pwdUser.Password != string(secret.Data["password"]) && pwdUser.Origin == spec.RoleOriginInfrastructure { + secret = generatedSecret + updateSecret = true + updateSecretMsg = fmt.Sprintf("updating the secret %s from the infrastructure roles", secretName) + } else { + // for non-infrastructure role - update the role with the password from the secret + pwdUser.Password = string(secret.Data["password"]) + userMap[userKey] = pwdUser + } + + if updateSecret { + c.logger.Debugln(updateSecretMsg) + if _, err = c.KubeClient.Secrets(secret.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("could not update secret %q: %v", secretName, err) + } + c.Secrets[secret.UID] = secret + } + return nil } diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index c6e2a8357..ea73fb97c 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -19,6 +19,7 @@ import ( "github.com/zalando/postgres-operator/mocks" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" + "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/patroni" @@ -26,6 +27,8 @@ import ( ) var patroniLogger = logrus.New().WithField("test", "patroni") +var acidClientSet = fakeacidv1.NewSimpleClientset() +var clientSet = fake.NewSimpleClientset() func newMockPod(ip string) *v1.Pod { return &v1.Pod{ @@ -36,9 +39,6 @@ func newMockPod(ip string) *v1.Pod { } func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) { - acidClientSet := fakeacidv1.NewSimpleClientset() - clientSet := fake.NewSimpleClientset() - return k8sutil.KubernetesClient{ PodsGetter: clientSet.CoreV1(), PostgresqlsGetter: acidClientSet.AcidV1(), @@ -46,6 +46,12 @@ func newFakeK8sSyncClient() (k8sutil.KubernetesClient, *fake.Clientset) { }, clientSet } +func newFakeK8sSyncSecretsClient() (k8sutil.KubernetesClient, *fake.Clientset) { + return k8sutil.KubernetesClient{ + SecretsGetter: clientSet.CoreV1(), + }, clientSet +} + func TestSyncStatefulSetsAnnotations(t *testing.T) { testName := "test syncing statefulsets annotations" client, _ := newFakeK8sSyncClient() @@ -196,17 +202,15 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) { cluster.patroni = p mockPod := newMockPod("192.168.100.1") - // simulate existing config that differs with cluster.Spec + // simulate existing config that differs from cluster.Spec tests := []struct { subtest string - pod *v1.Pod patroni acidv1.Patroni pgParams map[string]string restartMaster bool }{ { subtest: "Patroni and Postgresql.Parameters differ - restart replica first", - pod: mockPod, patroni: acidv1.Patroni{ TTL: 30, // desired 20 }, @@ -218,7 +222,6 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) { }, { subtest: "multiple Postgresql.Parameters differ - restart replica first", - pod: mockPod, patroni: acidv1.Patroni{ TTL: 20, }, @@ -230,7 +233,6 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) { }, { subtest: "desired max_connections bigger - restart replica first", - pod: mockPod, patroni: acidv1.Patroni{ TTL: 20, }, @@ -242,7 +244,6 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) { }, { subtest: "desired max_connections smaller - restart master first", - pod: mockPod, patroni: acidv1.Patroni{ TTL: 20, }, @@ -255,10 +256,112 @@ func TestCheckAndSetGlobalPostgreSQLConfiguration(t *testing.T) { } for _, tt := range tests { - requireMasterRestart, err := cluster.checkAndSetGlobalPostgreSQLConfiguration(tt.pod, tt.patroni, tt.pgParams) + requireMasterRestart, err := cluster.checkAndSetGlobalPostgreSQLConfiguration(mockPod, tt.patroni, cluster.Spec.Patroni, tt.pgParams, cluster.Spec.Parameters) assert.NoError(t, err) if requireMasterRestart != tt.restartMaster { t.Errorf("%s - %s: unexpect master restart strategy, got %v, expected %v", testName, tt.subtest, requireMasterRestart, tt.restartMaster) } } } + +func TestUpdateSecret(t *testing.T) { + testName := "test syncing secrets" + client, _ := newFakeK8sSyncSecretsClient() + + clusterName := "acid-test-cluster" + namespace := "default" + dbname := "app" + dbowner := "appowner" + secretTemplate := config.StringTemplate("{username}.{cluster}.credentials") + rotationUsers := make(spec.PgUserMap) + retentionUsers := make([]string, 0) + + // define manifest users and enable rotation for dbowner + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Databases: map[string]string{dbname: dbowner}, + Users: map[string]acidv1.UserFlags{"foo": {}, dbowner: {}}, + UsersWithInPlaceSecretRotation: []string{dbowner}, + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + // new cluster with enabled password rotation + var cluster = New( + Config{ + OpConfig: config.Config{ + Auth: config.Auth{ + SecretNameTemplate: secretTemplate, + EnablePasswordRotation: true, + PasswordRotationInterval: 1, + PasswordRotationUserRetention: 3, + }, + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + }, + }, + }, client, pg, logger, eventRecorder) + + cluster.Name = clusterName + cluster.Namespace = namespace + cluster.pgUsers = map[string]spec.PgUser{} + cluster.initRobotUsers() + + // create secrets + cluster.syncSecrets() + // initialize rotation with current time + cluster.syncSecrets() + + dayAfterTomorrow := time.Now().AddDate(0, 0, 2) + + for username := range cluster.Spec.Users { + pgUser := cluster.pgUsers[username] + + // first, get the secret + secret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{}) + assert.NoError(t, err) + secretPassword := string(secret.Data["password"]) + + // now update the secret setting a next rotation date (tomorrow + interval) + cluster.updateSecret(username, secret, &rotationUsers, &retentionUsers, dayAfterTomorrow) + updatedSecret, err := cluster.KubeClient.Secrets(namespace).Get(context.TODO(), secretTemplate.Format("username", username, "cluster", clusterName), metav1.GetOptions{}) + assert.NoError(t, err) + + // check that passwords are different + rotatedPassword := string(updatedSecret.Data["password"]) + if secretPassword == rotatedPassword { + t.Errorf("%s: password unchanged in updated secret for %s", testName, username) + } + + // check that next rotation date is tomorrow + interval, not date in secret + interval + nextRotation := string(updatedSecret.Data["nextRotation"]) + _, nextRotationDate := cluster.getNextRotationDate(dayAfterTomorrow) + if nextRotation != nextRotationDate { + t.Errorf("%s: updated secret of %s does not contain correct rotation date: expected %s, got %s", testName, username, nextRotationDate, nextRotation) + } + + // compare username, when it's dbowner they should be equal because of UsersWithInPlaceSecretRotation + secretUsername := string(updatedSecret.Data["username"]) + if pgUser.IsDbOwner { + if secretUsername != username { + t.Errorf("%s: username differs in updated secret: expected %s, got %s", testName, username, secretUsername) + } + } else { + rotatedUsername := username + dayAfterTomorrow.Format("060102") + if secretUsername != rotatedUsername { + t.Errorf("%s: updated secret does not contain correct username: expected %s, got %s", testName, rotatedUsername, secretUsername) + } + + if len(rotationUsers) != 1 && len(retentionUsers) != 1 { + t.Errorf("%s: unexpected number of users to rotate - expected only %s, found %d", testName, username, len(rotationUsers)) + } + } + } +} diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 199914ccc..67b4ee395 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -14,11 +14,14 @@ import ( type PostgresRole string const ( - // Master role - Master PostgresRole = "master" - - // Replica role + // spilo roles + Master PostgresRole = "master" Replica PostgresRole = "replica" + + // roles returned by Patroni cluster endpoint + Leader PostgresRole = "leader" + StandbyLeader PostgresRole = "standby_leader" + SyncStandby PostgresRole = "sync_standby" ) // PodEventType represents the type of a pod-related event diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 4350f9d56..0f71d2d64 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -6,7 +6,7 @@ import ( "encoding/gob" "encoding/json" "fmt" - "math/rand" + "net/http" "reflect" "sort" "strings" @@ -273,17 +273,21 @@ func (c *Cluster) getTeamMembers(teamID string) ([]string, error) { return nil, fmt.Errorf("could not get oauth token to authenticate to team service API: %v", err) } - teamInfo, err := c.teamsAPIClient.TeamInfo(teamID, token) - if err != nil { - return nil, fmt.Errorf("could not get team info for team %q: %v", teamID, err) - } + teamInfo, statusCode, err := c.teamsAPIClient.TeamInfo(teamID, token) - for _, member := range teamInfo.Members { - if !(util.SliceContains(members, member)) { - members = append(members, member) + if err != nil { + if statusCode == http.StatusNotFound { + c.logger.Warningf("could not get team info for team %q: %v", teamID, err) + } else { + return nil, fmt.Errorf("could not get team info for team %q: %v", teamID, err) + } + } else { + for _, member := range teamInfo.Members { + if !(util.SliceContains(members, member)) { + members = append(members, member) + } } } - return members, nil } @@ -525,10 +529,6 @@ func (c *Cluster) credentialSecretNameForCluster(username string, clusterName st "tprgroup", acidzalando.GroupName) } -func masterCandidate(replicas []spec.NamespacedName) spec.NamespacedName { - return replicas[rand.Intn(len(replicas))] -} - func cloneSpec(from *acidv1.Postgresql) (*acidv1.Postgresql, error) { var ( buf bytes.Buffer @@ -599,3 +599,15 @@ func trimCronjobName(name string) string { } return name } + +func parseResourceRequirements(resourcesRequirement v1.ResourceRequirements) (acidv1.Resources, error) { + var resources acidv1.Resources + resourcesJSON, err := json.Marshal(resourcesRequirement) + if err != nil { + return acidv1.Resources{}, fmt.Errorf("could not marshal K8s resources requirements") + } + if err = json.Unmarshal(resourcesJSON, &resources); err != nil { + return acidv1.Resources{}, fmt.Errorf("could not unmarshal K8s resources requirements into acidv1.Resources struct") + } + return resources, nil +} diff --git a/pkg/cluster/util_test.go b/pkg/cluster/util_test.go index 20214a085..9ed644ea2 100644 --- a/pkg/cluster/util_test.go +++ b/pkg/cluster/util_test.go @@ -57,7 +57,7 @@ func TestInheritedAnnotations(t *testing.T) { ConnectionPoolerDefaultCPULimit: "100m", ConnectionPoolerDefaultMemoryRequest: "100Mi", ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + NumberOfInstances: k8sutil.Int32ToPointer(1), }, PodManagementPolicy: "ordered_ready", Resources: config.Resources{ @@ -98,8 +98,8 @@ func TestInheritedAnnotations(t *testing.T) { t.Errorf("%s: pod template %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations) } // pvc template - if util.MapContains(sts.Spec.VolumeClaimTemplates[0].Annotations, inheritedAnnotations) { - t.Errorf("%s: PVC template %v not expected to have inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations) + if !(util.MapContains(sts.Spec.VolumeClaimTemplates[0].Annotations, inheritedAnnotations)) { + t.Errorf("%s: PVC template %v not inherited annotations %#v, got %#v", testName, sts.ObjectMeta.Name, inheritedAnnotations, sts.ObjectMeta.Annotations) } } diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go index 4962319ed..6b22ed54c 100644 --- a/pkg/cluster/volumes.go +++ b/pkg/cluster/volumes.go @@ -88,7 +88,7 @@ func (c *Cluster) syncUnderlyingEBSVolume() error { awsGp3 := aws.String("gp3") awsIo2 := aws.String("io2") - errors := []string{} + errors := make([]string, 0) for _, volume := range c.EBSVolumes { var modifyIops *int64 @@ -137,7 +137,6 @@ func (c *Cluster) syncUnderlyingEBSVolume() error { for _, s := range errors { c.logger.Warningf(s) } - // c.logger.Errorf("failed to modify %d of %d volumes", len(c.EBSVolumes), len(errors)) } return nil } @@ -149,7 +148,11 @@ func (c *Cluster) populateVolumeMetaData() error { if err != nil { return fmt.Errorf("could not list persistent volumes: %v", err) } - c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes)) + if len(pvs) == 0 { + c.EBSVolumes = make(map[string]volumes.VolumeProperties) + return fmt.Errorf("no persistent volumes found") + } + c.logger.Debugf("found %d persistent volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes)) volumeIds := []string{} var volumeID string @@ -167,7 +170,7 @@ func (c *Cluster) populateVolumeMetaData() error { return err } - if len(currentVolumes) != len(c.EBSVolumes) { + if len(currentVolumes) != len(c.EBSVolumes) && len(c.EBSVolumes) > 0 { c.logger.Debugf("number of ebs volumes (%d) discovered differs from already known volumes (%d)", len(currentVolumes), len(c.EBSVolumes)) } @@ -205,7 +208,7 @@ func (c *Cluster) syncVolumeClaims() error { // syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset. func (c *Cluster) syncEbsVolumes() error { - c.setProcessName("syncing EBS and Claims volumes") + c.setProcessName("syncing EBS volumes") act, err := c.volumesNeedResizing() if err != nil { @@ -451,29 +454,17 @@ func quantityToGigabyte(q resource.Quantity) int64 { } func (c *Cluster) executeEBSMigration() error { - if !c.OpConfig.EnableEBSGp3Migration { - return nil - } - c.logger.Infof("starting EBS gp2 to gp3 migration") - pvs, err := c.listPersistentVolumes() if err != nil { return fmt.Errorf("could not list persistent volumes: %v", err) } + if len(pvs) == 0 { + c.logger.Warningf("no persistent volumes found - skipping EBS migration") + return nil + } c.logger.Debugf("found %d volumes, size of known volumes %d", len(pvs), len(c.EBSVolumes)) - volumeIds := []string{} - var volumeID string - for _, pv := range pvs { - volumeID, err = c.VolumeResizer.ExtractVolumeID(pv.Spec.AWSElasticBlockStore.VolumeID) - if err != nil { - continue - } - - volumeIds = append(volumeIds, volumeID) - } - - if len(volumeIds) == len(c.EBSVolumes) { + if len(pvs) == len(c.EBSVolumes) { hasGp2 := false for _, v := range c.EBSVolumes { if v.VolumeType == "gp2" { @@ -487,15 +478,10 @@ func (c *Cluster) executeEBSMigration() error { } } - awsVolumes, err := c.VolumeResizer.DescribeVolumes(volumeIds) - if nil != err { - return err - } - var i3000 int64 = 3000 var i125 int64 = 125 - for _, volume := range awsVolumes { + for _, volume := range c.EBSVolumes { if volume.VolumeType == "gp2" && volume.Size < c.OpConfig.EnableEBSGp3MigrationMaxSize { c.logger.Infof("modifying EBS volume %s to type gp3 migration (%d)", volume.VolumeID, volume.Size) err = c.VolumeResizer.ModifyVolume(volume.VolumeID, aws.String("gp3"), &volume.Size, &i3000, &i125) diff --git a/pkg/cluster/volumes_test.go b/pkg/cluster/volumes_test.go index 204ea8aab..4ef94fcfb 100644 --- a/pkg/cluster/volumes_test.go +++ b/pkg/cluster/volumes_test.go @@ -224,7 +224,10 @@ func TestMigrateEBS(t *testing.T) { resizer.EXPECT().ModifyVolume(gomock.Eq("ebs-volume-1"), gomock.Eq(aws.String("gp3")), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) cluster.VolumeResizer = resizer - cluster.executeEBSMigration() + err := cluster.populateVolumeMetaData() + assert.NoError(t, err) + err = cluster.executeEBSMigration() + assert.NoError(t, err) } func initTestVolumesAndPods(client k8sutil.KubernetesClient, namespace, clustername string, labels labels.Set, volumes []testVolume) { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index f992ff782..de0dec69f 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -95,7 +95,9 @@ func NewController(controllerConfig *spec.ControllerConfig, controllerId string) // disabling the sending of events also to the logoutput // the operator currently duplicates a lot of log entries with this setup // eventBroadcaster.StartLogging(logger.Infof) - recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: myComponentName}) + scheme := scheme.Scheme + acidv1.AddToScheme(scheme) + recorder := eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: myComponentName}) c := &Controller{ config: *controllerConfig, @@ -307,8 +309,10 @@ func (c *Controller) initController() { c.controllerID = os.Getenv("CONTROLLER_ID") if configObjectName := os.Getenv("POSTGRES_OPERATOR_CONFIGURATION_OBJECT"); configObjectName != "" { - if err := c.createConfigurationCRD(c.opConfig.EnableCRDValidation); err != nil { - c.logger.Fatalf("could not register Operator Configuration CustomResourceDefinition: %v", err) + if c.opConfig.EnableCRDRegistration != nil && *c.opConfig.EnableCRDRegistration { + if err := c.createConfigurationCRD(); err != nil { + c.logger.Fatalf("could not register Operator Configuration CustomResourceDefinition: %v", err) + } } if cfg, err := c.readOperatorConfigurationFromCRD(spec.GetOperatorNamespace(), configObjectName); err != nil { c.logger.Fatalf("unable to read operator configuration: %v", err) @@ -323,8 +327,10 @@ func (c *Controller) initController() { c.modifyConfigFromEnvironment() - if err := c.createPostgresCRD(c.opConfig.EnableCRDValidation); err != nil { - c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err) + if c.opConfig.EnableCRDRegistration != nil && *c.opConfig.EnableCRDRegistration { + if err := c.createPostgresCRD(); err != nil { + c.logger.Fatalf("could not register Postgres CustomResourceDefinition: %v", err) + } } c.initSharedInformers() diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index fc56dbf96..029685729 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -10,6 +10,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -24,22 +25,20 @@ func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, con return config, nil } -func int32ToPointer(value int32) *int32 { - return &value -} - // importConfigurationFromCRD is a transitional function that converts CRD configuration to the one based on the configmap func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigurationData) *config.Config { result := &config.Config{} // general config + result.EnableCRDRegistration = util.CoalesceBool(fromCRD.EnableCRDRegistration, util.True()) result.EnableCRDValidation = util.CoalesceBool(fromCRD.EnableCRDValidation, util.True()) + result.CRDCategories = util.CoalesceStrArr(fromCRD.CRDCategories, []string{"all"}) result.EnableLazySpiloUpgrade = fromCRD.EnableLazySpiloUpgrade result.EnablePgVersionEnvVar = fromCRD.EnablePgVersionEnvVar result.EnableSpiloWalPathCompat = fromCRD.EnableSpiloWalPathCompat result.EtcdHost = fromCRD.EtcdHost result.KubernetesUseConfigMaps = fromCRD.KubernetesUseConfigMaps - result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-14:2.1-p3") + result.DockerImage = util.Coalesce(fromCRD.DockerImage, "registry.opensource.zalan.do/acid/spilo-14:2.1-p5") result.Workers = util.CoalesceUInt32(fromCRD.Workers, 8) result.MinInstances = fromCRD.MinInstances result.MaxInstances = fromCRD.MaxInstances @@ -53,9 +52,14 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // user config result.SuperUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.SuperUsername, "postgres") result.ReplicationUsername = util.Coalesce(fromCRD.PostgresUsersConfiguration.ReplicationUsername, "standby") + result.AdditionalOwnerRoles = fromCRD.PostgresUsersConfiguration.AdditionalOwnerRoles + result.EnablePasswordRotation = fromCRD.PostgresUsersConfiguration.EnablePasswordRotation + result.PasswordRotationInterval = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.PasswordRotationInterval, 90) + result.PasswordRotationUserRetention = util.CoalesceUInt32(fromCRD.PostgresUsersConfiguration.DeepCopy().PasswordRotationUserRetention, 180) // major version upgrade config result.MajorVersionUpgradeMode = util.Coalesce(fromCRD.MajorVersionUpgrade.MajorVersionUpgradeMode, "off") + result.MajorVersionUpgradeTeamAllowList = fromCRD.MajorVersionUpgrade.MajorVersionUpgradeTeamAllowList result.MinimalMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.MinimalMajorVersion, "9.6") result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "14") @@ -104,10 +108,12 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels result.InheritedAnnotations = fromCRD.Kubernetes.InheritedAnnotations result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations + result.IgnoredAnnotations = fromCRD.Kubernetes.IgnoredAnnotations result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey result.DeleteAnnotationNameKey = fromCRD.Kubernetes.DeleteAnnotationNameKey result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel + result.NodeReadinessLabelMerge = fromCRD.Kubernetes.NodeReadinessLabelMerge result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready") result.MasterPodMoveTimeout = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.MasterPodMoveTimeout), "10m") @@ -130,11 +136,15 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodDeletionWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PodDeletionWaitTimeout), "10m") result.ReadyWaitInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitInterval), "4s") result.ReadyWaitTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.ReadyWaitTimeout), "30s") + result.PatroniAPICheckInterval = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PatroniAPICheckInterval), "1s") + result.PatroniAPICheckTimeout = util.CoalesceDuration(time.Duration(fromCRD.Timeouts.PatroniAPICheckTimeout), "5s") // load balancer config result.DbHostedZone = util.Coalesce(fromCRD.LoadBalancer.DbHostedZone, "db.example.com") result.EnableMasterLoadBalancer = fromCRD.LoadBalancer.EnableMasterLoadBalancer + result.EnableMasterPoolerLoadBalancer = fromCRD.LoadBalancer.EnableMasterPoolerLoadBalancer result.EnableReplicaLoadBalancer = fromCRD.LoadBalancer.EnableReplicaLoadBalancer + result.EnableReplicaPoolerLoadBalancer = fromCRD.LoadBalancer.EnableReplicaPoolerLoadBalancer result.CustomServiceAnnotations = fromCRD.LoadBalancer.CustomServiceAnnotations result.MasterDNSNameFormat = fromCRD.LoadBalancer.MasterDNSNameFormat result.ReplicaDNSNameFormat = fromCRD.LoadBalancer.ReplicaDNSNameFormat @@ -155,7 +165,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // logical backup config result.LogicalBackupSchedule = util.Coalesce(fromCRD.LogicalBackup.Schedule, "30 00 * * *") - result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup:v1.7.1") + result.LogicalBackupDockerImage = util.Coalesce(fromCRD.LogicalBackup.DockerImage, "registry.opensource.zalan.do/acid/logical-backup:v1.8.0") result.LogicalBackupProvider = util.Coalesce(fromCRD.LogicalBackup.BackupProvider, "s3") result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region @@ -163,6 +173,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE + result.LogicalBackupS3RetentionTime = fromCRD.LogicalBackup.RetentionTime result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials result.LogicalBackupJobPrefix = util.Coalesce(fromCRD.LogicalBackup.JobPrefix, "logical-backup-") @@ -179,7 +190,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.TeamAdminRole = fromCRD.TeamsAPI.TeamAdminRole result.PamRoleName = util.Coalesce(fromCRD.TeamsAPI.PamRoleName, "zalandos") result.PamConfiguration = util.Coalesce(fromCRD.TeamsAPI.PamConfiguration, "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees") - result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin"}) + result.ProtectedRoles = util.CoalesceStrArr(fromCRD.TeamsAPI.ProtectedRoles, []string{"admin", "cron_admin"}) result.PostgresSuperuserTeams = fromCRD.TeamsAPI.PostgresSuperuserTeams result.EnablePostgresTeamCRD = fromCRD.TeamsAPI.EnablePostgresTeamCRD result.EnablePostgresTeamCRDSuperusers = fromCRD.TeamsAPI.EnablePostgresTeamCRDSuperusers @@ -204,11 +215,11 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // so ensure default values here. result.ConnectionPooler.NumberOfInstances = util.CoalesceInt32( fromCRD.ConnectionPooler.NumberOfInstances, - int32ToPointer(2)) + k8sutil.Int32ToPointer(2)) result.ConnectionPooler.NumberOfInstances = util.MaxInt32( result.ConnectionPooler.NumberOfInstances, - int32ToPointer(2)) + k8sutil.Int32ToPointer(2)) result.ConnectionPooler.Schema = util.Coalesce( fromCRD.ConnectionPooler.Schema, @@ -219,7 +230,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur constants.ConnectionPoolerUserName) if result.ConnectionPooler.User == result.SuperUsername { - msg := "Connection pool user is not allowed to be the same as super user, username: %s" + msg := "connection pool user is not allowed to be the same as super user, username: %s" panic(fmt.Errorf(msg, result.ConnectionPooler.User)) } @@ -249,7 +260,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ConnectionPooler.MaxDBConnections = util.CoalesceInt32( fromCRD.ConnectionPooler.MaxDBConnections, - int32ToPointer(constants.ConnectionPoolerMaxDBConnections)) + k8sutil.Int32ToPointer(constants.ConnectionPoolerMaxDBConnections)) return result } diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index b3c4d8414..aac078933 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -505,8 +505,6 @@ func (c *Controller) postgresqlAdd(obj interface{}) { // We will not get multiple Add events for the same cluster c.queueClusterEvent(nil, pg, EventAdd) } - - return } func (c *Controller) postgresqlUpdate(prev, cur interface{}) { @@ -521,8 +519,6 @@ func (c *Controller) postgresqlUpdate(prev, cur interface{}) { } c.queueClusterEvent(pgOld, pgNew, EventUpdate) } - - return } func (c *Controller) postgresqlDelete(obj interface{}) { @@ -530,8 +526,6 @@ func (c *Controller) postgresqlDelete(obj interface{}) { if pg != nil { c.queueClusterEvent(pg, nil, EventDelete) } - - return } func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql { @@ -550,7 +544,8 @@ func (c *Controller) postgresqlCheck(obj interface{}) *acidv1.Postgresql { Ensures the pod service account and role bindings exists in a namespace before a PG cluster is created there so that a user does not have to deploy these credentials manually. StatefulSets require the service account to - create pods; Patroni requires relevant RBAC bindings to access endpoints. + create pods; Patroni requires relevant RBAC bindings to access endpoints + or config maps. The operator does not sync accounts/role bindings after creation. */ diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 8aa891c09..bca8082f6 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -53,28 +53,35 @@ func (c *Controller) clusterWorkerID(clusterName spec.NamespacedName) uint32 { return c.clusterWorkers[clusterName] } -func (c *Controller) createOperatorCRD(crd *apiextv1.CustomResourceDefinition) error { - if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil { - if k8sutil.ResourceAlreadyExists(err) { - c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name) - - patch, err := json.Marshal(crd) - if err != nil { - return fmt.Errorf("could not marshal new customResourceDefintion: %v", err) - } - if _, err := c.KubeClient.CustomResourceDefinitions().Patch( - context.TODO(), crd.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { - return fmt.Errorf("could not update customResourceDefinition: %v", err) - } - } else { - c.logger.Errorf("could not create customResourceDefinition %q: %v", crd.Name, err) +func (c *Controller) createOperatorCRD(desiredCrd *apiextv1.CustomResourceDefinition) error { + crd, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), desiredCrd.Name, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + if _, err := c.KubeClient.CustomResourceDefinitions().Create(context.TODO(), desiredCrd, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("could not create customResourceDefinition %q: %v", desiredCrd.Name, err) + } + } + if err != nil { + c.logger.Errorf("could not get customResourceDefinition %q: %v", desiredCrd.Name, err) + } + if crd != nil { + c.logger.Infof("customResourceDefinition %q is already registered and will only be updated", crd.Name) + // copy annotations and labels from existing CRD since we do not define them + desiredCrd.Annotations = crd.Annotations + desiredCrd.Labels = crd.Labels + patch, err := json.Marshal(desiredCrd) + if err != nil { + return fmt.Errorf("could not marshal new customResourceDefintion %q: %v", desiredCrd.Name, err) + } + if _, err := c.KubeClient.CustomResourceDefinitions().Patch( + context.TODO(), crd.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("could not update customResourceDefinition %q: %v", crd.Name, err) } } else { c.logger.Infof("customResourceDefinition %q has been registered", crd.Name) } return wait.Poll(c.config.CRDReadyWaitInterval, c.config.CRDReadyWaitTimeout, func() (bool, error) { - c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{}) + c, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), desiredCrd.Name, metav1.GetOptions{}) if err != nil { return false, err } @@ -96,12 +103,12 @@ func (c *Controller) createOperatorCRD(crd *apiextv1.CustomResourceDefinition) e }) } -func (c *Controller) createPostgresCRD(enableValidation *bool) error { - return c.createOperatorCRD(acidv1.PostgresCRD(enableValidation)) +func (c *Controller) createPostgresCRD() error { + return c.createOperatorCRD(acidv1.PostgresCRD(c.opConfig.CRDCategories)) } -func (c *Controller) createConfigurationCRD(enableValidation *bool) error { - return c.createOperatorCRD(acidv1.ConfigurationCRD(enableValidation)) +func (c *Controller) createConfigurationCRD() error { + return c.createOperatorCRD(acidv1.ConfigurationCRD(c.opConfig.CRDCategories)) } func readDecodedRole(s string) (*spec.PgUser, error) { @@ -195,13 +202,12 @@ func (c *Controller) getInfrastructureRoleDefinitions() []*config.Infrastructure func (c *Controller) getInfrastructureRoles( rolesSecrets []*config.InfrastructureRole) ( - map[string]spec.PgUser, []error) { - - var errors []error - var noRolesProvided = true + map[string]spec.PgUser, error) { + errors := make([]string, 0) + noRolesProvided := true roles := []spec.PgUser{} - uniqRoles := map[string]spec.PgUser{} + uniqRoles := make(map[string]spec.PgUser) // To be compatible with the legacy implementation we need to return nil if // the provided secret name is empty. The equivalent situation in the @@ -214,37 +220,39 @@ func (c *Controller) getInfrastructureRoles( } if noRolesProvided { - return nil, nil + return uniqRoles, nil } for _, secret := range rolesSecrets { infraRoles, err := c.getInfrastructureRole(secret) if err != nil || infraRoles == nil { - c.logger.Debugf("Cannot get infrastructure role: %+v", *secret) + c.logger.Debugf("cannot get infrastructure role: %+v", *secret) if err != nil { - errors = append(errors, err) + errors = append(errors, fmt.Sprintf("%v", err)) } continue } - for _, r := range infraRoles { - roles = append(roles, r) - } + roles = append(roles, infraRoles...) } for _, r := range roles { if _, exists := uniqRoles[r.Name]; exists { - msg := "Conflicting infrastructure roles: roles[%s] = (%q, %q)" + msg := "conflicting infrastructure roles: roles[%s] = (%q, %q)" c.logger.Debugf(msg, r.Name, uniqRoles[r.Name], r) } uniqRoles[r.Name] = r } - return uniqRoles, errors + if len(errors) > 0 { + return uniqRoles, fmt.Errorf(strings.Join(errors, `', '`)) + } + + return uniqRoles, nil } // Generate list of users representing one infrastructure role based on its diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go index d8e4c3782..a4ca17728 100644 --- a/pkg/controller/util_test.go +++ b/pkg/controller/util_test.go @@ -7,6 +7,7 @@ import ( b64 "encoding/base64" + "github.com/stretchr/testify/assert" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" @@ -90,21 +91,21 @@ func TestClusterWorkerID(t *testing.T) { // not exist, or empty) and the old format. func TestOldInfrastructureRoleFormat(t *testing.T) { var testTable = []struct { - secretName spec.NamespacedName - expectedRoles map[string]spec.PgUser - expectedErrors []error + secretName spec.NamespacedName + expectedRoles map[string]spec.PgUser + expectedError error }{ { // empty secret name spec.NamespacedName{}, - nil, + map[string]spec.PgUser{}, nil, }, { // secret does not exist spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: "null"}, map[string]spec.PgUser{}, - []error{fmt.Errorf(`could not get infrastructure roles secret default/null: NotFound`)}, + fmt.Errorf(`could not get infrastructure roles secret default/null: NotFound`), }, { spec.NamespacedName{ @@ -129,7 +130,7 @@ func TestOldInfrastructureRoleFormat(t *testing.T) { }, } for _, test := range testTable { - roles, errors := utilTestController.getInfrastructureRoles( + roles, err := utilTestController.getInfrastructureRoles( []*config.InfrastructureRole{ &config.InfrastructureRole{ SecretName: test.secretName, @@ -140,22 +141,9 @@ func TestOldInfrastructureRoleFormat(t *testing.T) { }, }) - if len(errors) != len(test.expectedErrors) { + if err != nil && err.Error() != test.expectedError.Error() { t.Errorf("expected error '%v' does not match the actual error '%v'", - test.expectedErrors, errors) - } - - for idx := range errors { - err := errors[idx] - expectedErr := test.expectedErrors[idx] - - if err != expectedErr { - if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() { - continue - } - t.Errorf("expected error '%v' does not match the actual error '%v'", - expectedErr, err) - } + test.expectedError, err) } if !reflect.DeepEqual(roles, test.expectedRoles) { @@ -169,9 +157,8 @@ func TestOldInfrastructureRoleFormat(t *testing.T) { // corresponding secrets. Here we test the new format. func TestNewInfrastructureRoleFormat(t *testing.T) { var testTable = []struct { - secrets []spec.NamespacedName - expectedRoles map[string]spec.PgUser - expectedErrors []error + secrets []spec.NamespacedName + expectedRoles map[string]spec.PgUser }{ // one secret with one configmap { @@ -196,7 +183,6 @@ func TestNewInfrastructureRoleFormat(t *testing.T) { Flags: []string{"createdb"}, }, }, - nil, }, // multiple standalone secrets { @@ -224,7 +210,6 @@ func TestNewInfrastructureRoleFormat(t *testing.T) { MemberOf: []string{"new-test-inrole2"}, }, }, - nil, }, } for _, test := range testTable { @@ -239,27 +224,8 @@ func TestNewInfrastructureRoleFormat(t *testing.T) { }) } - roles, errors := utilTestController.getInfrastructureRoles(definitions) - if len(errors) != len(test.expectedErrors) { - t.Errorf("expected error does not match the actual error:\n%+v\n%+v", - test.expectedErrors, errors) - - // Stop and do not do any further checks - return - } - - for idx := range errors { - err := errors[idx] - expectedErr := test.expectedErrors[idx] - - if err != expectedErr { - if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() { - continue - } - t.Errorf("expected error '%v' does not match the actual error '%v'", - expectedErr, err) - } - } + roles, err := utilTestController.getInfrastructureRoles(definitions) + assert.NoError(t, err) if !reflect.DeepEqual(roles, test.expectedRoles) { t.Errorf("expected roles output/the actual:\n%#v\n%#v", diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index ab4a88735..50f0ac841 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ import ( "fmt" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" @@ -36,13 +37,15 @@ import ( type Interface interface { Discovery() discovery.DiscoveryInterface AcidV1() acidv1.AcidV1Interface + ZalandoV1() zalandov1.ZalandoV1Interface } // Clientset contains the clients for groups. Each group has exactly one // version included in a Clientset. type Clientset struct { *discovery.DiscoveryClient - acidV1 *acidv1.AcidV1Client + acidV1 *acidv1.AcidV1Client + zalandoV1 *zalandov1.ZalandoV1Client } // AcidV1 retrieves the AcidV1Client @@ -50,6 +53,11 @@ func (c *Clientset) AcidV1() acidv1.AcidV1Interface { return c.acidV1 } +// ZalandoV1 retrieves the ZalandoV1Client +func (c *Clientset) ZalandoV1() zalandov1.ZalandoV1Interface { + return c.zalandoV1 +} + // Discovery retrieves the DiscoveryClient func (c *Clientset) Discovery() discovery.DiscoveryInterface { if c == nil { @@ -75,6 +83,10 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { if err != nil { return nil, err } + cs.zalandoV1, err = zalandov1.NewForConfig(&configShallowCopy) + if err != nil { + return nil, err + } cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) if err != nil { @@ -88,6 +100,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { func NewForConfigOrDie(c *rest.Config) *Clientset { var cs Clientset cs.acidV1 = acidv1.NewForConfigOrDie(c) + cs.zalandoV1 = zalandov1.NewForConfigOrDie(c) cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) return &cs @@ -97,6 +110,7 @@ func NewForConfigOrDie(c *rest.Config) *Clientset { func New(c rest.Interface) *Clientset { var cs Clientset cs.acidV1 = acidv1.New(c) + cs.zalandoV1 = zalandov1.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) return &cs diff --git a/pkg/generated/clientset/versioned/doc.go b/pkg/generated/clientset/versioned/doc.go index ae87609f6..ebb4ab535 100644 --- a/pkg/generated/clientset/versioned/doc.go +++ b/pkg/generated/clientset/versioned/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 65d918801..4c94d23a6 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,8 @@ import ( clientset "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake" + zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" + fakezalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" @@ -89,3 +91,8 @@ var ( func (c *Clientset) AcidV1() acidv1.AcidV1Interface { return &fakeacidv1.FakeAcidV1{Fake: &c.Fake} } + +// ZalandoV1 retrieves the ZalandoV1Client +func (c *Clientset) ZalandoV1() zalandov1.ZalandoV1Interface { + return &fakezalandov1.FakeZalandoV1{Fake: &c.Fake} +} diff --git a/pkg/generated/clientset/versioned/fake/doc.go b/pkg/generated/clientset/versioned/fake/doc.go index bc1c91a11..b9e99ecf2 100644 --- a/pkg/generated/clientset/versioned/fake/doc.go +++ b/pkg/generated/clientset/versioned/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index c4d383aab..313eeacc2 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ package fake import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,6 +39,7 @@ var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ acidv1.AddToScheme, + zalandov1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/generated/clientset/versioned/scheme/doc.go b/pkg/generated/clientset/versioned/scheme/doc.go index cd594164b..387784624 100644 --- a/pkg/generated/clientset/versioned/scheme/doc.go +++ b/pkg/generated/clientset/versioned/scheme/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go index 8be969eb5..823909bcb 100644 --- a/pkg/generated/clientset/versioned/scheme/register.go +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ package scheme import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,6 +39,7 @@ var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ acidv1.AddToScheme, + zalandov1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go index 5666201d4..d36078922 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go index eb8fcf1f4..ba729b8ca 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go index c5fd1c04b..380725443 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go index 03e7dda94..c235b2761 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go index c03ea7d94..fbea4930c 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go index 01a0ed7a4..cb39ea290 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go index b333ae046..59cf600aa 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go index b4e99cbc8..a40f2bdb0 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go index be22e075d..a73fb0df6 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go index 5241cfb54..c58ed03ed 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go index 96fbb882a..002254ef1 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go new file mode 100644 index 000000000..ba729b8ca --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go @@ -0,0 +1,26 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1 diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go new file mode 100644 index 000000000..26c3431f2 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go @@ -0,0 +1,184 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + "time" + + v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// FabricEventStreamsGetter has a method to return a FabricEventStreamInterface. +// A group's client should implement this interface. +type FabricEventStreamsGetter interface { + FabricEventStreams(namespace string) FabricEventStreamInterface +} + +// FabricEventStreamInterface has methods to work with FabricEventStream resources. +type FabricEventStreamInterface interface { + Create(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.CreateOptions) (*v1.FabricEventStream, error) + Update(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.UpdateOptions) (*v1.FabricEventStream, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.FabricEventStream, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.FabricEventStreamList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.FabricEventStream, err error) + FabricEventStreamExpansion +} + +// fabricEventStreams implements FabricEventStreamInterface +type fabricEventStreams struct { + client rest.Interface + ns string +} + +// newFabricEventStreams returns a FabricEventStreams +func newFabricEventStreams(c *ZalandoV1Client, namespace string) *fabricEventStreams { + return &fabricEventStreams{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the fabricEventStream, and returns the corresponding fabricEventStream object, and an error if there is any. +func (c *fabricEventStreams) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.FabricEventStream, err error) { + result = &v1.FabricEventStream{} + err = c.client.Get(). + Namespace(c.ns). + Resource("fabriceventstreams"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of FabricEventStreams that match those selectors. +func (c *fabricEventStreams) List(ctx context.Context, opts metav1.ListOptions) (result *v1.FabricEventStreamList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1.FabricEventStreamList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("fabriceventstreams"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested fabricEventStreams. +func (c *fabricEventStreams) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("fabriceventstreams"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a fabricEventStream and creates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. +func (c *fabricEventStreams) Create(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.CreateOptions) (result *v1.FabricEventStream, err error) { + result = &v1.FabricEventStream{} + err = c.client.Post(). + Namespace(c.ns). + Resource("fabriceventstreams"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(fabricEventStream). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a fabricEventStream and updates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. +func (c *fabricEventStreams) Update(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.UpdateOptions) (result *v1.FabricEventStream, err error) { + result = &v1.FabricEventStream{} + err = c.client.Put(). + Namespace(c.ns). + Resource("fabriceventstreams"). + Name(fabricEventStream.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(fabricEventStream). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the fabricEventStream and deletes it. Returns an error if one occurs. +func (c *fabricEventStreams) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("fabriceventstreams"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *fabricEventStreams) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("fabriceventstreams"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched fabricEventStream. +func (c *fabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.FabricEventStream, err error) { + result = &v1.FabricEventStream{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("fabriceventstreams"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go new file mode 100644 index 000000000..380725443 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go @@ -0,0 +1,26 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go new file mode 100644 index 000000000..8d76e06de --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go @@ -0,0 +1,136 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeFabricEventStreams implements FabricEventStreamInterface +type FakeFabricEventStreams struct { + Fake *FakeZalandoV1 + ns string +} + +var fabriceventstreamsResource = schema.GroupVersionResource{Group: "zalando.org", Version: "v1", Resource: "fabriceventstreams"} + +var fabriceventstreamsKind = schema.GroupVersionKind{Group: "zalando.org", Version: "v1", Kind: "FabricEventStream"} + +// Get takes name of the fabricEventStream, and returns the corresponding fabricEventStream object, and an error if there is any. +func (c *FakeFabricEventStreams) Get(ctx context.Context, name string, options v1.GetOptions) (result *zalandoorgv1.FabricEventStream, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(fabriceventstreamsResource, c.ns, name), &zalandoorgv1.FabricEventStream{}) + + if obj == nil { + return nil, err + } + return obj.(*zalandoorgv1.FabricEventStream), err +} + +// List takes label and field selectors, and returns the list of FabricEventStreams that match those selectors. +func (c *FakeFabricEventStreams) List(ctx context.Context, opts v1.ListOptions) (result *zalandoorgv1.FabricEventStreamList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(fabriceventstreamsResource, fabriceventstreamsKind, c.ns, opts), &zalandoorgv1.FabricEventStreamList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &zalandoorgv1.FabricEventStreamList{ListMeta: obj.(*zalandoorgv1.FabricEventStreamList).ListMeta} + for _, item := range obj.(*zalandoorgv1.FabricEventStreamList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested fabricEventStreams. +func (c *FakeFabricEventStreams) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(fabriceventstreamsResource, c.ns, opts)) + +} + +// Create takes the representation of a fabricEventStream and creates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. +func (c *FakeFabricEventStreams) Create(ctx context.Context, fabricEventStream *zalandoorgv1.FabricEventStream, opts v1.CreateOptions) (result *zalandoorgv1.FabricEventStream, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &zalandoorgv1.FabricEventStream{}) + + if obj == nil { + return nil, err + } + return obj.(*zalandoorgv1.FabricEventStream), err +} + +// Update takes the representation of a fabricEventStream and updates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. +func (c *FakeFabricEventStreams) Update(ctx context.Context, fabricEventStream *zalandoorgv1.FabricEventStream, opts v1.UpdateOptions) (result *zalandoorgv1.FabricEventStream, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &zalandoorgv1.FabricEventStream{}) + + if obj == nil { + return nil, err + } + return obj.(*zalandoorgv1.FabricEventStream), err +} + +// Delete takes name of the fabricEventStream and deletes it. Returns an error if one occurs. +func (c *FakeFabricEventStreams) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(fabriceventstreamsResource, c.ns, name), &zalandoorgv1.FabricEventStream{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeFabricEventStreams) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(fabriceventstreamsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &zalandoorgv1.FabricEventStreamList{}) + return err +} + +// Patch applies the patch and returns the patched fabricEventStream. +func (c *FakeFabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *zalandoorgv1.FabricEventStream, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(fabriceventstreamsResource, c.ns, name, pt, data, subresources...), &zalandoorgv1.FabricEventStream{}) + + if obj == nil { + return nil, err + } + return obj.(*zalandoorgv1.FabricEventStream), err +} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go new file mode 100644 index 000000000..17786c9a7 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go @@ -0,0 +1,46 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeZalandoV1 struct { + *testing.Fake +} + +func (c *FakeZalandoV1) FabricEventStreams(namespace string) v1.FabricEventStreamInterface { + return &FakeFabricEventStreams{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeZalandoV1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go new file mode 100644 index 000000000..6a776af84 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go @@ -0,0 +1,27 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +type FabricEventStreamExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go new file mode 100644 index 000000000..937fd6b7e --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go @@ -0,0 +1,95 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type ZalandoV1Interface interface { + RESTClient() rest.Interface + FabricEventStreamsGetter +} + +// ZalandoV1Client is used to interact with features provided by the zalando.org group. +type ZalandoV1Client struct { + restClient rest.Interface +} + +func (c *ZalandoV1Client) FabricEventStreams(namespace string) FabricEventStreamInterface { + return newFabricEventStreams(c, namespace) +} + +// NewForConfig creates a new ZalandoV1Client for the given config. +func NewForConfig(c *rest.Config) (*ZalandoV1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientFor(&config) + if err != nil { + return nil, err + } + return &ZalandoV1Client{client}, nil +} + +// NewForConfigOrDie creates a new ZalandoV1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *ZalandoV1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new ZalandoV1Client for the given RESTClient. +func New(c rest.Interface) *ZalandoV1Client { + return &ZalandoV1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *ZalandoV1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go index 6f77564fa..b2b85d87d 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go index 5c05e6d68..347c53f99 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go index 1453af276..61d88fb22 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go index a19e4726f..eb15e4fad 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go index e4b1efdc6..0842aa5ae 100644 --- a/pkg/generated/informers/externalversions/factory.go +++ b/pkg/generated/informers/externalversions/factory.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -32,6 +32,7 @@ import ( versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidzalando "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do" internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + zalandoorg "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -179,8 +180,13 @@ type SharedInformerFactory interface { WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool Acid() acidzalando.Interface + Zalando() zalandoorg.Interface } func (f *sharedInformerFactory) Acid() acidzalando.Interface { return acidzalando.New(f, f.namespace, f.tweakListOptions) } + +func (f *sharedInformerFactory) Zalando() zalandoorg.Interface { + return zalandoorg.New(f, f.namespace, f.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 5fd693558..afa4c6ac2 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ import ( "fmt" v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) @@ -64,6 +65,10 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource case v1.SchemeGroupVersion.WithResource("postgresqls"): return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().Postgresqls().Informer()}, nil + // Group=zalando.org, Version=v1 + case zalandoorgv1.SchemeGroupVersion.WithResource("fabriceventstreams"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Zalando().V1().FabricEventStreams().Informer()}, nil + } return nil, fmt.Errorf("no informer found for %v", resource) diff --git a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go index 6d1b334bf..1046f8f48 100644 --- a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/zalando.org/interface.go b/pkg/generated/informers/externalversions/zalando.org/interface.go new file mode 100644 index 000000000..afece8ac3 --- /dev/null +++ b/pkg/generated/informers/externalversions/zalando.org/interface.go @@ -0,0 +1,52 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package zalando + +import ( + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + v1 "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org/v1" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1 provides access to shared informers for resources in V1. + V1() v1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1 returns a new v1.Interface. +func (g *group) V1() v1.Interface { + return v1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go b/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go new file mode 100644 index 000000000..13733e1d1 --- /dev/null +++ b/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go @@ -0,0 +1,96 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + time "time" + + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + v1 "github.com/zalando/postgres-operator/pkg/generated/listers/zalando.org/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// FabricEventStreamInformer provides access to a shared informer and lister for +// FabricEventStreams. +type FabricEventStreamInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.FabricEventStreamLister +} + +type fabricEventStreamInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewFabricEventStreamInformer constructs a new informer for FabricEventStream type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFabricEventStreamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredFabricEventStreamInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredFabricEventStreamInformer constructs a new informer for FabricEventStream type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredFabricEventStreamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ZalandoV1().FabricEventStreams(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ZalandoV1().FabricEventStreams(namespace).Watch(context.TODO(), options) + }, + }, + &zalandoorgv1.FabricEventStream{}, + resyncPeriod, + indexers, + ) +} + +func (f *fabricEventStreamInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredFabricEventStreamInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *fabricEventStreamInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&zalandoorgv1.FabricEventStream{}, f.defaultInformer) +} + +func (f *fabricEventStreamInformer) Lister() v1.FabricEventStreamLister { + return v1.NewFabricEventStreamLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/zalando.org/v1/interface.go b/pkg/generated/informers/externalversions/zalando.org/v1/interface.go new file mode 100644 index 000000000..329f15503 --- /dev/null +++ b/pkg/generated/informers/externalversions/zalando.org/v1/interface.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // FabricEventStreams returns a FabricEventStreamInformer. + FabricEventStreams() FabricEventStreamInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// FabricEventStreams returns a FabricEventStreamInformer. +func (v *version) FabricEventStreams() FabricEventStreamInformer { + return &fabricEventStreamInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go index cc3e578b2..3189f5948 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go +++ b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go index d2258bd01..650f8d579 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go index 38073e92d..c63b8da58 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/zalando.org/v1/expansion_generated.go b/pkg/generated/listers/zalando.org/v1/expansion_generated.go new file mode 100644 index 000000000..5a90e7828 --- /dev/null +++ b/pkg/generated/listers/zalando.org/v1/expansion_generated.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +// FabricEventStreamListerExpansion allows custom methods to be added to +// FabricEventStreamLister. +type FabricEventStreamListerExpansion interface{} + +// FabricEventStreamNamespaceListerExpansion allows custom methods to be added to +// FabricEventStreamNamespaceLister. +type FabricEventStreamNamespaceListerExpansion interface{} diff --git a/pkg/generated/listers/zalando.org/v1/fabriceventstream.go b/pkg/generated/listers/zalando.org/v1/fabriceventstream.go new file mode 100644 index 000000000..b4c895c74 --- /dev/null +++ b/pkg/generated/listers/zalando.org/v1/fabriceventstream.go @@ -0,0 +1,105 @@ +/* +Copyright 2022 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// FabricEventStreamLister helps list FabricEventStreams. +// All objects returned here must be treated as read-only. +type FabricEventStreamLister interface { + // List lists all FabricEventStreams in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) + // FabricEventStreams returns an object that can list and get FabricEventStreams. + FabricEventStreams(namespace string) FabricEventStreamNamespaceLister + FabricEventStreamListerExpansion +} + +// fabricEventStreamLister implements the FabricEventStreamLister interface. +type fabricEventStreamLister struct { + indexer cache.Indexer +} + +// NewFabricEventStreamLister returns a new FabricEventStreamLister. +func NewFabricEventStreamLister(indexer cache.Indexer) FabricEventStreamLister { + return &fabricEventStreamLister{indexer: indexer} +} + +// List lists all FabricEventStreams in the indexer. +func (s *fabricEventStreamLister) List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1.FabricEventStream)) + }) + return ret, err +} + +// FabricEventStreams returns an object that can list and get FabricEventStreams. +func (s *fabricEventStreamLister) FabricEventStreams(namespace string) FabricEventStreamNamespaceLister { + return fabricEventStreamNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// FabricEventStreamNamespaceLister helps list and get FabricEventStreams. +// All objects returned here must be treated as read-only. +type FabricEventStreamNamespaceLister interface { + // List lists all FabricEventStreams in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) + // Get retrieves the FabricEventStream from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1.FabricEventStream, error) + FabricEventStreamNamespaceListerExpansion +} + +// fabricEventStreamNamespaceLister implements the FabricEventStreamNamespaceLister +// interface. +type fabricEventStreamNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all FabricEventStreams in the indexer for a given namespace. +func (s fabricEventStreamNamespaceLister) List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1.FabricEventStream)) + }) + return ret, err +} + +// Get retrieves the FabricEventStream from the indexer for a given namespace and name. +func (s fabricEventStreamNamespaceLister) Get(name string) (*v1.FabricEventStream, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1.Resource("fabriceventstream"), name) + } + return obj.(*v1.FabricEventStream), nil +} diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 533aae79f..e594a7978 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -30,6 +30,7 @@ const ( RoleOriginManifest RoleOriginInfrastructure RoleOriginTeamsAPI + RoleOriginSpilo RoleOriginSystem RoleOriginBootstrap RoleConnectionPooler @@ -55,6 +56,7 @@ type PgUser struct { MemberOf []string `yaml:"inrole"` Parameters map[string]string `yaml:"db_parameters"` AdminRole string `yaml:"admin_role"` + IsDbOwner bool `yaml:"is_db_owner"` Deleted bool `yaml:"deleted"` } @@ -117,6 +119,7 @@ type ControllerConfig struct { CRDReadyWaitTimeout time.Duration ConfigMapName NamespacedName Namespace string + IgnoredAnnotations []string EnableJsonLogging bool } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 78e0a6c49..4d88db85d 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -14,11 +14,13 @@ import ( // CRD describes CustomResourceDefinition specific configuration parameters type CRD struct { - ReadyWaitInterval time.Duration `name:"ready_wait_interval" default:"4s"` - ReadyWaitTimeout time.Duration `name:"ready_wait_timeout" default:"30s"` - ResyncPeriod time.Duration `name:"resync_period" default:"30m"` - RepairPeriod time.Duration `name:"repair_period" default:"5m"` - EnableCRDValidation *bool `name:"enable_crd_validation" default:"true"` + ReadyWaitInterval time.Duration `name:"ready_wait_interval" default:"4s"` + ReadyWaitTimeout time.Duration `name:"ready_wait_timeout" default:"30s"` + ResyncPeriod time.Duration `name:"resync_period" default:"30m"` + RepairPeriod time.Duration `name:"repair_period" default:"5m"` + EnableCRDRegistration *bool `name:"enable_crd_registration" default:"true"` + EnableCRDValidation *bool `name:"enable_crd_validation" default:"true"` + CRDCategories []string `name:"crd_categories" default:"all"` } // Resources describes kubernetes resource specific configuration parameters @@ -40,6 +42,7 @@ type Resources struct { InheritedLabels []string `name:"inherited_labels" default:""` InheritedAnnotations []string `name:"inherited_annotations" default:""` DownscalerAnnotations []string `name:"downscaler_annotations"` + IgnoredAnnotations []string `name:"ignored_annotations"` ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` DeleteAnnotationDateKey string `name:"delete_annotation_date_key"` DeleteAnnotationNameKey string `name:"delete_annotation_name_key"` @@ -54,6 +57,7 @@ type Resources struct { PodEnvironmentConfigMap spec.NamespacedName `name:"pod_environment_configmap"` PodEnvironmentSecret string `name:"pod_environment_secret"` NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""` + NodeReadinessLabelMerge string `name:"node_readiness_label_merge" default:"OR"` MaxInstances int32 `name:"max_instances" default:"-1"` MinInstances int32 `name:"min_instances" default:"-1"` ShmVolume *bool `name:"enable_shm_volume" default:"true"` @@ -98,6 +102,10 @@ type Auth struct { InfrastructureRolesDefs string `name:"infrastructure_roles_secrets"` SuperUsername string `name:"super_username" default:"postgres"` ReplicationUsername string `name:"replication_username" default:"standby"` + AdditionalOwnerRoles []string `name:"additional_owner_roles" default:""` + EnablePasswordRotation bool `name:"enable_password_rotation" default:"false"` + PasswordRotationInterval uint32 `name:"password_rotation_interval" default:"90"` + PasswordRotationUserRetention uint32 `name:"password_rotation_user_retention" default:"180"` } // Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping: @@ -114,7 +122,7 @@ type Scalyr struct { // LogicalBackup defines configuration for logical backup type LogicalBackup struct { LogicalBackupSchedule string `name:"logical_backup_schedule" default:"30 00 * * *"` - LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup:v1.7.1"` + LogicalBackupDockerImage string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup:v1.8.0"` LogicalBackupProvider string `name:"logical_backup_provider" default:"s3"` LogicalBackupS3Bucket string `name:"logical_backup_s3_bucket" default:""` LogicalBackupS3Region string `name:"logical_backup_s3_region" default:""` @@ -122,6 +130,7 @@ type LogicalBackup struct { LogicalBackupS3AccessKeyID string `name:"logical_backup_s3_access_key_id" default:""` LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""` LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:""` + LogicalBackupS3RetentionTime string `name:"logical_backup_s3_retention_time" default:""` LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""` LogicalBackupJobPrefix string `name:"logical_backup_job_prefix" default:"logical-backup-"` } @@ -152,7 +161,7 @@ type Config struct { WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-14:2.1-p3"` + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-14:2.1-p5"` SidecarImages map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers SidecarContainers []v1.Container `name:"sidecars"` PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` @@ -183,7 +192,9 @@ type Config struct { EnablePostgresTeamCRD bool `name:"enable_postgres_team_crd" default:"false"` EnablePostgresTeamCRDSuperusers bool `name:"enable_postgres_team_crd_superusers" default:"false"` EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` + EnableMasterPoolerLoadBalancer bool `name:"enable_master_pooler_load_balancer" default:"false"` EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` + EnableReplicaPoolerLoadBalancer bool `name:"enable_replica_pooler_load_balancer" default:"false"` CustomServiceAnnotations map[string]string `name:"custom_service_annotations"` CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` @@ -204,7 +215,7 @@ type Config struct { TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"` PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` PodManagementPolicy string `name:"pod_management_policy" default:"ordered_ready"` - ProtectedRoles []string `name:"protected_role_names" default:"admin"` + ProtectedRoles []string `name:"protected_role_names" default:"admin,cron_admin"` PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` @@ -212,8 +223,11 @@ type Config struct { EnablePgVersionEnvVar bool `name:"enable_pgversion_env_var" default:"true"` EnableSpiloWalPathCompat bool `name:"enable_spilo_wal_path_compat" default:"false"` MajorVersionUpgradeMode string `name:"major_version_upgrade_mode" default:"off"` + MajorVersionUpgradeTeamAllowList []string `name:"major_version_upgrade_team_allow_list" default:""` MinimalMajorVersion string `name:"minimal_major_version" default:"9.6"` TargetMajorVersion string `name:"target_major_version" default:"14"` + PatroniAPICheckInterval time.Duration `name:"patroni_api_check_interval" default:"1s"` + PatroniAPICheckTimeout time.Duration `name:"patroni_api_check_timeout" default:"5s"` } // MustMarshal marshals the config or panics @@ -280,7 +294,7 @@ func validate(cfg *Config) (err error) { } if cfg.ConnectionPooler.User == cfg.SuperUsername { - msg := "Connection pool user is not allowed to be the same as super user, username: %s" + msg := "connection pool user is not allowed to be the same as super user, username: %s" err = fmt.Errorf(msg, cfg.ConnectionPooler.User) } diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go new file mode 100644 index 000000000..bd70b719b --- /dev/null +++ b/pkg/util/constants/streams.go @@ -0,0 +1,14 @@ +package constants + +// PostgreSQL specific constants +const ( + EventStreamCRDApiVersion = "zalando.org/v1" + EventStreamCRDKind = "FabricEventStream" + EventStreamCRDName = "fabriceventstreams.zalando.org" + EventStreamSourcePGType = "PostgresLogicalReplication" + EventStreamSourceSlotPrefix = "fes" + EventStreamSourcePluginType = "pgoutput" + EventStreamSourceAuthType = "DatabaseAuthenticationSecret" + EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" + EventStreamSinkNakadiType = "Nakadi" +) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index dd6ec1e8b..f4ea014f2 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -12,8 +12,9 @@ import ( clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" apiacidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - acidv1client "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" + zalandoclient "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" "github.com/zalando/postgres-operator/pkg/spec" apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -36,6 +37,14 @@ func Int32ToPointer(value int32) *int32 { return &value } +func UInt32ToPointer(value uint32) *uint32 { + return &value +} + +func StringToPointer(str string) *string { + return &str +} + // KubernetesClient describes getters for Kubernetes objects type KubernetesClient struct { corev1.SecretsGetter @@ -58,9 +67,11 @@ type KubernetesClient struct { acidv1.OperatorConfigurationsGetter acidv1.PostgresTeamsGetter acidv1.PostgresqlsGetter + zalandov1.FabricEventStreamsGetter - RESTClient rest.Interface - AcidV1ClientSet *acidv1client.Clientset + RESTClient rest.Interface + AcidV1ClientSet *zalandoclient.Clientset + Zalandov1ClientSet *zalandoclient.Clientset } type mockSecret struct { @@ -158,14 +169,19 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.CustomResourceDefinitionsGetter = apiextClient.ApiextensionsV1() - kubeClient.AcidV1ClientSet = acidv1client.NewForConfigOrDie(cfg) + kubeClient.AcidV1ClientSet = zalandoclient.NewForConfigOrDie(cfg) if err != nil { return kubeClient, fmt.Errorf("could not create acid.zalan.do clientset: %v", err) } + kubeClient.Zalandov1ClientSet = zalandoclient.NewForConfigOrDie(cfg) + if err != nil { + return kubeClient, fmt.Errorf("could not create zalando.org clientset: %v", err) + } kubeClient.OperatorConfigurationsGetter = kubeClient.AcidV1ClientSet.AcidV1() kubeClient.PostgresTeamsGetter = kubeClient.AcidV1ClientSet.AcidV1() kubeClient.PostgresqlsGetter = kubeClient.AcidV1ClientSet.AcidV1() + kubeClient.FabricEventStreamsGetter = kubeClient.Zalandov1ClientSet.ZalandoV1() return kubeClient, nil } @@ -197,57 +213,6 @@ func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.Namespaced return pg, nil } -// SameService compares the Services -func SameService(cur, new *v1.Service) (match bool, reason string) { - //TODO: improve comparison - if cur.Spec.Type != new.Spec.Type { - return false, fmt.Sprintf("new service's type %q does not match the current one %q", - new.Spec.Type, cur.Spec.Type) - } - - oldSourceRanges := cur.Spec.LoadBalancerSourceRanges - newSourceRanges := new.Spec.LoadBalancerSourceRanges - - /* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */ - if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) { - if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) { - return false, "new service's LoadBalancerSourceRange does not match the current one" - } - } - - match = true - - reasonPrefix := "new service's annotations does not match the current one:" - for ann := range cur.Annotations { - if _, ok := new.Annotations[ann]; !ok { - match = false - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" Removed '%s'.", ann) - } - } - - for ann := range new.Annotations { - v, ok := cur.Annotations[ann] - if !ok { - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" Added '%s' with value '%s'.", ann, new.Annotations[ann]) - match = false - } else if v != new.Annotations[ann] { - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", ann, v, new.Annotations[ann]) - match = false - } - } - - return match, reason -} - // SamePDB compares the PodDisruptionBudgets func SamePDB(cur, new *policybeta1.PodDisruptionBudget) (match bool, reason string) { //TODO: improve comparison diff --git a/pkg/util/k8sutil/k8sutil_test.go b/pkg/util/k8sutil/k8sutil_test.go deleted file mode 100644 index b3e768501..000000000 --- a/pkg/util/k8sutil/k8sutil_test.go +++ /dev/null @@ -1,311 +0,0 @@ -package k8sutil - -import ( - "strings" - "testing" - - "github.com/zalando/postgres-operator/pkg/util/constants" - - v1 "k8s.io/api/core/v1" -) - -func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service { - svc := &v1.Service{ - Spec: v1.ServiceSpec{ - Type: svcT, - LoadBalancerSourceRanges: lbSr, - }, - } - svc.Annotations = ann - return svc -} - -func TestSameService(t *testing.T) { - tests := []struct { - about string - current *v1.Service - new *v1.Service - reason string - match bool - }{ - { - about: "two equal services", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: true, - }, - { - about: "services differ on service type", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeClusterIP, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's type "LoadBalancer" does not match the current one "ClusterIP"`, - }, - { - about: "services differ on lb source ranges", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"185.249.56.0/22"}), - match: false, - reason: `new service's LoadBalancerSourceRange does not match the current one`, - }, - { - about: "new service doesn't have lb source ranges", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{}), - match: false, - reason: `new service's LoadBalancerSourceRange does not match the current one`, - }, - { - about: "services differ on DNS annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`, - }, - { - about: "services differ on AWS ELB annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: "1800", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`, - }, - { - about: "service changes existing annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "baz", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: 'foo' changed from 'bar' to 'baz'.`, - }, - { - about: "service changes multiple existing annotations", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - "bar": "foo", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "baz", - "bar": "fooz", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations does not match the current one:`, - }, - { - about: "service adds a new custom annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: Added 'foo' with value 'bar'.`, - }, - { - about: "service removes a custom annotation", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: Removed 'foo'.`, - }, - { - about: "service removes a custom annotation and adds a new one", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "bar": "foo", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - reason: `new service's annotations does not match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`, - }, - { - about: "service removes a custom annotation, adds a new one and change another", - current: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "foo": "bar", - "zalan": "do", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - "bar": "foo", - "zalan": "do.com", - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations does not match the current one: Removed 'foo'.`, - }, - { - about: "service add annotations", - current: newsService( - map[string]string{}, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - new: newsService( - map[string]string{ - constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do", - constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue, - }, - v1.ServiceTypeLoadBalancer, - []string{"128.141.0.0/16", "137.138.0.0/16"}), - match: false, - // Test just the prefix to avoid flakiness and map sorting - reason: `new service's annotations does not match the current one: Added `, - }, - } - for _, tt := range tests { - t.Run(tt.about, func(t *testing.T) { - match, reason := SameService(tt.current, tt.new) - if match && !tt.match { - t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new) - return - } - if !match && tt.match { - t.Errorf("expected services to be the same: '%q' and '%q'", tt.current, tt.new) - return - } - if !match && !tt.match { - if !strings.HasPrefix(reason, tt.reason) { - t.Errorf("expected reason prefix '%s', found '%s'", tt.reason, reason) - return - } - } - }) - } -} diff --git a/pkg/util/patroni/patroni.go b/pkg/util/patroni/patroni.go index 4de3b8201..8126eddc7 100644 --- a/pkg/util/patroni/patroni.go +++ b/pkg/util/patroni/patroni.go @@ -16,19 +16,22 @@ import ( "github.com/sirupsen/logrus" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) const ( failoverPath = "/failover" configPath = "/config" + clusterPath = "/cluster" statusPath = "/patroni" restartPath = "/restart" - apiPort = 8008 + ApiPort = 8008 timeout = 30 * time.Second ) // Interface describe patroni methods type Interface interface { + GetClusterMembers(master *v1.Pod) ([]ClusterMember, error) Switchover(master *v1.Pod, candidate string) error SetPostgresParameters(server *v1.Pod, options map[string]string) error GetMemberData(server *v1.Pod) (MemberData, error) @@ -71,7 +74,7 @@ func apiURL(masterPod *v1.Pod) (string, error) { return "", fmt.Errorf("%s is not a valid IPv4/IPv6 address", masterPod.Status.PodIP) } } - return fmt.Sprintf("http://%s", net.JoinHostPort(ip.String(), strconv.Itoa(apiPort))), nil + return fmt.Sprintf("http://%s", net.JoinHostPort(ip.String(), strconv.Itoa(ApiPort))), nil } func (p *Patroni) httpPostOrPatch(method string, url string, body *bytes.Buffer) (err error) { @@ -175,6 +178,20 @@ func (p *Patroni) SetConfig(server *v1.Pod, config map[string]interface{}) error return p.httpPostOrPatch(http.MethodPatch, apiURLString+configPath, buf) } +// ClusterMembers array of cluster members from Patroni API +type ClusterMembers struct { + Members []ClusterMember `json:"members"` +} + +// ClusterMember cluster member data from Patroni API +type ClusterMember struct { + Name string `json:"name"` + Role string `json:"role"` + State string `json:"state"` + Timeline int `json:"timeline"` + Lag intstr.IntOrString `json:"lag,omitempty"` +} + // MemberDataPatroni child element type MemberDataPatroni struct { Version string `json:"version"` @@ -246,6 +263,27 @@ func (p *Patroni) Restart(server *v1.Pod) error { return nil } +// GetClusterMembers read cluster data from patroni API +func (p *Patroni) GetClusterMembers(server *v1.Pod) ([]ClusterMember, error) { + + apiURLString, err := apiURL(server) + if err != nil { + return []ClusterMember{}, err + } + body, err := p.httpGet(apiURLString + clusterPath) + if err != nil { + return []ClusterMember{}, err + } + + data := ClusterMembers{} + err = json.Unmarshal([]byte(body), &data) + if err != nil { + return []ClusterMember{}, err + } + + return data.Members, nil +} + // GetMemberData read member data from patroni API func (p *Patroni) GetMemberData(server *v1.Pod) (MemberData, error) { diff --git a/pkg/util/patroni/patroni_test.go b/pkg/util/patroni/patroni_test.go index 5a6b2657c..aa6ad9206 100644 --- a/pkg/util/patroni/patroni_test.go +++ b/pkg/util/patroni/patroni_test.go @@ -15,6 +15,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) var logger = logrus.New().WithField("test", "patroni") @@ -35,17 +36,17 @@ func TestApiURL(t *testing.T) { }{ { "127.0.0.1", - fmt.Sprintf("http://127.0.0.1:%d", apiPort), + fmt.Sprintf("http://127.0.0.1:%d", ApiPort), nil, }, { "0000:0000:0000:0000:0000:0000:0000:0001", - fmt.Sprintf("http://[::1]:%d", apiPort), + fmt.Sprintf("http://[::1]:%d", ApiPort), nil, }, { "::1", - fmt.Sprintf("http://[::1]:%d", apiPort), + fmt.Sprintf("http://[::1]:%d", ApiPort), nil, }, { @@ -85,6 +86,54 @@ func TestApiURL(t *testing.T) { } } +func TestGetClusterMembers(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + expectedClusterMemberData := []ClusterMember{ + { + Name: "acid-test-cluster-0", + Role: "leader", + State: "running", + Timeline: 1, + }, { + Name: "acid-test-cluster-1", + Role: "sync_standby", + State: "running", + Timeline: 1, + Lag: intstr.IntOrString{IntVal: 0}, + }, { + Name: "acid-test-cluster-2", + Role: "replica", + State: "running", + Timeline: 1, + Lag: intstr.IntOrString{Type: 1, StrVal: "unknown"}, + }} + + json := `{"members": [{"name": "acid-test-cluster-0", "role": "leader", "state": "running", "api_url": "http://192.168.100.1:8008/patroni", "host": "192.168.100.1", "port": 5432, "timeline": 1}, {"name": "acid-test-cluster-1", "role": "sync_standby", "state": "running", "api_url": "http://192.168.100.2:8008/patroni", "host": "192.168.100.2", "port": 5432, "timeline": 1, "lag": 0}, {"name": "acid-test-cluster-2", "role": "replica", "state": "running", "api_url": "http://192.168.100.3:8008/patroni", "host": "192.168.100.3", "port": 5432, "timeline": 1, "lag": "unknown"}]}` + r := ioutil.NopCloser(bytes.NewReader([]byte(json))) + + response := http.Response{ + StatusCode: 200, + Body: r, + } + + mockClient := mocks.NewMockHTTPClient(ctrl) + mockClient.EXPECT().Get(gomock.Any()).Return(&response, nil) + + p := New(logger, mockClient) + + clusterMemberData, err := p.GetClusterMembers(newMockPod("192.168.100.1")) + + if !reflect.DeepEqual(expectedClusterMemberData, clusterMemberData) { + t.Errorf("Patroni cluster members differ: expected: %#v, got: %#v", expectedClusterMemberData, clusterMemberData) + } + + if err != nil { + t.Errorf("Could not read Patroni data: %v", err) + } +} + func TestGetMemberData(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -136,7 +185,7 @@ func TestGetConfig(t *testing.T) { Slots: map[string]map[string]string{ "cdc": { "database": "foo", - "plugin": "wal2json", + "plugin": "pgoutput", "type": "logical", }, }, @@ -169,7 +218,7 @@ func TestGetConfig(t *testing.T) { "wal_log_hints": "on", } - configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}` + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) response := http.Response{ @@ -204,7 +253,7 @@ func TestSetPostgresParameters(t *testing.T) { "wal_level": "logical", } - configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}` + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) response := http.Response{ diff --git a/pkg/util/teams/teams.go b/pkg/util/teams/teams.go index d7413ab9c..bf1260d6e 100644 --- a/pkg/util/teams/teams.go +++ b/pkg/util/teams/teams.go @@ -45,7 +45,7 @@ type httpClient interface { // Interface to the TeamsAPIClient type Interface interface { - TeamInfo(teamID, token string) (tm *Team, err error) + TeamInfo(teamID, token string) (tm *Team, statusCode int, err error) } // API describes teams API @@ -67,7 +67,7 @@ func NewTeamsAPI(url string, log *logrus.Entry) *API { } // TeamInfo returns information about a given team using its ID and a token to authenticate to the API service. -func (t *API) TeamInfo(teamID, token string) (tm *Team, err error) { +func (t *API) TeamInfo(teamID, token string) (tm *Team, statusCode int, err error) { var ( req *http.Request resp *http.Response @@ -77,37 +77,38 @@ func (t *API) TeamInfo(teamID, token string) (tm *Team, err error) { t.logger.Debugf("request url: %s", url) req, err = http.NewRequest("GET", url, nil) if err != nil { - return nil, err + return nil, http.StatusBadRequest, err } req.Header.Add("Authorization", "Bearer "+token) if resp, err = t.httpClient.Do(req); err != nil { - return nil, err + return nil, http.StatusUnauthorized, err } defer func() { if closeErr := resp.Body.Close(); closeErr != nil { err = fmt.Errorf("error when closing response: %v", closeErr) } }() - if resp.StatusCode != 200 { + statusCode = resp.StatusCode + if statusCode != http.StatusOK { var raw map[string]json.RawMessage d := json.NewDecoder(resp.Body) err = d.Decode(&raw) if err != nil { - return nil, fmt.Errorf("team API query failed with status code %d and malformed response: %v", resp.StatusCode, err) + return nil, statusCode, fmt.Errorf("team API query failed with status code %d and malformed response: %v", statusCode, err) } if errMessage, ok := raw["error"]; ok { - return nil, fmt.Errorf("team API query failed with status code %d and message: '%v'", resp.StatusCode, string(errMessage)) + return nil, statusCode, fmt.Errorf("team API query failed with status code %d and message: '%v'", statusCode, string(errMessage)) } - return nil, fmt.Errorf("team API query failed with status code %d", resp.StatusCode) + return nil, statusCode, fmt.Errorf("team API query failed with status code %d", statusCode) } tm = &Team{} d := json.NewDecoder(resp.Body) if err = d.Decode(tm); err != nil { - return nil, fmt.Errorf("could not parse team API response: %v", err) + return nil, statusCode, fmt.Errorf("could not parse team API response: %v", err) } - return tm, nil + return tm, statusCode, nil } diff --git a/pkg/util/teams/teams_test.go b/pkg/util/teams/teams_test.go index 33d01b75b..da9f497c1 100644 --- a/pkg/util/teams/teams_test.go +++ b/pkg/util/teams/teams_test.go @@ -1,225 +1,243 @@ -package teams - -import ( - "fmt" - "net/http" - "net/http/httptest" - "reflect" - "testing" - - "github.com/sirupsen/logrus" -) - -var ( - logger = logrus.New().WithField("pkg", "teamsapi") - token = "ec45b1cfbe7100c6315d183a3eb6cec0M2U1LWJkMzEtZDgzNzNmZGQyNGM3IiwiYXV0aF90aW1lIjoxNDkzNzMwNzQ1LCJpc3MiOiJodHRwcz" -) - -var teamsAPItc = []struct { - in string - inCode int - out *Team - err error -}{ - {`{ -"dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", -"id": "acid", -"id_name": "acid", -"team_id": "111222", -"type": "official", -"name": "Acid team name", -"mail": [ -"email1@example.com", -"email2@example.com" -], -"alias": [ -"acid" -], -"member": [ - "member1", - "member2", - "member3" -], -"infrastructure-accounts": [ -{ - "id": "1234512345", - "name": "acid", - "provider": "aws", - "type": "aws", - "description": "", - "owner": "acid", - "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - "disabled": false -}, -{ - "id": "5432154321", - "name": "db", - "provider": "aws", - "type": "aws", - "description": "", - "owner": "acid", - "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - "disabled": false -} -], -"cost_center": "00099999", -"delivery_lead": "member4", -"parent_team_id": "111221" -}`, - 200, - &Team{ - Dn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - ID: "acid", - TeamName: "acid", - TeamID: "111222", - Type: "official", - FullName: "Acid team name", - Aliases: []string{"acid"}, - Mails: []string{"email1@example.com", "email2@example.com"}, - Members: []string{"member1", "member2", "member3"}, - CostCenter: "00099999", - DeliveryLead: "member4", - ParentTeamID: "111221", - InfrastructureAccounts: []infrastructureAccount{ - { - ID: "1234512345", - Name: "acid", - Provider: "aws", - Type: "aws", - Description: "", - Owner: "acid", - OwnerDn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - Disabled: false}, - { - ID: "5432154321", - Name: "db", - Provider: "aws", - Type: "aws", - Description: "", - Owner: "acid", - OwnerDn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", - Disabled: false}, - }, - }, - nil}, { - `{"error": "Access Token not valid"}`, - 401, - nil, - fmt.Errorf(`team API query failed with status code 401 and message: '"Access Token not valid"'`), - }, - { - `{"status": "I'm a teapot'"}`, - 418, - nil, - fmt.Errorf(`team API query failed with status code 418`), - }, - { - `{"status": "I'm a teapot`, - 418, - nil, - fmt.Errorf(`team API query failed with status code 418 and malformed response: unexpected EOF`), - }, - { - `{"status": "I'm a teapot`, - 200, - nil, - fmt.Errorf(`could not parse team API response: unexpected EOF`), - }, -} - -var requestsURLtc = []struct { - url string - err error -}{ - { - "coffee://localhost/", - fmt.Errorf(`Get "coffee://localhost/teams/acid": unsupported protocol scheme "coffee"`), - }, - { - "http://192.168.0.%31/", - fmt.Errorf(`parse "http://192.168.0.%%31/teams/acid": invalid URL escape "%%31"`), - }, -} - -func TestInfo(t *testing.T) { - for _, tc := range teamsAPItc { - func() { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Authorization") != "Bearer "+token { - t.Errorf("authorization token is wrong or not provided") - } - w.WriteHeader(tc.inCode) - if _, err := fmt.Fprint(w, tc.in); err != nil { - t.Errorf("error writing teams api response %v", err) - } - })) - defer ts.Close() - api := NewTeamsAPI(ts.URL, logger) - - actual, err := api.TeamInfo("acid", token) - if err != nil && err.Error() != tc.err.Error() { - t.Errorf("expected error: %v, got: %v", tc.err, err) - return - } - - if !reflect.DeepEqual(actual, tc.out) { - t.Errorf("expected %#v, got: %#v", tc.out, actual) - } - }() - } -} - -type mockHTTPClient struct { -} - -type mockBody struct { -} - -func (b *mockBody) Read(p []byte) (n int, err error) { - return 2, nil -} - -func (b *mockBody) Close() error { - return fmt.Errorf("close error") -} - -func (c *mockHTTPClient) Do(req *http.Request) (*http.Response, error) { - resp := http.Response{ - Status: "200 OK", - StatusCode: 200, - ContentLength: 2, - Close: false, - Request: req, - } - resp.Body = &mockBody{} - - return &resp, nil -} - -func TestHttpClientClose(t *testing.T) { - ts := httptest.NewServer(nil) - - api := NewTeamsAPI(ts.URL, logger) - api.httpClient = &mockHTTPClient{} - - _, err := api.TeamInfo("acid", token) - expError := fmt.Errorf("error when closing response: close error") - if err.Error() != expError.Error() { - t.Errorf("expected error: %v, got: %v", expError, err) - } -} - -func TestRequest(t *testing.T) { - for _, tc := range requestsURLtc { - api := NewTeamsAPI(tc.url, logger) - resp, err := api.TeamInfo("acid", token) - if resp != nil { - t.Errorf("response expected to be nil") - continue - } - - if err.Error() != tc.err.Error() { - t.Errorf("expected error: %v, got: %v", tc.err, err) - } - } -} +package teams + +import ( + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/sirupsen/logrus" +) + +var ( + logger = logrus.New().WithField("pkg", "teamsapi") + token = "ec45b1cfbe7100c6315d183a3eb6cec0M2U1LWJkMzEtZDgzNzNmZGQyNGM3IiwiYXV0aF90aW1lIjoxNDkzNzMwNzQ1LCJpc3MiOiJodHRwcz" + input = `{ + "dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + "id": "acid", + "id_name": "acid", + "team_id": "111222", + "type": "official", + "name": "Acid team name", + "mail": [ + "email1@example.com", + "email2@example.com" + ], + "alias": [ + "acid" + ], + "member": [ + "member1", + "member2", + "member3" + ], + "infrastructure-accounts": [ + { + "id": "1234512345", + "name": "acid", + "provider": "aws", + "type": "aws", + "description": "", + "owner": "acid", + "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + "disabled": false + }, + { + "id": "5432154321", + "name": "db", + "provider": "aws", + "type": "aws", + "description": "", + "owner": "acid", + "owner_dn": "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + "disabled": false + } + ], + "cost_center": "00099999", + "delivery_lead": "member4", + "parent_team_id": "111221" + }` +) +var teamsAPItc = []struct { + in string + inCode int + inTeam string + out *Team + err error +}{ + { + input, + 200, + "acid", + &Team{ + Dn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + ID: "acid", + TeamName: "acid", + TeamID: "111222", + Type: "official", + FullName: "Acid team name", + Aliases: []string{"acid"}, + Mails: []string{"email1@example.com", "email2@example.com"}, + Members: []string{"member1", "member2", "member3"}, + CostCenter: "00099999", + DeliveryLead: "member4", + ParentTeamID: "111221", + InfrastructureAccounts: []infrastructureAccount{ + { + ID: "1234512345", + Name: "acid", + Provider: "aws", + Type: "aws", + Description: "", + Owner: "acid", + OwnerDn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + Disabled: false}, + { + ID: "5432154321", + Name: "db", + Provider: "aws", + Type: "aws", + Description: "", + Owner: "acid", + OwnerDn: "cn=100100,ou=official,ou=foobar,dc=zalando,dc=net", + Disabled: false}, + }, + }, + nil}, { + `{"error": "Access Token not valid"}`, + 401, + "acid", + nil, + fmt.Errorf(`team API query failed with status code 401 and message: '"Access Token not valid"'`), + }, + { + `{"status": "I'm a teapot'"}`, + 418, + "acid", + nil, + fmt.Errorf(`team API query failed with status code 418`), + }, + { + `{"status": "I'm a teapot`, + 418, + "acid", + nil, + fmt.Errorf(`team API query failed with status code 418 and malformed response: unexpected EOF`), + }, + { + `{"status": "I'm a teapot`, + 200, + "acid", + nil, + fmt.Errorf(`could not parse team API response: unexpected EOF`), + }, + { + input, + 404, + "banana", + nil, + fmt.Errorf(`team API query failed with status code 404`), + }, +} + +var requestsURLtc = []struct { + url string + err error +}{ + { + "coffee://localhost/", + fmt.Errorf(`Get "coffee://localhost/teams/acid": unsupported protocol scheme "coffee"`), + }, + { + "http://192.168.0.%31/", + fmt.Errorf(`parse "http://192.168.0.%%31/teams/acid": invalid URL escape "%%31"`), + }, +} + +func TestInfo(t *testing.T) { + for _, tc := range teamsAPItc { + func() { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+token { + t.Errorf("authorization token is wrong or not provided") + } + w.WriteHeader(tc.inCode) + if _, err := fmt.Fprint(w, tc.in); err != nil { + t.Errorf("error writing teams api response %v", err) + } + })) + defer ts.Close() + api := NewTeamsAPI(ts.URL, logger) + + actual, statusCode, err := api.TeamInfo(tc.inTeam, token) + if err != nil && err.Error() != tc.err.Error() { + t.Errorf("expected error: %v, got: %v", tc.err, err) + return + } + + if !reflect.DeepEqual(actual, tc.out) { + t.Errorf("expected %#v, got: %#v", tc.out, actual) + } + + if statusCode != tc.inCode { + t.Errorf("expected %d, got: %d", tc.inCode, statusCode) + } + }() + } +} + +type mockHTTPClient struct { +} + +type mockBody struct { +} + +func (b *mockBody) Read(p []byte) (n int, err error) { + return 2, nil +} + +func (b *mockBody) Close() error { + return fmt.Errorf("close error") +} + +func (c *mockHTTPClient) Do(req *http.Request) (*http.Response, error) { + resp := http.Response{ + Status: "200 OK", + StatusCode: 200, + ContentLength: 2, + Close: false, + Request: req, + } + resp.Body = &mockBody{} + + return &resp, nil +} + +func TestHttpClientClose(t *testing.T) { + ts := httptest.NewServer(nil) + + api := NewTeamsAPI(ts.URL, logger) + api.httpClient = &mockHTTPClient{} + + _, _, err := api.TeamInfo("acid", token) + expError := fmt.Errorf("error when closing response: close error") + if err.Error() != expError.Error() { + t.Errorf("expected error: %v, got: %v", expError, err) + } +} + +func TestRequest(t *testing.T) { + for _, tc := range requestsURLtc { + api := NewTeamsAPI(tc.url, logger) + resp, _, err := api.TeamInfo("acid", token) + if resp != nil { + t.Errorf("response expected to be nil") + continue + } + + if err.Error() != tc.err.Error() { + t.Errorf("expected error: %v, got: %v", tc.err, err) + } + } +} diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go index 3da933644..5007268d2 100644 --- a/pkg/util/users/users.go +++ b/pkg/util/users/users.go @@ -18,6 +18,7 @@ const ( alterUserRenameSQL = `ALTER ROLE "%s" RENAME TO "%s%s"` alterRoleResetAllSQL = `ALTER ROLE "%s" RESET ALL` alterRoleSetSQL = `ALTER ROLE "%s" SET %s TO %s` + dropUserSQL = `SET LOCAL synchronous_commit = 'local'; DROP ROLE "%s";` grantToUserSQL = `GRANT %s TO "%s"` doBlockStmt = `SET LOCAL synchronous_commit = 'local'; DO $$ BEGIN %s; END;$$;` passwordTemplate = "ENCRYPTED PASSWORD '%s'" @@ -52,25 +53,31 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM } } else { r := spec.PgSyncUserRequest{} + r.User = dbUser newMD5Password := util.NewEncryptor(strategy.PasswordEncryption).PGUserPassword(newUser) - if dbUser.Password != newMD5Password { - r.User.Password = newMD5Password - r.Kind = spec.PGsyncUserAlter + // do not compare for roles coming from docker image + if newUser.Origin != spec.RoleOriginSpilo { + if dbUser.Password != newMD5Password { + r.User.Password = newMD5Password + r.Kind = spec.PGsyncUserAlter + } + if addNewFlags, equal := util.SubstractStringSlices(newUser.Flags, dbUser.Flags); !equal { + r.User.Flags = addNewFlags + r.Kind = spec.PGsyncUserAlter + } } if addNewRoles, equal := util.SubstractStringSlices(newUser.MemberOf, dbUser.MemberOf); !equal { r.User.MemberOf = addNewRoles r.Kind = spec.PGsyncUserAlter } - if addNewFlags, equal := util.SubstractStringSlices(newUser.Flags, dbUser.Flags); !equal { - r.User.Flags = addNewFlags - r.Kind = spec.PGsyncUserAlter - } if r.Kind == spec.PGsyncUserAlter { r.User.Name = newUser.Name reqs = append(reqs, r) } - if len(newUser.Parameters) > 0 && !reflect.DeepEqual(dbUser.Parameters, newUser.Parameters) { + if newUser.Origin != spec.RoleOriginSpilo && + len(newUser.Parameters) > 0 && + !reflect.DeepEqual(dbUser.Parameters, newUser.Parameters) { reqs = append(reqs, spec.PgSyncUserRequest{Kind: spec.PGSyncAlterSet, User: newUser}) } } @@ -101,7 +108,7 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM // ExecuteSyncRequests makes actual database changes from the requests passed in its arguments. func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSyncUserRequest, db *sql.DB) error { var reqretries []spec.PgSyncUserRequest - var errors []string + errors := make([]string, 0) for _, request := range requests { switch request.Kind { case spec.PGSyncUserAdd: @@ -138,7 +145,7 @@ func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSy return err } } else { - return fmt.Errorf("could not execute sync requests for users: %v", errors) + return fmt.Errorf("could not execute sync requests for users: %v", strings.Join(errors, `', '`)) } } @@ -288,3 +295,13 @@ func quoteParameterValue(name, val string) string { } return fmt.Sprintf(`'%s'`, strings.Trim(val, " ")) } + +// DropPgUser to remove user created by the operator e.g. for password rotation +func DropPgUser(user string, db *sql.DB) error { + query := fmt.Sprintf(dropUserSQL, user) + if _, err := db.Exec(query); err != nil { // TODO: Try several times + return err + } + + return nil +} diff --git a/pkg/util/util.go b/pkg/util/util.go index a52925583..688153b89 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -8,6 +8,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "math" "math/big" "math/rand" "reflect" @@ -19,6 +20,7 @@ import ( "github.com/motomux/pretty" resource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "github.com/zalando/postgres-operator/pkg/spec" "golang.org/x/crypto/pbkdf2" @@ -322,6 +324,20 @@ func testNil(values ...*int32) bool { return false } +// Convert int to IntOrString type +func ToIntStr(val int) *intstr.IntOrString { + b := intstr.FromInt(val) + return &b +} + +// Get int from IntOrString and return max int if string +func IntFromIntStr(intOrStr intstr.IntOrString) int { + if intOrStr.Type == 1 { + return math.MaxInt + } + return intOrStr.IntValue() +} + // MaxInt32 : Return maximum of two integers provided via pointers. If one value // is not defined, return the other one. If both are not defined, result is also // undefined, caller needs to check for that. diff --git a/ui/app/package.json b/ui/app/package.json index d0528e0bd..a1838df0f 100644 --- a/ui/app/package.json +++ b/ui/app/package.json @@ -1,6 +1,6 @@ { "name": "postgres-operator-ui", - "version": "1.3.0", + "version": "1.8.0", "description": "PostgreSQL Operator UI", "main": "src/app.js", "config": { diff --git a/ui/app/src/edit.tag.pug b/ui/app/src/edit.tag.pug index c1d94e589..bb536a8e4 100644 --- a/ui/app/src/edit.tag.pug +++ b/ui/app/src/edit.tag.pug @@ -126,6 +126,7 @@ edit if (i.metadata.selfLink) { delete i.metadata.selfLink } if (i.metadata.uid) { delete i.metadata.uid } if (i.metadata.resourceVersion) { delete i.metadata.resourceVersion } + if (i.metadata.managedFields) { delete i.metadata.managedFields } this.update() this.refs.yamlNice.innerHTML = yamlParser.safeDump(i.postgresql, {sortKeys: true}) @@ -138,7 +139,15 @@ edit o.spec.enableMasterLoadBalancer = i.spec.enableMasterLoadBalancer || false o.spec.enableReplicaLoadBalancer = i.spec.enableReplicaLoadBalancer || false o.spec.enableConnectionPooler = i.spec.enableConnectionPooler || false - o.spec.volume = { size: i.spec.volume.size } + + o.spec.volume = { + size: i.spec.volume.size, + throughput: i.spec.volume.throughput || 125, + iops: i.spec.volume.iops || 3000 + } + + o.spec.postgresql = {} + o.spec.postgresql.version = i.spec.postgresql.version if ('users' in i.spec && typeof i.spec.users === 'object') { o.spec.users = Object.mapValues(i.spec.users, roleFlags => @@ -166,7 +175,7 @@ edit ].forEach(resourceType => { if (resourceType in resources) { const resourceClaim = resources[resourceType] - if (typeof resourceClaim === '') { + if (typeof resourceClaim === 'string') { o.spec.resources[section][resourceType] = resources[resourceType] } } diff --git a/ui/app/src/help-general.tag.pug b/ui/app/src/help-general.tag.pug index 1af25f1a8..7771fcebd 100644 --- a/ui/app/src/help-general.tag.pug +++ b/ui/app/src/help-general.tag.pug @@ -13,6 +13,6 @@ help-general h3 Basics p. - The PostgreSQL operator will use your definition to create a new + The Postgres Operator will use your definition to create a new PostgreSQL cluster for you. You can either copy the yaml definition - to the repositiory or you can just hit create cluster. + to a repositiory or hit create cluster (not available in prod). diff --git a/ui/app/src/new.tag.pug b/ui/app/src/new.tag.pug index 6293a6c7a..a2b0506da 100644 --- a/ui/app/src/new.tag.pug +++ b/ui/app/src/new.tag.pug @@ -266,6 +266,37 @@ new ) .input-group-addon .input-units Gi + tr + td + td Specify Iops and Throughput only if you need more than the default 3000 Iops and 125Mb/s EBS provides. + + tr + td Iops + td + .input-group + input.form-control( + ref='iops' + type='number' + value='{ iops }' + onchange='{ iopsChange }' + onkeyup='{ iopsChange }' + ) + .input-group-addon + .input-units + + tr + td Througput + td + .input-group + input.form-control( + ref='throughput' + type='number' + value='{ throughput }' + onchange='{ throughputChange }' + onkeyup='{ throughputChange }' + ) + .input-group-addon + .input-units MB/s tr(if='{ config.users_visible }') td @@ -509,7 +540,9 @@ new enableConnectionPooler: true {{/if}} volume: - size: "{{ volumeSize }}Gi" + size: "{{ volumeSize }}Gi"{{#if iops}} + iops: {{ iops }}{{/if}}{{#if throughput}} + throughput: {{ throughput }}{{/if}} {{#if users}} users:{{#each users}} {{ state }}: []{{/each}}{{/if}} @@ -560,6 +593,8 @@ new enableReplicaLoadBalancer: this.enableReplicaLoadBalancer, enableConnectionPooler: this.enableConnectionPooler, volumeSize: this.volumeSize, + iops: this.iops, + throughput: this.throughput, users: this.users.valids, databases: this.databases.valids, ranges: this.ranges, @@ -624,6 +659,14 @@ new this.volumeSize = +e.target.value } + this.iopsChange = e => { + this.iops = +e.target.value + } + + this.throughputChange = e => { + this.throughput = +e.target.value + } + this.updateDNSName = () => { this.dnsName = this.config.dns_format_string.format( this.name, diff --git a/ui/app/src/postgresqls.tag.pug b/ui/app/src/postgresqls.tag.pug index 38e5fcd9d..fabf769a0 100644 --- a/ui/app/src/postgresqls.tag.pug +++ b/ui/app/src/postgresqls.tag.pug @@ -69,7 +69,7 @@ postgresqls td { cpu } / { cpu_limit } td { memory } / { memory_limit } td { volume_size } - td { calcCosts(nodes, cpu, memory, volume_size) }$ + td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput) }$ td @@ -152,7 +152,7 @@ postgresqls td { cpu } / { cpu_limit } td { memory } / { memory_limit } td { volume_size } - td { calcCosts(nodes, cpu, memory, volume_size) }$ + td { calcCosts(nodes, cpu, memory, volume_size, iops, throughput) }$ td @@ -227,9 +227,22 @@ postgresqls + '/' + encodeURI(cluster.name) ) - const calcCosts = this.calcCosts = (nodes, cpu, memory, disk) => { - costs = Math.max(nodes, opts.config.min_pods) * (toCores(cpu) * opts.config.cost_core + toMemory(memory) * opts.config.cost_memory + toDisk(disk) * opts.config.cost_ebs) - return costs.toFixed(2) + const calcCosts = this.calcCosts = (nodes, cpu, memory, disk, iops, throughput) => { + podcount = Math.max(nodes, opts.config.min_pods) + corecost = toCores(cpu) * opts.config.cost_core + memorycost = toMemory(memory) * opts.config.cost_memory + diskcost = toDisk(disk) * opts.config.cost_ebs + iopscost = 0 + if (iops !== undefined && iops > 3000) { + iopscost = (iops - 3000) * opts.config.cost_iops + } + throughputcost = 0 + if (throughput !== undefined && throughput > 125) { + throughputcost = (throughput - 125) * opts.config.cost_throughput + } + + costs = podcount * (corecost + memorycost + diskcost + iopscost + throughputcost) + return costs.toFixed(2) } const toDisk = this.toDisk = value => { @@ -253,6 +266,11 @@ postgresqls value = Number(value) return value } + else if(value.endsWith("Ti")) { + value = value.substring(0, value.length-2) + value = Number(value) * 1000 + return value + } return value } diff --git a/ui/manifests/deployment.yaml b/ui/manifests/deployment.yaml index 865acd6b9..18d33d323 100644 --- a/ui/manifests/deployment.yaml +++ b/ui/manifests/deployment.yaml @@ -18,7 +18,7 @@ spec: serviceAccountName: postgres-operator-ui containers: - name: "service" - image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.7.1 + image: registry.opensource.zalan.do/acid/postgres-operator-ui:v1.8.0 ports: - containerPort: 8081 protocol: "TCP" @@ -62,7 +62,9 @@ spec: "replica_load_balancer_visible": true, "resources_visible": true, "users_visible": true, - "cost_ebs": 0.119, + "cost_ebs": 0.0952, + "cost_iops": 0.006, + "cost_throughput": 0.0476, "cost_core": 0.0575, "cost_memory": 0.014375, "postgresql_versions": [ diff --git a/ui/manifests/ingress.yaml b/ui/manifests/ingress.yaml index a5e6f0fab..53be1f45b 100644 --- a/ui/manifests/ingress.yaml +++ b/ui/manifests/ingress.yaml @@ -6,6 +6,7 @@ metadata: labels: application: "postgres-operator-ui" spec: + # ingressClassName: "ingress-nginx" rules: - host: "ui.example.org" http: diff --git a/ui/operator_ui/main.py b/ui/operator_ui/main.py index 5fbb6d24e..579f0e7d8 100644 --- a/ui/operator_ui/main.py +++ b/ui/operator_ui/main.py @@ -89,8 +89,10 @@ TARGET_NAMESPACE = getenv('TARGET_NAMESPACE') GOOGLE_ANALYTICS = getenv('GOOGLE_ANALYTICS', False) MIN_PODS= getenv('MIN_PODS', 2) -# storage pricing, i.e. https://aws.amazon.com/ebs/pricing/ -COST_EBS = float(getenv('COST_EBS', 0.119)) # GB per month +# storage pricing, i.e. https://aws.amazon.com/ebs/pricing/ (e.g. Europe - Franfurt) +COST_EBS = float(getenv('COST_EBS', 0.0952)) # GB per month +COST_IOPS = float(getenv('COST_IOPS', 0.006)) # IOPS per month above 3000 baseline +COST_THROUGHPUT = float(getenv('COST_THROUGHPUT', 0.0476)) # MB/s per month above 125 MB/s baseline # compute costs, i.e. https://www.ec2instances.info/?region=eu-central-1&selected=m5.2xlarge COST_CORE = 30.5 * 24 * float(getenv('COST_CORE', 0.0575)) # Core per hour m5.2xlarge / 8. @@ -98,7 +100,7 @@ COST_MEMORY = 30.5 * 24 * float(getenv('COST_MEMORY', 0.014375)) # Memory GB m5 WALE_S3_ENDPOINT = getenv( 'WALE_S3_ENDPOINT', - 'https+path://s3-eu-central-1.amazonaws.com:443', + 'https+path://s3.eu-central-1.amazonaws.com:443', ) USE_AWS_INSTANCE_PROFILE = ( @@ -308,6 +310,8 @@ DEFAULT_UI_CONFIG = { 'pgui_link': '', 'static_network_whitelist': {}, 'cost_ebs': COST_EBS, + 'cost_iops': COST_IOPS, + 'cost_throughput': COST_THROUGHPUT, 'cost_core': COST_CORE, 'cost_memory': COST_MEMORY, 'min_pods': MIN_PODS @@ -487,6 +491,8 @@ def get_postgresqls(): 'cpu': spec.get('resources', {}).get('requests', {}).get('cpu', 0), 'cpu_limit': spec.get('resources', {}).get('limits', {}).get('cpu', 0), 'volume_size': spec.get('volume', {}).get('size', 0), + 'iops': spec.get('volume', {}).get('iops', 3000), + 'throughput': spec.get('volume', {}).get('throughput', 125), 'team': ( spec.get('teamId') or metadata.get('labels', {}).get('team', '') @@ -614,6 +620,28 @@ def update_postgresql(namespace: str, cluster: str): spec['volume'] = {'size': size} + if ( + 'volume' in postgresql['spec'] + and 'iops' in postgresql['spec']['volume'] + and postgresql['spec']['volume']['iops'] != None + ): + iops = int(postgresql['spec']['volume']['iops']) + if not 'volume' in spec: + spec['volume'] = {} + + spec['volume']['iops'] = iops + + if ( + 'volume' in postgresql['spec'] + and 'throughput' in postgresql['spec']['volume'] + and postgresql['spec']['volume']['throughput'] != None + ): + throughput = int(postgresql['spec']['volume']['throughput']) + if not 'volume' in spec: + spec['volume'] = {} + + spec['volume']['throughput'] = throughput + if 'enableConnectionPooler' in postgresql['spec']: cp = postgresql['spec']['enableConnectionPooler'] if not cp: @@ -758,6 +786,27 @@ def update_postgresql(namespace: str, cluster: str): owner_username=owner_username, ) + resource_types = ["cpu","memory"] + resource_constraints = ["requests","limits"] + if "resources" in postgresql["spec"]: + spec["resources"] = {} + + res = postgresql["spec"]["resources"] + for rt in resource_types: + for rc in resource_constraints: + if rc in res: + if rt in res[rc]: + if not rc in spec["resources"]: + spec["resources"][rc] = {} + spec["resources"][rc][rt] = res[rc][rt] + + if "postgresql" in postgresql["spec"]: + if "version" in postgresql["spec"]["postgresql"]: + if "postgresql" not in spec: + spec["postgresql"]={} + + spec["postgresql"]["version"] = postgresql["spec"]["postgresql"]["version"] + o['spec'].update(spec) apply_postgresql(get_cluster(), namespace, cluster, o) @@ -912,8 +961,13 @@ def get_operator_get_logs(worker: int): @app.route('/operator/clusters///logs') @authorize def get_operator_get_logs_per_cluster(namespace: str, cluster: str): - team, clustername = cluster.split('-', 1) - return proxy_operator(f'/clusters/{team}/{namespace}/{clustername}/logs/') + user_teams = get_teams_for_user(session.get('user_name', '')) + for user_team in user_teams: + if cluster.find(user_team) == 0: + team = cluster[:len(user_team)] + cluster_name = cluster[len(user_team)+1:] + break + return proxy_operator(f'/clusters/{team}/{namespace}/{cluster_name}/logs/') @app.route('/login') diff --git a/ui/requirements.txt b/ui/requirements.txt index 8f612d554..c84450466 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -1,15 +1,15 @@ -Flask-OAuthlib==0.9.5 -Flask==1.1.2 +Flask-OAuthlib==0.9.6 +Flask==2.1.0 backoff==1.10.0 boto3==1.16.52 boto==2.49.0 -click==7.1.2 +click==8.1.2 furl==2.1.0 gevent==20.12.1 -jq==1.1.1 +jq==1.2.2 json_delta>=2.0 kubernetes==3.0.0 requests==2.25.1 stups-tokens>=1.1.19 wal_e==1.1.1 -werkzeug==0.16.1 +werkzeug==2.1.1 diff --git a/ui/run_local.sh b/ui/run_local.sh index 33c0abf27..e23b43423 100755 --- a/ui/run_local.sh +++ b/ui/run_local.sh @@ -19,7 +19,9 @@ default_operator_ui_config='{ "nat_gateways_visible": false, "resources_visible": true, "users_visible": true, - "cost_ebs": 0.119, + "cost_ebs": 0.0952, + "cost_iops": 0.006, + "cost_throughput": 0.0476, "cost_core": 0.0575, "cost_memory": 0.014375, "postgresql_versions": [