diff --git a/.github/workflows/run_e2e.yaml b/.github/workflows/run_e2e.yaml index a8e76f0ca..f7d984519 100644 --- a/.github/workflows/run_e2e.yaml +++ b/.github/workflows/run_e2e.yaml @@ -20,6 +20,6 @@ jobs: - name: Compile run: make linux - name: Run unit tests - run: go test ./... + run: make test - name: Run end-2-end tests run: make e2e diff --git a/Makefile b/Makefile index b83f7156b..07605c0a3 100644 --- a/Makefile +++ b/Makefile @@ -103,5 +103,8 @@ test: hack/verify-codegen.sh GO111MODULE=on go test ./... +codegen: + hack/update-codegen.sh + e2e: docker # build operator image to be tested cd e2e; make e2etest diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 1b5f36db8..d6e1dd94f 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -464,11 +464,11 @@ spec: type: integer standby: type: object - required: - - s3_wal_path properties: s3_wal_path: type: string + gs_wal_path: + type: string teamId: type: string tls: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index bf01b2a5c..3b6393550 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -366,12 +366,16 @@ under the `clone` top-level key and do not affect the already running cluster. ## Standby cluster On startup, an existing `standby` top-level key creates a standby Postgres -cluster streaming from a remote location. So far only streaming from a S3 WAL -archive is supported. +cluster streaming from a remote location. So far streaming from S3 and GCS WAL +archives is supported. * **s3_wal_path** the url to S3 bucket containing the WAL archive of the remote primary. - Required when the `standby` section is present. + Optional, but `s3_wal_path` or `gs_wal_path` is required. + +* **gs_wal_path** + the url to GS bucket containing the WAL archive of the remote primary. + Optional, but `s3_wal_path` or `gs_wal_path` is required. ## Volume properties diff --git a/docs/user.md b/docs/user.md index a2a65e63f..572d832ab 100644 --- a/docs/user.md +++ b/docs/user.md @@ -798,8 +798,8 @@ different location than its source database. Unlike cloning, the PostgreSQL version between source and target cluster has to be the same. To start a cluster as standby, add the following `standby` section in the YAML -file and specify the S3 bucket path. An empty path will result in an error and -no statefulset will be created. +file. Specify the S3/GS bucket path. Omitting both settings will result in an error +and no statefulset will be created. ```yaml spec: @@ -807,6 +807,12 @@ spec: s3_wal_path: "s3:///spilo///wal/" ``` +```yaml +spec: + standby: + gs_wal_path: "gs:///spilo///wal/" +``` + At the moment, the operator only allows to stream from the WAL archive of the master. Thus, it is recommended to deploy standby clusters with only [one pod](https://github.com/zalando/postgres-operator/blob/master/manifests/standby-manifest.yaml#L10). You can raise the instance count when detaching. Note, that the same pod role diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 280da9385..d304004dc 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -4,10 +4,22 @@ set -o errexit set -o nounset set -o pipefail +GENERATED_PACKAGE_ROOT="github.com" +OPERATOR_PACKAGE_ROOT="${GENERATED_PACKAGE_ROOT}/zalando/postgres-operator" SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. -CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)} +TARGET_CODE_DIR=${1-${SCRIPT_ROOT}/pkg} +CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo "${GOPATH}"/src/k8s.io/code-generator)} + +cleanup() { + rm -rf "${GENERATED_PACKAGE_ROOT}" +} +trap "cleanup" EXIT SIGINT bash "${CODEGEN_PKG}/generate-groups.sh" all \ - github.com/zalando/postgres-operator/pkg/generated github.com/zalando/postgres-operator/pkg/apis \ + "${OPERATOR_PACKAGE_ROOT}/pkg/generated" "${OPERATOR_PACKAGE_ROOT}/pkg/apis" \ "acid.zalan.do:v1" \ --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt + +cp -r "${OPERATOR_PACKAGE_ROOT}"/pkg/* "${TARGET_CODE_DIR}" + +cleanup diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh index 68710015e..451ac76b7 100755 --- a/hack/verify-codegen.sh +++ b/hack/verify-codegen.sh @@ -19,15 +19,14 @@ cleanup mkdir -p "${TMP_DIFFROOT}" cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}" -"${SCRIPT_ROOT}/hack/update-codegen.sh" +"${SCRIPT_ROOT}/hack/update-codegen.sh" "${TMP_DIFFROOT}" echo "diffing ${DIFFROOT} against freshly generated codegen" ret=0 diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$? -cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}" if [[ $ret -eq 0 ]] then echo "${DIFFROOT} up to date." else - echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh" + echo "${DIFFROOT} is out of date. Please run 'make codegen'" exit 1 fi diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 76fbbfa48..fae5a09f2 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -649,12 +649,14 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "integer", }, "standby": { - Type: "object", - Required: []string{"s3_wal_path"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "s3_wal_path": { Type: "string", }, + "gs_wal_path": { + Type: "string", + }, }, }, "teamId": { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 079cb8b98..57f9ae04a 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -166,6 +166,7 @@ type Patroni struct { // StandbyDescription contains s3 wal path type StandbyDescription struct { S3WalPath string `json:"s3_wal_path,omitempty"` + GSWalPath string `json:"gs_wal_path,omitempty"` } // TLSDescription specs TLS properties diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 7a8984ce4..61bbfde9c 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 366570c3c..e7d8ea376 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1058,8 +1058,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef sort.Slice(customPodEnvVarsList, func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name }) - if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" { - return nil, fmt.Errorf("s3_wal_path is empty for standby cluster") + if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" && + spec.StandbyCluster.GSWalPath == "" { + return nil, fmt.Errorf("one of s3_wal_path or gs_wal_path must be set for standby cluster") } // backward compatible check for InitContainers @@ -1874,17 +1875,36 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescription) []v1.EnvVar { result := make([]v1.EnvVar, 0) - if description.S3WalPath == "" { + if description.S3WalPath == "" && description.GSWalPath == "" { return nil } - // standby with S3, find out the bucket to setup standby - msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s " - c.logger.Infof(msg, description.S3WalPath) - result = append(result, v1.EnvVar{ - Name: "STANDBY_WALE_S3_PREFIX", - Value: description.S3WalPath, - }) + if description.S3WalPath != "" { + // standby with S3, find out the bucket to setup standby + msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s " + c.logger.Infof(msg, description.S3WalPath) + + result = append(result, v1.EnvVar{ + Name: "STANDBY_WALE_S3_PREFIX", + Value: description.S3WalPath, + }) + } else if description.GSWalPath != "" { + msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s " + c.logger.Infof(msg, description.GSWalPath) + + envs := []v1.EnvVar{ + { + Name: "STANDBY_WALE_GS_PREFIX", + Value: description.GSWalPath, + }, + { + Name: "STANDBY_GOOGLE_APPLICATION_CREDENTIALS", + Value: c.OpConfig.GCPCredentials, + }, + } + result = append(result, envs...) + + } result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"}) result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""})