Merge branch 'master' into switchover-return-err

.github/workflows/run_e2e.yaml

@@ -20,6 +20,6 @@ jobs:
     - name: Compile
       run: make linux
     - name: Run unit tests
-      run: go test ./...
+      run: make test
     - name: Run end-2-end tests
       run: make e2e

Makefile

@@ -103,5 +103,8 @@ test:
 	hack/verify-codegen.sh
 	GO111MODULE=on go test ./...
 
+codegen:
+	hack/update-codegen.sh
+
 e2e: docker # build operator image to be tested
 	cd e2e; make e2etest

charts/postgres-operator/crds/postgresqls.yaml

@@ -464,11 +464,11 @@ spec:
                 type: integer
               standby:
                 type: object
-                required:
-                  - s3_wal_path
                 properties:
                   s3_wal_path:
                     type: string
+                  gs_wal_path:
+                    type: string
               teamId:
                 type: string
               tls:

docs/reference/cluster_manifest.md

@@ -366,12 +366,16 @@ under the `clone` top-level key and do not affect the already running cluster.
 ## Standby cluster
 
 On startup, an existing `standby` top-level key creates a standby Postgres
-cluster streaming from a remote location. So far only streaming from a S3 WAL
+cluster streaming from a remote location. So far streaming from S3 and GCS WAL
-archive is supported.
+archives is supported.
 
 * **s3_wal_path**
   the url to S3 bucket containing the WAL archive of the remote primary.
-  Required when the `standby` section is present.
+  Optional, but `s3_wal_path`  or `gs_wal_path` is required.
+
+* **gs_wal_path**
+  the url to GS bucket containing the WAL archive of the remote primary.
+  Optional, but `s3_wal_path`  or `gs_wal_path` is required.
 
 ## Volume properties
 

docs/user.md

@@ -798,8 +798,8 @@ different location than its source database. Unlike cloning, the PostgreSQL
 version between source and target cluster has to be the same.
 
 To start a cluster as standby, add the following `standby` section in the YAML
-file and specify the S3 bucket path. An empty path will result in an error and
+file. Specify the S3/GS bucket path. Omitting both settings will result in an error
-no statefulset will be created.
+and no statefulset will be created.
 
 ```yaml
 spec:
@@ -807,6 +807,12 @@ spec:
     s3_wal_path: "s3://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
 ```
 
+```yaml
+spec:
+  standby:
+    gs_wal_path: "gs://<bucketname>/spilo/<source_db_cluster>/<UID>/wal/<PGVERSION>"
+```
+
 At the moment, the operator only allows to stream from the WAL archive of the
 master. Thus, it is recommended to deploy standby clusters with only [one pod](https://github.com/zalando/postgres-operator/blob/master/manifests/standby-manifest.yaml#L10).
 You can raise the instance count when detaching. Note, that the same pod role

hack/update-codegen.sh

@@ -4,10 +4,22 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+GENERATED_PACKAGE_ROOT="github.com"
+OPERATOR_PACKAGE_ROOT="${GENERATED_PACKAGE_ROOT}/zalando/postgres-operator"
 SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
-CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)}
+TARGET_CODE_DIR=${1-${SCRIPT_ROOT}/pkg}
+CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo "${GOPATH}"/src/k8s.io/code-generator)}
+
+cleanup() {
+    rm -rf "${GENERATED_PACKAGE_ROOT}"
+}
+trap "cleanup" EXIT SIGINT
 
 bash "${CODEGEN_PKG}/generate-groups.sh" all \
-  github.com/zalando/postgres-operator/pkg/generated github.com/zalando/postgres-operator/pkg/apis \
+  "${OPERATOR_PACKAGE_ROOT}/pkg/generated" "${OPERATOR_PACKAGE_ROOT}/pkg/apis" \
   "acid.zalan.do:v1" \
   --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt
+
+cp -r "${OPERATOR_PACKAGE_ROOT}"/pkg/* "${TARGET_CODE_DIR}"
+
+cleanup

hack/verify-codegen.sh

@@ -19,15 +19,14 @@ cleanup
 mkdir -p "${TMP_DIFFROOT}"
 cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}"
 
-"${SCRIPT_ROOT}/hack/update-codegen.sh"
+"${SCRIPT_ROOT}/hack/update-codegen.sh" "${TMP_DIFFROOT}"
 echo "diffing ${DIFFROOT} against freshly generated codegen"
 ret=0
 diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$?
-cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}"
 if [[ $ret -eq 0 ]]
 then
     echo "${DIFFROOT} up to date."
 else
-    echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh"
+    echo "${DIFFROOT} is out of date. Please run 'make codegen'"
     exit 1
 fi

pkg/apis/acid.zalan.do/v1/crds.go

@@ -649,12 +649,14 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 						Type: "integer",
 					},
 					"standby": {
-						Type:     "object",
+						Type: "object",
-						Required: []string{"s3_wal_path"},
 						Properties: map[string]apiextv1.JSONSchemaProps{
 							"s3_wal_path": {
 								Type: "string",
 							},
+							"gs_wal_path": {
+								Type: "string",
+							},
 						},
 					},
 					"teamId": {

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -166,6 +166,7 @@ type Patroni struct {
 // StandbyDescription contains s3 wal path
 type StandbyDescription struct {
 	S3WalPath string `json:"s3_wal_path,omitempty"`
+	GSWalPath string `json:"gs_wal_path,omitempty"`
 }
 
 // TLSDescription specs TLS properties

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*

pkg/cluster/k8sres.go

@@ -1058,8 +1058,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
 	sort.Slice(customPodEnvVarsList,
 		func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name })
 
-	if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" {
+	if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" &&
-		return nil, fmt.Errorf("s3_wal_path is empty for standby cluster")
+		spec.StandbyCluster.GSWalPath == "" {
+		return nil, fmt.Errorf("one of s3_wal_path or gs_wal_path must be set for standby cluster")
 	}
 
 	// backward compatible check for InitContainers
@@ -1874,17 +1875,36 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription)
 func (c *Cluster) generateStandbyEnvironment(description *acidv1.StandbyDescription) []v1.EnvVar {
 	result := make([]v1.EnvVar, 0)
 
-	if description.S3WalPath == "" {
+	if description.S3WalPath == "" && description.GSWalPath == "" {
 		return nil
 	}
-	// standby with S3, find out the bucket to setup standby
-	msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
-	c.logger.Infof(msg, description.S3WalPath)
 
-	result = append(result, v1.EnvVar{
+	if description.S3WalPath != "" {
-		Name:  "STANDBY_WALE_S3_PREFIX",
+		// standby with S3, find out the bucket to setup standby
-		Value: description.S3WalPath,
+		msg := "Standby from S3 bucket using custom parsed S3WalPath from the manifest %s "
-	})
+		c.logger.Infof(msg, description.S3WalPath)
+
+		result = append(result, v1.EnvVar{
+			Name:  "STANDBY_WALE_S3_PREFIX",
+			Value: description.S3WalPath,
+		})
+	} else if description.GSWalPath != "" {
+		msg := "Standby from GS bucket using custom parsed GSWalPath from the manifest %s "
+		c.logger.Infof(msg, description.GSWalPath)
+
+		envs := []v1.EnvVar{
+			{
+				Name:  "STANDBY_WALE_GS_PREFIX",
+				Value: description.GSWalPath,
+			},
+			{
+				Name:  "STANDBY_GOOGLE_APPLICATION_CREDENTIALS",
+				Value: c.OpConfig.GCPCredentials,
+			},
+		}
+		result = append(result, envs...)
+
+	}
 
 	result = append(result, v1.EnvVar{Name: "STANDBY_METHOD", Value: "STANDBY_WITH_WALE"})
 	result = append(result, v1.EnvVar{Name: "STANDBY_WAL_BUCKET_SCOPE_PREFIX", Value: ""})