make run.sh executable from within e2e (#619)

.travis.yml

@@ -15,9 +15,9 @@ before_install:
   - go get github.com/mattn/goveralls
 
 install:
-  - make deps e2e-tools e2e-build
+  - make deps
 
 script:
   - hack/verify-codegen.sh
   - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v
-  - make e2e-run
+  - make e2e

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean local test linux macos docker push scm-source.json e2e-run e2e-tools e2e-build
+.PHONY: clean local test linux macos docker push scm-source.json e2e
 
 BINARY ?= postgres-operator
 BUILD_FLAGS ?= -v
@@ -34,7 +34,10 @@ ifdef CDP_PULL_REQUEST_NUMBER
 	CDP_TAG := -${CDP_BUILD_VERSION}
 endif
 
-KIND_PATH := $(GOPATH)/bin
+ifndef GOPATH
+	GOPATH := $(HOME)/go
+endif
+
 PATH := $(GOPATH)/bin:$(PATH)
 SHELL := env PATH=$(PATH) $(SHELL)
 
@@ -92,15 +95,5 @@ test:
 	hack/verify-codegen.sh
 	@go test ./...
 
-e2e-build:
+e2e:
-	docker build --tag="postgres-operator-e2e-tests" -f e2e/Dockerfile .
+	cd e2e; make tools test
-
-e2e-tools:
-	# install pinned version of 'kind' 
-	# leave the name as is to avoid overwriting official binary named `kind`
-	wget https://github.com/kubernetes-sigs/kind/releases/download/v0.3.0/kind-linux-amd64
-	chmod +x kind-linux-amd64
-	mv kind-linux-amd64 $(KIND_PATH)
-
-e2e-run: docker
-	e2e/run.sh

README.md

@@ -64,7 +64,7 @@ There is a browser-friendly version of this documentation at
 * [The Postgres experience on K8s](docs/user.md)
 * [The Postgres Operator UI](docs/operator-ui.md)
 * [DBA options - from RBAC to backup](docs/administrator.md)
-* [Debug and extend the operator](docs/developer.md)
+* [Build, debug and extend the operator](docs/developer.md)
 * [Configuration options](docs/reference/operator_parameters.md)
 * [Postgres manifest reference](docs/reference/cluster_manifest.md)
 * [Command-line options and environment variables](docs/reference/command_line_and_environment.md)

delivery.yaml

@@ -44,7 +44,7 @@ pipeline:
         - desc: 'Run e2e tests'
           cmd: |
             cd $OPERATOR_TOP_DIR/postgres-operator
-            make e2e-tools e2e-build e2e-run
+            make e2e
         - desc: 'Push docker image'
           cmd: |
             export PATH=$PATH:$HOME/go/bin

docs/administrator.md

@@ -98,7 +98,7 @@ on `configmaps` resources). This is also done intentionally to avoid breaking
 things if someone decides to configure the same service account in the
 operator's ConfigMap to run Postgres clusters.
 
-### Give K8S users access to create/list `postgresqls`
+### Give K8s users access to create/list `postgresqls`
 
 By default `postgresql` custom resources can only be listed and changed by
 cluster admins. To allow read and/or write access to other human users apply
@@ -363,7 +363,7 @@ used internally in K8s.
 
 ## Logical backups
 
-The operator can manage k8s cron jobs to run logical backups of Postgres
+The operator can manage K8s cron jobs to run logical backups of Postgres
 clusters. The cron job periodically spawns a batch job that runs a single pod.
 The backup script within this pod's container can connect to a DB for a logical
 backup. The operator updates cron jobs during Sync if the job schedule changes;

docs/developer.md

@@ -96,7 +96,7 @@ kubectl get pod -l name=postgres-operator
 The operator employs K8s-provided code generation to obtain deep copy methods
 and K8s-like APIs for its custom resource definitions, namely the
 Postgres CRD and the operator CRD. The usage of the code generation follows
-conventions from the k8s community. Relevant scripts live in the `hack`
+conventions from the K8s community. Relevant scripts live in the `hack`
 directory:
 * `update-codegen.sh` triggers code generation for the APIs defined in `pkg/apis/acid.zalan.do/`,
 * `verify-codegen.sh` checks if the generated code is up-to-date (to be used within CI).
@@ -247,23 +247,20 @@ kubectl logs acid-minimal-cluster-0
 
 ## End-to-end tests
 
-The operator provides reference e2e (end-to-end) tests to ensure various infra
+The operator provides reference end-to-end tests (e2e) (as Docker image) to
-parts work smoothly together. Each e2e execution tests a Postgres Operator image
+ensure various infrastructure parts work smoothly together. Each e2e execution
-built from the current git branch. The test runner starts a [kind](https://kind.sigs.k8s.io/)
+tests a Postgres Operator image built from the current git branch. The test
-(local k8s) cluster and Docker container with tests. The k8s API client from
+runner creates a new local K8s cluster using [kind](https://kind.sigs.k8s.io/),
-within the container connects to the `kind` cluster using the standard Docker
+utilizes provided manifest examples, and runs e2e tests contained in the `tests`
-`bridge` network. The tests utilize examples from `/manifests` (ConfigMap is
+folder. The K8s API client in the container connects to the `kind` cluster via
-used for the operator configuration) to avoid maintaining yet another set of
+the standard Docker `bridge` network. The kind cluster is deleted if tests
-configuration files. The kind cluster is deleted if tests complete successfully.
+finish successfully or on each new run in case it still exists.
 
-End-to-end tests are executed automatically during builds:
+End-to-end tests are executed automatically during builds (for more details,
+see the [README](../e2e/README.md) in the `e2e` folder):
 
 ```bash
-# invoke them from the project's top directory
+make e2e
-make e2e-run
-
-# install kind and build test image before first run
-make e2e-tools e2e-build
 ```
 
 End-to-end tests are written in Python and use `flake8` for code quality.

docs/diagrams/pod.tex

@@ -38,7 +38,7 @@
       node[k8s-label] (app-label) {App}
       node[k8s-label, right=.25cm of app-label] (role-label) {Role}
       node[k8s-label, right=.25cm of role-label] (custom-label) {Custom}
-      node[label, below of=role-label] (k8s-label-label) {K8S Labels}
+      node[label, below of=role-label] (k8s-label-label) {K8s Labels}
       node[border, behind path,
            fit=(app-label)(role-label)(custom-label)(k8s-label-label)
       ] (k8s-labels) {};  \& \&

docs/index.md

@@ -46,7 +46,7 @@ used to complement it.
 Here is a diagram, that summarizes what would be created by the operator, when a
 new Postgres cluster CRD is submitted:
 
-![postgresql-operator](diagrams/operator.png "K8S resources, created by operator")
+![postgresql-operator](diagrams/operator.png "K8s resources, created by operator")
 
 This picture is not complete without an overview of what is inside a single
 cluster pod, so let's zoom in:

docs/reference/cluster_manifest.md

@@ -135,7 +135,7 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   to S3. Default: false. Optional.
 
 * **logicalBackupSchedule**
-  Schedule for the logical backup k8s cron job. Please take
+  Schedule for the logical backup K8s cron job. Please take
   [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)
   into account. Optional. Default is: "30 00 \* \* \*"
 

docs/reference/operator_parameters.md

@@ -158,8 +158,8 @@ configuration they are grouped under the `kubernetes` key.
 
 * **pod_service_account_role_binding_definition**
   This definition must bind pod service account to a role with permission
-  sufficient for the pods to start and for Patroni to access k8s endpoints;
+  sufficient for the pods to start and for Patroni to access K8s endpoints;
-  service account on its own lacks any such rights starting with k8s v1.8. If
+  service account on its own lacks any such rights starting with K8s v1.8. If
   not explicitly defined by the user, a simple definition that binds the
   account to the operator's own 'zalando-postgres-operator' cluster role will
   be used. The default is empty.
@@ -416,7 +416,7 @@ yet officially supported.
 
 ## Logical backup
 
-These parameters configure a k8s cron job managed by the operator to produce
+These parameters configure a K8s cron job managed by the operator to produce
 Postgres logical backups. In the CRD-based configuration those parameters are
 grouped under the `logical_backup` key.
 

e2e/Dockerfile

@@ -1,10 +1,8 @@
 FROM ubuntu:18.04
 LABEL maintainer="Team ACID @ Zalando <team-acid@zalando.de>"
 
-WORKDIR /e2e
-
 COPY manifests ./manifests
-COPY e2e/requirements.txt e2e/tests ./
+COPY requirements.txt tests ./
 
 RUN apt-get update \
     && apt-get install --no-install-recommends -y \
@@ -19,4 +17,7 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+ARG VERSION=dev
+RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" ./__init__.py
+
 CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"]

e2e/Makefile

@@ -0,0 +1,52 @@
+.PHONY: clean copy docker push tools test
+
+BINARY ?= postgres-operator-e2e-tests
+BUILD_FLAGS ?= -v
+CGO_ENABLED ?= 0
+ifeq ($(RACE),1)
+	BUILD_FLAGS += -race -a
+    CGO_ENABLED=1
+endif
+
+LOCAL_BUILD_FLAGS ?= $(BUILD_FLAGS)
+LDFLAGS ?= -X=main.version=$(VERSION)
+
+IMAGE            ?= registry.opensource.zalan.do/acid/$(BINARY)
+VERSION          ?= $(shell git describe --tags --always --dirty)
+TAG              ?= $(VERSION)
+GITHEAD          = $(shell git rev-parse --short HEAD)
+GITURL           = $(shell git config --get remote.origin.url)
+GITSTATU         = $(shell git status --porcelain || echo 'no changes')
+TTYFLAGS         = $(shell test -t 0 && echo '-it')
+
+ifndef GOPATH
+	GOPATH := $(HOME)/go
+endif
+
+KIND_PATH := $(GOPATH)/bin
+PATH := $(GOPATH)/bin:$(PATH)
+
+default: tools
+
+clean:
+	rm -fr manifests
+
+copy: clean
+	mkdir manifests
+	cp ../manifests -r .
+
+docker: copy
+	docker build --build-arg "VERSION=$(VERSION)" -t "$(IMAGE):$(TAG)" .
+
+push: docker
+	docker push "$(IMAGE):$(TAG)"
+
+tools: docker
+	# install pinned version of 'kind'
+	# leave the name as is to avoid overwriting official binary named `kind`
+	wget https://github.com/kubernetes-sigs/kind/releases/download/v0.4.0/kind-linux-amd64
+	chmod +x kind-linux-amd64
+	mv kind-linux-amd64 $(KIND_PATH)
+
+test:
+	./run.sh

e2e/README.md

@@ -0,0 +1,46 @@
+# Postgres Operator end-to-end tests
+
+End-to-end tests shall ensure that the Postgres Operator does its job when
+applying manifests against a Kubernetes (K8s) environment. A test runner
+Dockerfile is provided to run e2e tests without the need to install K8s and
+its runtime `kubectl` in advance. The test runner uses
+[kind](https://kind.sigs.k8s.io/) to create a local K8s cluster which runs on
+Docker.
+
+## Prerequisites
+
+Docker
+Go
+
+## Build test runner
+
+In the directory of the cloned Postgres Operator repository change to the e2e
+folder and run:
+
+```bash
+make
+```
+
+This will build the `postgres-operator-e2e-tests` image and download the kind
+runtime.
+
+## Run tests
+
+In the e2e folder you can invoke tests either with `make test` or with:
+
+```bash
+./run.sh
+```
+
+To run both the build and test step you can invoke `make e2e` from the parent
+directory.
+
+## Covered use cases
+
+The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py):
+
+* support for multiple namespaces
+* scale Postgres cluster up and down
+* taint-based eviction of Postgres pods
+* invoking logical backup cron job
+* uniqueness of master pod

e2e/run.sh

@@ -6,11 +6,26 @@ set -o nounset
 set -o pipefail
 IFS=$'\n\t'
 
+cd $(dirname "$0");
+
 readonly cluster_name="postgres-operator-e2e-tests"
-readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}"  | head -1)
-readonly e2e_test_image=${cluster_name}
 readonly kubeconfig_path="/tmp/kind-config-${cluster_name}"
 
+function pull_images(){
+
+  operator_tag=$(git describe --tags --always --dirty)
+  if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator:${operator_tag}) ]]
+  then
+    docker pull registry.opensource.zalan.do/acid/postgres-operator:latest
+  fi
+  if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:${operator_tag}) ]]
+  then
+    docker pull registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:latest
+  fi
+
+  operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1)
+  e2e_test_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests" --format "{{.Repository}}:{{.Tag}}" | head -1)
+}
 
 function start_kind(){
 
@@ -20,8 +35,9 @@ function start_kind(){
     kind-linux-amd64 delete cluster --name ${cluster_name}
   fi
 
-  kind-linux-amd64 create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml
+  kind-linux-amd64 create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml
   kind-linux-amd64 load docker-image "${operator_image}" --name ${cluster_name}
+  kind-linux-amd64 load docker-image "${e2e_test_image}" --name ${cluster_name}
   KUBECONFIG="$(kind-linux-amd64 get kubeconfig-path --name=${cluster_name})"
   export KUBECONFIG
 }
@@ -36,6 +52,7 @@ function set_kind_api_server_ip(){
 }
 
 function run_tests(){
+
   docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}"
 }
 
@@ -49,6 +66,7 @@ function main(){
 
   trap "clean_up" QUIT TERM EXIT
 
+  pull_images
   start_kind
   set_kind_api_server_ip
   run_tests

e2e/tests/__init__.py

@@ -0,0 +1,2 @@
+# This version is replaced during release process.
+__version__ = '2019.0.dev1'

e2e/tests/test_e2e.py

@@ -11,7 +11,7 @@ from kubernetes import client, config
 
 class EndToEndTestCase(unittest.TestCase):
     '''
-    Test interaction of the operator with multiple k8s components.
+    Test interaction of the operator with multiple K8s components.
     '''
 
     # `kind` pods may stuck in the `Terminating` phase for a few minutes; hence high test timeout
@@ -21,15 +21,15 @@ class EndToEndTestCase(unittest.TestCase):
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def setUpClass(cls):
         '''
-        Deploy operator to a "kind" cluster created by /e2e/run.sh using examples from /manifests.
+        Deploy operator to a "kind" cluster created by run.sh using examples from /manifests.
         This operator deployment is to be shared among all tests.
 
-        /e2e/run.sh deletes the 'kind' cluster after successful run along with all operator-related entities.
+        run.sh deletes the 'kind' cluster after successful run along with all operator-related entities.
         In the case of test failure the cluster will stay to enable manual examination;
-        next invocation of "make e2e-run" will re-create it.
+        next invocation of "make test" will re-create it.
         '''
 
-        # set a single k8s wrapper for all tests
+        # set a single K8s wrapper for all tests
         k8s = cls.k8s = K8s()
 
         # operator deploys pod service account there on start up

pkg/util/config/config.go

@@ -84,7 +84,7 @@ type Config struct {
 	LogicalBackup
 
 	WatchedNamespace string            `name:"watched_namespace"`    // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
-	EtcdHost         string            `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use k8s as a DCS
+	EtcdHost         string            `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS
 	DockerImage      string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-11:1.5-p9"`
 	Sidecars         map[string]string `name:"sidecar_docker_images"`
 	// default name `operator` enables backward compatibility with the older ServiceAccountName field

run_operator_locally.sh

@@ -4,7 +4,7 @@
 # Optionally re-build the operator binary beforehand to test local changes
 
 # Known limitations:
-# 1) minikube provides a single node k8s cluster. That is, you will not be able test functions like pod
+# 1) minikube provides a single node K8s cluster. That is, you will not be able test functions like pod
 #    migration between multiple nodes locally
 # 2) this script configures the operator via configmap, not the operator CRD
 

ui/Makefile

@@ -10,7 +10,6 @@ endif
 
 LOCAL_BUILD_FLAGS ?= $(BUILD_FLAGS)
 LDFLAGS ?= -X=main.version=$(VERSION)
-DOCKERDIR = docker
 
 IMAGE            ?= registry.opensource.zalan.do/acid/$(BINARY)
 VERSION          ?= $(shell git describe --tags --always --dirty)

ui/operator_ui/spiloutils.py

@@ -206,7 +206,7 @@ def create_postgresql(cluster, namespace, definition):
         r.raise_for_status()
         return True
     except Exception as ex:
-        logger.exception("K8S create request failed")
+        logger.exception("K8s create request failed")
         return False
 
 
@@ -221,7 +221,7 @@ def apply_postgresql(cluster, namespace, resource_name, definition):
         r.raise_for_status()
         return True
     except Exception as ex:
-        logger.exception("K8S create request failed")
+        logger.exception("K8s create request failed")
         return False
 
 
@@ -236,7 +236,7 @@ def remove_postgresql(cluster, namespace, resource_name):
         r.raise_for_status()
         return True
     except Exception as ex:
-        logger.exception("K8S delete request failed")
+        logger.exception("K8s delete request failed")
         return False