Improve e2e tests (#1111)

* icnrease vm size

* cache deps

* switch to the absolute cache path as cdp does not support shell expansion

* do not pull non-existing image

* manually install kind

* add alias to kind

* use full kind name

* one more name change

* install kind with other tools

* add bind mounts instead of copying files

* test fetching the runner image

* build image for pierone

* bump up the client-go version to match the master

* bump up go version

* install pinned version of kind before any test run

* do not overwrite local ./manifests during test run

* update the docs

* fix kind name

* update go.* files

* fix deps

* avoid unnecessary image upload

* properly install kind

* Change network to host to make it reachable within e2e runner. May not be the right solution though.

* Small changes. Also use entrypoint vs cmd.

* Bumping spilo. Load before test.

* undo incorrect merge from the master

Co-authored-by: Sergey Dudoladov <sergey.dudoladov@zalando.de>
Co-authored-by: Jan Mußler <janm81@gmail.com>

e2e/Dockerfile

@@ -1,11 +1,12 @@
-# An image to perform the actual test. Do not forget to copy all necessary test
-# files here.
-FROM ubuntu:18.04
+# An image to run e2e tests.
+# The image does not include the tests; all necessary files are bind-mounted when a container starts.
+FROM ubuntu:20.04
 LABEL maintainer="Team ACID @ Zalando <team-acid@zalando.de>"
 
-COPY manifests ./manifests
-COPY exec.sh ./exec.sh
-COPY requirements.txt tests ./
+ENV TERM xterm-256color
+
+COPY requirements.txt ./
+COPY scm-source.json ./
 
 RUN apt-get update \
     && apt-get install --no-install-recommends -y \
@@ -14,13 +15,10 @@ RUN apt-get update \
            python3-pip \
            curl \
     && pip3 install --no-cache-dir -r requirements.txt \
-    && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \
+    && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl \
     && chmod +x ./kubectl \
     && mv ./kubectl /usr/local/bin/kubectl \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-ARG VERSION=dev
-RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" ./__init__.py
-
-CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"]
+ENTRYPOINT ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"]

e2e/Makefile

@@ -1,6 +1,6 @@
 .PHONY: clean copy docker push tools test
 
-BINARY ?= postgres-operator-e2e-tests
+BINARY ?= postgres-operator-e2e-tests-runner
 BUILD_FLAGS ?= -v
 CGO_ENABLED ?= 0
 ifeq ($(RACE),1)
@@ -34,15 +34,20 @@ copy: clean
 	mkdir manifests
 	cp ../manifests -r .
 
-docker: copy
-	docker build --build-arg "VERSION=$(VERSION)" -t "$(IMAGE):$(TAG)" .
+docker: scm-source.json
+	docker build -t "$(IMAGE):$(TAG)" .
+
+scm-source.json: ../.git
+	echo '{\n "url": "git:$(GITURL)",\n "revision": "$(GITHEAD)",\n "author": "$(USER)",\n "status": "$(GITSTATUS)"\n}' > scm-source.json
 
 push: docker
 	docker push "$(IMAGE):$(TAG)"
 
-tools: docker
+tools:
 	# install pinned version of 'kind'
-	GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1
+	# go get must run outside of a dir with a (module-based) Go project !
+	# otherwise go get updates project's dependencies and/or behaves differently
+	cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1
 
-e2etest:
-	./run.sh
+e2etest: tools copy clean
+	./run.sh main

e2e/kind-cluster-postgres-operator-e2e-tests.yaml

@@ -1,5 +1,5 @@
 kind: Cluster
-apiVersion: kind.sigs.k8s.io/v1alpha3
+apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
 - role: worker

e2e/run.sh

@@ -6,29 +6,29 @@ set -o nounset
 set -o pipefail
 IFS=$'\n\t'
 
-cd $(dirname "$0");
-
 readonly cluster_name="postgres-operator-e2e-tests"
 readonly kubeconfig_path="/tmp/kind-config-${cluster_name}"
+readonly spilo_image="registry.opensource.zalan.do/acid/spilo-12:1.6-p5"
+
+echo "Clustername: ${cluster_name}"
+echo "Kubeconfig path: ${kubeconfig_path}"
 
 function pull_images(){
-
   operator_tag=$(git describe --tags --always --dirty)
   if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator:${operator_tag}) ]]
   then
     docker pull registry.opensource.zalan.do/acid/postgres-operator:latest
   fi
-  if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:${operator_tag}) ]]
-  then
-    docker pull registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:latest
-  fi
 
   operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1)
-  e2e_test_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests" --format "{{.Repository}}:{{.Tag}}" | head -1)
+
+  # this image does not contain the tests; a container mounts them from a local "./tests" dir at start time
+  e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:latest"
+  docker pull ${e2e_test_runner_image}
 }
 
 function start_kind(){
-
+  echo "Starting kind for e2e tests"
   # avoid interference with previous test runs
   if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]]
   then
@@ -38,10 +38,12 @@ function start_kind(){
   export KUBECONFIG="${kubeconfig_path}"
   kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml
   kind load docker-image "${operator_image}" --name ${cluster_name}
-  kind load docker-image "${e2e_test_image}" --name ${cluster_name}
+  docker pull "${spilo_image}"
+  kind load docker-image "${spilo_image}" --name ${cluster_name}
 }
 
 function set_kind_api_server_ip(){
+  echo "Setting up kind API server ip"
   # use the actual kubeconfig to connect to the 'kind' API server
   # but update the IP address of the API server to the one from the Docker 'bridge' network
   readonly local kind_api_server_port=6443 # well-known in the 'kind' codebase
@@ -50,10 +52,21 @@ function set_kind_api_server_ip(){
 }
 
 function run_tests(){
-  docker run --rm --network kind --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}"
+  echo "Running tests..."
+
+  # tests modify files in ./manifests, so we mount a copy of this directory done by the e2e Makefile
+
+  docker run --rm --network=host -e "TERM=xterm-256color" \
+  --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \
+  --mount type=bind,source="$(readlink -f manifests)",target=/manifests \
+  --mount type=bind,source="$(readlink -f tests)",target=/tests \
+  --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \
+  -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}"
+  
 }
 
 function clean_up(){
+  echo "Executing cleanup"
   unset KUBECONFIG
   kind delete cluster --name ${cluster_name}
   rm -rf ${kubeconfig_path}
@@ -63,11 +76,11 @@ function main(){
 
   trap "clean_up" QUIT TERM EXIT
 
-  pull_images
-  start_kind
-  set_kind_api_server_ip
+  time pull_images
+  time start_kind
+  time set_kind_api_server_ip
   run_tests
   exit 0
 }
 
-main "$@"
+"$@"

e2e/tests/test_e2e.py

@@ -34,6 +34,7 @@ class EndToEndTestCase(unittest.TestCase):
         In the case of test failure the cluster will stay to enable manual examination;
         next invocation of "make test" will re-create it.
         '''
+        print("Test Setup being executed")
 
         # set a single K8s wrapper for all tests
         k8s = cls.k8s = K8s()
@@ -216,7 +217,8 @@ class EndToEndTestCase(unittest.TestCase):
         k8s = self.k8s
         # update infrastructure roles description
         secret_name = "postgresql-infrastructure-roles"
-        roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon"
+        roles = "secretname: postgresql-infrastructure-roles-new, \
+            userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon"
         patch_infrastructure_roles = {
             "data": {
                 "infrastructure_roles_secret_name": secret_name,
@@ -313,7 +315,8 @@ class EndToEndTestCase(unittest.TestCase):
             image0 = k8s.get_effective_pod_image(pod0)
             image1 = k8s.get_effective_pod_image(pod1)
 
-            assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1)
+            assert_msg = "Disabling lazy upgrade failed: pods still have different \
+                images {} and {}".format(image0, image1)
             self.assertEqual(image0, image1, assert_msg)
 
         except timeout_decorator.TimeoutError:
@@ -710,11 +713,11 @@ class EndToEndTestCase(unittest.TestCase):
             time.sleep(10)
 
             # add annotations to manifest
-            deleteDate = datetime.today().strftime('%Y-%m-%d')
+            delete_date = datetime.today().strftime('%Y-%m-%d')
             pg_patch_delete_annotations = {
                 "metadata": {
                     "annotations": {
-                        "delete-date": deleteDate,
+                        "delete-date": delete_date,
                         "delete-clustername": "acid-minimal-cluster",
                     }
                 }