From 2066256a17f0d0cfb2455fa8d67bdb05e4ce3969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Mu=C3=9Fler?= Date: Wed, 21 Oct 2020 15:23:17 +0200 Subject: [PATCH] Progressing on faster e2e tests. --- Makefile | 10 +++++++--- e2e/Dockerfile | 8 +++++--- e2e/{run_tests_image.sh => exec_into_env.sh} | 2 +- e2e/run.sh | 10 +++++++--- e2e/tests/test_e2e.py | 7 +++---- pkg/cluster/pod.go | 9 ++++++++- 6 files changed, 31 insertions(+), 15 deletions(-) rename e2e/{run_tests_image.sh => exec_into_env.sh} (94%) diff --git a/Makefile b/Makefile index 29bbb47e6..bcd9ea02a 100644 --- a/Makefile +++ b/Makefile @@ -24,12 +24,16 @@ PKG := `go list ./... | grep -v /vendor/` ifeq ($(DEBUG),1) DOCKERFILE = DebugDockerfile - DEBUG_POSTFIX := -debug + DEBUG_POSTFIX := -debug-$(shell date hhmmss) BUILD_FLAGS += -gcflags "-N -l" else DOCKERFILE = Dockerfile endif +ifeq ($(FRESH),1) + DEBUG_FRESH=$(shell date +"%H-%M-%S") +endif + ifdef CDP_PULL_REQUEST_NUMBER CDP_TAG := -${CDP_BUILD_VERSION} endif @@ -66,7 +70,7 @@ docker: ${DOCKERDIR}/${DOCKERFILE} docker-context echo "Version ${VERSION}" echo "CDP tag ${CDP_TAG}" echo "git describe $(shell git describe --tags --always --dirty)" - cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" . + cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_FRESH)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" . indocker-race: docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.8.1 bash -c "make linux" @@ -97,4 +101,4 @@ test: GO111MODULE=on go test ./... e2e: docker # build operator image to be tested - cd e2e; make e2etest + cd e2e; make e2etest \ No newline at end of file diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 5881ca5f2..3eb8c9d70 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -14,13 +14,15 @@ RUN apt-get update \ python3-setuptools \ python3-pip \ curl \ + vim \ && pip3 install --no-cache-dir -r requirements.txt \ && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl \ - && apt-get vim && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ENTRYPOINT ["python3", "-m", "unittest", "--failfast", "-v"] -CMD ["discover","--start-directory","/tests"] \ No newline at end of file +# working line +# python3 -m unittest discover -v --failfast -k test_e2e.EndToEndTestCase.test_lazy_spilo_upgrade --start-directory tests +ENTRYPOINT ["python3", "-m", "unittest"] +CMD ["discover","-v","--failfast","--start-directory","/tests"] \ No newline at end of file diff --git a/e2e/run_tests_image.sh b/e2e/exec_into_env.sh similarity index 94% rename from e2e/run_tests_image.sh rename to e2e/exec_into_env.sh index 713af1dda..ca212c543 100755 --- a/e2e/run_tests_image.sh +++ b/e2e/exec_into_env.sh @@ -3,7 +3,7 @@ export cluster_name="postgres-operator-e2e-tests" export kubeconfig_path="/tmp/kind-config-${cluster_name}" export operator_image="registry.opensource.zalan.do/acid/postgres-operator:latest" -export e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:latest" +export e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3" docker run -it --entrypoint /bin/bash --network=host -e "TERM=xterm-256color" \ --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \ diff --git a/e2e/run.sh b/e2e/run.sh index a96a1add5..354b3760e 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -9,7 +9,10 @@ IFS=$'\n\t' readonly cluster_name="postgres-operator-e2e-tests" readonly kubeconfig_path="/tmp/kind-config-${cluster_name}" readonly spilo_image="registry.opensource.zalan.do/acid/spilo-12:1.6-p5" -readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:latest" +readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3" + +export GOPATH=${GOPATH-~/go} +export PATH=${GOPATH}/bin:$PATH echo "Clustername: ${cluster_name}" echo "Kubeconfig path: ${kubeconfig_path}" @@ -38,6 +41,7 @@ function start_kind(){ } function load_operator_image() { + echo "Loading operator image" export KUBECONFIG="${kubeconfig_path}" kind load docker-image "${operator_image}" --name ${cluster_name} } @@ -53,7 +57,6 @@ function set_kind_api_server_ip(){ function run_tests(){ echo "Running tests... image: ${e2e_test_runner_image}" - echo $@ # tests modify files in ./manifests, so we mount a copy of this directory done by the e2e Makefile docker run --rm --network=host -e "TERM=xterm-256color" \ @@ -61,7 +64,7 @@ function run_tests(){ --mount type=bind,source="$(readlink -f manifests)",target=/manifests \ --mount type=bind,source="$(readlink -f tests)",target=/tests \ --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \ - -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" $@ + -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" ${E2E_TEST_CASE-} $@ } function clean_up(){ @@ -78,6 +81,7 @@ function main(){ [[ ! -f ${kubeconfig_path} ]] && start_kind load_operator_image set_kind_api_server_ip + shift run_tests $@ exit 0 diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f06ebef88..1306e398f 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -10,7 +10,7 @@ import yaml from datetime import datetime from kubernetes import client, config -from k8s_api import K8s +from tests.k8s_api import K8s SPILO_CURRENT = "registry.opensource.zalan.do/acid/spilo-12:1.6-p5" SPILO_LAZY = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" @@ -262,7 +262,7 @@ class EndToEndTestCase(unittest.TestCase): k8s = self.k8s # update infrastructure roles description secret_name = "postgresql-infrastructure-roles" - roles = "secretname: postgresql-infrastructure-roles-new,userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" + roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" patch_infrastructure_roles = { "data": { "infrastructure_roles_secret_name": secret_name, @@ -336,8 +336,7 @@ class EndToEndTestCase(unittest.TestCase): } k8s.update_config(patch_lazy_spilo_upgrade, step="Init baseline image version") - self.eventuallyEqual(lambda: k8s.get_statefulset_image(), SPILO_CURRENT, "Stagefulset not updated initially") - + self.eventuallyEqual(lambda: k8s.get_statefulset_image(), SPILO_CURRENT, "Stagefulset not updated initially") self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members(pod0)), 2, "Postgres status did not enter running") diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index 44b2222e0..a13eb479c 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -304,9 +304,16 @@ func (c *Cluster) isSafeToRecreatePods(pods *v1.PodList) bool { after this check succeeds but before a pod is re-created */ + for _, pod := range pods.Items { + c.logger.Debugf("name=%s phase=%s ip=%s", pod.Name, pod.Status.Phase, pod.Status.PodIP) + } + for _, pod := range pods.Items { state, err := c.patroni.GetPatroniMemberState(&pod) - if err != nil || state == "creating replica" { + if err != nil { + c.logger.Errorf("failed to get Patroni state for pod: %s", err) + return false + } else if state == "creating replica" { c.logger.Warningf("cannot re-create replica %s: it is currently being initialized", pod.Name) return false }