From 407b2cab41e67b1e1d0cdfe269299804d7a86b6b Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 10 May 2019 17:00:16 +0200 Subject: [PATCH] move tests to a Docker container --- .travis.yml | 3 ++- Makefile | 14 +++++------ delivery.yaml | 5 +--- docs/developer.md | 17 +++---------- e2e/Dockerfile | 18 ++++++------- ...-cluster-postgres-operator-e2e-tests.yaml} | 0 ...d-config-kind-smoke-test-postgres-operator | 19 -------------- e2e/requirements.txt | 6 ++--- e2e/run.sh | 25 +++++++++++-------- e2e/tests/test_smoke.py | 12 +++------ 10 files changed, 41 insertions(+), 78 deletions(-) rename e2e/{kind-config-smoke-tests.yaml => kind-cluster-postgres-operator-e2e-tests.yaml} (100%) delete mode 100644 e2e/kind-config-kind-smoke-test-postgres-operator diff --git a/.travis.yml b/.travis.yml index 0fd48a9ca..7e77ffb41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,9 @@ before_install: - go get github.com/mattn/goveralls install: - - make deps + - make deps e2e-tools e2e-build script: - hack/verify-codegen.sh - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v + - make e2e-run diff --git a/Makefile b/Makefile index 378aa0082..fe7a5af00 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean local test linux macos docker push scm-source.json e2e e2e-tools +.PHONY: clean local test linux macos docker push scm-source.json e2e-run e2e-tools e2e-build BINARY ?= postgres-operator BUILD_FLAGS ?= -v @@ -92,11 +92,11 @@ test: hack/verify-codegen.sh @go test ./... -e2e-tools: - @pip3 install -r e2e/requirements.txt - @go get -u sigs.k8s.io/kind - # assumes kubectl is already isntalled +e2e-build: + docker build --tag="postgres-operator-e2e-tests" -f e2e/Dockerfile . -e2e: docker +e2e-tools: + @go get -u sigs.k8s.io/kind + +e2e-run: docker e2e/run.sh - flake8 --exit-zero diff --git a/delivery.yaml b/delivery.yaml index cd221229d..ee0306d86 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -46,10 +46,7 @@ pipeline: export PATH=$PATH:$HOME/go/bin cd $OPERATOR_TOP_DIR/postgres-operator echo "INFO install kubectl to test 'kind' from client side" - (curl -LO --silent https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && - chmod +x ./kubectl && - mv ./kubectl /usr/local/bin/kubectl) || exit 1 - make e2e-tools e2e + make e2e-tools e2e-build e2e-run - desc: 'Push docker image' cmd: | export PATH=$PATH:$HOME/go/bin diff --git a/docs/developer.md b/docs/developer.md index efdd1df17..561b437e6 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -317,20 +317,11 @@ kubectl logs acid-minimal-cluster-0 ## End-to-end tests -The operator provides e2e (end-to-end) tests to ensure various infra parts work smoothly together. -Such tests employ [kind](https://kind.sigs.k8s.io/) to start a local k8s cluster. -We intend to execute e2e tests during builds, but you can invoke them locally with `make e2e` from the project's top directory; run `make e2e-tools` to install `kind` and Python dependencies before the first run. Each e2e execution tests a Postgres operator image built from the current git branch. - -### Smoke tests - -The first provided collection of [smoke tests](../e2e/tests/test_smoke.py) covers the most basic operator capabilities. These tests utilize examples from `/manifests` (ConfigMap is used for the operator configuration) to avoid maintaining yet another set of configuration files. The `kind-smoke-test-postgres-operator` cluster is deleted if tests complete successfully. - -### Contributing tests suites - -1. Consider using a separate `kind` cluster per a logically separate collection of tests. This approach enables simple clean up after test success by deleting the relevant cluster. -2. Make tests time out; for now we use the [timeout-decorator](https://github.com/pnpnpn/timeout-decorator) for that purpose. -3. Please briefly describe the use case for the new test collection here and in the comments. +The operator provides reference e2e (end-to-end) tests to ensure various infra parts work smoothly together. +Each e2e execution tests a Postgres operator image built from the current git branch. The test runner starts a [kind](https://kind.sigs.k8s.io/) (local k8s) cluster and Docker container with tests. The k8s API client from within the container connects to the `kind` cluster using the standard Docker `bridge` network. +The tests utilize examples from `/manifests` (ConfigMap is used for the operator configuration) to avoid maintaining yet another set of configuration files. The kind cluster is deleted if tests complete successfully. +End-to-end tests are executed automatically during builds; to invoke them locally use `make e2e-run` from the project's top directory. Run `make e2e-tools e2e-build` to install `kind` and build the tests' image locally before the first run. ## Introduce additional configuration parameters diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 6c3011d02..e8b0c3303 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,18 +1,16 @@ FROM ubuntu:18.04 -MAINTAINER Team ACID @ Zalando +LABEL maintainer="Team ACID @ Zalando " WORKDIR /e2e -COPY e2e/requirements.txt ./ -COPY e2e/manifests ./manifests -COPY e2e/tests ./tests -COPY e2e/kind-config-kind-smoke-test-postgres-operator /root/.kube/config +COPY manifests ./manifests +COPY e2e/requirements.txt e2e/tests ./ -RUN apt-get update -RUN apt-get install -y python3 python3-pip curl \ - && pip3 install --no-cache-dir -r requirements.txt -RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ +RUN apt-get update \ + && apt-get install -y python3 python3-pip curl \ + && pip3 install --no-cache-dir -r requirements.txt \ + && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl -CMD ["python3", "-m", "unittest", "discover", "--start-directory", "tests", "-v"] \ No newline at end of file +CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"] \ No newline at end of file diff --git a/e2e/kind-config-smoke-tests.yaml b/e2e/kind-cluster-postgres-operator-e2e-tests.yaml similarity index 100% rename from e2e/kind-config-smoke-tests.yaml rename to e2e/kind-cluster-postgres-operator-e2e-tests.yaml diff --git a/e2e/kind-config-kind-smoke-test-postgres-operator b/e2e/kind-config-kind-smoke-test-postgres-operator deleted file mode 100644 index e91770a72..000000000 --- a/e2e/kind-config-kind-smoke-test-postgres-operator +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1EVXdPVEUxTlRVeU1Gb1hEVEk1TURVd05qRTFOVFV5TUZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWhZCjhTSVNNejlYb3VDMG9HQkRkNmVFdk5QQS9VZVJldHBucFRiUkNNU2tObUNkSFgzVWQ3SC9tTVdJcnVwdVhUeDQKWUV5dmJqWUt3YnJ5bUpTRU5xQ0h2RHE1SlV4TWE0dEZWRFBTNUZIaThiRUVHYW1UVUM3alo0Q2hma2hPN0ZSRQpKVW1xeDBLVDJ5SVVySDZTT203ajc3SDlOL0J5SGhhdnRJNXJsSlp3emdsbGw2eCtsV0RGbEExMndmYVVUaTdlCmpnd1Mzcjl4N0h3R1JQejVHNmhmaGlmSzZMVVhsNXAzYjB0RTZ1QVllNFBlWlNtQmJuS1UwdzAzQlJoYmpmTmIKSEdleUJIYm43RWhWZmJ4Y1dIV2RnaEMzV1dWRDdSa05NTXQvREtPbUdHR3Bab1RCeEpTSXR3UEFyMS9tUGVMRQp5VkxubjJmYzhDZ0dXT01SS0MwQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFEb2RRS1Nzd2NJa0NRMFVGUWNJRi9yNmtMQ2YKSjFHL3ZkQ1FkTW1taU9rcWRsMW1tQjAyL2d1UDZtL3AxbUltRmRmWjZJNGlPNGpwUCsvaGllMDB5aUMwMS84MgpLaUE2a292ckpMVzZnQ05rR2Nyb3hhdHRITklEYlF1TkVoZ3RJYWgxK2cxclo4eStTblU4b2pSRVFXYTZJUGJQClJKQjQyZjZ4dFFxZ1BJanRBT2EyeGpuMFdDNGpRTXNiSUx0M2Z0S2RLOTFmNG5BTFBmZGRSdyt6N3NxcWZNeFEKQU9UL3NycVo0eVJqT3d1Q2w4clBsTFVEUzBMTkRuM2FST0VDNVF6dm1nUElKY2k1ZWIrV1JGWEthblpjU2FXMwp6VFlVck9oTStSNEEyUGRBWWFSbm5qdnRGZ3JuY1cwRHVSb2tWUnFHL0Rvd2xHbzdZRUxQNEg4eDVHST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - server: https://172.17.0.3:6443 - name: kind-smoke-test-postgres-operator -contexts: -- context: - cluster: kind-smoke-test-postgres-operator - user: kubernetes-admin - name: kubernetes-admin@kind-smoke-test-postgres-operator -current-context: kubernetes-admin@kind-smoke-test-postgres-operator -kind: Config -preferences: {} -users: -- name: kubernetes-admin - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJWTVVRkRiKzc1c1l3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T1RBMU1Ea3hOVFUxTWpCYUZ3MHlNREExTURneE5UVTFNakJhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQW5OVkEvMFcrQmxqOENQcjYKZ1EvZUU0aWhMSnh3dlNuSEQ1VE1DYllRSFZFMTN5NmRiclRzTUhyU2duMlZBMmtaSGFEbmpPRXFieG0zMmt5aQowZmJhMzNZVm5BdGwrVmtyVzc2VW1USUFhb1RxaFNkRlpFczlrT3N5WUgweGluSXRMMExrWE9IeWwwVlVSUXVHCm94TVFxU2duaGpSMWJJVjdKbzYvYnhSbU42ZWJwb1FSRHN0QWk5Z2QramlHSU93Q05SWHVhM1B4Nm1DK04rVFoKTnZzRjNieHdIWURwTGo3aU1mNGUvRlM4MjQzVEU4eUo0Ymp1ejNyOHBsR2t4RkU0ZjZJbEtVQVpDK1dKN01KZAp2MDdOMDZVVFg3WlZxekE3R1l6QkROb2UrSmQ0S3JwcDFHbVdIWFgxaU9yRnhHNE9FSlFvbXE0c205VFhha25PCmlSVUQyUUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDMHdMSDRjTnBNeHlTQlRiVmlXYzd0Yk8rT2dyRlYzMmltNQpCa3pyMDBrd05nYmw5VFFUNFpkZDlHRnFTRVFpUDNXT2lWNjErMHpNdEpUcEthL0hITnBhNXg5MmhLc2FKZHdpCnErRm5TRisyVytIK0pnTy9tM0l6Z1VQK25kUVNNUXdwWmQ3bWQ0TnE0Wnh6VThrOTYwWWRMYlhHRXdTQTF5Ty8KN3RVQUJWWHVWdFFFaTQvQ1NSTXFjNndLT3NFNUU3QUNiYkU2MXhFTThsUzg5ZVhyaSs0aDZGenBpbkZWa1ZiRwpXaTAyYUV6cVJuTGFqV1NOZThuYnh2ZG1sRGhUZjFxYjkvWGJ0cWxIQWRMc1ZpdU5GT2Jxa3Yzb3FwSW5kQVNBClVRUDMxWXZ4VERPaEpaY3k2dkVyK2tqT2RpZjgrUytJa05BRnhzeVFPK0Vmc05zWUdvdz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBbk5WQS8wVytCbGo4Q1ByNmdRL2VFNGloTEp4d3ZTbkhENVRNQ2JZUUhWRTEzeTZkCmJyVHNNSHJTZ24yVkEya1pIYURuak9FcWJ4bTMya3lpMGZiYTMzWVZuQXRsK1Zrclc3NlVtVElBYW9UcWhTZEYKWkVzOWtPc3lZSDB4aW5JdEwwTGtYT0h5bDBWVVJRdUdveE1RcVNnbmhqUjFiSVY3Sm82L2J4Um1ONmVicG9RUgpEc3RBaTlnZCtqaUdJT3dDTlJYdWEzUHg2bUMrTitUWk52c0YzYnh3SFlEcExqN2lNZjRlL0ZTODI0M1RFOHlKCjRianV6M3I4cGxHa3hGRTRmNklsS1VBWkMrV0o3TUpkdjA3TjA2VVRYN1pWcXpBN0dZekJETm9lK0pkNEtycHAKMUdtV0hYWDFpT3JGeEc0T0VKUW9tcTRzbTlUWGFrbk9pUlVEMlFJREFRQUJBb0lCQVFDTTM3R0dteXJab015agpkRzNYeUZ6K3h0ZWZydFpGMUdVT1JlWVJRd3l1aU9nUEZWd3N1UzcvVFJRU1NxT3pjSkF5NFBtY3ZoVFR2eEk2CmNHUkFuYkIwMFNrUUJkMFBZVjFsQjRlTEpETGplNGo5R2cxbXpYNzcwWWhxeTRuWWhqNjRHU252bExYSDAycWkKcW52QnQ3cGJkOG9vN3E0YlVMc1NJMThwYy9WdFB5U1lmd3h3OVpFTEdRb3ZNNklRTGNBb1RxdHdKYXZDZmlKQQpKYjZFaXdROUljOTB1MDJxaGE3RGw4K3lrWmRaSnJmNmYwditoZDUrdGFRK3RkNnZMd3RvUGZCa2licXhBNkx1CkxId2IvWnBSOGlKbXJFTjkwdFNMTFpMWVAzQkhkYUNNcXdJYllXUkRsS2p2cE1DUUp2NDgrdDhuMERNcUVzSFUKQi9XK2EwdEJBb0dCQU02b3VHTmhUMWpkTTlSenBITzg3enpiaHh2T0VWQ3BDN29zdGl1dmJnUnVGcFl2WloxTQpzTHY5dVBrTFlmanAxais3VzJvWncvQ25Zcjh0ZGVYTHVBKzNpcXk5bzZlblhjVEExM0wwTFNHY0tuZXAwenpxCmtqUmNGYmZEOXNpM2Jsc0FmT0hnSzY5MFByZmdINjF1bW5rYTVjWEhSc3dTcThBbVBHVU5EQWxsQW9HQkFNSkgKR1Z3WWhlN2hlbE5FQ1FNTWxENW90c01zWGJtWEFzbUFCZ2dxQXhRVkw0dmVUTk9XZHV2b2t1eDFVVWZhMVhqdgpUNEduYTZmeCtvMDUyRlNkU1FLTkEwNlhwVWhxZUNIcHU1ZmV3MHV0WVJhblhrS2MzRnZ5NEJlbEd3aHZUUVJRCnIxVmdlWWc0V3A2bTFuaHJwRDEydGh2WDE2Rm9uN0I4Skp2clVxTmxBb0dCQU1vcVk3ZFV5cnEwS3EvN01UWEgKN29JcWY5SERsVXpERXFYZWQ1Zmsxa3VmSnBsbFpKS3RJM2ZFamQrVU14TytMY25MRDNLTUloS2FyUTg0K2MwRApyZHd5UVljYlBhNFZITFlOc0xiVUNCS0pJMEpNOEVqM2NHK29aZGFQN2l3TXhmaGdVY3JsOGRhQ2NaaVB1RzJCCmRieGpnOFFuWGlybFdQOXdhRVN5cnNQQkFvR0FmRkxwYktFWTNHU1lWajZja2NIei8vZ2N0TXRvY3dLck91MWQKYnM0THlFZENkUHhlSjYwTER5NTNEekNJUWpaTkU2WDVPQncrYld3UmpWeXVEbi9Vbi9oRFhJRDR1VjNBNE5ybApQR3ZHaUdBOFdEWGt3VFlHWWlVTHVMWGtsY0k4QS8zcUpmV2w4RUUzNUgwWmxGZzE4MHRMZ0lmZ3FwNzhTZ0UzCm9EdTRWMjBDZ1lCNER0MDNoNG4zWGxPMW8rZlBKMGJETnZnb1JGZnZYYWk2YjBVa1VpcXNKclRTZVFrRVEzenUKNnFPRnBLNUgzRDlWc3lweWF0N3JJNmk4bFQzbHlsaUU1dGdzMEV5ektvSjJCYVFEOE5MRVJueHVyRHFkV2paaQp6NXNJSjU0c0N6NDJjTytQSkVTUjRCTlBZU1ZNUEhDZURWd1diL1QwNXdOU1NxemIrRWJQN1E9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/e2e/requirements.txt b/e2e/requirements.txt index 40fb9efa1..c46b334ad 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -1,4 +1,2 @@ -kubernetes -flake8 -timeout_decorator -docker +kubernetes==9.0.0 +timeout_decorator==0.4.1 diff --git a/e2e/run.sh b/e2e/run.sh index 0471afd8a..e78a0b5bb 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -6,7 +6,11 @@ set -o nounset set -o pipefail IFS=$'\n\t' -readonly cluster_name="kind-smoke-test-postgres-operator" +readonly cluster_name="postgres-operator-e2e-tests" +readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) +readonly e2e_test_image=${cluster_name} +readonly kind_api_server_port=6443 # well-known in the 'kind' codebase +readonly kubeconfig_path="./e2e/kind-config-${cluster_name}" # avoid interference with previous test runs if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] @@ -14,20 +18,19 @@ then kind delete cluster --name ${cluster_name} fi -kind create cluster --name ${cluster_name} --config ./e2e/kind-config-smoke-tests.yaml +kind create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" + +kind load docker-image ${operator_image} --name ${cluster_name} kubectl cluster-info -image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) -kind load docker-image ${image} --name ${cluster_name} - -cp -r ./manifests ./e2e/manifests +# use the actual kubeconfig to connect to the 'kind' API server +# but update the IP address of the API server to the one from the Docker 'bridge' network cp $KUBECONFIG ./e2e -d=$(docker inspect -f "{{ .NetworkSettings.IPAddress }}:6443" ${cluster_name}-control-plane) -sed -i "s/server.*$/server: https:\/\/$d/g" e2e/kind-config-${cluster_name} +kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" ${cluster_name}-control-plane) +sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" ${kubeconfig_path} -docker build --tag=postgres-operator-e2e-tests -f e2e/Dockerfile . && docker run postgres-operator-e2e-tests -#python3 -m unittest discover --start-directory e2e/tests/ -v && +docker run --rm --mount type=bind,source="$(realpath ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image} kind delete cluster --name ${cluster_name} -rm -rf ./e2e/manifests ./e2e/kind-smoke-test-postgres-operator.yaml +rm -rf ${kubeconfig_path} diff --git a/e2e/tests/test_smoke.py b/e2e/tests/test_smoke.py index 63b6b7376..72998a00a 100755 --- a/e2e/tests/test_smoke.py +++ b/e2e/tests/test_smoke.py @@ -3,8 +3,7 @@ import time import timeout_decorator import subprocess import warnings -import docker -#from halo import Halo +import os from kubernetes import client, config, utils @@ -40,10 +39,7 @@ class SmokeTestCase(unittest.TestCase): for filename in ["configmap.yaml", "postgres-operator.yaml"]: utils.create_from_yaml(k8s_api.k8s_client, "manifests/" + filename) - # submit the most recent operator image built locally - # HACK assumes "images list" returns the most recent image at index 0 - # docker_client = docker.from_env() - image = "registry.opensource.zalan.do/acid/postgres-operator:v1.1.0-67-g52e53f1-dirty" # docker_client.images.list(name="registry.opensource.zalan.do/acid/postgres-operator")[0].tags[0] + # submit the most recent operator image built on the Docker host body = { "spec": { "template": { @@ -51,7 +47,7 @@ class SmokeTestCase(unittest.TestCase): "containers": [ { "name": "postgres-operator", - "image": image + "image": os.environ['OPERATOR_IMAGE'] } ] } @@ -114,7 +110,6 @@ class Utils: @staticmethod def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): pod_phase = 'No pod running' - #with Halo(text="Wait for the pod '{}' to start. Pod phase: {}".format(pod_labels, pod_phase), spinner='dots'): while pod_phase != 'Running': pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items if pods: @@ -133,7 +128,6 @@ class Utils: "v1", "default", "postgresqls", "acid-minimal-cluster", body) labels = 'version=acid-minimal-cluster' - #with Halo(text="Waiting for the cluster to scale to {} pods.".format(number_of_instances), spinner='dots'): while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: time.sleep(retry_timeout_sec)