Update README with more detailed test instructions (#503)

- You can now use `make acceptance/run` to run only a specific acceptance test case
- Add note about Ubuntu 20.04 users / snap-provided docker
- Add instruction to run Ginkgo tests
- Extract acceptance/load from acceptance/kind
- Make `acceptance/pull` not depend on `docker-build`, so that you can do `make docker-build acceptance/load` for faster image reload

Makefile

@@ -7,6 +7,7 @@ DOCKER_USER ?= $(shell echo ${NAME} | cut -d / -f1)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 VERSION ?= latest
 TEST_REPO ?= ${DOCKER_USER}/actions-runner-controller
+SYNC_PERIOD ?= 5m
 
 # From https://github.com/VictoriaMetrics/operator/pull/44
 YAML_DROP=$(YQ) delete --inplace
@@ -145,14 +146,21 @@ release/clean:
 	rm -rf release
 
 .PHONY: acceptance
-acceptance: release/clean acceptance/pull release
+acceptance: release/clean acceptance/pull docker-build release
-	ACCEPTANCE_TEST_SECRET_TYPE=token make acceptance/kind acceptance/setup acceptance/tests acceptance/teardown
+	ACCEPTANCE_TEST_SECRET_TYPE=token make acceptance/run
-	ACCEPTANCE_TEST_SECRET_TYPE=app make acceptance/kind acceptance/setup acceptance/tests acceptance/teardown
+	ACCEPTANCE_TEST_SECRET_TYPE=app make acceptance/run
-	ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=token make acceptance/kind acceptance/setup acceptance/tests acceptance/teardown
+	ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=token make acceptance/run
-	ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=app make acceptance/kind acceptance/setup acceptance/tests acceptance/teardown
+	ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=app make acceptance/run
+
+acceptance/run: acceptance/kind acceptance/load acceptance/setup acceptance/deploy acceptance/tests acceptance/teardown
 
 acceptance/kind:
 	kind create cluster --name acceptance
+
+# Set TMPDIR to somewhere under $HOME when you use docker installed with Ubuntu snap
+# Otherwise `load docker-image` fail while running `docker save`.
+# See https://kind.sigs.k8s.io/docs/user/known-issues/#docker-installed-with-snap
+acceptance/load:
 	kind load docker-image ${NAME}:${VERSION} --name acceptance
 	kind load docker-image quay.io/brancz/kube-rbac-proxy:v0.8.0 --name acceptance
 	kind load docker-image ${RUNNER_NAME}:${VERSION} --name acceptance
@@ -163,7 +171,7 @@ acceptance/kind:
 	kubectl cluster-info --context kind-acceptance
 
 # Pull the docker images for acceptance
-acceptance/pull: docker-build
+acceptance/pull:
 	docker pull quay.io/brancz/kube-rbac-proxy:v0.8.0
 	docker pull docker:dind
 	docker pull quay.io/jetstack/cert-manager-controller:v1.0.4
@@ -182,8 +190,10 @@ acceptance/setup:
 acceptance/teardown:
 	kind delete cluster --name acceptance
 
-acceptance/tests:
+acceptance/deploy:
 	NAME=${NAME} RUNNER_NAME=${RUNNER_NAME} DOCKER_USER=${DOCKER_USER} VERSION=${VERSION} TEST_REPO=${TEST_REPO} acceptance/deploy.sh
+
+acceptance/tests:
 	acceptance/checks.sh
 
 # Upload release file to GitHub.

README.md

@@ -811,6 +811,15 @@ DOCKER_USER=*** \
   make acceptance
 ```
 
+> **Notes for Ubuntu 20.04+ users**
+>
+> If you're using Ubuntu 20.04 or greater, you might have installed `docker` with `snap`.
+>
+> If you want to stick with `snap`-provided `docker`, do not forget to set `TMPDIR` to
+> somewhere under `$HOME`.
+> Otherwise `kind load docker-image` fail while running `docker save`.
+> See https://kind.sigs.k8s.io/docs/user/known-issues/#docker-installed-with-snap for more information.
+
 Please follow the instructions explained in [Using Personal Access Token](#using-personal-access-token) to obtain
 `GITHUB_TOKEN`, and those in [Using GitHub App](#using-github-app) to obtain `APP_ID`, `INSTALLATION_ID`, and
 `PRIAVTE_KEY_FILE_PATH`.
@@ -819,6 +828,26 @@ The test creates a one-off `kind` cluster, deploys `cert-manager` and `actions-r
 creates a `RunnerDeployment` custom resource for a public Git repository to confirm that the
 controller is able to bring up a runner pod with the actions runner registration token installed.
 
+**Rerunning a failed test**
+
+When one of tests run by `make acceptance` failed, you'd probably like to rerun only the failed one.
+
+It can be done by `make acceptance/run` and by setting the combination of `ACCEPTANCE_TEST_DEPLOYMENT_TOOL` and `ACCEPTANCE_TEST_SECRET_TYPE` values that failed.
+
+In the example below, we rerun the test for the combination `ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=token` only:
+
+```
+DOCKER_USER=*** \
+  GITHUB_TOKEN=*** \
+  APP_ID=*** \
+  PRIVATE_KEY_FILE_PATH=path/to/pem/file \
+  INSTALLATION_ID=*** \
+  ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=token \
+  make acceptance/run
+```
+
+**Testing in a non-kind cluster**
+
 If you prefer to test in a non-kind cluster, you can instead run:
 
 ```shell script
@@ -830,6 +859,7 @@ KUBECONFIG=path/to/kubeconfig \
   INSTALLATION_ID=*** \
   ACCEPTANCE_TEST_SECRET_TYPE=token \
   make docker-build acceptance/setup \
+       acceptance/deploy \
        acceptance/tests
 ```
 
@@ -839,7 +869,7 @@ If you've already deployed actions-runner-controller and only want to recreate p
 
 ```
 NAME=$DOCKER_USER/actions-runner-controller \
-  make docker-build docker-push && \
+  make docker-build acceptance/load && \
   kubectl -n actions-runner-system delete po $(kubectl -n actions-runner-system get po -ojsonpath={.items[*].metadata.name})
 ```
 

acceptance/deploy.sh

@@ -5,9 +5,11 @@ set -e
 tpe=${ACCEPTANCE_TEST_SECRET_TYPE}
 
 if [ "${tpe}" == "token" ]; then
+  if ! kubectl get secret controller-manager -n actions-runner-system >/dev/null; then
     kubectl create secret generic controller-manager \
       -n actions-runner-system \
       --from-literal=github_token=${GITHUB_TOKEN:?GITHUB_TOKEN must not be empty}
+  fi
 elif [ "${tpe}" == "app" ]; then
   kubectl create secret generic controller-manager \
     -n actions-runner-system \
@@ -26,7 +28,7 @@ if [ "${tool}" == "helm" ]; then
     charts/actions-runner-controller \
     -n actions-runner-system \
     --create-namespace \
-    --set syncPeriod=5m \
+    --set syncPeriod=${SYNC_PERIOD} \
     --set authSecret.create=false \
     --set image.repository=${NAME} \
     --set image.tag=${VERSION}