From 2a475f25c738e9d413400f7afbcc9d6e163f7366 Mon Sep 17 00:00:00 2001
From: Yusuke Kuoka <ykuoka@gmail.com>
Date: Thu, 7 Jul 2022 11:27:27 +0900
Subject: [PATCH] Use Argo Tunnel for exposing the autoscaler's webhook server
 (#1595)

I've been manually setting up Argo Tunnel to expose the webhook server while running E2E tests so that I can cover the webhook-based autoscaling. This automates the setup process so that we can automatiaclly bring up and down cloudflared before/after the test run, so that it can be a part of our upcoming automated E2E test.
---
 acceptance/argotunnel.sh | 97 ++++++++++++++++++++++++++++++++++++++++
 test/e2e/e2e_test.go     | 43 ++++++++++++++++++
 2 files changed, 140 insertions(+)
 create mode 100755 acceptance/argotunnel.sh

diff --git a/acceptance/argotunnel.sh b/acceptance/argotunnel.sh
new file mode 100755
index 00000000..c4fc0a4d
--- /dev/null
+++ b/acceptance/argotunnel.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+# See https://developers.cloudflare.com/cloudflare-one/tutorials/many-cfd-one-tunnel/
+
+kubectl create ns tunnel || :
+
+kubectl -n tunnel delete secret tunnel-credentials || :
+
+kubectl -n tunnel create secret generic tunnel-credentials \
+  --from-file=credentials.json=$HOME/.cloudflared/${TUNNEL_ID}.json || :
+
+cat <<MANIFEST | kubectl -n tunnel ${OP} -f -
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudflared
+spec:
+  selector:
+    matchLabels:
+      app: cloudflared
+  replicas: 2 # You could also consider elastic scaling for this deployment
+  template:
+    metadata:
+      labels:
+        app: cloudflared
+    spec:
+      containers:
+      - name: cloudflared
+        image: cloudflare/cloudflared:latest
+        args:
+        - tunnel
+        # Points cloudflared to the config file, which configures what
+        # cloudflared will actually do. This file is created by a ConfigMap
+        # below.
+        - --config
+        - /etc/cloudflared/config/config.yaml
+        - run
+        livenessProbe:
+          httpGet:
+            # Cloudflared has a /ready endpoint which returns 200 if and only if
+            # it has an active connection to the edge.
+            path: /ready
+            port: 2000
+          failureThreshold: 1
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        volumeMounts:
+        - name: config
+          mountPath: /etc/cloudflared/config
+          readOnly: true
+        # Each tunnel has an associated "credentials file" which authorizes machines
+        # to run the tunnel. cloudflared will read this file from its local filesystem,
+        # and it'll be stored in a k8s secret.
+        - name: creds
+          mountPath: /etc/cloudflared/creds
+          readOnly: true
+      volumes:
+      - name: creds
+        secret:
+          secretName: tunnel-credentials
+      # Create a config.yaml file from the ConfigMap below.
+      - name: config
+        configMap:
+          name: cloudflared
+          items:
+          - key: config.yaml
+            path: config.yaml
+---
+# This ConfigMap is just a way to define the cloudflared config.yaml file in k8s.
+# It's useful to define it in k8s, rather than as a stand-alone .yaml file, because
+# this lets you use various k8s templating solutions (e.g. Helm charts) to
+# parameterize your config, instead of just using string literals.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloudflared
+data:
+  config.yaml: |
+    # Name of the tunnel you want to run
+    tunnel: ${TUNNEL_NAME}
+    credentials-file: /etc/cloudflared/creds/credentials.json
+    # Serves the metrics server under /metrics and the readiness server under /ready
+    metrics: 0.0.0.0:2000
+    # Autoupdates applied in a k8s pod will be lost when the pod is removed or restarted, so
+    # autoupdate doesn't make sense in Kubernetes. However, outside of Kubernetes, we strongly
+    # recommend using autoupdate.
+    no-autoupdate: true
+    ingress:
+    # The first rule proxies traffic to the httpbin sample Service defined in app.yaml
+    - hostname: ${TUNNEL_HOSTNAME}
+      service: http://actions-runner-controller-github-webhook-server.actions-runner-system:80
+    # This rule matches any traffic which didn't match a previous rule, and responds with HTTP 404.
+    - service: http_status:404
+MANIFEST
+
+kubectl -n tunnel delete po -l app=cloudflared || :
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
index ab964fab..2ea6bd44 100644
--- a/test/e2e/e2e_test.go
+++ b/test/e2e/e2e_test.go
@@ -126,6 +126,7 @@ func TestE2E(t *testing.T) {
 	skipRunnerCleanUp := os.Getenv("ARC_E2E_SKIP_RUNNER_CLEANUP") != ""
 	retainCluster := os.Getenv("ARC_E2E_RETAIN_CLUSTER") != ""
 	skipTestIDCleanUp := os.Getenv("ARC_E2E_SKIP_TEST_ID_CLEANUP") != ""
+	skipArgoTunnelCleanUp := os.Getenv("ARC_E2E_SKIP_ARGO_TUNNEL_CLEAN_UP") != ""
 
 	env := initTestEnv(t, k8sMinorVer)
 
@@ -160,6 +161,16 @@ func TestE2E(t *testing.T) {
 			env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
 		})
 
+		t.Run("install argo-tunnel", func(t *testing.T) {
+			env.installArgoTunnel(t)
+		})
+
+		if !skipArgoTunnelCleanUp {
+			t.Cleanup(func() {
+				env.uninstallArgoTunnel(t)
+			})
+		}
+
 		t.Run("deploy runners", func(t *testing.T) {
 			env.deploy(t, RunnerSets, testID)
 		})
@@ -210,6 +221,16 @@ func TestE2E(t *testing.T) {
 			env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
 		})
 
+		t.Run("install argo-tunnel", func(t *testing.T) {
+			env.installArgoTunnel(t)
+		})
+
+		if !skipArgoTunnelCleanUp {
+			t.Cleanup(func() {
+				env.uninstallArgoTunnel(t)
+			})
+		}
+
 		t.Run("deploy runners", func(t *testing.T) {
 			env.deploy(t, RunnerDeployments, testID)
 		})
@@ -425,6 +446,28 @@ func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
 	e.RunScript(t, "../../acceptance/deploy_runners.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
 }
 
+func (e *env) installArgoTunnel(t *testing.T) {
+	e.doArgoTunnel(t, "apply")
+}
+
+func (e *env) uninstallArgoTunnel(t *testing.T) {
+	e.doArgoTunnel(t, "delete")
+}
+
+func (e *env) doArgoTunnel(t *testing.T, op string) {
+	t.Helper()
+
+	scriptEnv := []string{
+		"KUBECONFIG=" + e.Kubeconfig(),
+		"OP=" + op,
+		"TUNNEL_ID=" + os.Getenv("TUNNEL_ID"),
+		"TUNNE_NAME=" + os.Getenv("TUNNEL_NAME"),
+		"TUNNEL_HOSTNAME=" + os.Getenv("TUNNEL_HOSTNAME"),
+	}
+
+	e.RunScript(t, "../../acceptance/argotunnel.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
+}
+
 func (e *env) runnerLabel(testID string) string {
 	return "test-" + testID
 }