e2e,acceptance: Cover enterprise runners (#1124)

Adds various code and changes I have used while testing #1062

acceptance/deploy.sh

@@ -64,7 +64,7 @@ sleep 20
 RUNNER_LABEL=${RUNNER_LABEL:-self-hosted}
 
 if [ -n "${TEST_REPO}" ]; then
-  if [ -n "USE_RUNNERSET" ]; then
+  if [ "${USE_RUNNERSET}" -ne "false" ]; then
       cat acceptance/testdata/repo.runnerset.yaml | envsubst | kubectl apply -f -
       cat acceptance/testdata/repo.runnerset.hra.yaml | envsubst | kubectl apply -f -
   else
@@ -77,13 +77,25 @@ else
 fi
 
 if [ -n "${TEST_ORG}" ]; then
-  cat acceptance/testdata/org.runnerdeploy.yaml | envsubst | kubectl apply -f -
+  cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= NAME=org-runnerdeploy envsubst | kubectl apply -f -
 
-  if [ -n "${TEST_ORG_REPO}" ]; then
+  if [ -n "${TEST_ORG_GROUP}" ]; then
-    cat acceptance/testdata/org.hra.yaml | envsubst | kubectl apply -f -
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerdeploy envsubst | kubectl apply -f -
   else
-    echo 'Skipped deploying organizational hra. Set TEST_ORG_REPO to "yourorg/yourrepo" to deploy.'
+    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ORG_GROUP to deploy.'
   fi
 else
   echo 'Skipped deploying organizational runnerdeployment. Set TEST_ORG to deploy.'
 fi
+
+if [ -n "${TEST_ENTERPRISE}" ]; then
+  cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= NAME=enterprise-runnerdeploy envsubst | kubectl apply -f -
+
+  if [ -n "${TEST_ENTERPRISE_GROUP}" ]; then
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerdeploy envsubst | kubectl apply -f -
+  else
+    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE_GROUP to deploy.'
+  fi
+else
+  echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE to deploy.'
+fi

acceptance/testdata/org.runnerdeploy.yaml

@@ -14,6 +14,11 @@ spec:
       image: ${RUNNER_NAME}:${RUNNER_TAG}
       imagePullPolicy: IfNotPresent
 
+      # Whether to pass --ephemeral (true) or --once (false, deprecated)
+      env:
+      - name: RUNNER_FEATURE_FLAG_EPHEMERAL
+        value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
+
       #
       # dockerd within runner container
       #
@@ -30,6 +35,8 @@ spec:
       # labels:
       # - "mylabel 1"
       # - "mylabel 2"
+      labels:
+      - "${RUNNER_LABEL}"
 
       #
       # Non-standard working directory

acceptance/testdata/repo.runnerdeploy.yaml

@@ -14,6 +14,11 @@ spec:
       image: ${RUNNER_NAME}:${RUNNER_TAG}
       imagePullPolicy: IfNotPresent
 
+      # Whether to pass --ephemeral (true) or --once (false, deprecated)
+      env:
+      - name: RUNNER_FEATURE_FLAG_EPHEMERAL
+        value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
+
       #
       # dockerd within runner container
       #
@@ -30,6 +35,8 @@ spec:
       # labels:
       # - "mylabel 1"
       # - "mylabel 2"
+      labels:
+      - "${RUNNER_LABEL}"
 
       #
       # Non-standard working directory

acceptance/testdata/runnerdeploy.envsubst.yaml

@@ -0,0 +1,61 @@
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: ${NAME}
+spec:
+#  replicas: 1
+  template:
+    spec:
+      enterprise: ${TEST_ENTERPRISE}
+      group: ${TEST_GROUP}
+      organization: ${TEST_ORG}
+      repository: ${TEST_REPO}
+
+      #
+      # Custom runner image
+      #
+      image: ${RUNNER_NAME}:${RUNNER_TAG}
+      imagePullPolicy: IfNotPresent
+
+      # Whether to pass --ephemeral (true) or --once (false, deprecated)
+      env:
+      - name: RUNNER_FEATURE_FLAG_EPHEMERAL
+        value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
+
+      #
+      # dockerd within runner container
+      #
+      ## Replace `mumoshu/actions-runner-dind:dev` with your dind image
+      #dockerdWithinRunnerContainer: true
+      #image: mumoshu/actions-runner-dind:dev
+
+      #
+      # Set the MTU used by dockerd-managed network interfaces (including docker-build-ubuntu)
+      #
+      #dockerMTU: 1450
+
+      #Runner group
+      # labels:
+      # - "mylabel 1"
+      # - "mylabel 2"
+      labels:
+      - "${RUNNER_LABEL}"
+
+      #
+      # Non-standard working directory
+      #
+      # workDir: "/"
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: ${NAME}
+spec:
+  scaleTargetRef:
+    name: ${NAME}
+  scaleUpTriggers:
+  - githubEvent: {}
+    amount: 1
+    duration: "1m"
+  minReplicas: 0
+  maxReplicas: 10

acceptance/values.yaml

@@ -1,12 +1,15 @@
 # Set actions-runner-controller settings for testing
 githubAPICacheDuration: 10s
 githubWebhookServer:
+  logLevel: debug
   enabled: true
   labels: {}
   replicaCount: 1
   syncPeriod: 10m
+  useRunnerGroupsVisibility: true
   secret:
-    create: true
+    enabled: true
+    # create: true
     name: "github-webhook-server"
     ### GitHub Webhook Configuration
     #github_webhook_secret_token: ""

test/e2e/cmd/main.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	var configMapNames []string
+
+	output, err := output()
+	if err != nil {
+		log.Printf("Command failed with output: %s", string(output))
+		return err
+	}
+
+	s := bufio.NewScanner(bytes.NewBuffer(output))
+
+	for s.Scan() {
+		if t := s.Text(); strings.Contains(t, "test-info") || strings.Contains(t, "test-result-") {
+			configMapNames = append(configMapNames, s.Text())
+		}
+	}
+
+	for _, n := range configMapNames {
+		println(n)
+
+		if output, err := delete(n); err != nil {
+			log.Printf("Command failed with output: %s", string(output))
+			return err
+		}
+	}
+
+	return nil
+}
+
+func output() ([]byte, error) {
+	cmd := exec.Command("kubectl", "get", "cm", "-o", `jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}`)
+	data, err := cmd.CombinedOutput()
+	return data, err
+}
+
+func delete(cmName string) ([]byte, error) {
+	cmd := exec.Command("kubectl", "delete", "cm", cmName)
+	return cmd.CombinedOutput()
+}
+
+func deleteControllerManagerSecret() ([]byte, error) {
+	cmd := exec.Command("kubectl", "-n", "actions-runner-system", "delete", "secret", "controller-manager")
+	return cmd.CombinedOutput()
+}

test/e2e/e2e_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/testing"
@@ -166,7 +167,11 @@ type env struct {
 	useRunnerSet bool
 
 	testID                                                   string
+	repoToCommit                                             string
 	runnerLabel, githubToken, testRepo, testOrg, testOrgRepo string
+	githubTokenWebhook                                       string
+	testEnterprise                                           string
+	featureFlagEphemeral                                     bool
 	testJobs                                                 []job
 }
 
@@ -186,10 +191,15 @@ func initTestEnv(t *testing.T) *env {
 	e.testID = testID
 	e.runnerLabel = "test-" + id
 	e.githubToken = testing.Getenv(t, "GITHUB_TOKEN")
-	e.testRepo = testing.Getenv(t, "TEST_REPO")
+	e.githubTokenWebhook = testing.Getenv(t, "WEBHOOK_GITHUB_TOKEN")
-	e.testOrg = testing.Getenv(t, "TEST_ORG")
+	e.repoToCommit = testing.Getenv(t, "TEST_COMMIT_REPO")
-	e.testOrgRepo = testing.Getenv(t, "TEST_ORG_REPO")
+	e.testRepo = testing.Getenv(t, "TEST_REPO", "")
-	e.testJobs = createTestJobs(id, testResultCMNamePrefix, 2)
+	e.testOrg = testing.Getenv(t, "TEST_ORG", "")
+	e.testOrgRepo = testing.Getenv(t, "TEST_ORG_REPO", "")
+	e.testEnterprise = testing.Getenv(t, "TEST_ENTERPRISE")
+	e.testJobs = createTestJobs(id, testResultCMNamePrefix, 10)
+	ephemeral, _ := strconv.ParseBool(testing.Getenv(t, "TEST_FEATURE_FLAG_EPHEMERAL"))
+	e.featureFlagEphemeral = ephemeral
 
 	return e
 }
@@ -237,11 +247,14 @@ func (e *env) installActionsRunnerController(t *testing.T) {
 	}
 
 	varEnv := []string{
+		"TEST_ENTERPRISE=" + e.testEnterprise,
 		"TEST_REPO=" + e.testRepo,
 		"TEST_ORG=" + e.testOrg,
 		"TEST_ORG_REPO=" + e.testOrgRepo,
 		"GITHUB_TOKEN=" + e.githubToken,
+		"WEBHOOK_GITHUB_TOKEN=" + e.githubTokenWebhook,
 		"RUNNER_LABEL=" + e.runnerLabel,
+		fmt.Sprintf("RUNNER_FEATURE_FLAG_EPHEMERAL=%v", e.featureFlagEphemeral),
 	}
 
 	scriptEnv = append(scriptEnv, varEnv...)
@@ -260,7 +273,7 @@ func (e *env) createControllerNamespaceAndServiceAccount(t *testing.T) {
 func (e *env) installActionsWorkflow(t *testing.T) {
 	t.Helper()
 
-	installActionsWorkflow(t, e.testID, e.runnerLabel, testResultCMNamePrefix, e.testRepo, e.testJobs)
+	installActionsWorkflow(t, e.testID, e.runnerLabel, testResultCMNamePrefix, e.repoToCommit, e.testJobs)
 }
 
 func (e *env) verifyActionsWorkflowRun(t *testing.T) {
@@ -287,6 +300,8 @@ func createTestJobs(id, testResultCMNamePrefix string, numJobs int) []job {
 	return testJobs
 }
 
+const Branch = "main"
+
 func installActionsWorkflow(t *testing.T, testID, runnerLabel, testResultCMNamePrefix, testRepo string, testJobs []job) {
 	t.Helper()
 
@@ -298,7 +313,7 @@ func installActionsWorkflow(t *testing.T, testID, runnerLabel, testResultCMNameP
 		Name: wfName,
 		On: testing.On{
 			Push: &testing.Push{
-				Branches: []string{"master"},
+				Branches: []string{Branch},
 			},
 		},
 		Jobs: map[string]testing.Job{},
@@ -346,6 +361,7 @@ kubectl create cm %s$id --from-literal=status=ok
 			".github/workflows/workflow.yaml": wfContent,
 			"test.sh":                         script,
 		},
+		Branch: Branch,
 	}
 
 	if err := g.Sync(ctx); err != nil {

testing/getenv.go

@@ -5,12 +5,15 @@ import (
 	"testing"
 )
 
-func Getenv(t *testing.T, name string) string {
+func Getenv(t *testing.T, name string, opts ...string) string {
 	t.Helper()
 
 	v := os.Getenv(name)
 	if v == "" {
-		t.Fatal(name + " must be set")
+		if len(opts) == 0 {
+			t.Fatal(name + " must be set")
+		}
+		v = opts[0]
 	}
 	return v
 }

testing/git.go

@@ -16,6 +16,7 @@ type GitRepo struct {
 	Name          string
 	CommitMessage string
 	Contents      map[string][]byte
+	Branch        string
 
 	runtime.Cmdr
 }
@@ -43,6 +44,11 @@ func (g *GitRepo) Sync(ctx context.Context) error {
 
 	for path, content := range g.Contents {
 		absPath := filepath.Join(dir, path)
+		d := filepath.Dir(absPath)
+
+		if err := os.MkdirAll(d, 0755); err != nil {
+			return fmt.Errorf("error creating dir %s: %v", d, err)
+		}
 
 		if err := os.WriteFile(absPath, content, 0755); err != nil {
 			return fmt.Errorf("error writing %s: %w", path, err)
@@ -89,7 +95,7 @@ func (g *GitRepo) gitCommitCmd(ctx context.Context, dir, msg string) *exec.Cmd {
 }
 
 func (g *GitRepo) gitPushCmd(ctx context.Context, dir string) *exec.Cmd {
-	cmd := exec.CommandContext(ctx, "git", "push", "origin", "master")
+	cmd := exec.CommandContext(ctx, "git", "push", "origin", g.Branch)
 	cmd.Dir = dir
 	return cmd
 }

testing/testing.go

@@ -323,6 +323,11 @@ func (k *Kind) Start(ctx context.Context) error {
 		kindConfig := []byte(fmt.Sprintf(`kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 name: %s
+networking:
+  apiServerAddress: 0.0.0.0
+nodes:
+  - role: control-plane
+  - role: worker
 `, k.Name))
 
 		if err := os.WriteFile(f.Name(), kindConfig, 0644); err != nil {