dockerfile,e2e: Use buildx and cache mounts for faster rebuilds in E2E

.dockerignore

@@ -11,3 +11,4 @@ charts
 *.md
 *.txt
 *.sh
+test/e2e/.docker-build

Dockerfile

@@ -8,22 +8,26 @@ WORKDIR /workspace
 ENV GO111MODULE=on \
   CGO_ENABLED=0
 
-# Copy the Go Modules manifests
+# # Copy the Go Modules manifests
-COPY go.mod go.sum ./
+# COPY go.mod go.sum ./
 
-# cache deps before building and copying source so that we don't need to re-download as much
+# # cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
+# # and so that source changes don't invalidate our downloaded layer
-RUN go mod download
+# RUN --mount=type=cache,target=/go/pkg/mod go mod download
 
 # Copy the go source
-COPY . .
+# COPY . .
+
+ARG TARGETOS
+ARG TARGETARCH
 
 # Build
-RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
+RUN --mount=target=. \
-  export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \
+  --mount=type=cache,mode=0777,target=/root/.cache/go-build \
-  GOARM=$(echo ${TARGETPLATFORM} | cut -d / -f3 | cut -c2-) && \
+  --mount=type=cache,mode=0777,target=/go/pkg/mod\
-  go build -a -o manager main.go && \
+  GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
-  go build -a -o github-webhook-server ./cmd/githubwebhookserver
+  GOARM=$(echo ${TARGETPLATFORM} | cut -d / -f3 | cut -c2-) \
+  go build -o /out/manager main.go && go build -o /out/github-webhook-server ./cmd/githubwebhookserver
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
@@ -31,8 +35,8 @@ FROM gcr.io/distroless/static:nonroot
 
 WORKDIR /
 
-COPY --from=builder /workspace/manager .
+COPY --from=builder /out/manager .
-COPY --from=builder /workspace/github-webhook-server .
+COPY --from=builder /out/github-webhook-server .
 
 USER nonroot:nonroot
 

test/e2e/e2e_test.go

@@ -30,9 +30,10 @@ var (
 
 	builds = []testing.DockerBuild{
 		{
-			Dockerfile: "../../Dockerfile",
+			Dockerfile:   "../../Dockerfile",
-			Args:       []testing.BuildArg{},
+			Args:         []testing.BuildArg{},
-			Image:      controllerImage,
+			Image:        controllerImage,
+			EnableBuildX: true,
 		},
 		{
 			Dockerfile: "../../runner/Dockerfile",

testing/docker.go

@@ -3,8 +3,11 @@ package testing
 import (
 	"context"
 	"fmt"
+	"log"
+	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 
 	"github.com/actions-runner-controller/actions-runner-controller/testing/runtime"
 )
@@ -14,9 +17,10 @@ type Docker struct {
 }
 
 type DockerBuild struct {
-	Dockerfile string
+	Dockerfile   string
-	Args       []BuildArg
+	Args         []BuildArg
-	Image      ContainerImage
+	Image        ContainerImage
+	EnableBuildX bool
 }
 
 type BuildArg struct {
@@ -25,12 +29,7 @@ type BuildArg struct {
 
 func (k *Docker) Build(ctx context.Context, builds []DockerBuild) error {
 	for _, build := range builds {
-		var args []string
+		_, err := k.dockerBuildCombinedOutput(ctx, build)
-		args = append(args, "--build-arg=TARGETPLATFORM="+"linux/amd64")
-		for _, buildArg := range build.Args {
-			args = append(args, "--build-arg="+buildArg.Name+"="+buildArg.Value)
-		}
-		_, err := k.CombinedOutput(k.dockerBuildCmd(ctx, build.Dockerfile, build.Image.Repo, build.Image.Tag, args))
 
 		if err != nil {
 			return fmt.Errorf("failed building %v: %w", build, err)
@@ -40,10 +39,35 @@ func (k *Docker) Build(ctx context.Context, builds []DockerBuild) error {
 	return nil
 }
 
-func (k *Docker) dockerBuildCmd(ctx context.Context, dockerfile, repo, tag string, args []string) *exec.Cmd {
+func (k *Docker) dockerBuildCombinedOutput(ctx context.Context, build DockerBuild) (string, error) {
+	var args []string
+
+	args = append(args, "--build-arg=TARGETPLATFORM="+"linux/amd64")
+	for _, buildArg := range build.Args {
+		args = append(args, "--build-arg="+buildArg.Name+"="+buildArg.Value)
+	}
+
+	dockerfile := build.Dockerfile
+	repo := build.Image.Repo
+	tag := build.Image.Tag
+
 	buildContext := filepath.Dir(dockerfile)
+
+	docker := "docker"
+	env := os.Environ()
 	args = append([]string{"build", "--tag", repo + ":" + tag, "-f", dockerfile, buildContext}, args...)
 
-	cmd := exec.CommandContext(ctx, "docker", args...)
+	if build.EnableBuildX {
-	return cmd
+		args = append([]string{"buildx"}, args...)
+		args = append(args, "--load")
+
+		env = append(env, "DOCKER_BUILDKIT=1")
+	}
+
+	cmd := exec.CommandContext(ctx, docker, args...)
+	cmd.Env = env
+
+	log.Printf("%s %s", docker, strings.Join(args, " "))
+
+	return k.CombinedOutput(cmd)
 }