Share the Go build cache when building in Dockerfiles (#1853)

On my machine this makes image rebuilds go from ~5m to 1.5s.

This also required setting DOCKER_BUILDKIT=1 in integration test image
builds.

I also took the opportunity to bump the Go version used in those tests
to Go 1.17 to match the image build processes, and tidied up the
workflow files a bit too (renaming for consistency, typos, whitespace,
etc.)

.github/workflows/integration-k8s-tests.yaml

@@ -0,0 +1,29 @@
+name: Integration tests (K8s)
+
+on:
+  pull_request:
+    branches: ['master']
+
+concurrency:
+  group: int-test-k8s-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    env:
+      IMAGE_REPO: 'localhost:5000'
+      REGISTRY: 'localhost:5000'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+    - uses: actions/checkout@v2
+    - uses: docker/setup-buildx-action@v1
+
+    - name: Run build-image-and-k8s-integration-test
+      run: |
+        make travis-setup
+        make minikube-setup
+        make k8s-executor-build-push
+        make integration-test-k8s

.github/workflows/integration-layers-tests.yaml

@@ -1,29 +1,25 @@
 name: Integration tests (Layers)
 
-# Triggers the workflow on push or pull request events
-on: [pull_request]
+on:
+  pull_request:
+    branches: ['master']
 
 concurrency:
   group: int-test-layers-${{ github.head_ref }}
   cancel-in-progress: true
 
 jobs:
-  build-executor:
+  tests:
     env:
       IMAGE_REPO: 'localhost:5000'
       REGISTRY: 'localhost:5000'
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go
-      uses: actions/setup-go@v2
+    - uses: actions/setup-go@v2
       with:
-        go-version: 1.14
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
+        go-version: 1.17
+    - uses: actions/checkout@v2
+    - uses: docker/setup-buildx-action@v1
 
     - name: Run integration-test-layers
       run: |

.github/workflows/integration-misc.yaml

@@ -1,14 +1,15 @@
 name: Integration Tests Misc
 
-# Triggers the workflow on push or pull request events
-on: [pull_request]
+on:
+  pull_request:
+    branches: ['master']
 
 concurrency:
   group: int-test-mis-${{ github.head_ref }}
   cancel-in-progress: true
 
 jobs:
-  build-executor:
+  tests:
     env:
       IMAGE_REPO: 'localhost:5000'
       REGISTRY: 'localhost:5000'
@@ -17,25 +18,20 @@ jobs:
       TRAVIS_PULL_REQUEST: ${{ github.event.number }}
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.14
-        id: go
+    - uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+    - uses: actions/checkout@v2
+    - uses: docker/setup-buildx-action@v1
 
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
+    - name: Print Travis ENV vars
+      run: |
+        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST: ${TRAVIS_PULL_REQUEST}"
+        echo "TRAVIS_REPO_SLUG: ${TRAVIS_REPO_SLUG}"
 
-      - name: Print Travis ENV vars
-        run: |
-          echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-          echo "TRAVIS_PULL_REQUEST: ${TRAVIS_PULL_REQUEST}"
-          echo "TRAVIS_REPO_SLUG: ${TRAVIS_REPO_SLUG}"
-
-      - name: Run integration-test-misc
-        run : |
-          make travis-setup
-          make minikube-setup
-          make integration-test-misc
+    - name: Run integration-test-misc
+      run : |
+        make travis-setup
+        make minikube-setup
+        make integration-test-misc

.github/workflows/integration-run-tests.yaml

@@ -1,31 +1,28 @@
 name: Integration tests (Run)
 
-# Triggers the workflow on push or pull request events
-on: [pull_request]
+on:
+  pull_request:
+    branches: ['master']
 
 concurrency:
   group: int-test-run-${{ github.head_ref }}
   cancel-in-progress: true
 
 jobs:
-  build-executor:
+  tests:
     env:
       IMAGE_REPO: 'localhost:5000'
       REGISTRY: 'localhost:5000'
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.14
-        id: go
+    - uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+    - uses: actions/checkout@v2
+    - uses: docker/setup-buildx-action@v1
 
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Run integration-test-run
-        run: |
-          make travis-setup
-          make minikube-setup
-          make integration-test-run
+    - name: Run integration-test-run
+      run: |
+        make travis-setup
+        make minikube-setup
+        make integration-test-run

.github/workflows/integration_k8_tests.yaml

@@ -1,33 +0,0 @@
-name: Integration tests (K8s)
-
-# Triggers the workflow on pull request events
-on: [pull_request]
-
-concurrency:
-  group: int-test-k8s${{ github.head_ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-executor:
-    env:
-      IMAGE_REPO: 'localhost:5000'
-      REGISTRY: 'localhost:5000'
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.14
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: Run build-image-and-k8s-integration-test
-      run: |
-        make travis-setup
-        make minikube-setup
-        make k8s-executor-build-push
-        make integration-test-k8s

.github/workflows/unit-tests.yaml

@@ -1,30 +1,20 @@
 name: Unit tests
 
-# Triggers the workflow on push or pull request events
-on: [pull_request]
+on:
+  pull_request:
+    branches: ['master']
 
 concurrency:
   group: unit-tests-${{ github.head_ref }}
   cancel-in-progress: true
 
 jobs:
-  build-executor:
-    env:
-      IMAGE_REPO: 'localhost:5000'
-      REGISTRY: 'localhost:5000'
+  tests:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.14
-        id: go
+    - uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+    - uses: actions/checkout@v2
 
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Run unit-test
-        run:
-          make test
+    - run: make test

Makefile

@@ -88,7 +88,7 @@ integration-test-misc:
 
 .PHONY: k8s-executor-build-push
 k8s-executor-build-push:
-	docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:latest -f deploy/Dockerfile .
+	DOCKER_BUILDKIT=1 docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:latest -f deploy/Dockerfile .
 	docker push $(REGISTRY)/executor:latest
 
 .PHONY: images

deploy/Dockerfile

@@ -35,7 +35,10 @@ RUN go install github.com/chrismellard/docker-credential-acr-env@09e2b5a8ac86c3e
 RUN mkdir -p /kaniko/.docker
 
 COPY . .
-RUN make GOARCH=$TARGETARCH
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg \
+    make GOARCH=$TARGETARCH
 
 # Generate latest ca-certificates
 

deploy/Dockerfile_debug

@@ -35,8 +35,11 @@ RUN go install github.com/chrismellard/docker-credential-acr-env@09e2b5a8ac86c3e
 RUN mkdir -p /kaniko/.docker
 
 COPY . .
-RUN make GOARCH=$TARGETARCH
-RUN make GOARCH=$TARGETARCH out/warmer
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg \
+    make GOARCH=$TARGETARCH && \
+    make GOARCH=$TARGETARCH out/warmer
 
 # Generate latest ca-certificates
 

deploy/Dockerfile_slim

@@ -21,7 +21,10 @@ ARG TARGETARCH
 WORKDIR /src
 COPY . .
 
-RUN make GOARCH=$TARGETARCH
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg \
+    make GOARCH=$TARGETARCH
 
 # Generate latest ca-certificates
 

deploy/Dockerfile_warmer

@@ -35,7 +35,10 @@ RUN go install github.com/chrismellard/docker-credential-acr-env@09e2b5a8ac86c3e
 RUN mkdir -p /kaniko/.docker
 
 COPY . .
-RUN make GOARCH=$TARGETARCH out/warmer
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg \
+    make GOARCH=$TARGETARCH out/warmer
 
 # Generate latest ca-certificates
 

integration/integration_test.go

@@ -164,6 +164,7 @@ func buildRequiredImages() error {
 	for _, setupCmd := range setupCommands {
 		fmt.Println(setupCmd.name)
 		cmd := exec.Command(setupCmd.command[0], setupCmd.command[1:]...)
+		cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1") // Build with buildkit enabled.
 		if out, err := RunCommandWithoutTest(cmd); err != nil {
 			return errors.Wrap(err, fmt.Sprintf("%s failed: %s", setupCmd.name, string(out)))
 		}

scripts/integration-test.sh

@@ -20,7 +20,6 @@ IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}"
 
 docker version
 
-# Sets up a kokoro (Google internal integration testing tool) environment
 echo "Running integration tests..."
 make out/executor
 make out/warmer