Persistent volume caching for base images (#383)

* comments

* initial commit for persisent volume caching

* cache warmer works

* general cleanup

* adding some debugging

* adding missing files

* Fixing up cache retrieval and cleanup

* fix tests

* removing auth since we only cache public images

* simplifying the caching logic

* fixing logic

* adding volume cache to integration tests. remove auth from cache warmer image.

* add building warmer to integration-test

* move sample yaml files to examples dir

* small test fix

Makefile

@@ -35,11 +35,15 @@ GO_LDFLAGS += -w -s # Drop debugging symbols.
 GO_LDFLAGS += '
 
 EXECUTOR_PACKAGE = $(REPOPATH)/cmd/executor
+WARMER_PACKAGE = $(REPOPATH)/cmd/warmer
 KANIKO_PROJECT = $(REPOPATH)/kaniko
 
 out/executor: $(GO_FILES)
 	GOARCH=$(GOARCH) GOOS=linux CGO_ENABLED=0 go build -ldflags $(GO_LDFLAGS) -o $@ $(EXECUTOR_PACKAGE)
 
+out/warmer: $(GO_FILES)
+	GOARCH=$(GOARCH) GOOS=linux CGO_ENABLED=0 go build -ldflags $(GO_LDFLAGS) -o $@ $(WARMER_PACKAGE)
+
 .PHONY: test
 test: out/executor
 	@ ./test.sh
@@ -52,3 +56,4 @@ integration-test:
 images:
 	docker build -t $(REGISTRY)/executor:latest -f deploy/Dockerfile .
 	docker build -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug .
+	docker build -t $(REGISTRY)/warmer:latest -f deploy/Dockerfile_warmer .

cmd/executor/cmd/root.go

@@ -99,6 +99,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) {
 	RootCmd.PersistentFlags().StringVarP(&opts.Target, "target", "", "", "Set the target build stage to build")
 	RootCmd.PersistentFlags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry")
 	RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided")
+	RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image")
 	RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end")
 }

cmd/warmer/cmd/root.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/cache"
+	"github.com/GoogleContainerTools/kaniko/pkg/config"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+var (
+	opts     = &config.WarmerOptions{}
+	logLevel string
+)
+
+func init() {
+	RootCmd.PersistentFlags().StringVarP(&logLevel, "verbosity", "v", constants.DefaultLogLevel, "Log level (debug, info, warn, error, fatal, panic")
+	addKanikoOptionsFlags(RootCmd)
+	addHiddenFlags(RootCmd)
+}
+
+var RootCmd = &cobra.Command{
+	Use: "cache warmer",
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		if err := util.ConfigureLogging(logLevel); err != nil {
+			return err
+		}
+		if len(opts.Images) == 0 {
+			return errors.New("You must select at least one image to cache")
+		}
+		return nil
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := cache.WarmCache(opts); err != nil {
+			exit(errors.Wrap(err, "Failed warming cache"))
+		}
+	},
+}
+
+// addKanikoOptionsFlags configures opts
+func addKanikoOptionsFlags(cmd *cobra.Command) {
+	RootCmd.PersistentFlags().VarP(&opts.Images, "image", "i", "Image to cache. Set it repeatedly for multiple images.")
+	RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "c", "/cache", "Directory of the cache.")
+}
+
+// addHiddenFlags marks certain flags as hidden from the executor help text
+func addHiddenFlags(cmd *cobra.Command) {
+	RootCmd.PersistentFlags().MarkHidden("azure-container-registry-config")
+}
+
+func exit(err error) {
+	fmt.Println(err)
+	os.Exit(1)
+}

cmd/warmer/main.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+
+	"github.com/GoogleContainerTools/kaniko/cmd/warmer/cmd"
+)
+
+func main() {
+	if err := cmd.RootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

deploy/Dockerfile_warmer

@@ -0,0 +1,32 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds the static Go image to execute in a Kubernetes job
+
+FROM golang:1.10
+WORKDIR /go/src/github.com/GoogleContainerTools/kaniko
+COPY . .
+RUN make
+
+FROM scratch
+COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/warmer /kaniko/warmer
+COPY files/ca-certificates.crt /kaniko/ssl/certs/
+COPY files/config.json /kaniko/.docker/
+ENV HOME /root
+ENV USER /root
+ENV PATH /usr/local/bin:/kaniko
+ENV SSL_CERT_DIR=/kaniko/ssl/certs
+ENV DOCKER_CONFIG /kaniko/.docker/
+WORKDIR /workspace
+ENTRYPOINT ["/kaniko/warmer"]

examples/kaniko-cache-claim.yaml

@@ -0,0 +1,11 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: kaniko-cache-claim
+spec:
+  storageClassName: manual
+  accessModes:
+    - ReadOnlyMany
+  resources:
+    requests:
+      storage: 8Gi

examples/kaniko-cache-volume.yaml

@@ -0,0 +1,14 @@
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: kaniko-cache-volume
+  labels:
+    type: local
+spec:
+  storageClassName: manual
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadOnlyMany
+  hostPath:
+    path: "/tmp/kaniko-cache"

examples/kaniko-test.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kaniko
+spec:
+  containers:
+  - name: kaniko
+    image: gcr.io/kaniko-project/executor:latest
+    args: ["--dockerfile=<dockerfile>",
+            "--context=<context>",
+            "--destination=<destination>",
+            "--cache",
+            "--cache-dir=/cache"]
+    volumeMounts:
+      - name: kaniko-secret
+        mountPath: /secret
+      - name: kaniko-cache
+        mountPath: /cache
+    env:
+      - name: GOOGLE_APPLICATION_CREDENTIALS
+        value: /secret/kaniko-secret.json
+  restartPolicy: Never
+  volumes:
+    - name: kaniko-secret
+      secret:
+        secretName: kaniko-secret
+    - name: kaniko-cache
+      persistentVolumeClaim:
+              claimName: kaniko-cache-claim
+

examples/kaniko-warmer.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kaniko-warmer
+spec:
+  containers:
+  - name: kaniko-warmer
+    image: gcr.io/kaniko-project/warmer:latest
+    args: ["--cache-dir=/cache",
+           "--image=gcr.io/google-appengine/debian9"]
+    volumeMounts:
+      - name: kaniko-secret
+        mountPath: /secret
+      - name: kaniko-cache
+        mountPath: /cache
+    env:
+      - name: GOOGLE_APPLICATION_CREDENTIALS
+        value: /secret/kaniko-secret.json
+  restartPolicy: Never
+  volumes:
+    - name: kaniko-secret
+      secret:
+        secretName: kaniko-secret
+    - name: kaniko-cache
+      persistentVolumeClaim:
+              claimName: kaniko-cache-claim
+

integration-test.sh

@@ -34,5 +34,6 @@ fi
 
 echo "Running integration tests..."
 make out/executor
+make out/warmer
 pushd integration
 go test -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 30m

integration/images.go

@@ -30,10 +30,13 @@ import (
 const (
 	// ExecutorImage is the name of the kaniko executor image
 	ExecutorImage = "executor-image"
+	WarmerImage   = "warmer-image"
 
 	dockerPrefix     = "docker-"
 	kanikoPrefix     = "kaniko-"
 	buildContextPath = "/workspace"
+	cacheDir         = "/workspace/cache"
+	baseImageToCache = "gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0"
 )
 
 // Arguments to build Dockerfiles with, used for both docker and kaniko builds
@@ -201,6 +204,26 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do
 	return nil
 }
 
+func populateVolumeCache() error {
+	_, ex, _, _ := runtime.Caller(0)
+	cwd := filepath.Dir(ex)
+	warmerCmd := exec.Command("docker",
+		append([]string{"run",
+			"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
+			"-v", cwd + ":/workspace",
+			WarmerImage,
+			"-c", cacheDir,
+			"-i", baseImageToCache},
+		)...,
+	)
+
+	if _, err := RunCommandWithoutTest(warmerCmd); err != nil {
+		return fmt.Errorf("Failed to warm kaniko cache: %s", err)
+	}
+
+	return nil
+}
+
 // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
 func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath string, version int) error {
 	_, ex, _, _ := runtime.Caller(0)
@@ -219,7 +242,8 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesP
 				"-d", kanikoImage,
 				"-c", buildContextPath,
 				cacheFlag,
-				"--cache-repo", cacheRepo})...,
+				"--cache-repo", cacheRepo,
+				"--cache-dir", cacheDir})...,
 		)
 
 		if _, err := RunCommandWithoutTest(kanikoCmd); err != nil {

integration/integration_test.go

@@ -145,6 +145,13 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 
+	fmt.Println("Building cache warmer image")
+	cmd = exec.Command("docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", "..")
+	if _, err = RunCommandWithoutTest(cmd); err != nil {
+		fmt.Printf("Building kaniko's cache warmer failed: %s", err)
+		os.Exit(1)
+	}
+
 	fmt.Println("Building onbuild base image")
 	buildOnbuildBase := exec.Command("docker", "build", "-t", config.onbuildBaseImage, "-f", "dockerfiles/Dockerfile_onbuild_base", ".")
 	if err := buildOnbuildBase.Run(); err != nil {
@@ -238,6 +245,7 @@ func TestLayers(t *testing.T) {
 
 // Build each image with kaniko twice, and then make sure they're exactly the same
 func TestCache(t *testing.T) {
+	populateVolumeCache()
 	for dockerfile := range imageBuilder.TestCacheDockerfiles {
 		t.Run("test_cache_"+dockerfile, func(t *testing.T) {
 			cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano()))

pkg/cache/cache.go

@@ -18,6 +18,7 @@ package cache
 
 import (
 	"fmt"
+	"path"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/google/go-containerregistry/pkg/authn"
@@ -25,6 +26,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -68,3 +70,20 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) {
 	}
 	return fmt.Sprintf("%s:%s", cache, cacheKey), nil
 }
+
+func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) {
+	cache := opts.CacheDir
+	if cache == "" {
+		return nil, nil
+	}
+
+	path := path.Join(cache, cacheKey)
+
+	imgTar, err := tarball.ImageFromPath(path, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting image from path")
+	}
+
+	logrus.Infof("Found %s in local cache", cacheKey)
+	return imgTar, nil
+}

pkg/cache/warm.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"fmt"
+	"path"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/config"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+func WarmCache(opts *config.WarmerOptions) error {
+	cacheDir := opts.CacheDir
+	images := opts.Images
+	logrus.Debugf("%s\n", cacheDir)
+	logrus.Debugf("%s\n", images)
+
+	for _, image := range images {
+		cacheRef, err := name.NewTag(image, name.WeakValidation)
+		if err != nil {
+			errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image))
+		}
+		img, err := remote.Image(cacheRef)
+		if err != nil {
+			errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image))
+		}
+
+		digest, err := img.Digest()
+		if err != nil {
+			errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image))
+		}
+		cachePath := path.Join(cacheDir, digest.String())
+		err = tarball.WriteToFile(cachePath, cacheRef, img)
+		if err != nil {
+			errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image))
+		} else {
+			logrus.Debugf("Wrote %s to cache", image)
+		}
+
+	}
+	return nil
+}

pkg/config/options.go

@@ -25,6 +25,7 @@ type KanikoOptions struct {
 	TarPath        string
 	Target         string
 	CacheRepo      string
+	CacheDir       string
 	Destinations   multiArg
 	BuildArgs      multiArg
 	InsecurePush   bool
@@ -35,3 +36,9 @@ type KanikoOptions struct {
 	Cache          bool
 	Cleanup        bool
 }
+
+// WarmerOptions are options that are set by command line arguments to the cache warmer.
+type WarmerOptions struct {
+	Images   multiArg
+	CacheDir string
+}

pkg/executor/build.go

@@ -54,7 +54,7 @@ type stageBuilder struct {
 
 // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage
 func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*stageBuilder, error) {
-	sourceImage, err := util.RetrieveSourceImage(stage, opts.BuildArgs)
+	sourceImage, err := util.RetrieveSourceImage(stage, opts.BuildArgs, opts)
 	if err != nil {
 		return nil, err
 	}

pkg/util/image_util.go

@@ -30,6 +30,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/sirupsen/logrus"
 
+	"github.com/GoogleContainerTools/kaniko/pkg/cache"
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 )
@@ -41,7 +42,7 @@ var (
 )
 
 // RetrieveSourceImage returns the base image of the stage at index
-func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string) (v1.Image, error) {
+func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string, opts *config.KanikoOptions) (v1.Image, error) {
 	currentBaseName, err := ResolveEnvironmentReplacement(stage.BaseName, buildArgs, false)
 	if err != nil {
 		return nil, err
@@ -57,6 +58,19 @@ func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string) (v1.Image
 		return retrieveTarImage(stage.BaseImageIndex)
 	}
 
+	// Next, check if local caching is enabled
+	// If so, look in the local cache before trying the remote registry
+	if opts.Cache && opts.CacheDir != "" {
+		cachedImage, err := cachedImage(opts, currentBaseName)
+		if cachedImage != nil {
+			return cachedImage, nil
+		}
+
+		if err != nil {
+			logrus.Warnf("Error while retrieving image from cache: %v", err)
+		}
+	}
+
 	// Otherwise, initialize image as usual
 	return retrieveRemoteImage(currentBaseName)
 }
@@ -92,3 +106,29 @@ func remoteImage(image string) (v1.Image, error) {
 	kc := authn.NewMultiKeychain(authn.DefaultKeychain, k8sc)
 	return remote.Image(ref, remote.WithAuthFromKeychain(kc))
 }
+
+func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) {
+	ref, err := name.ParseReference(image, name.WeakValidation)
+	if err != nil {
+		return nil, err
+	}
+
+	var cacheKey string
+	if d, ok := ref.(name.Digest); ok {
+		cacheKey = d.DigestStr()
+	} else {
+		img, err := remote.Image(ref)
+		if err != nil {
+			return nil, err
+		}
+
+		d, err := img.Digest()
+		if err != nil {
+			return nil, err
+		}
+
+		cacheKey = d.String()
+	}
+
+	return cache.LocalSource(opts, cacheKey)
+}

pkg/util/image_util_test.go

@@ -57,7 +57,7 @@ func Test_StandardImage(t *testing.T) {
 	retrieveRemoteImage = mock
 	actual, err := RetrieveSourceImage(config.KanikoStage{
 		Stage: stages[0],
-	}, nil)
+	}, nil, &config.KanikoOptions{})
 	testutil.CheckErrorAndDeepEqual(t, false, err, nil, actual)
 }
 func Test_ScratchImage(t *testing.T) {
@@ -67,7 +67,7 @@ func Test_ScratchImage(t *testing.T) {
 	}
 	actual, err := RetrieveSourceImage(config.KanikoStage{
 		Stage: stages[1],
-	}, nil)
+	}, nil, &config.KanikoOptions{})
 	expected := empty.Image
 	testutil.CheckErrorAndDeepEqual(t, false, err, expected, actual)
 }
@@ -89,7 +89,7 @@ func Test_TarImage(t *testing.T) {
 		BaseImageStoredLocally: true,
 		BaseImageIndex:         0,
 		Stage:                  stages[2],
-	}, nil)
+	}, nil, &config.KanikoOptions{})
 	testutil.CheckErrorAndDeepEqual(t, false, err, nil, actual)
 }