From effac9dfc34f3bbe7217d37221688b71a72ac33f Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 11 Oct 2018 13:38:05 -0700 Subject: [PATCH] Persistent volume caching for base images (#383) * comments * initial commit for persisent volume caching * cache warmer works * general cleanup * adding some debugging * adding missing files * Fixing up cache retrieval and cleanup * fix tests * removing auth since we only cache public images * simplifying the caching logic * fixing logic * adding volume cache to integration tests. remove auth from cache warmer image. * add building warmer to integration-test * move sample yaml files to examples dir * small test fix --- Makefile | 5 +++ cmd/executor/cmd/root.go | 1 + cmd/warmer/cmd/root.go | 74 +++++++++++++++++++++++++++++++ cmd/warmer/main.go | 29 ++++++++++++ deploy/Dockerfile_warmer | 32 +++++++++++++ examples/kaniko-cache-claim.yaml | 11 +++++ examples/kaniko-cache-volume.yaml | 14 ++++++ examples/kaniko-test.yaml | 30 +++++++++++++ examples/kaniko-warmer.yaml | 27 +++++++++++ integration-test.sh | 1 + integration/images.go | 26 ++++++++++- integration/integration_test.go | 8 ++++ pkg/cache/cache.go | 19 ++++++++ pkg/cache/warm.go | 61 +++++++++++++++++++++++++ pkg/config/options.go | 7 +++ pkg/executor/build.go | 2 +- pkg/util/image_util.go | 42 +++++++++++++++++- pkg/util/image_util_test.go | 6 +-- 18 files changed, 389 insertions(+), 6 deletions(-) create mode 100644 cmd/warmer/cmd/root.go create mode 100644 cmd/warmer/main.go create mode 100644 deploy/Dockerfile_warmer create mode 100644 examples/kaniko-cache-claim.yaml create mode 100644 examples/kaniko-cache-volume.yaml create mode 100644 examples/kaniko-test.yaml create mode 100644 examples/kaniko-warmer.yaml create mode 100644 pkg/cache/warm.go diff --git a/Makefile b/Makefile index 8d20c2b03..94c18ad40 100644 --- a/Makefile +++ b/Makefile @@ -35,11 +35,15 @@ GO_LDFLAGS += -w -s # Drop debugging symbols. GO_LDFLAGS += ' EXECUTOR_PACKAGE = $(REPOPATH)/cmd/executor +WARMER_PACKAGE = $(REPOPATH)/cmd/warmer KANIKO_PROJECT = $(REPOPATH)/kaniko out/executor: $(GO_FILES) GOARCH=$(GOARCH) GOOS=linux CGO_ENABLED=0 go build -ldflags $(GO_LDFLAGS) -o $@ $(EXECUTOR_PACKAGE) +out/warmer: $(GO_FILES) + GOARCH=$(GOARCH) GOOS=linux CGO_ENABLED=0 go build -ldflags $(GO_LDFLAGS) -o $@ $(WARMER_PACKAGE) + .PHONY: test test: out/executor @ ./test.sh @@ -52,3 +56,4 @@ integration-test: images: docker build -t $(REGISTRY)/executor:latest -f deploy/Dockerfile . docker build -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug . + docker build -t $(REGISTRY)/warmer:latest -f deploy/Dockerfile_warmer . diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index bdd4e52c6..55d256f28 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -99,6 +99,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().StringVarP(&opts.Target, "target", "", "", "Set the target build stage to build") RootCmd.PersistentFlags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry") RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided") + RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.") RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end") } diff --git a/cmd/warmer/cmd/root.go b/cmd/warmer/cmd/root.go new file mode 100644 index 000000000..0e4908d2b --- /dev/null +++ b/cmd/warmer/cmd/root.go @@ -0,0 +1,74 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "os" + + "github.com/GoogleContainerTools/kaniko/pkg/cache" + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/pkg/constants" + "github.com/GoogleContainerTools/kaniko/pkg/util" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var ( + opts = &config.WarmerOptions{} + logLevel string +) + +func init() { + RootCmd.PersistentFlags().StringVarP(&logLevel, "verbosity", "v", constants.DefaultLogLevel, "Log level (debug, info, warn, error, fatal, panic") + addKanikoOptionsFlags(RootCmd) + addHiddenFlags(RootCmd) +} + +var RootCmd = &cobra.Command{ + Use: "cache warmer", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if err := util.ConfigureLogging(logLevel); err != nil { + return err + } + if len(opts.Images) == 0 { + return errors.New("You must select at least one image to cache") + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + if err := cache.WarmCache(opts); err != nil { + exit(errors.Wrap(err, "Failed warming cache")) + } + }, +} + +// addKanikoOptionsFlags configures opts +func addKanikoOptionsFlags(cmd *cobra.Command) { + RootCmd.PersistentFlags().VarP(&opts.Images, "image", "i", "Image to cache. Set it repeatedly for multiple images.") + RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "c", "/cache", "Directory of the cache.") +} + +// addHiddenFlags marks certain flags as hidden from the executor help text +func addHiddenFlags(cmd *cobra.Command) { + RootCmd.PersistentFlags().MarkHidden("azure-container-registry-config") +} + +func exit(err error) { + fmt.Println(err) + os.Exit(1) +} diff --git a/cmd/warmer/main.go b/cmd/warmer/main.go new file mode 100644 index 000000000..c91eba44a --- /dev/null +++ b/cmd/warmer/main.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + + "github.com/GoogleContainerTools/kaniko/cmd/warmer/cmd" +) + +func main() { + if err := cmd.RootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer new file mode 100644 index 000000000..2cee0968c --- /dev/null +++ b/deploy/Dockerfile_warmer @@ -0,0 +1,32 @@ +# Copyright 2018 Google, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds the static Go image to execute in a Kubernetes job + +FROM golang:1.10 +WORKDIR /go/src/github.com/GoogleContainerTools/kaniko +COPY . . +RUN make + +FROM scratch +COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/warmer /kaniko/warmer +COPY files/ca-certificates.crt /kaniko/ssl/certs/ +COPY files/config.json /kaniko/.docker/ +ENV HOME /root +ENV USER /root +ENV PATH /usr/local/bin:/kaniko +ENV SSL_CERT_DIR=/kaniko/ssl/certs +ENV DOCKER_CONFIG /kaniko/.docker/ +WORKDIR /workspace +ENTRYPOINT ["/kaniko/warmer"] diff --git a/examples/kaniko-cache-claim.yaml b/examples/kaniko-cache-claim.yaml new file mode 100644 index 000000000..dc30c3a8d --- /dev/null +++ b/examples/kaniko-cache-claim.yaml @@ -0,0 +1,11 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kaniko-cache-claim +spec: + storageClassName: manual + accessModes: + - ReadOnlyMany + resources: + requests: + storage: 8Gi diff --git a/examples/kaniko-cache-volume.yaml b/examples/kaniko-cache-volume.yaml new file mode 100644 index 000000000..700aa2999 --- /dev/null +++ b/examples/kaniko-cache-volume.yaml @@ -0,0 +1,14 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: kaniko-cache-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadOnlyMany + hostPath: + path: "/tmp/kaniko-cache" diff --git a/examples/kaniko-test.yaml b/examples/kaniko-test.yaml new file mode 100644 index 000000000..d6f990433 --- /dev/null +++ b/examples/kaniko-test.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kaniko +spec: + containers: + - name: kaniko + image: gcr.io/kaniko-project/executor:latest + args: ["--dockerfile=", + "--context=", + "--destination=", + "--cache", + "--cache-dir=/cache"] + volumeMounts: + - name: kaniko-secret + mountPath: /secret + - name: kaniko-cache + mountPath: /cache + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/kaniko-secret.json + restartPolicy: Never + volumes: + - name: kaniko-secret + secret: + secretName: kaniko-secret + - name: kaniko-cache + persistentVolumeClaim: + claimName: kaniko-cache-claim + diff --git a/examples/kaniko-warmer.yaml b/examples/kaniko-warmer.yaml new file mode 100644 index 000000000..318f62878 --- /dev/null +++ b/examples/kaniko-warmer.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kaniko-warmer +spec: + containers: + - name: kaniko-warmer + image: gcr.io/kaniko-project/warmer:latest + args: ["--cache-dir=/cache", + "--image=gcr.io/google-appengine/debian9"] + volumeMounts: + - name: kaniko-secret + mountPath: /secret + - name: kaniko-cache + mountPath: /cache + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/kaniko-secret.json + restartPolicy: Never + volumes: + - name: kaniko-secret + secret: + secretName: kaniko-secret + - name: kaniko-cache + persistentVolumeClaim: + claimName: kaniko-cache-claim + diff --git a/integration-test.sh b/integration-test.sh index 51fcb6e20..3217f767a 100755 --- a/integration-test.sh +++ b/integration-test.sh @@ -34,5 +34,6 @@ fi echo "Running integration tests..." make out/executor +make out/warmer pushd integration go test -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 30m diff --git a/integration/images.go b/integration/images.go index cf1f90105..464db8cc8 100644 --- a/integration/images.go +++ b/integration/images.go @@ -30,10 +30,13 @@ import ( const ( // ExecutorImage is the name of the kaniko executor image ExecutorImage = "executor-image" + WarmerImage = "warmer-image" dockerPrefix = "docker-" kanikoPrefix = "kaniko-" buildContextPath = "/workspace" + cacheDir = "/workspace/cache" + baseImageToCache = "gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0" ) // Arguments to build Dockerfiles with, used for both docker and kaniko builds @@ -201,6 +204,26 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do return nil } +func populateVolumeCache() error { + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + warmerCmd := exec.Command("docker", + append([]string{"run", + "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", + "-v", cwd + ":/workspace", + WarmerImage, + "-c", cacheDir, + "-i", baseImageToCache}, + )..., + ) + + if _, err := RunCommandWithoutTest(warmerCmd); err != nil { + return fmt.Errorf("Failed to warm kaniko cache: %s", err) + } + + return nil +} + // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath string, version int) error { _, ex, _, _ := runtime.Caller(0) @@ -219,7 +242,8 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesP "-d", kanikoImage, "-c", buildContextPath, cacheFlag, - "--cache-repo", cacheRepo})..., + "--cache-repo", cacheRepo, + "--cache-dir", cacheDir})..., ) if _, err := RunCommandWithoutTest(kanikoCmd); err != nil { diff --git a/integration/integration_test.go b/integration/integration_test.go index e6e4f8959..313d8f053 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -145,6 +145,13 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Println("Building cache warmer image") + cmd = exec.Command("docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", "..") + if _, err = RunCommandWithoutTest(cmd); err != nil { + fmt.Printf("Building kaniko's cache warmer failed: %s", err) + os.Exit(1) + } + fmt.Println("Building onbuild base image") buildOnbuildBase := exec.Command("docker", "build", "-t", config.onbuildBaseImage, "-f", "dockerfiles/Dockerfile_onbuild_base", ".") if err := buildOnbuildBase.Run(); err != nil { @@ -238,6 +245,7 @@ func TestLayers(t *testing.T) { // Build each image with kaniko twice, and then make sure they're exactly the same func TestCache(t *testing.T) { + populateVolumeCache() for dockerfile := range imageBuilder.TestCacheDockerfiles { t.Run("test_cache_"+dockerfile, func(t *testing.T) { cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano())) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 69682ba48..fca66e631 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -18,6 +18,7 @@ package cache import ( "fmt" + "path" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/google/go-containerregistry/pkg/authn" @@ -25,6 +26,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -68,3 +70,20 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { } return fmt.Sprintf("%s:%s", cache, cacheKey), nil } + +func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { + cache := opts.CacheDir + if cache == "" { + return nil, nil + } + + path := path.Join(cache, cacheKey) + + imgTar, err := tarball.ImageFromPath(path, nil) + if err != nil { + return nil, errors.Wrap(err, "getting image from path") + } + + logrus.Infof("Found %s in local cache", cacheKey) + return imgTar, nil +} diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go new file mode 100644 index 000000000..fc7b6b6c8 --- /dev/null +++ b/pkg/cache/warm.go @@ -0,0 +1,61 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "fmt" + "path" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func WarmCache(opts *config.WarmerOptions) error { + cacheDir := opts.CacheDir + images := opts.Images + logrus.Debugf("%s\n", cacheDir) + logrus.Debugf("%s\n", images) + + for _, image := range images { + cacheRef, err := name.NewTag(image, name.WeakValidation) + if err != nil { + errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image)) + } + img, err := remote.Image(cacheRef) + if err != nil { + errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image)) + } + + digest, err := img.Digest() + if err != nil { + errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image)) + } + cachePath := path.Join(cacheDir, digest.String()) + err = tarball.WriteToFile(cachePath, cacheRef, img) + if err != nil { + errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image)) + } else { + logrus.Debugf("Wrote %s to cache", image) + } + + } + return nil +} diff --git a/pkg/config/options.go b/pkg/config/options.go index 26fecec29..fc15c1f21 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -25,6 +25,7 @@ type KanikoOptions struct { TarPath string Target string CacheRepo string + CacheDir string Destinations multiArg BuildArgs multiArg InsecurePush bool @@ -35,3 +36,9 @@ type KanikoOptions struct { Cache bool Cleanup bool } + +// WarmerOptions are options that are set by command line arguments to the cache warmer. +type WarmerOptions struct { + Images multiArg + CacheDir string +} diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 1f90037d8..7d991069d 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -54,7 +54,7 @@ type stageBuilder struct { // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*stageBuilder, error) { - sourceImage, err := util.RetrieveSourceImage(stage, opts.BuildArgs) + sourceImage, err := util.RetrieveSourceImage(stage, opts.BuildArgs, opts) if err != nil { return nil, err } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index d19e684f6..6eb41aa57 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -30,6 +30,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/sirupsen/logrus" + "github.com/GoogleContainerTools/kaniko/pkg/cache" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/constants" ) @@ -41,7 +42,7 @@ var ( ) // RetrieveSourceImage returns the base image of the stage at index -func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string) (v1.Image, error) { +func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string, opts *config.KanikoOptions) (v1.Image, error) { currentBaseName, err := ResolveEnvironmentReplacement(stage.BaseName, buildArgs, false) if err != nil { return nil, err @@ -57,6 +58,19 @@ func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string) (v1.Image return retrieveTarImage(stage.BaseImageIndex) } + // Next, check if local caching is enabled + // If so, look in the local cache before trying the remote registry + if opts.Cache && opts.CacheDir != "" { + cachedImage, err := cachedImage(opts, currentBaseName) + if cachedImage != nil { + return cachedImage, nil + } + + if err != nil { + logrus.Warnf("Error while retrieving image from cache: %v", err) + } + } + // Otherwise, initialize image as usual return retrieveRemoteImage(currentBaseName) } @@ -92,3 +106,29 @@ func remoteImage(image string) (v1.Image, error) { kc := authn.NewMultiKeychain(authn.DefaultKeychain, k8sc) return remote.Image(ref, remote.WithAuthFromKeychain(kc)) } + +func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { + ref, err := name.ParseReference(image, name.WeakValidation) + if err != nil { + return nil, err + } + + var cacheKey string + if d, ok := ref.(name.Digest); ok { + cacheKey = d.DigestStr() + } else { + img, err := remote.Image(ref) + if err != nil { + return nil, err + } + + d, err := img.Digest() + if err != nil { + return nil, err + } + + cacheKey = d.String() + } + + return cache.LocalSource(opts, cacheKey) +} diff --git a/pkg/util/image_util_test.go b/pkg/util/image_util_test.go index dbd7b8ee1..44897fe72 100644 --- a/pkg/util/image_util_test.go +++ b/pkg/util/image_util_test.go @@ -57,7 +57,7 @@ func Test_StandardImage(t *testing.T) { retrieveRemoteImage = mock actual, err := RetrieveSourceImage(config.KanikoStage{ Stage: stages[0], - }, nil) + }, nil, &config.KanikoOptions{}) testutil.CheckErrorAndDeepEqual(t, false, err, nil, actual) } func Test_ScratchImage(t *testing.T) { @@ -67,7 +67,7 @@ func Test_ScratchImage(t *testing.T) { } actual, err := RetrieveSourceImage(config.KanikoStage{ Stage: stages[1], - }, nil) + }, nil, &config.KanikoOptions{}) expected := empty.Image testutil.CheckErrorAndDeepEqual(t, false, err, expected, actual) } @@ -89,7 +89,7 @@ func Test_TarImage(t *testing.T) { BaseImageStoredLocally: true, BaseImageIndex: 0, Stage: stages[2], - }, nil) + }, nil, &config.KanikoOptions{}) testutil.CheckErrorAndDeepEqual(t, false, err, nil, actual) }