From c216fbf91b8d6771c416104e7537d37a3920ad6c Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 13 Sep 2018 18:01:43 -0700 Subject: [PATCH 1/5] Add layer caching to kaniko To add layer caching to kaniko, I added two flags: --cache and --use-cache. If --use-cache is set, then the cache will be used, and if --cache is specified then that repo will be used to store cached layers. If --cache isn't set, a cache will be inferred from the destination provided. Currently, caching only works for RUN commands. Before executing the command, kaniko checks if the cached layer exists. If it does, it pulls it and extracts it. It then adds those files to the snapshotter and append a layer to the config history. If the cached layer does not exist, kaniko executes the command and pushes the newly created layer to the cache. All cached layers are tagged with a stable key, which is built based off of: 1. The base image digest 2. The current state of the filesystem 3. The current command being run 4. The current config file (to account for metadata changes) I also added two integration tests to make sure caching works 1. Dockerfile_test_cache runs 'date', which should be exactly the same the second time the image is built 2. Dockerfile_test_cache_install makes sure apt-get install can be reproduced --- cmd/executor/cmd/root.go | 2 + integration/dockerfiles/Dockerfile_test_cache | 22 +++++ .../dockerfiles/Dockerfile_test_cache_install | 20 +++++ integration/images.go | 53 ++++++++++-- integration/integration_test.go | 84 ++++++++++++++----- pkg/cache/cache.go | 70 ++++++++++++++++ pkg/config/options.go | 2 + pkg/executor/build.go | 75 ++++++++++++++--- pkg/executor/push.go | 30 +++++++ pkg/util/fs_util.go | 26 +++--- 10 files changed, 338 insertions(+), 46 deletions(-) create mode 100644 integration/dockerfiles/Dockerfile_test_cache create mode 100644 integration/dockerfiles/Dockerfile_test_cache_install create mode 100644 pkg/cache/cache.go diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 97511e092..32c99af91 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -92,6 +92,8 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().BoolVarP(&opts.Reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible") RootCmd.PersistentFlags().StringVarP(&opts.Target, "target", "", "", "Set the target build stage to build") RootCmd.PersistentFlags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry") + RootCmd.PersistentFlags().StringVarP(&opts.Cache, "cache", "", "", "Specify a registry to use as a chace, otherwise one will be inferred from the destination provided") + RootCmd.PersistentFlags().BoolVarP(&opts.UseCache, "use-cache", "", true, "Use cache when building image") } // addHiddenFlags marks certain flags as hidden from the executor help text diff --git a/integration/dockerfiles/Dockerfile_test_cache b/integration/dockerfiles/Dockerfile_test_cache new file mode 100644 index 000000000..215da54e7 --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_cache @@ -0,0 +1,22 @@ +# Copyright 2018 Google, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test to make sure the cache works properly +# /date should be the same regardless of when this image is built +# if the cache is implemented correctly + +FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +RUN date > /date +COPY context/foo /foo +RUN echo hey diff --git a/integration/dockerfiles/Dockerfile_test_cache_install b/integration/dockerfiles/Dockerfile_test_cache_install new file mode 100644 index 000000000..8a6a9a58e --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_cache_install @@ -0,0 +1,20 @@ +# Copyright 2018 Google, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test to make sure the cache works properly +# /date should be the same regardless of when this image is built +# if the cache is implemented correctly + +FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 +RUN apt-get update && apt-get install -y make diff --git a/integration/images.go b/integration/images.go index af48d2553..6cc8930db 100644 --- a/integration/images.go +++ b/integration/images.go @@ -23,6 +23,7 @@ import ( "path" "path/filepath" "runtime" + "strconv" "strings" ) @@ -77,12 +78,16 @@ func GetKanikoImage(imageRepo, dockerfile string) string { return strings.ToLower(imageRepo + kanikoPrefix + dockerfile) } +// GetVersionedKanikoImage versions constructs the name of the kaniko image that would be built +// with the dockerfile and versions it for cache testing +func GetVersionedKanikoImage(imageRepo, dockerfile string, version int) string { + return strings.ToLower(imageRepo + kanikoPrefix + dockerfile + strconv.Itoa(version)) +} + // FindDockerFiles will look for test docker files in the directory dockerfilesPath. // These files must start with `Dockerfile_test`. If the file is one we are intentionally // skipping, it will not be included in the returned list. func FindDockerFiles(dockerfilesPath string) ([]string, error) { - // TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed - testsToIgnore := map[string]bool{"Dockerfile_test_user_run": true} allDockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*")) if err != nil { return []string{}, fmt.Errorf("Failed to find docker files at %s: %s", dockerfilesPath, err) @@ -92,9 +97,8 @@ func FindDockerFiles(dockerfilesPath string) ([]string, error) { for _, dockerfile := range allDockerfiles { // Remove the leading directory from the path dockerfile = dockerfile[len("dockerfiles/"):] - if !testsToIgnore[dockerfile] { - dockerfiles = append(dockerfiles, dockerfile) - } + dockerfiles = append(dockerfiles, dockerfile) + } return dockerfiles, err } @@ -103,7 +107,9 @@ func FindDockerFiles(dockerfilesPath string) ([]string, error) { // keeps track of which files have been built. type DockerFileBuilder struct { // Holds all available docker files and whether or not they've been built - FilesBuilt map[string]bool + FilesBuilt map[string]bool + DockerfilesToIgnore map[string]struct{} + TestCacheDockerfiles map[string]struct{} } // NewDockerFileBuilder will create a DockerFileBuilder initialized with dockerfiles, which @@ -113,6 +119,14 @@ func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder { for _, f := range dockerfiles { d.FilesBuilt[f] = false } + d.DockerfilesToIgnore = map[string]struct{}{ + // TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed + "Dockerfile_test_user_run": {}, + } + d.TestCacheDockerfiles = map[string]struct{}{ + "Dockerfile_test_cache": {}, + "Dockerfile_test_cache_install": {}, + } return &d } @@ -164,6 +178,7 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do } } + cacheFlag := "--use-cache=false" // build kaniko image additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) kanikoImage := GetKanikoImage(imageRepo, dockerfile) @@ -174,6 +189,7 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do ExecutorImage, "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), "-d", kanikoImage, reproducibleFlag, + cacheFlag, contextFlag, contextPath}, additionalFlags...)..., ) @@ -186,3 +202,28 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do d.FilesBuilt[dockerfile] = true return nil } + +// buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built +func (d *DockerFileBuilder) buildCachedImages(imageRepo, cache, dockerfilesPath, dockerfile string, version int) error { + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + + for dockerfile := range d.TestCacheDockerfiles { + kanikoImage := GetVersionedKanikoImage(imageRepo, dockerfile, version) + kanikoCmd := exec.Command("docker", + append([]string{"run", + "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", + "-v", cwd + ":/workspace", + ExecutorImage, + "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), + "-d", kanikoImage, + "-c", buildContextPath, + "--cache", cache})..., + ) + + if _, err := RunCommandWithoutTest(kanikoCmd); err != nil { + return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %s", kanikoImage, kanikoCmd.Args, err) + } + } + return nil +} diff --git a/integration/integration_test.go b/integration/integration_test.go index 20fd22480..328fd39f8 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -24,8 +24,10 @@ import ( "math" "os" "os/exec" + "path/filepath" "strings" "testing" + "time" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/daemon" @@ -148,6 +150,7 @@ func TestMain(m *testing.M) { fmt.Printf("error building onbuild base: %v", err) os.Exit(1) } + pushOnbuildBase := exec.Command("docker", "push", config.onbuildBaseImage) if err := pushOnbuildBase.Run(); err != nil { fmt.Printf("error pushing onbuild base %s: %v", config.onbuildBaseImage, err) @@ -165,7 +168,6 @@ func TestMain(m *testing.M) { fmt.Printf("error pushing hardlink base %s: %v", config.hardlinkBaseImage, err) os.Exit(1) } - dockerfiles, err := FindDockerFiles(dockerfilesPath) if err != nil { fmt.Printf("Coudn't create map of dockerfiles: %s", err) @@ -177,6 +179,12 @@ func TestMain(m *testing.M) { func TestRun(t *testing.T) { for dockerfile, built := range imageBuilder.FilesBuilt { t.Run("test_"+dockerfile, func(t *testing.T) { + if _, ok := imageBuilder.DockerfilesToIgnore[dockerfile]; ok { + t.SkipNow() + } + if _, ok := imageBuilder.TestCacheDockerfiles[dockerfile]; ok { + t.SkipNow() + } if !built { err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile) if err != nil { @@ -195,25 +203,8 @@ func TestRun(t *testing.T) { t.Logf("diff = %s", string(diff)) expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) + checkContainerDiffOutput(t, diff, expected) - // Let's compare the json objects themselves instead of strings to avoid - // issues with spaces and indents - var diffInt interface{} - var expectedInt interface{} - - err := json.Unmarshal(diff, &diffInt) - if err != nil { - t.Error(err) - t.Fail() - } - - err = json.Unmarshal([]byte(expected), &expectedInt) - if err != nil { - t.Error(err) - t.Fail() - } - - testutil.CheckErrorAndDeepEqual(t, false, nil, expectedInt, diffInt) }) } } @@ -228,6 +219,9 @@ func TestLayers(t *testing.T) { } for dockerfile, built := range imageBuilder.FilesBuilt { t.Run("test_layer_"+dockerfile, func(t *testing.T) { + if _, ok := imageBuilder.DockerfilesToIgnore[dockerfile]; ok { + t.SkipNow() + } if !built { err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile) if err != nil { @@ -244,6 +238,58 @@ func TestLayers(t *testing.T) { } } +// Build each image with kaniko twice, and then make sure they're exactly the same +func TestCache(t *testing.T) { + for dockerfile := range imageBuilder.TestCacheDockerfiles { + t.Run("test_cache_"+dockerfile, func(t *testing.T) { + cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano())) + // Build the initial image which will cache layers + if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, dockerfile, 0); err != nil { + t.Fatalf("error building cached image for the first time: %v", err) + } + // Build the second image which should pull from the cache + if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, dockerfile, 1); err != nil { + t.Fatalf("error building cached image for the first time: %v", err) + } + // Make sure both images are the same + kanikoVersion0 := GetVersionedKanikoImage(config.imageRepo, dockerfile, 0) + kanikoVersion1 := GetVersionedKanikoImage(config.imageRepo, dockerfile, 1) + + // container-diff + containerdiffCmd := exec.Command("container-diff", "diff", + kanikoVersion0, kanikoVersion1, + "-q", "--type=file", "--type=metadata", "--json") + + diff := RunCommand(containerdiffCmd, t) + t.Logf("diff = %s", diff) + + expected := fmt.Sprintf(emptyContainerDiff, kanikoVersion0, kanikoVersion1, kanikoVersion0, kanikoVersion1) + checkContainerDiffOutput(t, diff, expected) + }) + } +} + +func checkContainerDiffOutput(t *testing.T, diff []byte, expected string) { + // Let's compare the json objects themselves instead of strings to avoid + // issues with spaces and indents + t.Helper() + + var diffInt interface{} + var expectedInt interface{} + + err := json.Unmarshal(diff, &diffInt) + if err != nil { + t.Error(err) + } + + err = json.Unmarshal([]byte(expected), &expectedInt) + if err != nil { + t.Error(err) + } + + testutil.CheckErrorAndDeepEqual(t, false, nil, expectedInt, diffInt) +} + func checkLayers(t *testing.T, image1, image2 string, offset int) { t.Helper() img1, err := getImageDetails(image1) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go new file mode 100644 index 000000000..08909ce94 --- /dev/null +++ b/pkg/cache/cache.go @@ -0,0 +1,70 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "fmt" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/authn/k8schain" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// RetrieveLayer checks the specified cache for a layer with the tag :cacheKey +func RetrieveLayer(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { + cache, err := Destination(opts, cacheKey) + if err != nil { + return nil, errors.Wrap(err, "getting cache destination") + } + logrus.Infof("Checking for cached layer %s...", cache) + + cacheRef, err := name.NewTag(cache, name.WeakValidation) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("getting reference for %s", cache)) + } + k8sc, err := k8schain.NewNoClient() + if err != nil { + return nil, err + } + kc := authn.NewMultiKeychain(authn.DefaultKeychain, k8sc) + img, err := remote.Image(cacheRef, remote.WithAuthFromKeychain(kc)) + if err != nil { + return nil, err + } + _, err = img.Layers() + return img, err +} + +// Destination returns the repo where the layer should be stored +// If no cache is specified, one is inferred from the destination provided +func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { + cache := opts.Cache + if cache == "" { + destination := opts.Destinations[0] + destRef, err := name.NewTag(destination, name.WeakValidation) + if err != nil { + return "", errors.Wrap(err, "getting tag for destination") + } + return fmt.Sprintf("%s/cache", destRef.Context()), nil + } + return fmt.Sprintf("%s:%s", cache, cacheKey), nil +} diff --git a/pkg/config/options.go b/pkg/config/options.go index 18e1d2028..3c84fa7bb 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -24,6 +24,7 @@ type KanikoOptions struct { Bucket string TarPath string Target string + Cache string Destinations multiArg BuildArgs multiArg InsecurePush bool @@ -31,4 +32,5 @@ type KanikoOptions struct { SingleSnapshot bool Reproducible bool NoPush bool + UseCache bool } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index b908a0beb..b4f794d0b 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -18,6 +18,7 @@ package executor import ( "bytes" + "encoding/json" "fmt" "io" "io/ioutil" @@ -33,6 +34,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/GoogleContainerTools/kaniko/pkg/cache" "github.com/GoogleContainerTools/kaniko/pkg/commands" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/constants" @@ -84,20 +86,52 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*sta } // key will return a string representation of the build at the cmd -// TODO: priyawadhwa@ to fill this out when implementing caching -// func (s *stageBuilder) key(cmd string) (string, error) { -// return "", nil -// } +func (s *stageBuilder) key(cmd string) (string, error) { + fsKey, err := s.snapshotter.Key() + if err != nil { + return "", err + } + c := bytes.NewBuffer([]byte{}) + enc := json.NewEncoder(c) + enc.Encode(s.cf) + cf, err := util.SHA256(c) + if err != nil { + return "", err + } + logrus.Debugf("%s\n%s\n%s\n%s\n", s.baseImageDigest, fsKey, cf, cmd) + return util.SHA256(bytes.NewReader([]byte(s.baseImageDigest + fsKey + cf + cmd))) +} // extractCachedLayer will extract the cached layer and append it to the config file -// TODO: priyawadhwa@ to fill this out when implementing caching -// func (s *stageBuilder) extractCachedLayer(layer v1.Image, createdBy string) error { -// return nil -// } +func (s *stageBuilder) extractCachedLayer(layer v1.Image, createdBy string) error { + logrus.Infof("Found cached layer, extracting to filesystem") + extractedFiles, err := util.GetFSFromImage(constants.RootDir, layer) + if err != nil { + return errors.Wrap(err, "extracting fs from image") + } + if _, err := s.snapshotter.TakeSnapshot(extractedFiles); err != nil { + return err + } + logrus.Infof("Appending cached layer to base image") + l, err := layer.Layers() + if err != nil { + return errors.Wrap(err, "getting cached layer from image") + } + s.image, err = mutate.Append(s.image, + mutate.Addendum{ + Layer: l[0], + History: v1.History{ + Author: constants.Author, + CreatedBy: createdBy, + }, + }, + ) + return err +} func (s *stageBuilder) build(opts *config.KanikoOptions) error { // Unpack file system to root - if err := util.GetFSFromImage(constants.RootDir, s.image); err != nil { + if _, err := util.GetFSFromImage(constants.RootDir, s.image); err != nil { return err } // Take initial snapshot @@ -115,6 +149,20 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { continue } logrus.Info(command.String()) + cacheKey, err := s.key(command.String()) + if err != nil { + return errors.Wrap(err, "getting key") + } + if command.CacheCommand() && opts.UseCache { + image, err := cache.RetrieveLayer(opts, cacheKey) + if err == nil { + if err := s.extractCachedLayer(image, command.String()); err != nil { + return errors.Wrap(err, "extracting cached layer") + } + continue + } + logrus.Info("No cached layer found, executing command...") + } if err := command.ExecuteCommand(&s.cf.Config, args); err != nil { return err } @@ -163,6 +211,12 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { if err != nil { return err } + // Push layer to cache now along with new config file + if command.CacheCommand() && opts.UseCache { + if err := pushLayerToCache(opts, cacheKey, layer, command.String()); err != nil { + return err + } + } s.image, err = mutate.Append(s.image, mutate.Addendum{ Layer: layer, @@ -233,7 +287,8 @@ func extractImageToDependecyDir(index int, image v1.Image) error { return err } logrus.Infof("trying to extract to %s", dependencyDir) - return util.GetFSFromImage(dependencyDir, image) + _, err := util.GetFSFromImage(dependencyDir, image) + return err } func saveStageAsTarball(stageIndex int, image v1.Image) error { diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 53799b71a..aa68c723c 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -21,12 +21,16 @@ import ( "fmt" "net/http" + "github.com/GoogleContainerTools/kaniko/pkg/cache" "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/version" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/authn/k8schain" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" @@ -100,3 +104,29 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { } return nil } + +// pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache +// if opts.Cache doesn't exist, infer the cache from the given destination +func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, layer v1.Layer, createdBy string) error { + cache, err := cache.Destination(opts, cacheKey) + if err != nil { + return errors.Wrap(err, "getting cache destination") + } + logrus.Infof("Pushing layer %s to cache now", cache) + empty := empty.Image + empty, err = mutate.Append(empty, + mutate.Addendum{ + Layer: layer, + History: v1.History{ + Author: constants.Author, + CreatedBy: createdBy, + }, + }, + ) + if err != nil { + return errors.Wrap(err, "appending layer onto empty image") + } + return DoPush(empty, &config.KanikoOptions{ + Destinations: []string{cache}, + }) +} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index d1136d7bc..3e45c69c1 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -46,22 +46,25 @@ var whitelist = []string{ } var volumeWhitelist = []string{} -func GetFSFromImage(root string, img v1.Image) error { +// GetFSFromImage extracts the layers of img to root +// It returns a list of all files extracted +func GetFSFromImage(root string, img v1.Image) ([]string, error) { whitelist, err := fileSystemWhitelist(constants.WhitelistPath) if err != nil { - return err + return nil, err } - logrus.Infof("Mounted directories: %v", whitelist) + logrus.Debugf("Mounted directories: %v", whitelist) layers, err := img.Layers() if err != nil { - return err + return nil, err } + extractedFiles := []string{} for i, l := range layers { logrus.Infof("Extracting layer %d", i) r, err := l.Uncompressed() if err != nil { - return err + return nil, err } tr := tar.NewReader(r) for { @@ -70,7 +73,7 @@ func GetFSFromImage(root string, img v1.Image) error { break } if err != nil { - return err + return nil, err } path := filepath.Join(root, filepath.Clean(hdr.Name)) base := filepath.Base(path) @@ -79,13 +82,13 @@ func GetFSFromImage(root string, img v1.Image) error { logrus.Debugf("Whiting out %s", path) name := strings.TrimPrefix(base, ".wh.") if err := os.RemoveAll(filepath.Join(dir, name)); err != nil { - return errors.Wrapf(err, "removing whiteout %s", hdr.Name) + return nil, errors.Wrapf(err, "removing whiteout %s", hdr.Name) } continue } whitelisted, err := CheckWhitelist(path) if err != nil { - return err + return nil, err } if whitelisted && !checkWhitelistRoot(root) { logrus.Debugf("Not adding %s because it is whitelisted", path) @@ -94,7 +97,7 @@ func GetFSFromImage(root string, img v1.Image) error { if hdr.Typeflag == tar.TypeSymlink { whitelisted, err := CheckWhitelist(hdr.Linkname) if err != nil { - return err + return nil, err } if whitelisted { logrus.Debugf("skipping symlink from %s to %s because %s is whitelisted", hdr.Linkname, path, hdr.Linkname) @@ -102,11 +105,12 @@ func GetFSFromImage(root string, img v1.Image) error { } } if err := extractFile(root, hdr, tr); err != nil { - return err + return nil, err } + extractedFiles = append(extractedFiles, filepath.Join(root, filepath.Clean(hdr.Name))) } } - return nil + return extractedFiles, nil } // DeleteFilesystem deletes the extracted image file system From eb7194a16538e44f0591454f73c6c55b2e45fb96 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 13 Sep 2018 22:06:38 -0700 Subject: [PATCH 2/5] Fix linting errors --- integration/images.go | 2 +- integration/integration_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/images.go b/integration/images.go index 6cc8930db..61fc6a7ac 100644 --- a/integration/images.go +++ b/integration/images.go @@ -204,7 +204,7 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do } // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built -func (d *DockerFileBuilder) buildCachedImages(imageRepo, cache, dockerfilesPath, dockerfile string, version int) error { +func (d *DockerFileBuilder) buildCachedImages(imageRepo, cache, dockerfilesPath string, version int) error { _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) diff --git a/integration/integration_test.go b/integration/integration_test.go index 328fd39f8..de2409ed7 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -244,11 +244,11 @@ func TestCache(t *testing.T) { t.Run("test_cache_"+dockerfile, func(t *testing.T) { cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano())) // Build the initial image which will cache layers - if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, dockerfile, 0); err != nil { + if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 0); err != nil { t.Fatalf("error building cached image for the first time: %v", err) } // Build the second image which should pull from the cache - if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, dockerfile, 1); err != nil { + if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 1); err != nil { t.Fatalf("error building cached image for the first time: %v", err) } // Make sure both images are the same From f7ba67ea2573ad504f1e1e7222ce1d9eb12a16ed Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 14 Sep 2018 09:53:03 -0700 Subject: [PATCH 3/5] Specify cache key to differentiate cache layers --- pkg/cache/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 08909ce94..198ec6651 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -64,7 +64,7 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { if err != nil { return "", errors.Wrap(err, "getting tag for destination") } - return fmt.Sprintf("%s/cache", destRef.Context()), nil + return fmt.Sprintf("%s/cache:%s", destRef.Context(), cacheKey), nil } return fmt.Sprintf("%s:%s", cache, cacheKey), nil } From 177bd4f40e433fac13cbe96db1077034cc3d7144 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 17 Sep 2018 11:05:57 +0100 Subject: [PATCH 4/5] Fix typo and update comments --- cmd/executor/cmd/root.go | 2 +- integration/dockerfiles/Dockerfile_test_cache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 32c99af91..351595966 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -92,7 +92,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().BoolVarP(&opts.Reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible") RootCmd.PersistentFlags().StringVarP(&opts.Target, "target", "", "", "Set the target build stage to build") RootCmd.PersistentFlags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry") - RootCmd.PersistentFlags().StringVarP(&opts.Cache, "cache", "", "", "Specify a registry to use as a chace, otherwise one will be inferred from the destination provided") + RootCmd.PersistentFlags().StringVarP(&opts.Cache, "cache", "", "", "Specify a registry to use as a cache, otherwise one will be inferred from the destination provided") RootCmd.PersistentFlags().BoolVarP(&opts.UseCache, "use-cache", "", true, "Use cache when building image") } diff --git a/integration/dockerfiles/Dockerfile_test_cache b/integration/dockerfiles/Dockerfile_test_cache index 215da54e7..e3ebe304d 100644 --- a/integration/dockerfiles/Dockerfile_test_cache +++ b/integration/dockerfiles/Dockerfile_test_cache @@ -13,7 +13,7 @@ # limitations under the License. # Test to make sure the cache works properly -# /date should be the same regardless of when this image is built +# If the image is built twice, /date should be the same in both images # if the cache is implemented correctly FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0 From e2ca1152f4b538d3623a5776e38294f1ecc6474b Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 24 Sep 2018 13:18:42 -0700 Subject: [PATCH 5/5] Rename flags and default caching to false Rename --use-cache to --cache, and --cache to --cache-repo to clarify what the flags are used for. Default caching to false. --- cmd/executor/cmd/root.go | 4 ++-- integration/images.go | 9 +++++---- pkg/cache/cache.go | 2 +- pkg/config/options.go | 4 ++-- pkg/executor/build.go | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 351595966..1b2cf06ff 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -92,8 +92,8 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().BoolVarP(&opts.Reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible") RootCmd.PersistentFlags().StringVarP(&opts.Target, "target", "", "", "Set the target build stage to build") RootCmd.PersistentFlags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry") - RootCmd.PersistentFlags().StringVarP(&opts.Cache, "cache", "", "", "Specify a registry to use as a cache, otherwise one will be inferred from the destination provided") - RootCmd.PersistentFlags().BoolVarP(&opts.UseCache, "use-cache", "", true, "Use cache when building image") + RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided") + RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") } // addHiddenFlags marks certain flags as hidden from the executor help text diff --git a/integration/images.go b/integration/images.go index 61fc6a7ac..cf1f90105 100644 --- a/integration/images.go +++ b/integration/images.go @@ -178,7 +178,6 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do } } - cacheFlag := "--use-cache=false" // build kaniko image additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) kanikoImage := GetKanikoImage(imageRepo, dockerfile) @@ -189,7 +188,6 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do ExecutorImage, "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), "-d", kanikoImage, reproducibleFlag, - cacheFlag, contextFlag, contextPath}, additionalFlags...)..., ) @@ -204,10 +202,12 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do } // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built -func (d *DockerFileBuilder) buildCachedImages(imageRepo, cache, dockerfilesPath string, version int) error { +func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath string, version int) error { _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) + cacheFlag := "--cache=true" + for dockerfile := range d.TestCacheDockerfiles { kanikoImage := GetVersionedKanikoImage(imageRepo, dockerfile, version) kanikoCmd := exec.Command("docker", @@ -218,7 +218,8 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cache, dockerfilesPath "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), "-d", kanikoImage, "-c", buildContextPath, - "--cache", cache})..., + cacheFlag, + "--cache-repo", cacheRepo})..., ) if _, err := RunCommandWithoutTest(kanikoCmd); err != nil { diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 198ec6651..69682ba48 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -57,7 +57,7 @@ func RetrieveLayer(opts *config.KanikoOptions, cacheKey string) (v1.Image, error // Destination returns the repo where the layer should be stored // If no cache is specified, one is inferred from the destination provided func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { - cache := opts.Cache + cache := opts.CacheRepo if cache == "" { destination := opts.Destinations[0] destRef, err := name.NewTag(destination, name.WeakValidation) diff --git a/pkg/config/options.go b/pkg/config/options.go index 3c84fa7bb..c9bad39e6 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -24,7 +24,7 @@ type KanikoOptions struct { Bucket string TarPath string Target string - Cache string + CacheRepo string Destinations multiArg BuildArgs multiArg InsecurePush bool @@ -32,5 +32,5 @@ type KanikoOptions struct { SingleSnapshot bool Reproducible bool NoPush bool - UseCache bool + Cache bool } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index b4f794d0b..49313900b 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -153,7 +153,7 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { if err != nil { return errors.Wrap(err, "getting key") } - if command.CacheCommand() && opts.UseCache { + if command.CacheCommand() && opts.Cache { image, err := cache.RetrieveLayer(opts, cacheKey) if err == nil { if err := s.extractCachedLayer(image, command.String()); err != nil { @@ -212,7 +212,7 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { return err } // Push layer to cache now along with new config file - if command.CacheCommand() && opts.UseCache { + if command.CacheCommand() && opts.Cache { if err := pushLayerToCache(opts, cacheKey, layer, command.String()); err != nil { return err }