From f6aa8f709bd41573d3564ee53bd9a30d1eb3d2f5 Mon Sep 17 00:00:00 2001 From: Will Ripley Date: Fri, 8 Nov 2019 12:59:25 -0600 Subject: [PATCH 01/28] Modified error message for writing image with digest file --- pkg/executor/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/executor/push.go b/pkg/executor/push.go index c5ea7c05b..3999c4973 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -151,7 +151,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { if opts.ImageNameDigestFile != "" { err := ioutil.WriteFile(opts.ImageNameDigestFile, []byte(builder.String()), 0644) if err != nil { - return errors.Wrap(err, "writing digest to file failed") + return errors.Wrap(err, "writing image name with digest to file failed") } } From 8eb05761ad59eb467fedd517ed9bc068f3750799 Mon Sep 17 00:00:00 2001 From: Balint Pato Date: Fri, 15 Nov 2019 09:50:44 -0800 Subject: [PATCH 02/28] nits --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 37ca66c27..56bf6254c 100644 --- a/README.md +++ b/README.md @@ -279,7 +279,7 @@ as a remote image destination: ### Caching #### Caching Layers -kaniko currently can cache layers created by `RUN` commands in a remote repository. +kaniko can cache layers created by `RUN` commands in a remote repository. Before executing a command, kaniko checks the cache for the layer. If it exists, kaniko will pull and extract the cached layer instead of executing the command. If not, kaniko will execute the command and then push the newly created layer to the cache. @@ -290,7 +290,7 @@ If this flag isn't provided, a cached repo will be inferred from the `--destinat #### Caching Base Images -kaniko can cache images in a local directory that can be volume mounted into the kaniko image. +kaniko can cache images in a local directory that can be volume mounted into the kaniko pod. To do so, the cache must first be populated, as it is read-only. We provide a kaniko cache warming image at `gcr.io/kaniko-project/warmer`: @@ -301,7 +301,7 @@ docker run -v $(pwd):/workspace gcr.io/kaniko-project/warmer:latest --cache-dir= `--image` can be specified for any number of desired images. This command will cache those images by digest in a local directory named `cache`. Once the cache is populated, caching is opted into with the same `--cache=true` flag as above. -The location of the local cache is provided via the `--cache-dir` flag, defaulting at `/cache` as with the cache warmer. +The location of the local cache is provided via the `--cache-dir` flag, defaulting to `/cache` as with the cache warmer. See the `examples` directory for how to use with kubernetes clusters and persistent cache volumes. ### Pushing to Different Registries From 8f66e8613f36e52daca4c42d08dff8961e793628 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Tue, 12 Nov 2019 14:37:48 -0800 Subject: [PATCH 03/28] Add new test for copy to symlink which should fail --- integration/dockerfiles/Dockerfile_test_add_dest_symlink_dir | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 integration/dockerfiles/Dockerfile_test_add_dest_symlink_dir diff --git a/integration/dockerfiles/Dockerfile_test_add_dest_symlink_dir b/integration/dockerfiles/Dockerfile_test_add_dest_symlink_dir new file mode 100644 index 000000000..a3e68e593 --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_add_dest_symlink_dir @@ -0,0 +1,2 @@ +FROM phusion/baseimage:0.11 +ADD context/foo /etc/service/foo From 50f1373837c2a8c3af2aff05a5eb7412426e5e77 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Tue, 12 Nov 2019 15:01:13 -0800 Subject: [PATCH 04/28] Update Add command RequiresUnpackedFS --- pkg/commands/add.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/commands/add.go b/pkg/commands/add.go index 7a3d6164b..3ade7ea69 100644 --- a/pkg/commands/add.go +++ b/pkg/commands/add.go @@ -141,3 +141,7 @@ func (a *AddCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfi func (a *AddCommand) MetadataOnly() bool { return false } + +func (a *AddCommand) RequiresUnpackedFS() bool { + return true +} From 2c13842451223261c5e585f527f1732e510db60d Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Tue, 12 Nov 2019 17:07:12 -0800 Subject: [PATCH 05/28] Resolve symlink paths --- pkg/commands/copy.go | 26 ++++++++++++++++++++++++++ pkg/util/fs_util.go | 8 +++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index 15f778e8d..ad75f6f3f 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -66,6 +66,14 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu if err != nil { return err } + + // If the destination dir is a symlink we need to resolve the path and use + // that instead of the symlink path + destPath, err = resolveIfSymlink(destPath) + if err != nil { + return err + } + if fi.IsDir() { if !filepath.IsAbs(dest) { // we need to add '/' to the end to indicate the destination is a directory @@ -178,3 +186,21 @@ func (cr *CachingCopyCommand) FilesToSnapshot() []string { func (cr *CachingCopyCommand) String() string { return cr.cmd.String() } + +func resolveIfSymlink(destPath string) (string, error) { + baseDir := filepath.Dir(destPath) + if info, err := os.Lstat(baseDir); err == nil { + switch mode := info.Mode(); { + case mode&os.ModeSymlink != 0: + linkPath, err := os.Readlink(baseDir) + if err != nil { + return "", errors.Wrap(err, "error reading symlink") + } + absLinkPath := filepath.Join(filepath.Dir(baseDir), linkPath) + newPath := filepath.Join(absLinkPath, filepath.Base(destPath)) + logrus.Tracef("Updating destination path from %v to %v due to symlink", destPath, newPath) + return newPath, nil + } + } + return destPath, nil +} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 00562e56f..10fccb7f7 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -424,11 +424,17 @@ func FilepathExists(path string) bool { func CreateFile(path string, reader io.Reader, perm os.FileMode, uid uint32, gid uint32) error { // Create directory path if it doesn't exist baseDir := filepath.Dir(path) - if _, err := os.Lstat(baseDir); os.IsNotExist(err) { + if info, err := os.Lstat(baseDir); os.IsNotExist(err) { logrus.Tracef("baseDir %s for file %s does not exist. Creating.", baseDir, path) if err := os.MkdirAll(baseDir, 0755); err != nil { return err } + } else { + switch mode := info.Mode(); { + case mode&os.ModeSymlink != 0: + logrus.Infof("destination cannot be a symlink %v", baseDir) + return errors.New("destination cannot be a symlink") + } } dest, err := os.Create(path) if err != nil { From 2b26dfea61d9f10f62422bf3f90e4b5abba84525 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Fri, 15 Nov 2019 10:28:23 -0800 Subject: [PATCH 06/28] Add unit tests for resolveIfSymlink --- pkg/commands/copy_test.go | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/pkg/commands/copy_test.go b/pkg/commands/copy_test.go index 9b16e9291..5f8fb7211 100644 --- a/pkg/commands/copy_test.go +++ b/pkg/commands/copy_test.go @@ -16,6 +16,7 @@ limitations under the License. package commands import ( + "fmt" "io" "io/ioutil" "os" @@ -165,3 +166,52 @@ func copySetUpBuildArgs() *dockerfile.BuildArgs { buildArgs.AddArg("buildArg2", &d) return buildArgs } + +func Test_resolveIfSymlink(t *testing.T) { + type testCase struct { + destPath string + expectedPath string + err error + } + + tmpDir, err := ioutil.TempDir("", "copy-test") + if err != nil { + t.Error(err) + } + + baseDir, err := ioutil.TempDir(tmpDir, "not-linked") + if err != nil { + t.Error(err) + } + + path, err := ioutil.TempFile(baseDir, "foo.txt") + if err != nil { + t.Error(err) + } + + thepath, err := filepath.Abs(filepath.Dir(path.Name())) + if err != nil { + t.Error(err) + } + cases := []testCase{{destPath: thepath, expectedPath: thepath, err: nil}} + + baseDir = tmpDir + symLink := filepath.Join(baseDir, "symlink") + if err := os.Symlink(filepath.Base(thepath), symLink); err != nil { + t.Error(err) + } + cases = append(cases, testCase{filepath.Join(symLink, "foo.txt"), filepath.Join(thepath, "foo.txt"), nil}) + + for i, c := range cases { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + res, e := resolveIfSymlink(c.destPath) + if e != c.err { + t.Errorf("%s: expected %v but got %v", c.destPath, c.err, e) + } + + if res != c.expectedPath { + t.Errorf("%s: expected %v but got %v", c.destPath, c.expectedPath, res) + } + }) + } +} From 02db3c18fad44b2d804e85d5328eb460c31cfeda Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Thu, 21 Nov 2019 12:32:37 -0800 Subject: [PATCH 07/28] Update readme * Know Issues * kaniko in non-official images * v1 Registry Schema --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d5219bd86..af571775d 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,10 @@ After each command, we append a layer of changed files to the base image (if the ## Known Issues -kaniko does not support building Windows containers. +* kaniko does not support building Windows containers. +* Running kaniko in any Docker image other than the official kaniko image is not supported (ie YMMV). + * This includes copying the kaniko executables from the official image into another image. +* kaniko does not support the v1 Registry API ([Registry v1 API Deprecation](https://engineering.docker.com/2019/03/registry-v1-api-deprecation/)) ## Demo From c49b4747bda70a811a68ec7d9655c208d300117b Mon Sep 17 00:00:00 2001 From: "tommaso.doninelli" Date: Fri, 22 Nov 2019 07:12:31 +0100 Subject: [PATCH 08/28] Invalid link to missing file config.json Link points to the AWS ECR Credentials Helper config that explain how to configure it --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d5219bd86..fb35ae2b2 100644 --- a/README.md +++ b/README.md @@ -342,7 +342,7 @@ Run kaniko with the `config.json` inside `/kaniko/.docker/config.json` The Amazon ECR [credential helper](https://github.com/awslabs/amazon-ecr-credential-helper) is built in to the kaniko executor image. To configure credentials, you will need to do the following: -1. Update the `credHelpers` section of [config.json](https://github.com/GoogleContainerTools/kaniko/blob/master/files/config.json) with the specific URI of your ECR registry: +1. Update the `credHelpers` section of [config.json](https://github.com/awslabs/amazon-ecr-credential-helper#configuration) with the specific URI of your ECR registry: ```json { From a6e458caf1aca5deda542269714cde53becf1544 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Fri, 22 Nov 2019 12:11:48 -0800 Subject: [PATCH 09/28] Update error handling and logging for cache Previously we returned a low level file system error when checking for a cached image. By adding a more human friendly log message and explicit error handling we improve upon the user experience. --- pkg/cache/cache.go | 5 +++-- pkg/util/image_util.go | 31 ++++++++++++++++++------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 0953a28f6..d8c7898ac 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -28,7 +28,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/creds" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" @@ -124,7 +124,8 @@ func LocalSource(opts *config.CacheOptions, cacheKey string) (v1.Image, error) { fi, err := os.Stat(path) if err != nil { - return nil, errors.Wrap(err, "getting file info") + logrus.Debugf("No file found for cache key %v %v", cacheKey, err) + return nil, nil } // A stale cache is a bad cache diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 01dc12fae..e6f653cee 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -74,15 +74,13 @@ func RetrieveSourceImage(stage config.KanikoStage, opts *config.KanikoOptions) ( // If so, look in the local cache before trying the remote registry if opts.CacheDir != "" { cachedImage, err := cachedImage(opts, currentBaseName) - if cachedImage != nil { + if err != nil { + logrus.Errorf("Error while retrieving image from cache: %v %v", currentBaseName, err) + } else if cachedImage != nil { return cachedImage, nil } - - if err != nil { - logrus.Infof("Error while retrieving image from cache: %v", err) - } } - + logrus.Infof("Image %v not found in cache", currentBaseName) // Otherwise, initialize image as usual return RetrieveRemoteImage(currentBaseName, opts) } @@ -93,13 +91,23 @@ func tarballImage(index int) (v1.Image, error) { return tarball.ImageFromPath(tarPath, nil) } +// Retrieves the manifest for the specified image from the specified registry func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { - logrus.Infof("Downloading base image %s", image) + logrus.Infof("Retrieving image manifest %s", image) ref, err := name.ParseReference(image, name.WeakValidation) if err != nil { return nil, err } + rOpts, err := prepareRemoteRequest(ref, opts) + if err != nil { + return nil, err + } + + return remote.Image(ref, rOpts...) +} + +func prepareRemoteRequest(ref name.Reference, opts *config.KanikoOptions) ([]remote.Option, error) { registryName := ref.Context().RegistryStr() if opts.InsecurePull || opts.InsecureRegistries.Contains(registryName) { newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) @@ -122,8 +130,7 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { InsecureSkipVerify: true, } } - - return remote.Image(ref, remote.WithTransport(tr), remote.WithAuthFromKeychain(creds.GetKeychain())) + return []remote.Option{remote.WithTransport(tr), remote.WithAuthFromKeychain(creds.GetKeychain())}, nil } func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { @@ -136,18 +143,16 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { if d, ok := ref.(name.Digest); ok { cacheKey = d.DigestStr() } else { - img, err := remoteImage(image, opts) + image, err := remoteImage(image, opts) if err != nil { return nil, err } - d, err := img.Digest() + d, err := image.Digest() if err != nil { return nil, err } - cacheKey = d.String() } - return cache.LocalSource(&opts.CacheOptions, cacheKey) } From c2a8b33f9c938b055181b9e71f66c26dab62b1eb Mon Sep 17 00:00:00 2001 From: Eduard Laur Date: Thu, 21 Nov 2019 15:31:35 +0200 Subject: [PATCH 10/28] Fix README.md anchor links --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index d5219bd86..4b59f46a3 100644 --- a/README.md +++ b/README.md @@ -46,26 +46,26 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [Pushing to Docker Hub](#pushing-to-docker-hub) - [Pushing to Amazon ECR](#pushing-to-amazon-ecr) - [Additional Flags](#additional-flags) - - [--build-arg](#build-arg) - - [--cache](#cache) - - [--cache-dir](#cache-dir) - - [--cache-repo](#cache-repo) - - [--digest-file](#digest-file) - - [--oci-layout-path](#oci-layout-path) - - [--insecure-registry](#insecure-registry) - - [--skip-tls-verify-registry](#skip-tls-verify-registry) - - [--cleanup](#cleanup) - - [--insecure](#insecure) - - [--insecure-pull](#insecure-pull) - - [--no-push](#no-push) - - [--reproducible](#reproducible) - - [--single-snapshot](#single-snapshot) - - [--skip-tls-verify](#skip-tls-verify) - - [--skip-tls-verify-pull](#skip-tls-verify-pull) - - [--snapshotMode](#snapshotmode) - - [--target](#target) - - [--tarPath](#tarpath) - - [--verbosity](#verbosity) + - [--build-arg](#--build-arg) + - [--cache](#--cache) + - [--cache-dir](#--cache-dir) + - [--cache-repo](#--cache-repo) + - [--digest-file](#--digest-file) + - [--oci-layout-path](#--oci-layout-path) + - [--insecure-registry](#--insecure-registry) + - [--skip-tls-verify-registry](#--skip-tls-verify-registry) + - [--cleanup](#--cleanup) + - [--insecure](#--insecure) + - [--insecure-pull](#--insecure-pull) + - [--no-push](#--no-push) + - [--reproducible](#--reproducible) + - [--single-snapshot](#--single-snapshot) + - [--skip-tls-verify](#--skip-tls-verify) + - [--skip-tls-verify-pull](#--skip-tls-verify-pull) + - [--snapshotMode](#--snapshotmode) + - [--target](#--target) + - [--tarPath](#--tarpath) + - [--verbosity](#--verbosity) - [Debug Image](#debug-image) - [Security](#security) - [Comparison with Other Tools](#comparison-with-other-tools) From 2755ae447062b86c752e093ab9d48e1ea5e5a7d0 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Wed, 27 Nov 2019 14:40:05 -0800 Subject: [PATCH 11/28] Final cachekey for stage Store the last cachekey generated for each stage If the base image for a stage is present in the map of digest and cachekeys use the retrieved cachekey instead of the base image digest in the compositecache --- pkg/executor/build.go | 53 +++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 45f23cca3..541557931 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -54,19 +54,21 @@ const emptyTarSize = 1024 // stageBuilder contains all fields necessary to build one stage of a Dockerfile type stageBuilder struct { - stage config.KanikoStage - image v1.Image - cf *v1.ConfigFile - snapshotter *snapshot.Snapshotter - baseImageDigest string - opts *config.KanikoOptions - cmds []commands.DockerCommand - args *dockerfile.BuildArgs - crossStageDeps map[int][]string + stage config.KanikoStage + image v1.Image + cf *v1.ConfigFile + snapshotter *snapshot.Snapshotter + baseImageDigest string + finalCacheKey string + opts *config.KanikoOptions + cmds []commands.DockerCommand + args *dockerfile.BuildArgs + crossStageDeps map[int][]string + digestToCacheKeyMap map[string]string } // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage -func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string) (*stageBuilder, error) { +func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string, dcm map[string]string) (*stageBuilder, error) { sourceImage, err := util.RetrieveSourceImage(stage, opts) if err != nil { return nil, err @@ -93,13 +95,14 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross return nil, err } s := &stageBuilder{ - stage: stage, - image: sourceImage, - cf: imageConfig, - snapshotter: snapshotter, - baseImageDigest: digest.String(), - opts: opts, - crossStageDeps: crossStageDeps, + stage: stage, + image: sourceImage, + cf: imageConfig, + snapshotter: snapshotter, + baseImageDigest: digest.String(), + opts: opts, + crossStageDeps: crossStageDeps, + digestToCacheKeyMap: dcm, } for _, cmd := range s.stage.Commands { @@ -163,6 +166,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro if err != nil { return err } + s.finalCacheKey = ck if command.ShouldCacheOutput() { img, err := layerCache.RetrieveLayer(ck) if err != nil { @@ -190,7 +194,12 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro func (s *stageBuilder) build() error { // Set the initial cache key to be the base image digest, the build args and the SrcContext. - compositeKey := NewCompositeCache(s.baseImageDigest) + var compositeKey *CompositeCache + if cacheKey, ok := s.digestToCacheKeyMap[s.baseImageDigest]; ok { + compositeKey = NewCompositeCache(cacheKey) + } else { + compositeKey = NewCompositeCache(s.baseImageDigest) + } compositeKey.AddKey(s.opts.BuildArgs...) // Apply optimizations to the instructions. @@ -422,6 +431,7 @@ func CalculateDependencies(opts *config.KanikoOptions) (map[int][]string, error) // DoBuild executes building the Dockerfile func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { t := timing.Start("Total Build Time") + digestToCacheKeyMap := make(map[string]string) // Parse dockerfile and unpack base image to root stages, err := dockerfile.Stages(opts) if err != nil { @@ -442,7 +452,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { logrus.Infof("Built cross stage deps: %v", crossStageDependencies) for index, stage := range stages { - sb, err := newStageBuilder(opts, stage, crossStageDependencies) + sb, err := newStageBuilder(opts, stage, crossStageDependencies, digestToCacheKeyMap) if err != nil { return nil, err } @@ -454,6 +464,11 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { if err != nil { return nil, err } + d, err := sourceImage.Digest() + if err != nil { + return nil, err + } + digestToCacheKeyMap[d.String()] = sb.finalCacheKey if stage.Final { sourceImage, err = mutate.CreatedAt(sourceImage, v1.Time{Time: time.Now()}) if err != nil { From 54635c3d39611d597bf2a7f46ff3ef92c29a77ce Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Wed, 27 Nov 2019 16:25:56 -0800 Subject: [PATCH 12/28] don't exit optimize early so we record cache keys --- pkg/executor/build.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 541557931..c1d987120 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -141,7 +141,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro layerCache := &cache.RegistryCache{ Opts: s.opts, } - + stopCache := false // Possibly replace commands with their cached implementations. // We walk through all the commands, running any commands that only operate on metadata. // We throw the metadata away after, but we need it to properly track command dependencies @@ -167,13 +167,14 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro return err } s.finalCacheKey = ck - if command.ShouldCacheOutput() { + if command.ShouldCacheOutput() && !stopCache { img, err := layerCache.RetrieveLayer(ck) if err != nil { logrus.Debugf("Failed to retrieve layer: %s", err) logrus.Infof("No cached layer found for cmd %s", command.String()) logrus.Debugf("Key missing was: %s", compositeKey.Key()) - break + stopCache = true + continue } if cacheCmd := command.CacheCommand(img); cacheCmd != nil { From 697037cbcffb178cb1c3327b9be8425243cdda6d Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Wed, 27 Nov 2019 10:17:03 -0800 Subject: [PATCH 13/28] Add unit tests for compositecache and stagebuilder * add mock types for testing * enhance error messaging * add tests --- pkg/commands/commands.go | 7 + pkg/commands/copy.go | 2 +- pkg/commands/fake_commands.go | 88 ++++++++ pkg/executor/build.go | 52 +++-- pkg/executor/build_test.go | 316 +++++++++++++++++++++++++++ pkg/executor/composite_cache_test.go | 121 ++++++++++ pkg/executor/fakes.go | 180 +++++++++++++++ pkg/snapshot/snapshot.go | 2 +- pkg/util/command_util.go | 17 +- 9 files changed, 756 insertions(+), 29 deletions(-) create mode 100644 pkg/commands/fake_commands.go create mode 100644 pkg/executor/composite_cache_test.go create mode 100644 pkg/executor/fakes.go diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index a2ea7788d..cc58d51c7 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -17,6 +17,7 @@ limitations under the License. package commands import ( + "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" @@ -24,6 +25,12 @@ import ( "github.com/sirupsen/logrus" ) +var RootDir string + +func init() { + RootDir = constants.RootDir +} + type CurrentCacheKey func() (string, error) type DockerCommand interface { diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index ad75f6f3f..4071f957b 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -171,7 +171,7 @@ type CachingCopyCommand struct { func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { logrus.Infof("Found cached layer, extracting to filesystem") var err error - cr.extractedFiles, err = util.GetFSFromImage(constants.RootDir, cr.img) + cr.extractedFiles, err = util.GetFSFromImage(RootDir, cr.img) logrus.Infof("extractedFiles: %s", cr.extractedFiles) if err != nil { return errors.Wrap(err, "extracting fs from image") diff --git a/pkg/commands/fake_commands.go b/pkg/commands/fake_commands.go new file mode 100644 index 000000000..8efee7c4d --- /dev/null +++ b/pkg/commands/fake_commands.go @@ -0,0 +1,88 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// used for testing in the commands package +package commands + +import ( + "bytes" + "io" + "io/ioutil" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type fakeLayer struct { + TarContent []byte +} + +func (f fakeLayer) Digest() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeLayer) DiffID() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeLayer) Compressed() (io.ReadCloser, error) { + return nil, nil +} +func (f fakeLayer) Uncompressed() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(f.TarContent)), nil +} +func (f fakeLayer) Size() (int64, error) { + return 0, nil +} +func (f fakeLayer) MediaType() (types.MediaType, error) { + return "", nil +} + +type fakeImage struct { + ImageLayers []v1.Layer +} + +func (f fakeImage) Layers() ([]v1.Layer, error) { + return f.ImageLayers, nil +} +func (f fakeImage) MediaType() (types.MediaType, error) { + return "", nil +} +func (f fakeImage) Size() (int64, error) { + return 0, nil +} +func (f fakeImage) ConfigName() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeImage) ConfigFile() (*v1.ConfigFile, error) { + return &v1.ConfigFile{}, nil +} +func (f fakeImage) RawConfigFile() ([]byte, error) { + return []byte{}, nil +} +func (f fakeImage) Digest() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeImage) Manifest() (*v1.Manifest, error) { + return &v1.Manifest{}, nil +} +func (f fakeImage) RawManifest() ([]byte, error) { + return []byte{}, nil +} +func (f fakeImage) LayerByDigest(v1.Hash) (v1.Layer, error) { + return fakeLayer{}, nil +} +func (f fakeImage) LayerByDiffID(v1.Hash) (v1.Layer, error) { + return fakeLayer{}, nil +} diff --git a/pkg/executor/build.go b/pkg/executor/build.go index c1d987120..e3c48b5e1 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -23,7 +23,7 @@ import ( "strconv" "time" - "github.com/otiai10/copy" + otiai10Cpy "github.com/otiai10/copy" "github.com/google/go-containerregistry/pkg/v1/partial" @@ -52,12 +52,21 @@ import ( // This is the size of an empty tar in Go const emptyTarSize = 1024 +type cachePusher func(*config.KanikoOptions, string, string, string) error +type snapShotter interface { + Init() error + TakeSnapshotFS() (string, error) + TakeSnapshot([]string) (string, error) +} + // stageBuilder contains all fields necessary to build one stage of a Dockerfile type stageBuilder struct { stage config.KanikoStage image v1.Image cf *v1.ConfigFile - snapshotter *snapshot.Snapshotter + snapshotter snapShotter + layerCache cache.LayerCache + pushCache cachePusher baseImageDigest string finalCacheKey string opts *config.KanikoOptions @@ -103,6 +112,10 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross opts: opts, crossStageDeps: crossStageDeps, digestToCacheKeyMap: dcm, + layerCache: &cache.RegistryCache{ + Opts: opts, + }, + pushCache: pushLayerToCache, } for _, cmd := range s.stage.Commands { @@ -138,9 +151,6 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro return nil } - layerCache := &cache.RegistryCache{ - Opts: s.opts, - } stopCache := false // Possibly replace commands with their cached implementations. // We walk through all the commands, running any commands that only operate on metadata. @@ -154,21 +164,21 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro // If the command uses files from the context, add them. files, err := command.FilesUsedFromContext(&cfg, s.args) if err != nil { - return err + return errors.Wrap(err, "failed to get files used from context") } for _, f := range files { if err := compositeKey.AddPath(f); err != nil { - return err + return errors.Wrap(err, "failed to add path to composite key") } } ck, err := compositeKey.Hash() if err != nil { - return err + return errors.Wrap(err, "failed to hash composite key") } s.finalCacheKey = ck if command.ShouldCacheOutput() && !stopCache { - img, err := layerCache.RetrieveLayer(ck) + img, err := s.layerCache.RetrieveLayer(ck) if err != nil { logrus.Debugf("Failed to retrieve layer: %s", err) logrus.Infof("No cached layer found for cmd %s", command.String()) @@ -205,7 +215,7 @@ func (s *stageBuilder) build() error { // Apply optimizations to the instructions. if err := s.optimize(*compositeKey, s.cf.Config); err != nil { - return err + return errors.Wrap(err, "failed to optimize instructions") } // Unpack file system to root if we need to. @@ -224,14 +234,14 @@ func (s *stageBuilder) build() error { if shouldUnpack { t := timing.Start("FS Unpacking") if _, err := util.GetFSFromImage(constants.RootDir, s.image); err != nil { - return err + return errors.Wrap(err, "failed to get filesystem from image") } timing.DefaultRun.Stop(t) } else { logrus.Info("Skipping unpacking as no commands require it.") } if err := util.DetectFilesystemWhitelist(constants.WhitelistPath); err != nil { - return err + return errors.Wrap(err, "failed to check filesystem whitelist") } // Take initial snapshot t := timing.Start("Initial FS snapshot") @@ -252,17 +262,17 @@ func (s *stageBuilder) build() error { // If the command uses files from the context, add them. files, err := command.FilesUsedFromContext(&s.cf.Config, s.args) if err != nil { - return err + return errors.Wrap(err, "failed to get files used from context") } for _, f := range files { if err := compositeKey.AddPath(f); err != nil { - return err + return errors.Wrap(err, fmt.Sprintf("failed to add path to composite key %v", f)) } } logrus.Info(command.String()) if err := command.ExecuteCommand(&s.cf.Config, s.args); err != nil { - return err + return errors.Wrap(err, "failed to execute command") } files = command.FilesToSnapshot() timing.DefaultRun.Stop(t) @@ -273,21 +283,21 @@ func (s *stageBuilder) build() error { tarPath, err := s.takeSnapshot(files) if err != nil { - return err + return errors.Wrap(err, "failed to take snapshot") } ck, err := compositeKey.Hash() if err != nil { - return err + return errors.Wrap(err, "failed to hash composite key") } // Push layer to cache (in parallel) now along with new config file if s.opts.Cache && command.ShouldCacheOutput() { cacheGroup.Go(func() error { - return pushLayerToCache(s.opts, ck, tarPath, command.String()) + return s.pushCache(s.opts, ck, tarPath, command.String()) }) } if err := s.saveSnapshotToImage(command.String(), tarPath); err != nil { - return err + return errors.Wrap(err, "failed to save snapshot to image") } } if err := cacheGroup.Wait(); err != nil { @@ -343,7 +353,7 @@ func (s *stageBuilder) saveSnapshotToImage(createdBy string, tarPath string) err } fi, err := os.Stat(tarPath) if err != nil { - return err + return errors.Wrap(err, "tar file path does not exist") } if fi.Size() <= emptyTarSize { logrus.Info("No files were changed, appending empty layer to config. No layer added to image.") @@ -505,7 +515,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { } for _, p := range filesToSave { logrus.Infof("Saving file %s for later use.", p) - copy.Copy(p, filepath.Join(dstDir, p)) + otiai10Cpy.Copy(p, filepath.Join(dstDir, p)) } // Delete the filesystem diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 44f6a1211..47a7fdacd 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -17,6 +17,8 @@ limitations under the License. package executor import ( + "archive/tar" + "bytes" "io/ioutil" "os" "path/filepath" @@ -24,6 +26,7 @@ import ( "sort" "testing" + "github.com/GoogleContainerTools/kaniko/pkg/commands" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/testutil" @@ -462,3 +465,316 @@ func TestInitializeConfig(t *testing.T) { testutil.CheckDeepEqual(t, tt.expected, actual.Config) } } + +func Test_stageBuilder_optimize(t *testing.T) { + testCases := []struct { + opts *config.KanikoOptions + retrieve bool + name string + }{ + { + name: "cache enabled and layer not present in cache", + opts: &config.KanikoOptions{Cache: true}, + }, + { + name: "cache enabled and layer present in cache", + opts: &config.KanikoOptions{Cache: true}, + retrieve: true, + }, + { + name: "cache disabled and layer not present in cache", + opts: &config.KanikoOptions{Cache: false}, + }, + { + name: "cache disabled and layer present in cache", + opts: &config.KanikoOptions{Cache: false}, + retrieve: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cf := &v1.ConfigFile{} + snap := fakeSnapShotter{} + lc := &fakeLayerCache{retrieve: tc.retrieve} + sb := &stageBuilder{opts: tc.opts, cf: cf, snapshotter: snap, layerCache: lc} + ck := CompositeCache{} + file, err := ioutil.TempFile("", "foo") + if err != nil { + t.Error(err) + } + command := MockDockerCommand{ + contextFiles: []string{file.Name()}, + cacheCommand: MockCachedDockerCommand{}, + } + sb.cmds = []commands.DockerCommand{command} + err = sb.optimize(ck, cf.Config) + if err != nil { + t.Errorf("Expected error to be nil but was %v", err) + } + + }) + } +} + +func Test_stageBuilder_build(t *testing.T) { + type testcase struct { + description string + opts *config.KanikoOptions + layerCache *fakeLayerCache + expectedCacheKeys []string + pushedCacheKeys []string + commands []commands.DockerCommand + fileName string + rootDir string + image v1.Image + config *v1.ConfigFile + } + // The two copy command test cases use the same filesystem content and should generate the same cache keys. If they don't then something is wrong. They share this variable to help ensure that. + copyCommandCacheKey := "7263908b66952551d89fd895ffb067e2e30f474be9f38a8f1792af2b6df7c6e3" + tempDirAndFile := func() (string, string) { + dir, err := ioutil.TempDir("", "foo") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + filename := "bar.txt" + filepath := filepath.Join(dir, filename) + err = ioutil.WriteFile(filepath, []byte(`meow`), 0777) + if err != nil { + t.Errorf("could not create temp file %v", err) + } + buf := bytes.NewBuffer([]byte{}) + writer := tar.NewWriter(buf) + defer writer.Close() + + return dir, filename + } + testCases := []testcase{ + { + description: "fake command cache enabled but key not in cache", + opts: &config.KanikoOptions{Cache: true}, + expectedCacheKeys: []string{"2cd95a0195a42f2873273b7e8c970e3a87970bd0e5d330b3c7d068e7419e5017"}, + pushedCacheKeys: []string{"2cd95a0195a42f2873273b7e8c970e3a87970bd0e5d330b3c7d068e7419e5017"}, + }, + { + description: "fake command cache enabled and key in cache", + opts: &config.KanikoOptions{Cache: true}, + layerCache: &fakeLayerCache{ + retrieve: true, + }, + expectedCacheKeys: []string{"2cd95a0195a42f2873273b7e8c970e3a87970bd0e5d330b3c7d068e7419e5017"}, + pushedCacheKeys: []string{"2cd95a0195a42f2873273b7e8c970e3a87970bd0e5d330b3c7d068e7419e5017"}, + }, + { + description: "fake command cache disabled and key not in cache", + opts: &config.KanikoOptions{Cache: false}, + }, + { + description: "fake command cache disabled and key in cache", + opts: &config.KanikoOptions{Cache: false}, + layerCache: &fakeLayerCache{ + retrieve: true, + }, + }, + func() testcase { + dir, filename := tempDirAndFile() + filepath := filepath.Join(dir, filename) + + buf := bytes.NewBuffer([]byte{}) + writer := tar.NewWriter(buf) + defer writer.Close() + + info, err := os.Stat(filepath) + if err != nil { + t.Errorf("could not get file info for temp file %v", err) + } + hdr, err := tar.FileInfoHeader(info, filename) + if err != nil { + t.Errorf("could not get tar header for temp file %v", err) + } + + if err := writer.WriteHeader(hdr); err != nil { + t.Errorf("could not write tar header %v", err) + } + + content, err := ioutil.ReadFile(filepath) + if err != nil { + t.Errorf("could not read tempfile %v", err) + } + + if _, err := writer.Write(content); err != nil { + t.Errorf("could not write file contents to tar") + } + tarContent := buf.Bytes() + return testcase{ + description: "copy command cache enabled and key in cache", + opts: &config.KanikoOptions{Cache: true}, + layerCache: &fakeLayerCache{ + retrieve: true, + img: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + }, + }, + rootDir: dir, + expectedCacheKeys: []string{copyCommandCacheKey}, + // CachingCopyCommand is not pushed to the cache + pushedCacheKeys: []string{}, + commands: func() []commands.DockerCommand { + cmd, err := commands.GetCommand( + &instructions.CopyCommand{ + SourcesAndDest: []string{ + filename, "foo.txt", + }, + }, + dir, + ) + if err != nil { + panic(err) + } + return []commands.DockerCommand{ + cmd, + } + }(), + fileName: filename, + } + }(), + func() testcase { + dir, filename := tempDirAndFile() + + tarContent := []byte{} + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + return testcase{ + description: "copy command cache enabled and key is not in cache", + opts: &config.KanikoOptions{Cache: true}, + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + layerCache: &fakeLayerCache{ + retrieve: false, + }, + image: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + }, + rootDir: dir, + expectedCacheKeys: []string{copyCommandCacheKey}, + pushedCacheKeys: []string{copyCommandCacheKey}, + commands: func() []commands.DockerCommand { + cmd, err := commands.GetCommand( + &instructions.CopyCommand{ + SourcesAndDest: []string{ + filename, "foo.txt", + }, + }, + dir, + ) + if err != nil { + panic(err) + } + return []commands.DockerCommand{ + cmd, + } + }(), + fileName: filename, + } + }(), + } + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + var fileName string + if tc.commands == nil { + file, err := ioutil.TempFile("", "foo") + if err != nil { + t.Error(err) + } + command := MockDockerCommand{ + contextFiles: []string{file.Name()}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{file.Name()}, + }, + } + tc.commands = []commands.DockerCommand{command} + fileName = file.Name() + } else { + fileName = tc.fileName + } + + cf := tc.config + if cf == nil { + cf = &v1.ConfigFile{ + Config: v1.Config{ + Env: make([]string, 0), + }, + } + } + + snap := fakeSnapShotter{file: fileName} + lc := tc.layerCache + if lc == nil { + lc = &fakeLayerCache{} + } + keys := []string{} + sb := &stageBuilder{ + args: &dockerfile.BuildArgs{}, //required or code will panic + image: tc.image, + opts: tc.opts, + cf: cf, + snapshotter: snap, + layerCache: lc, + pushCache: func(_ *config.KanikoOptions, cacheKey, _, _ string) error { + keys = append(keys, cacheKey) + return nil + }, + } + sb.cmds = tc.commands + tmp := commands.RootDir + if tc.rootDir != "" { + commands.RootDir = tc.rootDir + } + err := sb.build() + if err != nil { + t.Errorf("Expected error to be nil but was %v", err) + } + + if len(tc.expectedCacheKeys) != len(lc.receivedKeys) { + t.Errorf("expected to receive %v keys but was %v", len(tc.expectedCacheKeys), len(lc.receivedKeys)) + } + for _, key := range tc.expectedCacheKeys { + match := false + for _, receivedKey := range lc.receivedKeys { + if key == receivedKey { + match = true + break + } + } + if !match { + t.Errorf("expected received keys to include %v but did not %v", key, lc.receivedKeys) + } + } + if len(tc.pushedCacheKeys) != len(keys) { + t.Errorf("expected to push %v keys but was %v", len(tc.pushedCacheKeys), len(keys)) + } + for _, key := range tc.pushedCacheKeys { + match := false + for _, pushedKey := range keys { + if key == pushedKey { + match = true + break + } + } + if !match { + t.Errorf("expected pushed keys to include %v but did not %v", key, keys) + } + } + commands.RootDir = tmp + + }) + } +} diff --git a/pkg/executor/composite_cache_test.go b/pkg/executor/composite_cache_test.go new file mode 100644 index 000000000..edd75c917 --- /dev/null +++ b/pkg/executor/composite_cache_test.go @@ -0,0 +1,121 @@ +package executor + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" +) + +func Test_NewCompositeCache(t *testing.T) { + r := NewCompositeCache() + if reflect.TypeOf(r).String() != "*executor.CompositeCache" { + t.Errorf("expected return to be *executor.CompositeCache but was %v", reflect.TypeOf(r).String()) + } +} + +func Test_CompositeCache_AddKey(t *testing.T) { + keys := []string{ + "meow", + "purr", + } + r := NewCompositeCache() + r.AddKey(keys...) + if len(r.keys) != 2 { + t.Errorf("expected keys to have length 2 but was %v", len(r.keys)) + } +} + +func Test_CompositeCache_Key(t *testing.T) { + r := NewCompositeCache("meow", "purr") + k := r.Key() + if k != "meow-purr" { + t.Errorf("expected result to equal meow-purr but was %v", k) + } +} + +func Test_CompositeCache_Hash(t *testing.T) { + r := NewCompositeCache("meow", "purr") + h, err := r.Hash() + if err != nil { + t.Errorf("expected error to be nil but was %v", err) + } + + expectedHash := "b4fd5a11af812a11a79d794007c842794cc668c8e7ebaba6d1e6d021b8e06c71" + if h != expectedHash { + t.Errorf("expected result to equal %v but was %v", expectedHash, h) + } +} + +func Test_CompositeCache_AddPath_dir(t *testing.T) { + tmpDir, err := ioutil.TempDir("/tmp", "foo") + if err != nil { + t.Errorf("got error setting up test %v", err) + } + + content := `meow meow meow` + if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo.txt"), []byte(content), 0777); err != nil { + t.Errorf("got error writing temp file %v", err) + } + + fn := func() string { + r := NewCompositeCache() + if err := r.AddPath(tmpDir); err != nil { + t.Errorf("expected error to be nil but was %v", err) + } + + if len(r.keys) != 1 { + t.Errorf("expected len of keys to be 1 but was %v", len(r.keys)) + } + hash, err := r.Hash() + if err != nil { + t.Errorf("couldnt generate hash from test cache") + } + return hash + } + + hash1 := fn() + hash2 := fn() + if hash1 != hash2 { + t.Errorf("expected hash %v to equal hash %v", hash1, hash2) + } +} +func Test_CompositeCache_AddPath_file(t *testing.T) { + tmpfile, err := ioutil.TempFile("/tmp", "foo.txt") + if err != nil { + t.Errorf("got error setting up test %v", err) + } + defer os.Remove(tmpfile.Name()) // clean up + + content := `meow meow meow` + if _, err := tmpfile.Write([]byte(content)); err != nil { + t.Errorf("got error writing temp file %v", err) + } + if err := tmpfile.Close(); err != nil { + t.Errorf("got error closing temp file %v", err) + } + + p := tmpfile.Name() + fn := func() string { + r := NewCompositeCache() + if err := r.AddPath(p); err != nil { + t.Errorf("expected error to be nil but was %v", err) + } + + if len(r.keys) != 1 { + t.Errorf("expected len of keys to be 1 but was %v", len(r.keys)) + } + hash, err := r.Hash() + if err != nil { + t.Errorf("couldnt generate hash from test cache") + } + return hash + } + + hash1 := fn() + hash2 := fn() + if hash1 != hash2 { + t.Errorf("expected hash %v to equal hash %v", hash1, hash2) + } +} diff --git a/pkg/executor/fakes.go b/pkg/executor/fakes.go new file mode 100644 index 000000000..86ee673ed --- /dev/null +++ b/pkg/executor/fakes.go @@ -0,0 +1,180 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// for use in tests +package executor + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + + "github.com/GoogleContainerTools/kaniko/pkg/commands" + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func fakeCachePush(_ *config.KanikoOptions, _, _, _ string) error { + return nil +} + +type fakeSnapShotter struct { + file string + tarPath string +} + +func (f fakeSnapShotter) Init() error { return nil } +func (f fakeSnapShotter) TakeSnapshotFS() (string, error) { + return f.tarPath, nil +} +func (f fakeSnapShotter) TakeSnapshot(_ []string) (string, error) { + return f.tarPath, nil +} + +type MockDockerCommand struct { + contextFiles []string + cacheCommand commands.DockerCommand +} + +func (m MockDockerCommand) ExecuteCommand(c *v1.Config, args *dockerfile.BuildArgs) error { return nil } +func (m MockDockerCommand) String() string { + return "meow" +} +func (m MockDockerCommand) FilesToSnapshot() []string { + return []string{"meow-snapshot-no-cache"} +} +func (m MockDockerCommand) CacheCommand(image v1.Image) commands.DockerCommand { + return m.cacheCommand +} +func (m MockDockerCommand) FilesUsedFromContext(c *v1.Config, args *dockerfile.BuildArgs) ([]string, error) { + return m.contextFiles, nil +} +func (m MockDockerCommand) MetadataOnly() bool { + return false +} +func (m MockDockerCommand) RequiresUnpackedFS() bool { + return false +} +func (m MockDockerCommand) ShouldCacheOutput() bool { + return true +} + +type MockCachedDockerCommand struct { + contextFiles []string +} + +func (m MockCachedDockerCommand) ExecuteCommand(c *v1.Config, args *dockerfile.BuildArgs) error { + return nil +} +func (m MockCachedDockerCommand) String() string { + return "meow" +} +func (m MockCachedDockerCommand) FilesToSnapshot() []string { + return []string{"meow-snapshot"} +} +func (m MockCachedDockerCommand) CacheCommand(image v1.Image) commands.DockerCommand { + return nil +} +func (m MockCachedDockerCommand) FilesUsedFromContext(c *v1.Config, args *dockerfile.BuildArgs) ([]string, error) { + return m.contextFiles, nil +} +func (m MockCachedDockerCommand) MetadataOnly() bool { + return false +} +func (m MockCachedDockerCommand) RequiresUnpackedFS() bool { + return false +} +func (m MockCachedDockerCommand) ShouldCacheOutput() bool { + return true +} + +type fakeLayerCache struct { + retrieve bool + receivedKeys []string + img v1.Image +} + +func (f *fakeLayerCache) RetrieveLayer(key string) (v1.Image, error) { + f.receivedKeys = append(f.receivedKeys, key) + if !f.retrieve { + return nil, errors.New("could not find layer") + } + return f.img, nil +} + +type fakeLayer struct { + TarContent []byte +} + +func (f fakeLayer) Digest() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeLayer) DiffID() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeLayer) Compressed() (io.ReadCloser, error) { + return nil, nil +} +func (f fakeLayer) Uncompressed() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(f.TarContent)), nil +} +func (f fakeLayer) Size() (int64, error) { + return 0, nil +} +func (f fakeLayer) MediaType() (types.MediaType, error) { + return "", nil +} + +type fakeImage struct { + ImageLayers []v1.Layer +} + +func (f fakeImage) Layers() ([]v1.Layer, error) { + return f.ImageLayers, nil +} +func (f fakeImage) MediaType() (types.MediaType, error) { + return "", nil +} +func (f fakeImage) Size() (int64, error) { + return 0, nil +} +func (f fakeImage) ConfigName() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeImage) ConfigFile() (*v1.ConfigFile, error) { + return &v1.ConfigFile{}, nil +} +func (f fakeImage) RawConfigFile() ([]byte, error) { + return []byte{}, nil +} +func (f fakeImage) Digest() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeImage) Manifest() (*v1.Manifest, error) { + return &v1.Manifest{}, nil +} +func (f fakeImage) RawManifest() ([]byte, error) { + return []byte{}, nil +} +func (f fakeImage) LayerByDigest(v1.Hash) (v1.Layer, error) { + return fakeLayer{}, nil +} +func (f fakeImage) LayerByDiffID(v1.Hash) (v1.Layer, error) { + return fakeLayer{}, nil +} diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 5577da087..b54897d59 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -176,7 +176,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { // Only add changed files. fileChanged, err := s.l.CheckFileChange(path) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("could not check if file has changed %s %s", path, err) } if fileChanged { logrus.Debugf("Adding %s to layer, because it was changed.", path) diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go index c4b3437ea..103d22e0b 100644 --- a/pkg/util/command_util.go +++ b/pkg/util/command_util.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + "fmt" "net/http" "net/url" "os" @@ -25,7 +26,7 @@ import ( "strings" "github.com/GoogleContainerTools/kaniko/pkg/constants" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/frontend/dockerfile/shell" @@ -77,13 +78,16 @@ func ResolveEnvAndWildcards(sd instructions.SourcesAndDest, buildcontext string, // First, resolve any environment replacement resolvedEnvs, err := ResolveEnvironmentReplacementList(sd, envs, true) if err != nil { - return nil, "", err + return nil, "", errors.Wrap(err, "failed to resolve environment") + } + if len(resolvedEnvs) == 0 { + return nil, "", errors.New("resolved envs is empty") } dest := resolvedEnvs[len(resolvedEnvs)-1] // Resolve wildcards and get a list of resolved sources srcs, err := ResolveSources(resolvedEnvs[0:len(resolvedEnvs)-1], buildcontext) if err != nil { - return nil, "", err + return nil, "", errors.Wrap(err, "failed to resolve sources") } err = IsSrcsValid(sd, srcs, buildcontext) return srcs, dest, err @@ -219,9 +223,10 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri if IsSrcRemoteFileURL(resolvedSources[0]) { return nil } - fi, err := os.Lstat(filepath.Join(root, resolvedSources[0])) + path := filepath.Join(root, resolvedSources[0]) + fi, err := os.Lstat(path) if err != nil { - return err + return errors.Wrap(err, fmt.Sprintf("failed to get fileinfo for %v", path)) } if fi.IsDir() { return nil @@ -237,7 +242,7 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri src = filepath.Clean(src) files, err := RelativeFiles(src, root) if err != nil { - return err + return errors.Wrap(err, "failed to get relative files") } for _, file := range files { if excludeFile(file, root) { From 33f3191b1730a5b70a8abd580247b260d0f21ab8 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Wed, 27 Nov 2019 15:03:46 -0800 Subject: [PATCH 14/28] Don't hardcode hashes for stagebuilder tests --- pkg/executor/build_test.go | 104 ++++++++++++++++++++++++++++++------- pkg/executor/fakes.go | 2 +- 2 files changed, 86 insertions(+), 20 deletions(-) diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 47a7fdacd..fac9bed40 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -35,6 +35,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/sirupsen/logrus" ) func Test_reviewConfig(t *testing.T) { @@ -529,8 +530,6 @@ func Test_stageBuilder_build(t *testing.T) { image v1.Image config *v1.ConfigFile } - // The two copy command test cases use the same filesystem content and should generate the same cache keys. If they don't then something is wrong. They share this variable to help ensure that. - copyCommandCacheKey := "7263908b66952551d89fd895ffb067e2e30f474be9f38a8f1792af2b6df7c6e3" tempDirAndFile := func() (string, string) { dir, err := ioutil.TempDir("", "foo") if err != nil { @@ -549,21 +548,71 @@ func Test_stageBuilder_build(t *testing.T) { return dir, filename } testCases := []testcase{ - { - description: "fake command cache enabled but key not in cache", - opts: &config.KanikoOptions{Cache: true}, - expectedCacheKeys: []string{"2cd95a0195a42f2873273b7e8c970e3a87970bd0e5d330b3c7d068e7419e5017"}, - pushedCacheKeys: []string{"2cd95a0195a42f2873273b7e8c970e3a87970bd0e5d330b3c7d068e7419e5017"}, - }, - { - description: "fake command cache enabled and key in cache", - opts: &config.KanikoOptions{Cache: true}, - layerCache: &fakeLayerCache{ - retrieve: true, - }, - expectedCacheKeys: []string{"2cd95a0195a42f2873273b7e8c970e3a87970bd0e5d330b3c7d068e7419e5017"}, - pushedCacheKeys: []string{"2cd95a0195a42f2873273b7e8c970e3a87970bd0e5d330b3c7d068e7419e5017"}, - }, + func() testcase { + dir, file := tempDirAndFile() + filePath := filepath.Join(dir, file) + ch := NewCompositeCache("", "meow") + + ch.AddPath(filePath) + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + command := MockDockerCommand{ + contextFiles: []string{filePath}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{filePath}, + }, + } + + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + return testcase{ + description: "fake command cache enabled but key not in cache", + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + opts: &config.KanikoOptions{Cache: true}, + expectedCacheKeys: []string{hash}, + pushedCacheKeys: []string{hash}, + commands: []commands.DockerCommand{command}, + rootDir: dir, + } + }(), + func() testcase { + dir, file := tempDirAndFile() + filePath := filepath.Join(dir, file) + ch := NewCompositeCache("", "meow") + + ch.AddPath(filePath) + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + command := MockDockerCommand{ + contextFiles: []string{filePath}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{filePath}, + }, + } + + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + return testcase{ + description: "fake command cache enabled and key in cache", + opts: &config.KanikoOptions{Cache: true}, + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + layerCache: &fakeLayerCache{ + retrieve: true, + }, + expectedCacheKeys: []string{hash}, + pushedCacheKeys: []string{}, + commands: []commands.DockerCommand{command}, + rootDir: dir, + } + }(), { description: "fake command cache disabled and key not in cache", opts: &config.KanikoOptions{Cache: false}, @@ -605,6 +654,15 @@ func Test_stageBuilder_build(t *testing.T) { t.Errorf("could not write file contents to tar") } tarContent := buf.Bytes() + + ch := NewCompositeCache("", "") + ch.AddPath(filepath) + logrus.SetLevel(logrus.DebugLevel) + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + copyCommandCacheKey := hash return testcase{ description: "copy command cache enabled and key in cache", opts: &config.KanikoOptions{Cache: true}, @@ -649,6 +707,14 @@ func Test_stageBuilder_build(t *testing.T) { if err != nil { t.Errorf("could not create temp dir %v", err) } + filePath := filepath.Join(dir, filename) + ch := NewCompositeCache("", "") + ch.AddPath(filePath) + logrus.SetLevel(logrus.DebugLevel) + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } return testcase{ description: "copy command cache enabled and key is not in cache", opts: &config.KanikoOptions{Cache: true}, @@ -664,8 +730,8 @@ func Test_stageBuilder_build(t *testing.T) { }, }, rootDir: dir, - expectedCacheKeys: []string{copyCommandCacheKey}, - pushedCacheKeys: []string{copyCommandCacheKey}, + expectedCacheKeys: []string{hash}, + pushedCacheKeys: []string{hash}, commands: func() []commands.DockerCommand { cmd, err := commands.GetCommand( &instructions.CopyCommand{ diff --git a/pkg/executor/fakes.go b/pkg/executor/fakes.go index 86ee673ed..e34a5ec23 100644 --- a/pkg/executor/fakes.go +++ b/pkg/executor/fakes.go @@ -101,7 +101,7 @@ func (m MockCachedDockerCommand) RequiresUnpackedFS() bool { return false } func (m MockCachedDockerCommand) ShouldCacheOutput() bool { - return true + return false } type fakeLayerCache struct { From 6d0c8da90eab42860dda748dc2bf26a239436477 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Wed, 27 Nov 2019 23:02:33 -0800 Subject: [PATCH 15/28] more stagebuilder caching tests --- pkg/commands/fake_commands.go | 88 ------------- pkg/executor/build_test.go | 234 +++++++++++++++++++++++++--------- pkg/executor/fakes.go | 14 +- 3 files changed, 185 insertions(+), 151 deletions(-) delete mode 100644 pkg/commands/fake_commands.go diff --git a/pkg/commands/fake_commands.go b/pkg/commands/fake_commands.go deleted file mode 100644 index 8efee7c4d..000000000 --- a/pkg/commands/fake_commands.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// used for testing in the commands package -package commands - -import ( - "bytes" - "io" - "io/ioutil" - - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/types" -) - -type fakeLayer struct { - TarContent []byte -} - -func (f fakeLayer) Digest() (v1.Hash, error) { - return v1.Hash{}, nil -} -func (f fakeLayer) DiffID() (v1.Hash, error) { - return v1.Hash{}, nil -} -func (f fakeLayer) Compressed() (io.ReadCloser, error) { - return nil, nil -} -func (f fakeLayer) Uncompressed() (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(f.TarContent)), nil -} -func (f fakeLayer) Size() (int64, error) { - return 0, nil -} -func (f fakeLayer) MediaType() (types.MediaType, error) { - return "", nil -} - -type fakeImage struct { - ImageLayers []v1.Layer -} - -func (f fakeImage) Layers() ([]v1.Layer, error) { - return f.ImageLayers, nil -} -func (f fakeImage) MediaType() (types.MediaType, error) { - return "", nil -} -func (f fakeImage) Size() (int64, error) { - return 0, nil -} -func (f fakeImage) ConfigName() (v1.Hash, error) { - return v1.Hash{}, nil -} -func (f fakeImage) ConfigFile() (*v1.ConfigFile, error) { - return &v1.ConfigFile{}, nil -} -func (f fakeImage) RawConfigFile() ([]byte, error) { - return []byte{}, nil -} -func (f fakeImage) Digest() (v1.Hash, error) { - return v1.Hash{}, nil -} -func (f fakeImage) Manifest() (*v1.Manifest, error) { - return &v1.Manifest{}, nil -} -func (f fakeImage) RawManifest() ([]byte, error) { - return []byte{}, nil -} -func (f fakeImage) LayerByDigest(v1.Hash) (v1.Layer, error) { - return fakeLayer{}, nil -} -func (f fakeImage) LayerByDiffID(v1.Hash) (v1.Layer, error) { - return fakeLayer{}, nil -} diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index fac9bed40..02d73a9d6 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -19,6 +19,7 @@ package executor import ( "archive/tar" "bytes" + "fmt" "io/ioutil" "os" "path/filepath" @@ -530,26 +531,60 @@ func Test_stageBuilder_build(t *testing.T) { image v1.Image config *v1.ConfigFile } - tempDirAndFile := func() (string, string) { + tempDirAndFile := func(filenames ...string) (string, []string) { + if len(filenames) == 0 { + filenames = []string{"bar.txt"} + } dir, err := ioutil.TempDir("", "foo") if err != nil { t.Errorf("could not create temp dir %v", err) } - filename := "bar.txt" - filepath := filepath.Join(dir, filename) - err = ioutil.WriteFile(filepath, []byte(`meow`), 0777) - if err != nil { - t.Errorf("could not create temp file %v", err) + for _, filename := range filenames { + filepath := filepath.Join(dir, filename) + err = ioutil.WriteFile(filepath, []byte(`meow`), 0777) + if err != nil { + t.Errorf("could not create temp file %v", err) + } } + + return dir, filenames + } + generateTar := func(dir string, fileNames ...string) []byte { buf := bytes.NewBuffer([]byte{}) writer := tar.NewWriter(buf) defer writer.Close() - return dir, filename + for _, filename := range fileNames { + filePath := filepath.Join(dir, filename) + info, err := os.Stat(filePath) + if err != nil { + t.Errorf("could not get file info for temp file %v", err) + } + hdr, err := tar.FileInfoHeader(info, filename) + if err != nil { + t.Errorf("could not get tar header for temp file %v", err) + } + + if err := writer.WriteHeader(hdr); err != nil { + t.Errorf("could not write tar header %v", err) + } + + content, err := ioutil.ReadFile(filePath) + if err != nil { + t.Errorf("could not read tempfile %v", err) + } + + if _, err := writer.Write(content); err != nil { + t.Errorf("could not write file contents to tar") + } + } + return buf.Bytes() } + testCases := []testcase{ func() testcase { - dir, file := tempDirAndFile() + dir, files := tempDirAndFile() + file := files[0] filePath := filepath.Join(dir, file) ch := NewCompositeCache("", "meow") @@ -580,7 +615,8 @@ func Test_stageBuilder_build(t *testing.T) { } }(), func() testcase { - dir, file := tempDirAndFile() + dir, files := tempDirAndFile() + file := files[0] filePath := filepath.Join(dir, file) ch := NewCompositeCache("", "meow") @@ -625,35 +661,11 @@ func Test_stageBuilder_build(t *testing.T) { }, }, func() testcase { - dir, filename := tempDirAndFile() + dir, filenames := tempDirAndFile() + filename := filenames[0] filepath := filepath.Join(dir, filename) - buf := bytes.NewBuffer([]byte{}) - writer := tar.NewWriter(buf) - defer writer.Close() - - info, err := os.Stat(filepath) - if err != nil { - t.Errorf("could not get file info for temp file %v", err) - } - hdr, err := tar.FileInfoHeader(info, filename) - if err != nil { - t.Errorf("could not get tar header for temp file %v", err) - } - - if err := writer.WriteHeader(hdr); err != nil { - t.Errorf("could not write tar header %v", err) - } - - content, err := ioutil.ReadFile(filepath) - if err != nil { - t.Errorf("could not read tempfile %v", err) - } - - if _, err := writer.Write(content); err != nil { - t.Errorf("could not write file contents to tar") - } - tarContent := buf.Bytes() + tarContent := generateTar(dir, filename) ch := NewCompositeCache("", "") ch.AddPath(filepath) @@ -700,8 +712,8 @@ func Test_stageBuilder_build(t *testing.T) { } }(), func() testcase { - dir, filename := tempDirAndFile() - + dir, filenames := tempDirAndFile() + filename := filenames[0] tarContent := []byte{} destDir, err := ioutil.TempDir("", "baz") if err != nil { @@ -751,6 +763,110 @@ func Test_stageBuilder_build(t *testing.T) { fileName: filename, } }(), + func() testcase { + dir, filenames := tempDirAndFile() + filename := filenames[0] + tarContent := generateTar(filename) + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + filePath := filepath.Join(dir, filename) + ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) + ch.AddPath(filePath) + logrus.SetLevel(logrus.DebugLevel) + logrus.Infof("test composite key %v", ch) + hash1, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) + ch.AddPath(filePath) + logrus.Infof("test composite key %v", ch) + hash2, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) + ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) + ch.AddPath(filePath) + logrus.Infof("test composite key %v", ch) + hash3, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + image := fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + } + + dockerFile := fmt.Sprintf(` +FROM ubuntu:16.04 +COPY %s foo.txt +COPY %s bar.txt +`, filename, filename) + f, _ := ioutil.TempFile("", "") + ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755) + opts := &config.KanikoOptions{ + DockerfilePath: f.Name(), + } + + stages, err := dockerfile.Stages(opts) + if err != nil { + t.Errorf("could not parse test dockerfile") + } + stage := stages[0] + cmds := stage.Commands + return testcase{ + description: "cached copy command followed by uncached copy command result in different read and write hashes", + opts: &config.KanikoOptions{Cache: true}, + rootDir: dir, + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + layerCache: &fakeLayerCache{ + keySequence: []string{hash1}, + img: image, + }, + image: image, + // hash1 is the read cachekey for the first layer + // hash2 is the read cachekey for the second layer + expectedCacheKeys: []string{hash1, hash2}, + // Due to CachingCopyCommand and CopyCommand returning different values the write cache key for the second copy command will never match the read cache key + // hash3 is the cachekey used to write to the cache for layer 2 + pushedCacheKeys: []string{hash3}, + commands: func() []commands.DockerCommand { + outCommands := make([]commands.DockerCommand, 0) + for _, c := range cmds { + cmd, err := commands.GetCommand( + c, + dir, + ) + if err != nil { + panic(err) + } + outCommands = append(outCommands, cmd) + } + return outCommands + //fn := func(srcAndDest []string) commands.DockerCommand { + // cmd, err := commands.GetCommand( + // &instructions.CopyCommand{ + // SourcesAndDest: srcAndDest, + // }, + // dir, + // ) + // if err != nil { + // panic(err) + // } + // return cmd + //} + //return []commands.DockerCommand{ + // fn([]string{filename, "foo.txt"}), fn([]string{filename, "bar.txt"}), + //} + }(), + } + }(), } for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { @@ -812,31 +928,33 @@ func Test_stageBuilder_build(t *testing.T) { if len(tc.expectedCacheKeys) != len(lc.receivedKeys) { t.Errorf("expected to receive %v keys but was %v", len(tc.expectedCacheKeys), len(lc.receivedKeys)) } - for _, key := range tc.expectedCacheKeys { - match := false - for _, receivedKey := range lc.receivedKeys { - if key == receivedKey { - match = true - break - } - } - if !match { - t.Errorf("expected received keys to include %v but did not %v", key, lc.receivedKeys) + expectedCached := tc.expectedCacheKeys + actualCached := lc.receivedKeys + sort.Slice(expectedCached, func(x, y int) bool { + return expectedCached[x] > expectedCached[y] + }) + sort.Slice(actualCached, func(x, y int) bool { + return actualCached[x] > actualCached[y] + }) + for i, key := range expectedCached { + if key != actualCached[i] { + t.Errorf("expected retrieved keys %d to be %v but was %v %v", i, key, actualCached[i], actualCached) } } if len(tc.pushedCacheKeys) != len(keys) { t.Errorf("expected to push %v keys but was %v", len(tc.pushedCacheKeys), len(keys)) } - for _, key := range tc.pushedCacheKeys { - match := false - for _, pushedKey := range keys { - if key == pushedKey { - match = true - break - } - } - if !match { - t.Errorf("expected pushed keys to include %v but did not %v", key, keys) + expectedPushed := tc.pushedCacheKeys + actualPushed := keys + sort.Slice(expectedPushed, func(x, y int) bool { + return expectedPushed[x] > expectedPushed[y] + }) + sort.Slice(actualPushed, func(x, y int) bool { + return actualPushed[x] > actualPushed[y] + }) + for i, key := range expectedPushed { + if key != actualPushed[i] { + t.Errorf("expected pushed keys %d to be %v but was %v %v", i, key, actualPushed[i], actualPushed) } } commands.RootDir = tmp diff --git a/pkg/executor/fakes.go b/pkg/executor/fakes.go index e34a5ec23..e9cfdc694 100644 --- a/pkg/executor/fakes.go +++ b/pkg/executor/fakes.go @@ -24,16 +24,11 @@ import ( "io/ioutil" "github.com/GoogleContainerTools/kaniko/pkg/commands" - "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/types" ) -func fakeCachePush(_ *config.KanikoOptions, _, _, _ string) error { - return nil -} - type fakeSnapShotter struct { file string tarPath string @@ -108,10 +103,19 @@ type fakeLayerCache struct { retrieve bool receivedKeys []string img v1.Image + keySequence []string } func (f *fakeLayerCache) RetrieveLayer(key string) (v1.Image, error) { f.receivedKeys = append(f.receivedKeys, key) + if len(f.keySequence) > 0 { + if f.keySequence[0] == key { + f.keySequence = f.keySequence[1:] + return f.img, nil + } + return f.img, errors.New("could not find layer") + } + if !f.retrieve { return nil, errors.New("could not find layer") } From 828e764b95456693704518126d334a01a9ce6586 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Thu, 28 Nov 2019 09:18:58 -0800 Subject: [PATCH 16/28] add boilerplate for composite_cache_test --- pkg/executor/composite_cache_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/executor/composite_cache_test.go b/pkg/executor/composite_cache_test.go index edd75c917..a3c7f55af 100644 --- a/pkg/executor/composite_cache_test.go +++ b/pkg/executor/composite_cache_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package executor import ( From 7ba65daf7f0d1dfa29507ad5d4a3ded0950b25c3 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Thu, 28 Nov 2019 09:35:41 -0800 Subject: [PATCH 17/28] cleanup executor/build_test.go --- pkg/executor/build_test.go | 257 +++++++++++++++---------------------- 1 file changed, 107 insertions(+), 150 deletions(-) diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 02d73a9d6..66c0ab5b5 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -531,59 +531,10 @@ func Test_stageBuilder_build(t *testing.T) { image v1.Image config *v1.ConfigFile } - tempDirAndFile := func(filenames ...string) (string, []string) { - if len(filenames) == 0 { - filenames = []string{"bar.txt"} - } - dir, err := ioutil.TempDir("", "foo") - if err != nil { - t.Errorf("could not create temp dir %v", err) - } - for _, filename := range filenames { - filepath := filepath.Join(dir, filename) - err = ioutil.WriteFile(filepath, []byte(`meow`), 0777) - if err != nil { - t.Errorf("could not create temp file %v", err) - } - } - - return dir, filenames - } - generateTar := func(dir string, fileNames ...string) []byte { - buf := bytes.NewBuffer([]byte{}) - writer := tar.NewWriter(buf) - defer writer.Close() - - for _, filename := range fileNames { - filePath := filepath.Join(dir, filename) - info, err := os.Stat(filePath) - if err != nil { - t.Errorf("could not get file info for temp file %v", err) - } - hdr, err := tar.FileInfoHeader(info, filename) - if err != nil { - t.Errorf("could not get tar header for temp file %v", err) - } - - if err := writer.WriteHeader(hdr); err != nil { - t.Errorf("could not write tar header %v", err) - } - - content, err := ioutil.ReadFile(filePath) - if err != nil { - t.Errorf("could not read tempfile %v", err) - } - - if _, err := writer.Write(content); err != nil { - t.Errorf("could not write file contents to tar") - } - } - return buf.Bytes() - } testCases := []testcase{ func() testcase { - dir, files := tempDirAndFile() + dir, files := tempDirAndFile(t) file := files[0] filePath := filepath.Join(dir, file) ch := NewCompositeCache("", "meow") @@ -615,7 +566,7 @@ func Test_stageBuilder_build(t *testing.T) { } }(), func() testcase { - dir, files := tempDirAndFile() + dir, files := tempDirAndFile(t) file := files[0] filePath := filepath.Join(dir, file) ch := NewCompositeCache("", "meow") @@ -661,11 +612,11 @@ func Test_stageBuilder_build(t *testing.T) { }, }, func() testcase { - dir, filenames := tempDirAndFile() + dir, filenames := tempDirAndFile(t) filename := filenames[0] filepath := filepath.Join(dir, filename) - tarContent := generateTar(dir, filename) + tarContent := generateTar(t, dir, filename) ch := NewCompositeCache("", "") ch.AddPath(filepath) @@ -692,27 +643,18 @@ func Test_stageBuilder_build(t *testing.T) { expectedCacheKeys: []string{copyCommandCacheKey}, // CachingCopyCommand is not pushed to the cache pushedCacheKeys: []string{}, - commands: func() []commands.DockerCommand { - cmd, err := commands.GetCommand( - &instructions.CopyCommand{ - SourcesAndDest: []string{ - filename, "foo.txt", - }, + commands: getCommands(dir, []instructions.Command{ + &instructions.CopyCommand{ + SourcesAndDest: []string{ + filename, "foo.txt", }, - dir, - ) - if err != nil { - panic(err) - } - return []commands.DockerCommand{ - cmd, - } - }(), + }, + }), fileName: filename, } }(), func() testcase { - dir, filenames := tempDirAndFile() + dir, filenames := tempDirAndFile(t) filename := filenames[0] tarContent := []byte{} destDir, err := ioutil.TempDir("", "baz") @@ -731,9 +673,7 @@ func Test_stageBuilder_build(t *testing.T) { description: "copy command cache enabled and key is not in cache", opts: &config.KanikoOptions{Cache: true}, config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, - layerCache: &fakeLayerCache{ - retrieve: false, - }, + layerCache: &fakeLayerCache{}, image: fakeImage{ ImageLayers: []v1.Layer{ fakeLayer{ @@ -744,29 +684,20 @@ func Test_stageBuilder_build(t *testing.T) { rootDir: dir, expectedCacheKeys: []string{hash}, pushedCacheKeys: []string{hash}, - commands: func() []commands.DockerCommand { - cmd, err := commands.GetCommand( - &instructions.CopyCommand{ - SourcesAndDest: []string{ - filename, "foo.txt", - }, + commands: getCommands(dir, []instructions.Command{ + &instructions.CopyCommand{ + SourcesAndDest: []string{ + filename, "foo.txt", }, - dir, - ) - if err != nil { - panic(err) - } - return []commands.DockerCommand{ - cmd, - } - }(), + }, + }), fileName: filename, } }(), func() testcase { - dir, filenames := tempDirAndFile() + dir, filenames := tempDirAndFile(t) filename := filenames[0] - tarContent := generateTar(filename) + tarContent := generateTar(t, filename) destDir, err := ioutil.TempDir("", "baz") if err != nil { t.Errorf("could not create temp dir %v", err) @@ -836,35 +767,7 @@ COPY %s bar.txt // Due to CachingCopyCommand and CopyCommand returning different values the write cache key for the second copy command will never match the read cache key // hash3 is the cachekey used to write to the cache for layer 2 pushedCacheKeys: []string{hash3}, - commands: func() []commands.DockerCommand { - outCommands := make([]commands.DockerCommand, 0) - for _, c := range cmds { - cmd, err := commands.GetCommand( - c, - dir, - ) - if err != nil { - panic(err) - } - outCommands = append(outCommands, cmd) - } - return outCommands - //fn := func(srcAndDest []string) commands.DockerCommand { - // cmd, err := commands.GetCommand( - // &instructions.CopyCommand{ - // SourcesAndDest: srcAndDest, - // }, - // dir, - // ) - // if err != nil { - // panic(err) - // } - // return cmd - //} - //return []commands.DockerCommand{ - // fn([]string{filename, "foo.txt"}), fn([]string{filename, "bar.txt"}), - //} - }(), + commands: getCommands(dir, cmds), } }(), } @@ -925,40 +828,94 @@ COPY %s bar.txt t.Errorf("Expected error to be nil but was %v", err) } - if len(tc.expectedCacheKeys) != len(lc.receivedKeys) { - t.Errorf("expected to receive %v keys but was %v", len(tc.expectedCacheKeys), len(lc.receivedKeys)) - } - expectedCached := tc.expectedCacheKeys - actualCached := lc.receivedKeys - sort.Slice(expectedCached, func(x, y int) bool { - return expectedCached[x] > expectedCached[y] - }) - sort.Slice(actualCached, func(x, y int) bool { - return actualCached[x] > actualCached[y] - }) - for i, key := range expectedCached { - if key != actualCached[i] { - t.Errorf("expected retrieved keys %d to be %v but was %v %v", i, key, actualCached[i], actualCached) - } - } - if len(tc.pushedCacheKeys) != len(keys) { - t.Errorf("expected to push %v keys but was %v", len(tc.pushedCacheKeys), len(keys)) - } - expectedPushed := tc.pushedCacheKeys - actualPushed := keys - sort.Slice(expectedPushed, func(x, y int) bool { - return expectedPushed[x] > expectedPushed[y] - }) - sort.Slice(actualPushed, func(x, y int) bool { - return actualPushed[x] > actualPushed[y] - }) - for i, key := range expectedPushed { - if key != actualPushed[i] { - t.Errorf("expected pushed keys %d to be %v but was %v %v", i, key, actualPushed[i], actualPushed) - } - } + assertCacheKeys(t, tc.expectedCacheKeys, lc.receivedKeys, "receive") + assertCacheKeys(t, tc.pushedCacheKeys, keys, "push") + commands.RootDir = tmp }) } } + +func assertCacheKeys(t *testing.T, expectedCacheKeys, actualCacheKeys []string, description string) { + if len(expectedCacheKeys) != len(actualCacheKeys) { + t.Errorf("expected to %v %v keys but was %v", description, len(expectedCacheKeys), len(actualCacheKeys)) + } + + sort.Slice(expectedCacheKeys, func(x, y int) bool { + return expectedCacheKeys[x] > expectedCacheKeys[y] + }) + sort.Slice(actualCacheKeys, func(x, y int) bool { + return actualCacheKeys[x] > actualCacheKeys[y] + }) + for i, key := range expectedCacheKeys { + if key != actualCacheKeys[i] { + t.Errorf("expected to %v keys %d to be %v but was %v %v", description, i, key, actualCacheKeys[i], actualCacheKeys) + } + } +} + +func getCommands(dir string, cmds []instructions.Command) []commands.DockerCommand { + outCommands := make([]commands.DockerCommand, 0) + for _, c := range cmds { + cmd, err := commands.GetCommand( + c, + dir, + ) + if err != nil { + panic(err) + } + outCommands = append(outCommands, cmd) + } + return outCommands + +} + +func tempDirAndFile(t *testing.T) (string, []string) { + filenames := []string{"bar.txt"} + + dir, err := ioutil.TempDir("", "foo") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + for _, filename := range filenames { + filepath := filepath.Join(dir, filename) + err = ioutil.WriteFile(filepath, []byte(`meow`), 0777) + if err != nil { + t.Errorf("could not create temp file %v", err) + } + } + + return dir, filenames +} +func generateTar(t *testing.T, dir string, fileNames ...string) []byte { + buf := bytes.NewBuffer([]byte{}) + writer := tar.NewWriter(buf) + defer writer.Close() + + for _, filename := range fileNames { + filePath := filepath.Join(dir, filename) + info, err := os.Stat(filePath) + if err != nil { + t.Errorf("could not get file info for temp file %v", err) + } + hdr, err := tar.FileInfoHeader(info, filename) + if err != nil { + t.Errorf("could not get tar header for temp file %v", err) + } + + if err := writer.WriteHeader(hdr); err != nil { + t.Errorf("could not write tar header %v", err) + } + + content, err := ioutil.ReadFile(filePath) + if err != nil { + t.Errorf("could not read tempfile %v", err) + } + + if _, err := writer.Write(content); err != nil { + t.Errorf("could not write file contents to tar") + } + } + return buf.Bytes() +} From 6734a9714dda340b26017c084fa74acb388d4290 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Thu, 28 Nov 2019 10:06:39 -0800 Subject: [PATCH 18/28] add golangci.yaml file matching current config --- .golangci.yaml | 191 +++++++++++++++++++++++++++++++++++++++++++++++++ hack/linter.sh | 13 +--- 2 files changed, 192 insertions(+), 12 deletions(-) create mode 100644 .golangci.yaml diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 000000000..bb2c7b5a9 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,191 @@ +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 4 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + deadline: 1m + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: true + + # list of build tags, all linters use it. Default is empty list. + build-tags: + + # which dirs to skip: they won't be analyzed; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but next dirs are always skipped independently + # from this option's value: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs: + + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + skip-files: + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number" + format: colored-line-number + + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + + +# all available settings of specific linters +linters-settings: + errcheck: + # report about not checking of errors in type assetions: `a := b.(MyStruct)`; + # default is false: such cases aren't reported by default. + check-type-assertions: false + + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: false + govet: + # report about shadowed variables + #check-shadowing: true + + # Obtain type information from installed (to $GOPATH/pkg) package files: + # golangci-lint will execute `go install -i` and `go test -i` for analyzed packages + # before analyzing them. + # By default this option is disabled and govet gets type information by loader from source code. + # Loading from source code is slow, but it's done only once for all linters. + # Go-installing of packages first time is much slower than loading them from source code, + # therefore this option is disabled by default. + # But repeated installation is fast in go >= 1.10 because of build caching. + # Enable this option only if all conditions are met: + # 1. you use only "fast" linters (--fast e.g.): no program loading occurs + # 2. you use go >= 1.10 + # 3. you do repeated runs (false for CI) or cache $GOPATH/pkg or `go env GOCACHE` dir in CI. + #use-installed-packages: false + golint: + # minimal confidence for issues, default is 0.8 + min-confidence: 0.8 + gofmt: + # simplify code: gofmt with `-s` option, true by default + simplify: true + #gocyclo: + # # minimal code complexity to report, 30 by default (but we recommend 10-20) + # min-complexity: 10 + maligned: + # print struct with more effective memory layout or not, false by default + suggest-new: true + #dupl: + # # tokens count to trigger issue, 150 by default + # threshold: 100 + goconst: + # minimal length of string constant, 3 by default + min-len: 3 + # minimal occurrences count to trigger, 3 by default + min-occurrences: 3 + #depguard: + # list-type: blacklist + # include-go-root: false + # packages: + # - github.com/davecgh/go-spew/spew + misspell: + # Correct spellings using locale preferences for US or UK. + # Default is to use a neutral variety of English. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + locale: US + #lll: + # # max line length, lines longer will be reported. Default is 120. + # # '\t' is counted as 1 character by default, and can be changed with the tab-width option + # line-length: 120 + # # tab width in spaces. Default to 1. + # tab-width: 1 + unused: + # treat code as a program (not a library) and report unused exported identifiers; default is false. + # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find funcs usages. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + unparam: + # call graph construction algorithm (cha, rta). In general, use cha for libraries, + # and rta for programs with main packages. Default is cha. + algo: cha + + # Inspect exported functions, default is false. Set to true if no external program/library imports your code. + # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find external interfaces. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + #nakedret: + # # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 + # max-func-lines: 30 + #prealloc: + # # XXX: we don't recommend using this linter before doing performance profiling. + # # For most programs usage of prealloc will be a premature optimization. + + # # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. + # # True by default. + # simple: true + # range-loops: true # Report preallocation suggestions on range loops, true by default + # for-loops: false # Report preallocation suggestions on for loops, false by default + + +linters: + enable: + - goconst + - goimports + - golint + - interfacer + - maligned + - misspell + - unconvert + - unparam + enable-all: false + disable: + - errcheck + - gas + disable-all: false + presets: + - bugs + - unused + fast: false + + +issues: + # List of regexps of issue texts to exclude, empty list by default. + # But independently from this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. To list all + # excluded by default patterns execute `golangci-lint run --help` + exclude: + + # Independently from option `exclude` we use default exclude patterns, + # it can be disabled by this option. To list all + # excluded by default patterns execute `golangci-lint run --help`. + # Default value for this option is true. + exclude-use-default: true + + # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + max-per-linter: 50 + + # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + max-same: 3 + + # Show only new issues: if there are unstaged changes or untracked files, + # only those changes are analyzed, else only changes in HEAD~ are analyzed. + # It's a super-useful option for integration of golangci-lint into existing + # large codebase. It's not practical to fix all existing issues at the moment + # of integration: much better don't allow issues in new code. + # Default is false. + new: false + + ## Show only new issues created after git revision `REV` + #new-from-rev: REV + + ## Show only new issues created in git patch with set file path. + #new-from-patch: path/to/patch/file diff --git a/hack/linter.sh b/hack/linter.sh index fb4713b08..8ae624696 100755 --- a/hack/linter.sh +++ b/hack/linter.sh @@ -23,15 +23,4 @@ if ! [ -x "$(command -v golangci-lint)" ]; then ${DIR}/install_golint.sh -b $GOPATH/bin v1.9.3 fi -golangci-lint run \ - --no-config \ - -E goconst \ - -E goimports \ - -E golint \ - -E interfacer \ - -E maligned \ - -E misspell \ - -E unconvert \ - -E unparam \ - -D errcheck \ - -D gas +golangci-lint run From c2645b2207495d42db0ff06798b7a58d989820d1 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Mon, 2 Dec 2019 11:40:11 -0800 Subject: [PATCH 19/28] Fix #897 - only build required docker images Only build the docker images required for the integration tests which will be executed rather than building all docker images every time. --- integration/integration_test.go | 139 ++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 59 deletions(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index 9fd89d717..0553b7fcd 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -32,7 +32,6 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/daemon" - "golang.org/x/sync/errgroup" "github.com/GoogleContainerTools/kaniko/pkg/timing" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -42,40 +41,6 @@ import ( var config = initGCPConfig() var imageBuilder *DockerFileBuilder -type gcpConfig struct { - gcsBucket string - imageRepo string - onbuildBaseImage string - hardlinkBaseImage string -} - -type imageDetails struct { - name string - numLayers int - digest string -} - -func (i imageDetails) String() string { - return fmt.Sprintf("Image: [%s] Digest: [%s] Number of Layers: [%d]", i.name, i.digest, i.numLayers) -} - -func initGCPConfig() *gcpConfig { - var c gcpConfig - flag.StringVar(&c.gcsBucket, "bucket", "gs://kaniko-test-bucket", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.") - flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo.") - flag.Parse() - - if c.gcsBucket == "" || c.imageRepo == "" { - log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo) - } - if !strings.HasSuffix(c.imageRepo, "/") { - c.imageRepo = c.imageRepo + "/" - } - c.onbuildBaseImage = c.imageRepo + "onbuild-base:latest" - c.hardlinkBaseImage = c.imageRepo + "hardlink-base:latest" - return &c -} - const ( daemonPrefix = "daemon://" dockerfilesPath = "dockerfiles" @@ -102,19 +67,6 @@ const ( ]` ) -func meetsRequirements() bool { - requiredTools := []string{"container-diff", "gsutil"} - hasRequirements := true - for _, tool := range requiredTools { - _, err := exec.LookPath(tool) - if err != nil { - fmt.Printf("You must have %s installed and on your PATH\n", tool) - hasRequirements = false - } - } - return hasRequirements -} - func TestMain(m *testing.M) { if !meetsRequirements() { fmt.Println("Missing required tools") @@ -188,19 +140,20 @@ func TestMain(m *testing.M) { } imageBuilder = NewDockerFileBuilder(dockerfiles) - g := errgroup.Group{} - for dockerfile := range imageBuilder.FilesBuilt { - df := dockerfile - g.Go(func() error { - return imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, df) - }) - } - if err := g.Wait(); err != nil { - fmt.Printf("Error building images: %s", err) - os.Exit(1) - } + //g := errgroup.Group{} + //for dockerfile := range imageBuilder.FilesBuilt { + // df := dockerfile + // g.Go(func() error { + // return imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, df) + // }) + //} + //if err := g.Wait(); err != nil { + // fmt.Printf("Error building images: %s", err) + // os.Exit(1) + //} os.Exit(m.Run()) } + func TestRun(t *testing.T) { for dockerfile := range imageBuilder.FilesBuilt { t.Run("test_"+dockerfile, func(t *testing.T) { @@ -212,6 +165,9 @@ func TestRun(t *testing.T) { if _, ok := imageBuilder.TestCacheDockerfiles[dockerfile]; ok { t.SkipNow() } + + imageBuilder.FilesBuilt[dockerfile] = buildImage(t, dockerfile, imageBuilder) + dockerImage := GetDockerImage(config.imageRepo, dockerfile) kanikoImage := GetKanikoImage(config.imageRepo, dockerfile) @@ -329,10 +285,14 @@ func TestLayers(t *testing.T) { for dockerfile := range imageBuilder.FilesBuilt { t.Run("test_layer_"+dockerfile, func(t *testing.T) { dockerfile := dockerfile + t.Parallel() if _, ok := imageBuilder.DockerfilesToIgnore[dockerfile]; ok { t.SkipNow() } + + imageBuilder.FilesBuilt[dockerfile] = buildImage(t, dockerfile, imageBuilder) + // Pull the kaniko image dockerImage := GetDockerImage(config.imageRepo, dockerfile) kanikoImage := GetKanikoImage(config.imageRepo, dockerfile) @@ -348,6 +308,20 @@ func TestLayers(t *testing.T) { } } +func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder) bool { + if imageBuilder.FilesBuilt[dockerfile] { + return true + } + + if err := imageBuilder.BuildImage( + config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile, + ); err != nil { + t.Errorf("Error building image: %s", err) + t.FailNow() + } + return true +} + // Build each image with kaniko twice, and then make sure they're exactly the same func TestCache(t *testing.T) { populateVolumeCache() @@ -522,3 +496,50 @@ func logBenchmarks(benchmark string) error { } return nil } + +type gcpConfig struct { + gcsBucket string + imageRepo string + onbuildBaseImage string + hardlinkBaseImage string +} + +type imageDetails struct { + name string + numLayers int + digest string +} + +func (i imageDetails) String() string { + return fmt.Sprintf("Image: [%s] Digest: [%s] Number of Layers: [%d]", i.name, i.digest, i.numLayers) +} + +func initGCPConfig() *gcpConfig { + var c gcpConfig + flag.StringVar(&c.gcsBucket, "bucket", "gs://kaniko-test-bucket", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.") + flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo.") + flag.Parse() + + if c.gcsBucket == "" || c.imageRepo == "" { + log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo) + } + if !strings.HasSuffix(c.imageRepo, "/") { + c.imageRepo = c.imageRepo + "/" + } + c.onbuildBaseImage = c.imageRepo + "onbuild-base:latest" + c.hardlinkBaseImage = c.imageRepo + "hardlink-base:latest" + return &c +} + +func meetsRequirements() bool { + requiredTools := []string{"container-diff", "gsutil"} + hasRequirements := true + for _, tool := range requiredTools { + _, err := exec.LookPath(tool) + if err != nil { + fmt.Printf("You must have %s installed and on your PATH\n", tool) + hasRequirements = false + } + } + return hasRequirements +} From d22a7608c2ce700d69c797d09ff4c0873a9a438d Mon Sep 17 00:00:00 2001 From: Ben Einaudi Date: Sat, 26 Oct 2019 20:04:20 +0200 Subject: [PATCH 20/28] Fix failure when using capital letters in image alias in 'FROM ... AS' instruction The third library moby/buildkit lowers the image alias used in 'FROM .. AS' instruction. This change fixes this issue by making the resolve of dependencies agnostic to case. Fixes #592 Fixes #770 --- pkg/dockerfile/dockerfile.go | 3 ++- pkg/dockerfile/dockerfile_test.go | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 9929f7f4d..1f1314fb4 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -203,6 +203,7 @@ func targetStage(stages []instructions.Stage, target string) (int, error) { // resolveStages resolves any calls to previous stages with names to indices // Ex. --from=second_stage should be --from=1 for easier processing later on +// As third party library lowers stage name in FROM instruction, this function resolves stage case insensitively. func resolveStages(stages []instructions.Stage) { nameToIndex := make(map[string]string) for i, stage := range stages { @@ -214,7 +215,7 @@ func resolveStages(stages []instructions.Stage) { switch c := cmd.(type) { case *instructions.CopyCommand: if c.From != "" { - if val, ok := nameToIndex[c.From]; ok { + if val, ok := nameToIndex[strings.ToLower(c.From)]; ok { c.From = val } diff --git a/pkg/dockerfile/dockerfile_test.go b/pkg/dockerfile/dockerfile_test.go index a5cdfc774..d903fdb52 100644 --- a/pkg/dockerfile/dockerfile_test.go +++ b/pkg/dockerfile/dockerfile_test.go @@ -197,8 +197,14 @@ func Test_resolveStages(t *testing.T) { FROM scratch AS second COPY --from=0 /hi /hi2 - FROM scratch + FROM scratch AS tHiRd COPY --from=second /hi2 /hi3 + COPY --from=1 /hi2 /hi3 + + FROM scratch + COPY --from=thIrD /hi3 /hi4 + COPY --from=third /hi3 /hi4 + COPY --from=2 /hi3 /hi4 ` stages, _, err := Parse([]byte(dockerfile)) if err != nil { @@ -209,11 +215,14 @@ func Test_resolveStages(t *testing.T) { if index == 0 { continue } - copyCmd := stage.Commands[0].(*instructions.CopyCommand) expectedStage := strconv.Itoa(index - 1) - if copyCmd.From != expectedStage { - t.Fatalf("unexpected copy command: %s resolved to stage %s, expected %s", copyCmd.String(), copyCmd.From, expectedStage) + for _, command := range stage.Commands { + copyCmd := command.(*instructions.CopyCommand) + if copyCmd.From != expectedStage { + t.Fatalf("unexpected copy command: %s resolved to stage %s, expected %s", copyCmd.String(), copyCmd.From, expectedStage) + } } + } } From 0a2f2957ec90064ebbd100b2392e47533188ce87 Mon Sep 17 00:00:00 2001 From: poy Date: Sun, 8 Dec 2019 00:29:25 -0700 Subject: [PATCH 21/28] when copying, skip files with the same name When using the COPY command, if the source and destination have the same the file should be skipped rather than copied. This is to prevent the file from being overwritten and therefore producing an empty file. fixes #904 --- pkg/util/fs_util.go | 6 ++++++ pkg/util/fs_util_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 10fccb7f7..77376228e 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -551,6 +551,12 @@ func CopyFile(src, dest, buildcontext string) (bool, error) { logrus.Debugf("%s found in .dockerignore, ignoring", src) return true, nil } + if src == dest { + // This is a no-op. Move on, but don't list it as ignored. + // We have to make sure we do this so we don't overwrite our own file. + // See iusse #904 for an example. + return false, nil + } fi, err := os.Stat(src) if err != nil { return false, err diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index 5a1f008f4..2a4b31c6f 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -825,3 +825,41 @@ func Test_correctDockerignoreFileIsUsed(t *testing.T) { } } } + +func Test_CopyFile_skips_self(t *testing.T) { + t.Parallel() + tempDir, err := ioutil.TempDir("", "kaniko_test") + if err != nil { + t.Fatal(err) + } + + tempFile := filepath.Join(tempDir, "foo") + expected := "bar" + + if err := ioutil.WriteFile( + tempFile, + []byte(expected), + 0755, + ); err != nil { + t.Fatal(err) + } + + ignored, err := CopyFile(tempFile, tempFile, "") + if err != nil { + t.Fatal(err) + } + + if ignored { + t.Fatal("expected file to NOT be ignored") + } + + // Ensure file has expected contents + actualData, err := ioutil.ReadFile(tempFile) + if err != nil { + t.Fatal(err) + } + + if actual := string(actualData); actual != expected { + t.Fatalf("expected file contents to be %q, but got %q", expected, actual) + } +} From 32e321af780cb30525e2f5c184747590ed4e9d6c Mon Sep 17 00:00:00 2001 From: Pweetoo <58683024+Pweetoo@users.noreply.github.com> Date: Mon, 9 Dec 2019 09:15:26 +0100 Subject: [PATCH 22/28] updated readme Added argument -n for echo command. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index af571775d..d37ecc559 100644 --- a/README.md +++ b/README.md @@ -322,7 +322,7 @@ kaniko comes with support for GCR, Docker `config.json` and Amazon ECR, but conf Get your docker registry user and password encoded in base64 - echo USER:PASSWORD | base64 + echo -n USER:PASSWORD | base64 Create a `config.json` file with your Docker registry url and the previous generated base64 string From 05447f1eaa559480ec4df503198c43b98d1ae531 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Mon, 9 Dec 2019 13:57:59 -0800 Subject: [PATCH 23/28] clean up unused code --- integration/integration_test.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index 0553b7fcd..de3900667 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -140,17 +140,6 @@ func TestMain(m *testing.M) { } imageBuilder = NewDockerFileBuilder(dockerfiles) - //g := errgroup.Group{} - //for dockerfile := range imageBuilder.FilesBuilt { - // df := dockerfile - // g.Go(func() error { - // return imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, df) - // }) - //} - //if err := g.Wait(); err != nil { - // fmt.Printf("Error building images: %s", err) - // os.Exit(1) - //} os.Exit(m.Run()) } From 7b4b768edf29ee718627a41dfe54fa4ab09dc20e Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Mon, 25 Nov 2019 14:25:20 -0800 Subject: [PATCH 24/28] Update copy command cache key logic Include the digest of the stage specified in the --from argument for COPY commands which use --from --- pkg/commands/copy.go | 8 +++++ pkg/config/stage.go | 4 ++- pkg/executor/build.go | 71 +++++++++++++++++++++++++++++--------- pkg/executor/build_test.go | 48 ++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 17 deletions(-) diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index 4071f957b..dab5917fa 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -161,6 +161,10 @@ func (c *CopyCommand) CacheCommand(img v1.Image) DockerCommand { } } +func (c *CopyCommand) From() string { + return c.cmd.From +} + type CachingCopyCommand struct { BaseCommand img v1.Image @@ -187,6 +191,10 @@ func (cr *CachingCopyCommand) String() string { return cr.cmd.String() } +func (cr *CachingCopyCommand) From() string { + return cr.cmd.From +} + func resolveIfSymlink(destPath string) (string, error) { baseDir := filepath.Dir(destPath) if info, err := os.Lstat(baseDir); err == nil { diff --git a/pkg/config/stage.go b/pkg/config/stage.go index 56c4a3f0f..fad18049c 100644 --- a/pkg/config/stage.go +++ b/pkg/config/stage.go @@ -16,7 +16,9 @@ limitations under the License. package config -import "github.com/moby/buildkit/frontend/dockerfile/instructions" +import ( + "github.com/moby/buildkit/frontend/dockerfile/instructions" +) // KanikoStage wraps a stage of the Dockerfile and provides extra information type KanikoStage struct { diff --git a/pkg/executor/build.go b/pkg/executor/build.go index e3c48b5e1..47c456359 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -64,9 +64,6 @@ type stageBuilder struct { stage config.KanikoStage image v1.Image cf *v1.ConfigFile - snapshotter snapShotter - layerCache cache.LayerCache - pushCache cachePusher baseImageDigest string finalCacheKey string opts *config.KanikoOptions @@ -74,6 +71,9 @@ type stageBuilder struct { args *dockerfile.BuildArgs crossStageDeps map[int][]string digestToCacheKeyMap map[string]string + snapshotter snapShotter + layerCache cache.LayerCache + pushCache cachePusher } // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage @@ -146,6 +146,38 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { return imageConfig, nil } +func (s *stageBuilder) populateCompositeKey(command commands.DockerCommand, files []string, compositeKey CompositeCache) (CompositeCache, error) { + // Add the next command to the cache key. + compositeKey.AddKey(command.String()) + switch v := command.(type) { + case *commands.CopyCommand: + if v.From() != "" { + digest, ok := s.digestMap[v.From()] + if ok { + ds := digest.String() + logrus.Debugf("adding digest %v from previous stage to composite key for %v", ds, command.String()) + compositeKey.AddKey(ds) + } + } + case *commands.CachingCopyCommand: + if v.From() != "" { + digest, ok := s.digestMap[v.From()] + if ok { + ds := digest.String() + logrus.Debugf("adding digest %v from previous stage to composite key for %v", ds, command.String()) + compositeKey.AddKey(ds) + } + } + } + + for _, f := range files { + if err := compositeKey.AddPath(f); err != nil { + return compositeKey, err + } + } + return compositeKey, nil +} + func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) error { if !s.opts.Cache { return nil @@ -160,16 +192,14 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro if command == nil { continue } - compositeKey.AddKey(command.String()) - // If the command uses files from the context, add them. files, err := command.FilesUsedFromContext(&cfg, s.args) if err != nil { return errors.Wrap(err, "failed to get files used from context") } - for _, f := range files { - if err := compositeKey.AddPath(f); err != nil { - return errors.Wrap(err, "failed to add path to composite key") - } + + compositeKey, err = s.populateCompositeKey(command, files, compositeKey) + if err != nil { + return err } ck, err := compositeKey.Hash() @@ -256,19 +286,19 @@ func (s *stageBuilder) build() error { continue } - // Add the next command to the cache key. - compositeKey.AddKey(command.String()) t := timing.Start("Command: " + command.String()) + // If the command uses files from the context, add them. files, err := command.FilesUsedFromContext(&s.cf.Config, s.args) if err != nil { return errors.Wrap(err, "failed to get files used from context") } - for _, f := range files { - if err := compositeKey.AddPath(f); err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to add path to composite key %v", f)) - } + + *compositeKey, err = s.populateCompositeKey(command, files, *compositeKey) + if err != nil { + return err } + logrus.Info(command.String()) if err := command.ExecuteCommand(&s.cf.Config, s.args); err != nil { @@ -303,6 +333,7 @@ func (s *stageBuilder) build() error { if err := cacheGroup.Wait(); err != nil { logrus.Warnf("error uploading layer to cache: %s", err) } + return nil } @@ -374,7 +405,6 @@ func (s *stageBuilder) saveSnapshotToImage(createdBy string, tarPath string) err }, ) return err - } func CalculateDependencies(opts *config.KanikoOptions) (map[int][]string, error) { @@ -443,6 +473,7 @@ func CalculateDependencies(opts *config.KanikoOptions) (map[int][]string, error) func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { t := timing.Start("Total Build Time") digestToCacheKeyMap := make(map[string]string) + // Parse dockerfile and unpack base image to root stages, err := dockerfile.Stages(opts) if err != nil { @@ -470,6 +501,14 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { if err := sb.build(); err != nil { return nil, errors.Wrap(err, "error building stage") } + + d, err := sb.image.Digest() + if err != nil { + return nil, err + } + + digestMap[fmt.Sprintf("%d", sb.stage.Index)] = d + reviewConfig(stage, &sb.cf.Config) sourceImage, err := mutate.Config(sb.image, sb.cf.Config) if err != nil { diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 66c0ab5b5..0b4a546f8 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -919,3 +919,51 @@ func generateTar(t *testing.T, dir string, fileNames ...string) []byte { } return buf.Bytes() } + +//======= +// testCases := []struct { +// opts *config.KanikoOptions +// retrieve bool +// }{ +// { +// opts: &config.KanikoOptions{Cache: true}, +// }, +// { +// opts: &config.KanikoOptions{Cache: true}, +// retrieve: true, +// }, +// { +// opts: &config.KanikoOptions{Cache: false}, +// }, +// { +// opts: &config.KanikoOptions{Cache: false}, +// retrieve: true, +// }, +// } +// for i, tc := range testCases { +// t.Run(fmt.Sprintf("Case %d", i), func(t *testing.T) { +// file, err := ioutil.TempFile("", "foo") +// if err != nil { +// t.Error(err) +// } + +// cf := &v1.ConfigFile{} +// snap := fakeSnapShotter{file: file.Name()} +// lc := fakeLayerCache{retrieve: tc.retrieve} +// sb := &stageBuilder{opts: tc.opts, cf: cf, snapshotter: snap, layerCache: lc, pushCache: fakeCachePush} + +// command := MockDockerCommand{ +// contextFiles: []string{file.Name()}, +// cacheCommand: MockCachedDockerCommand{ +// contextFiles: []string{file.Name()}, +// }, +// } +// sb.cmds = []commands.DockerCommand{command} +// err = sb.build() +// if err != nil { +// t.Errorf("Expected error to be nil but was %v", err) +// } +// }) +// } +//} +//>>>>>>> Add unit tests for stagebuilder build and optimize From b19214ad1ebe8e113fdcbdd8fe413992ae5f0cd3 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Wed, 27 Nov 2019 16:35:52 -0800 Subject: [PATCH 25/28] Use cachekey not digest for COPY --from src * use the cachekey of the src stage rather than the digest for COPY --from commands as they are reproducible unlike digests * track digest to cache keys and stage indexes to digest * add extra debug logging for troubleshooting cachekey building issues * convert Sha256 hashes to hex encoded strings rather than plain strings for easier human reading --- pkg/executor/build.go | 109 +++++++++++++++++--------------- pkg/executor/build_test.go | 49 +------------- pkg/executor/composite_cache.go | 5 +- 3 files changed, 63 insertions(+), 100 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 47c456359..caa1cf063 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -61,23 +61,24 @@ type snapShotter interface { // stageBuilder contains all fields necessary to build one stage of a Dockerfile type stageBuilder struct { - stage config.KanikoStage - image v1.Image - cf *v1.ConfigFile - baseImageDigest string - finalCacheKey string - opts *config.KanikoOptions - cmds []commands.DockerCommand - args *dockerfile.BuildArgs - crossStageDeps map[int][]string - digestToCacheKeyMap map[string]string - snapshotter snapShotter - layerCache cache.LayerCache - pushCache cachePusher + stage config.KanikoStage + image v1.Image + cf *v1.ConfigFile + baseImageDigest string + finalCacheKey string + opts *config.KanikoOptions + cmds []commands.DockerCommand + args *dockerfile.BuildArgs + crossStageDeps map[int][]string + digestToCacheKey map[string]string + stageIdxToDigest map[string]string + snapshotter snapShotter + layerCache cache.LayerCache + pushCache cachePusher } // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage -func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string, dcm map[string]string) (*stageBuilder, error) { +func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string, dcm map[string]string, sid map[string]string) (*stageBuilder, error) { sourceImage, err := util.RetrieveSourceImage(stage, opts) if err != nil { return nil, err @@ -104,14 +105,15 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross return nil, err } s := &stageBuilder{ - stage: stage, - image: sourceImage, - cf: imageConfig, - snapshotter: snapshotter, - baseImageDigest: digest.String(), - opts: opts, - crossStageDeps: crossStageDeps, - digestToCacheKeyMap: dcm, + stage: stage, + image: sourceImage, + cf: imageConfig, + snapshotter: snapshotter, + baseImageDigest: digest.String(), + opts: opts, + crossStageDeps: crossStageDeps, + digestToCacheKey: dcm, + stageIdxToDigest: sid, layerCache: &cache.RegistryCache{ Opts: opts, }, @@ -146,28 +148,14 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { return imageConfig, nil } -func (s *stageBuilder) populateCompositeKey(command commands.DockerCommand, files []string, compositeKey CompositeCache) (CompositeCache, error) { +func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string, compositeKey CompositeCache) (CompositeCache, error) { // Add the next command to the cache key. compositeKey.AddKey(command.String()) switch v := command.(type) { case *commands.CopyCommand: - if v.From() != "" { - digest, ok := s.digestMap[v.From()] - if ok { - ds := digest.String() - logrus.Debugf("adding digest %v from previous stage to composite key for %v", ds, command.String()) - compositeKey.AddKey(ds) - } - } + compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey) case *commands.CachingCopyCommand: - if v.From() != "" { - digest, ok := s.digestMap[v.From()] - if ok { - ds := digest.String() - logrus.Debugf("adding digest %v from previous stage to composite key for %v", ds, command.String()) - compositeKey.AddKey(ds) - } - } + compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey) } for _, f := range files { @@ -178,6 +166,22 @@ func (s *stageBuilder) populateCompositeKey(command commands.DockerCommand, file return compositeKey, nil } +func (s *stageBuilder) populateCopyCmdCompositeKey(command fmt.Stringer, from string, compositeKey CompositeCache) CompositeCache { + if from != "" { + digest, ok := s.stageIdxToDigest[from] + if ok { + ds := digest + cacheKey, ok := s.digestToCacheKey[ds] + if ok { + logrus.Debugf("adding digest %v from previous stage to composite key for %v", ds, command.String()) + compositeKey.AddKey(cacheKey) + } + } + } + + return compositeKey +} + func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) error { if !s.opts.Cache { return nil @@ -202,10 +206,12 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro return err } + logrus.Debugf("optimize: composite key for command %v %v", command.String(), compositeKey) ck, err := compositeKey.Hash() if err != nil { return errors.Wrap(err, "failed to hash composite key") } + logrus.Debugf("optimize: cache key for command %v %v", command.String(), ck) s.finalCacheKey = ck if command.ShouldCacheOutput() && !stopCache { img, err := s.layerCache.RetrieveLayer(ck) @@ -236,7 +242,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro func (s *stageBuilder) build() error { // Set the initial cache key to be the base image digest, the build args and the SrcContext. var compositeKey *CompositeCache - if cacheKey, ok := s.digestToCacheKeyMap[s.baseImageDigest]; ok { + if cacheKey, ok := s.digestToCacheKey[s.baseImageDigest]; ok { compositeKey = NewCompositeCache(cacheKey) } else { compositeKey = NewCompositeCache(s.baseImageDigest) @@ -316,6 +322,7 @@ func (s *stageBuilder) build() error { return errors.Wrap(err, "failed to take snapshot") } + logrus.Debugf("build: composite key for command %v %v", command.String(), compositeKey) ck, err := compositeKey.Hash() if err != nil { return errors.Wrap(err, "failed to hash composite key") @@ -472,7 +479,8 @@ func CalculateDependencies(opts *config.KanikoOptions) (map[int][]string, error) // DoBuild executes building the Dockerfile func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { t := timing.Start("Total Build Time") - digestToCacheKeyMap := make(map[string]string) + digestToCacheKey := make(map[string]string) + stageIdxToDigest := make(map[string]string) // Parse dockerfile and unpack base image to root stages, err := dockerfile.Stages(opts) @@ -494,7 +502,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { logrus.Infof("Built cross stage deps: %v", crossStageDependencies) for index, stage := range stages { - sb, err := newStageBuilder(opts, stage, crossStageDependencies, digestToCacheKeyMap) + sb, err := newStageBuilder(opts, stage, crossStageDependencies, digestToCacheKey, stageIdxToDigest) if err != nil { return nil, err } @@ -502,23 +510,24 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { return nil, errors.Wrap(err, "error building stage") } - d, err := sb.image.Digest() - if err != nil { - return nil, err - } - - digestMap[fmt.Sprintf("%d", sb.stage.Index)] = d - reviewConfig(stage, &sb.cf.Config) + sourceImage, err := mutate.Config(sb.image, sb.cf.Config) if err != nil { return nil, err } + d, err := sourceImage.Digest() if err != nil { return nil, err } - digestToCacheKeyMap[d.String()] = sb.finalCacheKey + + stageIdxToDigest[fmt.Sprintf("%d", sb.stage.Index)] = d.String() + logrus.Debugf("mapping stage idx %v to digest %v", sb.stage.Index, d.String()) + + digestToCacheKey[d.String()] = sb.finalCacheKey + logrus.Debugf("mapping digest %v to cachekey %v", d.String(), sb.finalCacheKey) + if stage.Final { sourceImage, err = mutate.CreatedAt(sourceImage, v1.Time{Time: time.Now()}) if err != nil { diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 0b4a546f8..d5c07acfc 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -888,6 +888,7 @@ func tempDirAndFile(t *testing.T) (string, []string) { return dir, filenames } + func generateTar(t *testing.T, dir string, fileNames ...string) []byte { buf := bytes.NewBuffer([]byte{}) writer := tar.NewWriter(buf) @@ -919,51 +920,3 @@ func generateTar(t *testing.T, dir string, fileNames ...string) []byte { } return buf.Bytes() } - -//======= -// testCases := []struct { -// opts *config.KanikoOptions -// retrieve bool -// }{ -// { -// opts: &config.KanikoOptions{Cache: true}, -// }, -// { -// opts: &config.KanikoOptions{Cache: true}, -// retrieve: true, -// }, -// { -// opts: &config.KanikoOptions{Cache: false}, -// }, -// { -// opts: &config.KanikoOptions{Cache: false}, -// retrieve: true, -// }, -// } -// for i, tc := range testCases { -// t.Run(fmt.Sprintf("Case %d", i), func(t *testing.T) { -// file, err := ioutil.TempFile("", "foo") -// if err != nil { -// t.Error(err) -// } - -// cf := &v1.ConfigFile{} -// snap := fakeSnapShotter{file: file.Name()} -// lc := fakeLayerCache{retrieve: tc.retrieve} -// sb := &stageBuilder{opts: tc.opts, cf: cf, snapshotter: snap, layerCache: lc, pushCache: fakeCachePush} - -// command := MockDockerCommand{ -// contextFiles: []string{file.Name()}, -// cacheCommand: MockCachedDockerCommand{ -// contextFiles: []string{file.Name()}, -// }, -// } -// sb.cmds = []commands.DockerCommand{command} -// err = sb.build() -// if err != nil { -// t.Errorf("Expected error to be nil but was %v", err) -// } -// }) -// } -//} -//>>>>>>> Add unit tests for stagebuilder build and optimize diff --git a/pkg/executor/composite_cache.go b/pkg/executor/composite_cache.go index 15a88a40b..ae8ff0609 100644 --- a/pkg/executor/composite_cache.go +++ b/pkg/executor/composite_cache.go @@ -18,6 +18,7 @@ package executor import ( "crypto/sha256" + "fmt" "os" "path/filepath" "strings" @@ -75,7 +76,7 @@ func (s *CompositeCache) AddPath(p string) error { return err } - s.keys = append(s.keys, string(sha.Sum(nil))) + s.keys = append(s.keys, fmt.Sprintf("%x", sha.Sum(nil))) return nil } @@ -98,5 +99,5 @@ func HashDir(p string) (string, error) { return "", err } - return string(sha.Sum(nil)), nil + return fmt.Sprintf("%x", sha.Sum(nil)), nil } From 2aa481c15e90b56e6927b0ac6cc9f3ac9b2f0e03 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Wed, 27 Nov 2019 09:35:53 -0800 Subject: [PATCH 26/28] add unit tests for caching run and copy --- pkg/commands/copy.go | 18 +++- pkg/commands/copy_test.go | 141 +++++++++++++++++++++++++ pkg/commands/fake_commands.go | 88 ++++++++++++++++ pkg/commands/run.go | 16 ++- pkg/commands/run_test.go | 193 ++++++++++++++++++++++++++++++++++ pkg/executor/build.go | 16 ++- pkg/util/fs_util.go | 16 ++- pkg/util/fs_util_test.go | 2 +- 8 files changed, 477 insertions(+), 13 deletions(-) create mode 100644 pkg/commands/fake_commands.go diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index dab5917fa..6bbff699a 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -17,6 +17,7 @@ limitations under the License. package commands import ( + "fmt" "os" "path/filepath" @@ -156,8 +157,9 @@ func (c *CopyCommand) ShouldCacheOutput() bool { func (c *CopyCommand) CacheCommand(img v1.Image) DockerCommand { return &CachingCopyCommand{ - img: img, - cmd: c.cmd, + img: img, + cmd: c.cmd, + extractFn: util.ExtractFile, } } @@ -170,16 +172,23 @@ type CachingCopyCommand struct { img v1.Image extractedFiles []string cmd *instructions.CopyCommand + extractFn util.ExtractFunction } func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { logrus.Infof("Found cached layer, extracting to filesystem") var err error - cr.extractedFiles, err = util.GetFSFromImage(RootDir, cr.img) + + if cr.img == nil { + return errors.New(fmt.Sprintf("cached command image is nil %v", cr.String())) + } + cr.extractedFiles, err = util.GetFSFromImage(RootDir, cr.img, cr.extractFn) + logrus.Infof("extractedFiles: %s", cr.extractedFiles) if err != nil { return errors.Wrap(err, "extracting fs from image") } + return nil } @@ -188,6 +197,9 @@ func (cr *CachingCopyCommand) FilesToSnapshot() []string { } func (cr *CachingCopyCommand) String() string { + if cr.cmd == nil { + return "nil command" + } return cr.cmd.String() } diff --git a/pkg/commands/copy_test.go b/pkg/commands/copy_test.go index 5f8fb7211..d03168059 100644 --- a/pkg/commands/copy_test.go +++ b/pkg/commands/copy_test.go @@ -16,6 +16,7 @@ limitations under the License. package commands import ( + "archive/tar" "fmt" "io" "io/ioutil" @@ -215,3 +216,143 @@ func Test_resolveIfSymlink(t *testing.T) { }) } } + +func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) { + tarContent, err := prepareTarFixture([]string{"foo.txt"}) + if err != nil { + t.Errorf("couldn't prepare tar fixture %v", err) + } + + config := &v1.Config{} + buildArgs := &dockerfile.BuildArgs{} + + type testCase struct { + desctiption string + expectLayer bool + expectErr bool + count *int + expectedCount int + command *CachingCopyCommand + extractedFiles []string + contextFiles []string + } + testCases := []testCase{ + func() testCase { + c := &CachingCopyCommand{ + img: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{TarContent: tarContent}, + }, + }, + cmd: &instructions.CopyCommand{ + SourcesAndDest: []string{ + "foo.txt", "foo.txt", + }, + }, + } + count := 0 + tc := testCase{ + desctiption: "with valid image and valid layer", + count: &count, + expectedCount: 1, + expectLayer: true, + extractedFiles: []string{"/foo.txt"}, + contextFiles: []string{"foo.txt"}, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + *tc.count++ + return nil + } + tc.command = c + return tc + }(), + func() testCase { + c := &CachingCopyCommand{} + tc := testCase{ + desctiption: "with no image", + expectErr: true, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + return nil + } + tc.command = c + return tc + }(), + func() testCase { + c := &CachingCopyCommand{ + img: fakeImage{}, + } + tc := testCase{ + desctiption: "with image containing no layers", + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + return nil + } + tc.command = c + return tc + }(), + func() testCase { + c := &CachingCopyCommand{ + img: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{}, + }, + }, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + return nil + } + tc := testCase{ + desctiption: "with image one layer which has no tar content", + expectErr: false, // this one probably should fail but doesn't because of how ExecuteCommand and util.GetFSFromLayers are implemented - cvgw- 2019-11-25 + expectLayer: true, + } + tc.command = c + return tc + }(), + } + + for _, tc := range testCases { + t.Run(tc.desctiption, func(t *testing.T) { + c := tc.command + err := c.ExecuteCommand(config, buildArgs) + if !tc.expectErr && err != nil { + t.Errorf("Expected err to be nil but was %v", err) + } else if tc.expectErr && err == nil { + t.Error("Expected err but was nil") + } + + if tc.count != nil { + if *tc.count != tc.expectedCount { + t.Errorf("Expected extractFn to be called %v times but was called %v times", tc.expectedCount, *tc.count) + } + for _, file := range tc.extractedFiles { + match := false + cFiles := c.FilesToSnapshot() + for _, cFile := range cFiles { + if file == cFile { + match = true + break + } + } + if !match { + t.Errorf("Expected extracted files to include %v but did not %v", file, cFiles) + } + } + // CachingCopyCommand does not override BaseCommand + // FilesUseFromContext so this will always return an empty slice and no error + // This seems like it might be a bug as it results in CopyCommands and CachingCopyCommands generating different cache keys - cvgw - 2019-11-27 + cmdFiles, err := c.FilesUsedFromContext( + config, buildArgs, + ) + if err != nil { + t.Errorf("failed to get files used from context from command") + } + if len(cmdFiles) != 0 { + t.Errorf("expected files used from context to be empty but was not") + } + } + + }) + } +} diff --git a/pkg/commands/fake_commands.go b/pkg/commands/fake_commands.go new file mode 100644 index 000000000..8efee7c4d --- /dev/null +++ b/pkg/commands/fake_commands.go @@ -0,0 +1,88 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// used for testing in the commands package +package commands + +import ( + "bytes" + "io" + "io/ioutil" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type fakeLayer struct { + TarContent []byte +} + +func (f fakeLayer) Digest() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeLayer) DiffID() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeLayer) Compressed() (io.ReadCloser, error) { + return nil, nil +} +func (f fakeLayer) Uncompressed() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(f.TarContent)), nil +} +func (f fakeLayer) Size() (int64, error) { + return 0, nil +} +func (f fakeLayer) MediaType() (types.MediaType, error) { + return "", nil +} + +type fakeImage struct { + ImageLayers []v1.Layer +} + +func (f fakeImage) Layers() ([]v1.Layer, error) { + return f.ImageLayers, nil +} +func (f fakeImage) MediaType() (types.MediaType, error) { + return "", nil +} +func (f fakeImage) Size() (int64, error) { + return 0, nil +} +func (f fakeImage) ConfigName() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeImage) ConfigFile() (*v1.ConfigFile, error) { + return &v1.ConfigFile{}, nil +} +func (f fakeImage) RawConfigFile() ([]byte, error) { + return []byte{}, nil +} +func (f fakeImage) Digest() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeImage) Manifest() (*v1.Manifest, error) { + return &v1.Manifest{}, nil +} +func (f fakeImage) RawManifest() ([]byte, error) { + return []byte{}, nil +} +func (f fakeImage) LayerByDigest(v1.Hash) (v1.Layer, error) { + return fakeLayer{}, nil +} +func (f fakeImage) LayerByDiffID(v1.Hash) (v1.Layer, error) { + return fakeLayer{}, nil +} diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 4867fd546..0dcd4ed52 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -164,8 +164,9 @@ func (r *RunCommand) FilesToSnapshot() []string { func (r *RunCommand) CacheCommand(img v1.Image) DockerCommand { return &CachingRunCommand{ - img: img, - cmd: r.cmd, + img: img, + cmd: r.cmd, + extractFn: util.ExtractFile, } } @@ -186,15 +187,21 @@ type CachingRunCommand struct { img v1.Image extractedFiles []string cmd *instructions.RunCommand + extractFn util.ExtractFunction } func (cr *CachingRunCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { logrus.Infof("Found cached layer, extracting to filesystem") var err error - cr.extractedFiles, err = util.GetFSFromImage(constants.RootDir, cr.img) + + if cr.img == nil { + return errors.New(fmt.Sprintf("command image is nil %v", cr.String())) + } + cr.extractedFiles, err = util.GetFSFromImage(constants.RootDir, cr.img, cr.extractFn) if err != nil { return errors.Wrap(err, "extracting fs from image") } + return nil } @@ -203,5 +210,8 @@ func (cr *CachingRunCommand) FilesToSnapshot() []string { } func (cr *CachingRunCommand) String() string { + if cr.cmd == nil { + return "nil command" + } return cr.cmd.String() } diff --git a/pkg/commands/run_test.go b/pkg/commands/run_test.go index 0e7c51e8c..36ae98bad 100644 --- a/pkg/commands/run_test.go +++ b/pkg/commands/run_test.go @@ -16,10 +16,19 @@ limitations under the License. package commands import ( + "archive/tar" + "bytes" + "io" + "io/ioutil" + "log" + "os" "os/user" + "path/filepath" "testing" + "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/testutil" + v1 "github.com/google/go-containerregistry/pkg/v1" ) func Test_addDefaultHOME(t *testing.T) { @@ -120,3 +129,187 @@ func Test_addDefaultHOME(t *testing.T) { }) } } + +func prepareTarFixture(fileNames []string) ([]byte, error) { + dir, err := ioutil.TempDir("/tmp", "tar-fixture") + if err != nil { + return nil, err + } + + content := ` +Meow meow meow meow +meow meow meow meow +` + for _, name := range fileNames { + if err := ioutil.WriteFile(filepath.Join(dir, name), []byte(content), 0777); err != nil { + return nil, err + } + } + writer := bytes.NewBuffer([]byte{}) + tw := tar.NewWriter(writer) + defer tw.Close() + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + if err := tw.WriteHeader(hdr); err != nil { + log.Fatal(err) + } + body, err := ioutil.ReadFile(path) + if err != nil { + return err + } + if _, err := tw.Write(body); err != nil { + log.Fatal(err) + } + + return nil + }) + + return writer.Bytes(), nil +} + +func Test_CachingRunCommand_ExecuteCommand(t *testing.T) { + tarContent, err := prepareTarFixture([]string{"foo.txt"}) + if err != nil { + t.Errorf("couldn't prepare tar fixture %v", err) + } + + config := &v1.Config{} + buildArgs := &dockerfile.BuildArgs{} + + type testCase struct { + desctiption string + expectLayer bool + expectErr bool + count *int + expectedCount int + command *CachingRunCommand + extractedFiles []string + contextFiles []string + } + testCases := []testCase{ + func() testCase { + c := &CachingRunCommand{ + img: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{TarContent: tarContent}, + }, + }, + } + count := 0 + tc := testCase{ + desctiption: "with valid image and valid layer", + count: &count, + expectedCount: 1, + expectLayer: true, + extractedFiles: []string{"/foo.txt"}, + contextFiles: []string{"foo.txt"}, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + *tc.count++ + return nil + } + tc.command = c + return tc + }(), + func() testCase { + c := &CachingRunCommand{} + tc := testCase{ + desctiption: "with no image", + expectErr: true, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + return nil + } + tc.command = c + return tc + }(), + func() testCase { + c := &CachingRunCommand{ + img: fakeImage{}, + } + tc := testCase{ + desctiption: "with image containing no layers", + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + return nil + } + tc.command = c + return tc + }(), + func() testCase { + c := &CachingRunCommand{ + img: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{}, + }, + }, + } + c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error { + return nil + } + tc := testCase{ + desctiption: "with image one layer which has no tar content", + expectErr: false, // this one probably should fail but doesn't because of how ExecuteCommand and util.GetFSFromLayers are implemented - cvgw- 2019-11-25 + expectLayer: true, + } + tc.command = c + return tc + }(), + } + + for _, tc := range testCases { + t.Run(tc.desctiption, func(t *testing.T) { + c := tc.command + err := c.ExecuteCommand(config, buildArgs) + if !tc.expectErr && err != nil { + t.Errorf("Expected err to be nil but was %v", err) + } else if tc.expectErr && err == nil { + t.Error("Expected err but was nil") + } + + if tc.count != nil { + if *tc.count != tc.expectedCount { + t.Errorf("Expected extractFn to be called %v times but was called %v times", 1, *tc.count) + } + for _, file := range tc.extractedFiles { + match := false + cmdFiles := c.extractedFiles + for _, f := range cmdFiles { + if file == f { + match = true + break + } + } + if !match { + t.Errorf("Expected extracted files to include %v but did not %v", file, cmdFiles) + } + } + + // CachingRunCommand does not override BaseCommand + // FilesUseFromContext so this will always return an empty slice and no error + // This seems like it might be a bug as it results in RunCommands and CachingRunCommands generating different cache keys - cvgw - 2019-11-27 + cmdFiles, err := c.FilesUsedFromContext( + config, buildArgs, + ) + if err != nil { + t.Errorf("failed to get files used from context from command") + } + + if len(cmdFiles) != 0 { + t.Errorf("expected files used from context to be empty but was not") + } + } + }) + } +} diff --git a/pkg/executor/build.go b/pkg/executor/build.go index caa1cf063..563a8f379 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -211,10 +211,13 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro if err != nil { return errors.Wrap(err, "failed to hash composite key") } + logrus.Debugf("optimize: cache key for command %v %v", command.String(), ck) s.finalCacheKey = ck + if command.ShouldCacheOutput() && !stopCache { img, err := s.layerCache.RetrieveLayer(ck) + if err != nil { logrus.Debugf("Failed to retrieve layer: %s", err) logrus.Infof("No cached layer found for cmd %s", command.String()) @@ -247,6 +250,7 @@ func (s *stageBuilder) build() error { } else { compositeKey = NewCompositeCache(s.baseImageDigest) } + compositeKey.AddKey(s.opts.BuildArgs...) // Apply optimizations to the instructions. @@ -269,21 +273,26 @@ func (s *stageBuilder) build() error { if shouldUnpack { t := timing.Start("FS Unpacking") - if _, err := util.GetFSFromImage(constants.RootDir, s.image); err != nil { + + if _, err := util.GetFSFromImage(constants.RootDir, s.image, util.ExtractFile); err != nil { return errors.Wrap(err, "failed to get filesystem from image") } + timing.DefaultRun.Stop(t) } else { logrus.Info("Skipping unpacking as no commands require it.") } + if err := util.DetectFilesystemWhitelist(constants.WhitelistPath); err != nil { return errors.Wrap(err, "failed to check filesystem whitelist") } + // Take initial snapshot t := timing.Start("Initial FS snapshot") if err := s.snapshotter.Init(); err != nil { return err } + timing.DefaultRun.Stop(t) cacheGroup := errgroup.Group{} @@ -327,6 +336,9 @@ func (s *stageBuilder) build() error { if err != nil { return errors.Wrap(err, "failed to hash composite key") } + + logrus.Debugf("build: cache key for command %v %v", command.String(), ck) + // Push layer to cache (in parallel) now along with new config file if s.opts.Cache && command.ShouldCacheOutput() { cacheGroup.Go(func() error { @@ -641,7 +653,7 @@ func extractImageToDependencyDir(name string, image v1.Image) error { return err } logrus.Debugf("trying to extract to %s", dependencyDir) - _, err := util.GetFSFromImage(dependencyDir, image) + _, err := util.GetFSFromImage(dependencyDir, image, util.ExtractFile) return err } diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 77376228e..f7022e8ec 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -69,9 +69,17 @@ var volumes = []string{} var excluded []string +type ExtractFunction func(string, *tar.Header, io.Reader) error + // GetFSFromImage extracts the layers of img to root // It returns a list of all files extracted -func GetFSFromImage(root string, img v1.Image) ([]string, error) { +func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]string, error) { + if extract == nil { + return nil, errors.New("must supply an extract function") + } + if img == nil { + return nil, errors.New("image cannot be nil") + } if err := DetectFilesystemWhitelist(constants.WhitelistPath); err != nil { return nil, err } @@ -114,7 +122,7 @@ func GetFSFromImage(root string, img v1.Image) ([]string, error) { } continue } - if err := extractFile(root, hdr, tr); err != nil { + if err := extract(root, hdr, tr); err != nil { return nil, err } extractedFiles = append(extractedFiles, filepath.Join(root, filepath.Clean(hdr.Name))) @@ -179,7 +187,7 @@ func unTar(r io.Reader, dest string) ([]string, error) { if err != nil { return nil, err } - if err := extractFile(dest, hdr, tr); err != nil { + if err := ExtractFile(dest, hdr, tr); err != nil { return nil, err } extractedFiles = append(extractedFiles, dest) @@ -187,7 +195,7 @@ func unTar(r io.Reader, dest string) ([]string, error) { return extractedFiles, nil } -func extractFile(dest string, hdr *tar.Header, tr io.Reader) error { +func ExtractFile(dest string, hdr *tar.Header, tr io.Reader) error { path := filepath.Join(dest, filepath.Clean(hdr.Name)) base := filepath.Base(path) dir := filepath.Dir(path) diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index 2a4b31c6f..1ea074f58 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -659,7 +659,7 @@ func TestExtractFile(t *testing.T) { defer os.RemoveAll(r) for _, hdr := range tc.hdrs { - if err := extractFile(r, hdr, bytes.NewReader(tc.contents)); err != nil { + if err := ExtractFile(r, hdr, bytes.NewReader(tc.contents)); err != nil { t.Fatal(err) } } From fe47e3f15145e227e4cfc6ac966ff142cbc91024 Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Wed, 11 Dec 2019 18:02:48 -0500 Subject: [PATCH 27/28] Fix contribution issue sentence --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a341217a..d715eabe9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ specifically: - Start with a subject line - Contain a body that explains _why_ you're making the change you're making -- Reference an issue number one exists, closing it if applicable (with text such as +- Reference an issue number if one exists, closing it if applicable (with text such as ["Fixes #245" or "Closes #111"](https://help.github.com/articles/closing-issues-using-keywords/)) Aim for [2 paragraphs in the body](https://www.youtube.com/watch?v=PJjmw9TRB7s). @@ -67,4 +67,4 @@ interesting to work on: - To find issues that we particularly would like contributors to tackle, look for [issues with the "help wanted" label](https://github.com/GoogleContainerTools/kaniko/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22). - Issues that are good for new folks will additionally be marked with - ["good first issue"](https://github.com/GoogleContainerTools/kaniko/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). \ No newline at end of file + ["good first issue"](https://github.com/GoogleContainerTools/kaniko/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). From 9e9b8a6e710a64f4d609e6b752b58bc3c17f7f87 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Sun, 15 Dec 2019 10:23:31 -0800 Subject: [PATCH 28/28] Fix #899 cached copy results in inconsistent key * Update cached copy command to return the same result for files used from context so that cached and uncached copy commands produce the same cache key * Update tests for fix * Add test for cached run command key consistency --- pkg/commands/copy.go | 60 +++++++++++++-------- pkg/commands/copy_test.go | 20 ++++--- pkg/executor/build_test.go | 103 +++++++++++++++++++++++++++++++------ 3 files changed, 141 insertions(+), 42 deletions(-) diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index 6bbff699a..fd4b18a04 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -121,24 +121,7 @@ func (c *CopyCommand) String() string { } func (c *CopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) { - // We don't use the context if we're performing a copy --from. - if c.cmd.From != "" { - return nil, nil - } - - replacementEnvs := buildArgs.ReplacementEnvs(config.Env) - srcs, _, err := util.ResolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs) - if err != nil { - return nil, err - } - - files := []string{} - for _, src := range srcs { - fullPath := filepath.Join(c.buildcontext, src) - files = append(files, fullPath) - } - logrus.Infof("Using files from context: %v", files) - return files, nil + return copyCmdFilesUsedFromContext(config, buildArgs, c.cmd, c.buildcontext) } func (c *CopyCommand) MetadataOnly() bool { @@ -157,9 +140,10 @@ func (c *CopyCommand) ShouldCacheOutput() bool { func (c *CopyCommand) CacheCommand(img v1.Image) DockerCommand { return &CachingCopyCommand{ - img: img, - cmd: c.cmd, - extractFn: util.ExtractFile, + img: img, + cmd: c.cmd, + buildcontext: c.buildcontext, + extractFn: util.ExtractFile, } } @@ -172,6 +156,7 @@ type CachingCopyCommand struct { img v1.Image extractedFiles []string cmd *instructions.CopyCommand + buildcontext string extractFn util.ExtractFunction } @@ -192,6 +177,10 @@ func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *docke return nil } +func (cr *CachingCopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) { + return copyCmdFilesUsedFromContext(config, buildArgs, cr.cmd, cr.buildcontext) +} + func (cr *CachingCopyCommand) FilesToSnapshot() []string { return cr.extractedFiles } @@ -224,3 +213,32 @@ func resolveIfSymlink(destPath string) (string, error) { } return destPath, nil } + +func copyCmdFilesUsedFromContext( + config *v1.Config, buildArgs *dockerfile.BuildArgs, cmd *instructions.CopyCommand, + buildcontext string, +) ([]string, error) { + // We don't use the context if we're performing a copy --from. + if cmd.From != "" { + return nil, nil + } + + replacementEnvs := buildArgs.ReplacementEnvs(config.Env) + + srcs, _, err := util.ResolveEnvAndWildcards( + cmd.SourcesAndDest, buildcontext, replacementEnvs, + ) + if err != nil { + return nil, err + } + + files := []string{} + for _, src := range srcs { + fullPath := filepath.Join(buildcontext, src) + files = append(files, fullPath) + } + + logrus.Debugf("Using files from context: %v", files) + + return files, nil +} diff --git a/pkg/commands/copy_test.go b/pkg/commands/copy_test.go index d03168059..492211951 100644 --- a/pkg/commands/copy_test.go +++ b/pkg/commands/copy_test.go @@ -218,6 +218,8 @@ func Test_resolveIfSymlink(t *testing.T) { } func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) { + tempDir := setupTestTemp() + tarContent, err := prepareTarFixture([]string{"foo.txt"}) if err != nil { t.Errorf("couldn't prepare tar fixture %v", err) @@ -238,12 +240,19 @@ func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) { } testCases := []testCase{ func() testCase { + err = ioutil.WriteFile(filepath.Join(tempDir, "foo.txt"), []byte("meow"), 0644) + if err != nil { + t.Errorf("couldn't write tempfile %v", err) + t.FailNow() + } + c := &CachingCopyCommand{ img: fakeImage{ ImageLayers: []v1.Layer{ fakeLayer{TarContent: tarContent}, }, }, + buildcontext: tempDir, cmd: &instructions.CopyCommand{ SourcesAndDest: []string{ "foo.txt", "foo.txt", @@ -339,17 +348,16 @@ func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) { t.Errorf("Expected extracted files to include %v but did not %v", file, cFiles) } } - // CachingCopyCommand does not override BaseCommand - // FilesUseFromContext so this will always return an empty slice and no error - // This seems like it might be a bug as it results in CopyCommands and CachingCopyCommands generating different cache keys - cvgw - 2019-11-27 + cmdFiles, err := c.FilesUsedFromContext( config, buildArgs, ) if err != nil { - t.Errorf("failed to get files used from context from command") + t.Errorf("failed to get files used from context from command %v", err) } - if len(cmdFiles) != 0 { - t.Errorf("expected files used from context to be empty but was not") + + if len(cmdFiles) != len(tc.contextFiles) { + t.Errorf("expected files used from context to equal %v but was %v", tc.contextFiles, cmdFiles) } } diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index d5c07acfc..db3f55154 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -36,7 +36,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/moby/buildkit/frontend/dockerfile/instructions" - "github.com/sirupsen/logrus" ) func Test_reviewConfig(t *testing.T) { @@ -620,7 +619,7 @@ func Test_stageBuilder_build(t *testing.T) { ch := NewCompositeCache("", "") ch.AddPath(filepath) - logrus.SetLevel(logrus.DebugLevel) + hash, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) @@ -664,7 +663,7 @@ func Test_stageBuilder_build(t *testing.T) { filePath := filepath.Join(dir, filename) ch := NewCompositeCache("", "") ch.AddPath(filePath) - logrus.SetLevel(logrus.DebugLevel) + hash, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) @@ -698,22 +697,24 @@ func Test_stageBuilder_build(t *testing.T) { dir, filenames := tempDirAndFile(t) filename := filenames[0] tarContent := generateTar(t, filename) + destDir, err := ioutil.TempDir("", "baz") if err != nil { t.Errorf("could not create temp dir %v", err) } + filePath := filepath.Join(dir, filename) - ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) - ch.AddPath(filePath) - logrus.SetLevel(logrus.DebugLevel) - logrus.Infof("test composite key %v", ch) + + ch := NewCompositeCache("", fmt.Sprintf("RUN foobar")) + hash1, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) } + ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) ch.AddPath(filePath) - logrus.Infof("test composite key %v", ch) + hash2, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) @@ -721,11 +722,78 @@ func Test_stageBuilder_build(t *testing.T) { ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) ch.AddPath(filePath) - logrus.Infof("test composite key %v", ch) - hash3, err := ch.Hash() + + image := fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + } + + dockerFile := fmt.Sprintf(` +FROM ubuntu:16.04 +RUN foobar +COPY %s bar.txt +`, filename) + f, _ := ioutil.TempFile("", "") + ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755) + opts := &config.KanikoOptions{ + DockerfilePath: f.Name(), + } + + stages, err := dockerfile.Stages(opts) + if err != nil { + t.Errorf("could not parse test dockerfile") + } + + stage := stages[0] + + cmds := stage.Commands + return testcase{ + description: "cached run command followed by uncached copy command result in consistent read and write hashes", + opts: &config.KanikoOptions{Cache: true}, + rootDir: dir, + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + layerCache: &fakeLayerCache{ + keySequence: []string{hash1}, + img: image, + }, + image: image, + // hash1 is the read cachekey for the first layer + // hash2 is the read cachekey for the second layer + expectedCacheKeys: []string{hash1, hash2}, + pushedCacheKeys: []string{hash2}, + commands: getCommands(dir, cmds), + } + }(), + func() testcase { + dir, filenames := tempDirAndFile(t) + filename := filenames[0] + tarContent := generateTar(t, filename) + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + filePath := filepath.Join(dir, filename) + ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) + ch.AddPath(filePath) + + hash1, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) } + ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) + ch.AddPath(filePath) + + hash2, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) + ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) + ch.AddPath(filePath) + image := fakeImage{ ImageLayers: []v1.Layer{ fakeLayer{ @@ -749,10 +817,12 @@ COPY %s bar.txt if err != nil { t.Errorf("could not parse test dockerfile") } + stage := stages[0] + cmds := stage.Commands return testcase{ - description: "cached copy command followed by uncached copy command result in different read and write hashes", + description: "cached copy command followed by uncached copy command result in consistent read and write hashes", opts: &config.KanikoOptions{Cache: true}, rootDir: dir, config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, @@ -764,10 +834,8 @@ COPY %s bar.txt // hash1 is the read cachekey for the first layer // hash2 is the read cachekey for the second layer expectedCacheKeys: []string{hash1, hash2}, - // Due to CachingCopyCommand and CopyCommand returning different values the write cache key for the second copy command will never match the read cache key - // hash3 is the cachekey used to write to the cache for layer 2 - pushedCacheKeys: []string{hash3}, - commands: getCommands(dir, cmds), + pushedCacheKeys: []string{hash2}, + commands: getCommands(dir, cmds), } }(), } @@ -848,6 +916,11 @@ func assertCacheKeys(t *testing.T, expectedCacheKeys, actualCacheKeys []string, sort.Slice(actualCacheKeys, func(x, y int) bool { return actualCacheKeys[x] > actualCacheKeys[y] }) + + if len(expectedCacheKeys) != len(actualCacheKeys) { + t.Errorf("expected %v to equal %v", actualCacheKeys, expectedCacheKeys) + } + for i, key := range expectedCacheKeys { if key != actualCacheKeys[i] { t.Errorf("expected to %v keys %d to be %v but was %v %v", description, i, key, actualCacheKeys[i], actualCacheKeys)