From 46e01340c23c6c31fc97f050d97b06cf0d284f15 Mon Sep 17 00:00:00 2001 From: Janosch Maier Date: Tue, 19 Oct 2021 09:14:06 +0200 Subject: [PATCH] Remove tarball.WithCompressedCaching flag to resolve OOM Killed error (#1722) * Remove tarball.WithCompressedCaching flag to resolve OOM Killed error Large images cannot be build as the kaniko container will be killed due to an OOM error. Removing the tarball compression drastically reduces the memory required to push large image layers. Fixes #1680 This change may increase the build time for smaller images. Therefore a command line option to trigger the compression or a more intelligent behaviour may be useful. * Add new command line flag to toggle compressed caching * Add unittest for build with --compressed-caching command line flag set to false --- README.md | 6 ++++++ cmd/executor/cmd/root.go | 1 + pkg/config/options.go | 1 + pkg/executor/build.go | 7 ++++++- pkg/executor/build_test.go | 36 ++++++++++++++++++++++++++++++++++++ pkg/executor/push.go | 8 +++++++- 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fe2c6fa82..827eee289 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [--cache-ttl duration](#--cache-ttl-duration) - [--cleanup](#--cleanup) - [--context-sub-path](#--context-sub-path) + - [--compressed-caching](#--compressed-caching) - [--customPlatform](#--customPlatform) - [--digest-file](#--digest-file) - [--dockerfile](#--dockerfile) @@ -613,6 +614,11 @@ Cache timeout in hours. Defaults to two weeks. Set this flag to clean the filesystem at the end of the build. +#### --compressed-caching + +Set this to false in order to prevent tar compression for cached layers. This will increase the runtime of the build, but decrease the memory usage especially for large builds. +Try to use `--compressed-caching=false` if your build fails with an out of memory error. Defaults to true. + #### --context-sub-path Set a sub path within the given `--context`. diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 0ff3dd50f..90ae82b4e 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -202,6 +202,7 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().StringVarP(&opts.ImageNameTagDigestFile, "image-name-tag-with-digest-file", "", "", "Specify a file to save the image name w/ image tag w/ digest of the built image to.") RootCmd.PersistentFlags().StringVarP(&opts.OCILayoutPath, "oci-layout-path", "", "", "Path to save the OCI image layout of the built image.") RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") + RootCmd.PersistentFlags().BoolVarP(&opts.CompressedCaching, "compressed-caching", "", true, "Compress the cached layers. Decreases build time, but increases memory usage.") RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end") RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.") RootCmd.PersistentFlags().VarP(&opts.InsecureRegistries, "insecure-registry", "", "Insecure registry using plain HTTP to push and pull. Set it repeatedly for multiple registries.") diff --git a/pkg/config/options.go b/pkg/config/options.go index 1af8a8fca..3482ead0e 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -70,6 +70,7 @@ type KanikoOptions struct { NoPush bool Cache bool Cleanup bool + CompressedCaching bool IgnoreVarRun bool SkipUnusedStages bool RunV2 bool diff --git a/pkg/executor/build.go b/pkg/executor/build.go index a37a7fc0e..1f2b3aee1 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -477,7 +477,12 @@ func (s *stageBuilder) saveSnapshotToLayer(tarPath string) (v1.Layer, error) { return nil, nil } - layer, err := tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching) + var layer v1.Layer + if s.opts.CompressedCaching == true { + layer, err = tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching) + } else { + layer, err = tarball.LayerFromFile(tarPath) + } if err != nil { return nil, err } diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index e2ee3ee83..4f59f1b42 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -784,6 +784,42 @@ func Test_stageBuilder_build(t *testing.T) { rootDir: dir, } }(), + func() testcase { + dir, files := tempDirAndFile(t) + file := files[0] + filePath := filepath.Join(dir, file) + ch := NewCompositeCache("", "meow") + + ch.AddPath(filePath, util.FileContext{}) + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + command := MockDockerCommand{ + command: "meow", + contextFiles: []string{filePath}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{filePath}, + }, + } + + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + return testcase{ + description: "fake command cache enabled with tar compression disabled and key in cache", + opts: &config.KanikoOptions{Cache: true, CompressedCaching: false}, + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + layerCache: &fakeLayerCache{ + retrieve: true, + }, + expectedCacheKeys: []string{hash}, + pushedCacheKeys: []string{}, + commands: []commands.DockerCommand{command}, + rootDir: dir, + } + }(), { description: "fake command cache disabled and key not in cache", opts: &config.KanikoOptions{Cache: false}, diff --git a/pkg/executor/push.go b/pkg/executor/push.go index d84a850bf..a989db920 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -327,7 +327,13 @@ func writeImageOutputs(image v1.Image, destRefs []name.Tag) error { // pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache // if opts.Cache doesn't exist, infer the cache from the given destination func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error { - layer, err := tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching) + var layer v1.Layer + var err error + if opts.CompressedCaching == true { + layer, err = tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching) + } else { + layer, err = tarball.LayerFromFile(tarPath) + } if err != nil { return err }