From 76afb70790c460346857e195fec49194d071adf7 Mon Sep 17 00:00:00 2001 From: Lavrenti Frobeen Date: Sun, 2 Apr 2023 04:25:25 +0200 Subject: [PATCH] Add support for zstd compression (#2313) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for configurable compression algorithm (gzip, zstd) and compression level We want to make the layer compression in kaniko configurable, so we have added two optional command line arguments “--compression” and “--compression-level”. The former allows the user to specify a compression algorithm (zstd, gzip) and the latter can be used to specify the compression level. Depending on the selected compression algorithm and level we modify the set of layerOptions that are used to create tarball layers in `push.go` and `build.go`. The actual implementation of the zstd support can be found in our fork of the go-containerregistry package for which we have filed this PR: google/go-containerregistry#1487 The changes should be fully backwards compatible. * Restrict inputs for compression flag to gzip and zstd This change will ensure that users can only specify supported compression algorithms (`zstd`, `gzip`) to the `--compression` flag. * Fix incorrect type for switch statements on config.Compression --- cmd/executor/cmd/root.go | 2 ++ pkg/config/options.go | 29 +++++++++++++++++++++++++++++ pkg/executor/build.go | 21 +++++++++++++++++---- pkg/executor/push.go | 22 +++++++++++++++++----- 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index eadda2eae..5597d33df 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -225,6 +225,8 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().StringVarP(&opts.ImageNameDigestFile, "image-name-with-digest-file", "", "", "Specify a file to save the image name w/ digest of the built image to.") RootCmd.PersistentFlags().StringVarP(&opts.ImageNameTagDigestFile, "image-name-tag-with-digest-file", "", "", "Specify a file to save the image name w/ image tag w/ digest of the built image to.") RootCmd.PersistentFlags().StringVarP(&opts.OCILayoutPath, "oci-layout-path", "", "", "Path to save the OCI image layout of the built image.") + RootCmd.PersistentFlags().VarP(&opts.Compression, "compression", "", "Compression algorithm (gzip, zstd)") + RootCmd.PersistentFlags().IntVarP(&opts.CompressionLevel, "compression-level", "", -1, "Compression level") RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") RootCmd.PersistentFlags().BoolVarP(&opts.CompressedCaching, "compressed-caching", "", true, "Compress the cached layers. Decreases build time, but increases memory usage.") RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end") diff --git a/pkg/config/options.go b/pkg/config/options.go index d368db139..b330f4a26 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -68,6 +68,8 @@ type KanikoOptions struct { ImageNameDigestFile string ImageNameTagDigestFile string OCILayoutPath string + Compression Compression + CompressionLevel int ImageFSExtractRetry int SingleSnapshot bool Reproducible bool @@ -125,6 +127,33 @@ func (k *KanikoGitOptions) Set(s string) error { return nil } +// Compression is an enumeration of the supported compression algorithms +type Compression string + +// The collection of known MediaType values. +const ( + GZip Compression = "gzip" + ZStd Compression = "zstd" +) + +func (c *Compression) String() string { + return string(*c) +} + +func (c *Compression) Set(v string) error { + switch v { + case "gzip", "zstd": + *c = Compression(v) + return nil + default: + return errors.New(`must be either "gzip" or "zstd"`) + } +} + +func (c *Compression) Type() string { + return "compression" +} + // WarmerOptions are options that are set by command line arguments to the cache warmer. type WarmerOptions struct { CacheOptions diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 286062608..02575776e 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -515,12 +515,25 @@ func (s *stageBuilder) saveSnapshotToLayer(tarPath string) (v1.Layer, error) { return nil, nil } - var layer v1.Layer + var layerOpts []tarball.LayerOption + if s.opts.CompressedCaching == true { - layer, err = tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching) - } else { - layer, err = tarball.LayerFromFile(tarPath) + layerOpts = append(layerOpts, tarball.WithCompressedCaching) } + + if s.opts.CompressionLevel > 0 { + layerOpts = append(layerOpts, tarball.WithCompressionLevel(s.opts.CompressionLevel)) + } + + switch s.opts.Compression { + case config.ZStd: + layerOpts = append(layerOpts, tarball.WithCompression("zstd")) + + case config.GZip: + // layer already gzipped by default + } + + layer, err := tarball.LayerFromFile(tarPath, layerOpts...) if err != nil { return nil, err } diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 7e2e156a2..f8620dbd6 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -296,16 +296,28 @@ func writeImageOutputs(image v1.Image, destRefs []name.Tag) error { // pushLayerToCache pushes layer (tagged with cacheKey) to opts.CacheRepo // if opts.CacheRepo doesn't exist, infer the cache from the given destination func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error { - var layer v1.Layer - var err error + var layerOpts []tarball.LayerOption if opts.CompressedCaching == true { - layer, err = tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching) - } else { - layer, err = tarball.LayerFromFile(tarPath) + layerOpts = append(layerOpts, tarball.WithCompressedCaching) } + + if opts.CompressionLevel > 0 { + layerOpts = append(layerOpts, tarball.WithCompressionLevel(opts.CompressionLevel)) + } + + switch opts.Compression { + case config.ZStd: + layerOpts = append(layerOpts, tarball.WithCompression("zstd")) + + case config.GZip: + // layer already gzipped by default + } + + layer, err := tarball.LayerFromFile(tarPath, layerOpts...) if err != nil { return err } + cache, err := cache.Destination(opts, cacheKey) if err != nil { return errors.Wrap(err, "getting cache destination")