From f9dcb92c38ec18626837a56e11c3148ab71baa73 Mon Sep 17 00:00:00 2001 From: Ramy Date: Tue, 9 Aug 2022 14:35:00 +0200 Subject: [PATCH] add cache run command option (#2032) Co-authored-by: Rami CHAABANE --- README.md | 6 +++++- cmd/executor/cmd/root.go | 1 + pkg/commands/commands.go | 6 +++--- pkg/commands/run.go | 5 +++-- pkg/commands/run_marker.go | 7 ++++--- pkg/config/options.go | 1 + pkg/executor/build.go | 2 +- pkg/executor/build_test.go | 17 +++++++++-------- 8 files changed, 27 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ca8d15711..4d574c1a6 100644 --- a/README.md +++ b/README.md @@ -400,7 +400,7 @@ as a remote image destination: ### Caching #### Caching Layers -kaniko can cache layers created by `RUN` (and `COPY`, configured by the `--cache-copy-layers` flag) commands in a remote repository. +kaniko can cache layers created by `RUN`(configured by flag `--cache-run-layers`) and `COPY` (configured by flag `--cache-copy-layers`) commands in a remote repository. Before executing a command, kaniko checks the cache for the layer. If it exists, kaniko will pull and extract the cached layer instead of executing the command. If not, kaniko will execute the command and then push the newly created layer to the cache. @@ -669,6 +669,10 @@ _This flag must be used in conjunction with the `--cache=true` flag._ Set this flag to cache copy layers. +#### --cache-run-layers + +Set this flag to cache run layers (default=true). + #### --cache-ttl duration Cache timeout in hours. Defaults to two weeks. diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 5acf9246c..43ae566df 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -221,6 +221,7 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().BoolVarP(&opts.RunV2, "use-new-run", "", false, "Use the experimental run implementation for detecting changes without requiring file system snapshots.") RootCmd.PersistentFlags().Var(&opts.Git, "git", "Branch to clone if build context is a git repository") RootCmd.PersistentFlags().BoolVarP(&opts.CacheCopyLayers, "cache-copy-layers", "", false, "Caches copy layers") + RootCmd.PersistentFlags().BoolVarP(&opts.CacheRunLayers, "cache-run-layers", "", true, "Caches run layers") RootCmd.PersistentFlags().VarP(&opts.IgnorePaths, "ignore-path", "", "Ignore these paths when taking a snapshot. Set it repeatedly for multiple paths.") RootCmd.PersistentFlags().BoolVarP(&opts.ForceBuildMetadata, "force-build-metadata", "", false, "Force add metadata layers to build image") diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index 2683c98f5..416fc64ab 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -60,13 +60,13 @@ type DockerCommand interface { ShouldDetectDeletedFiles() bool } -func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool, cacheCopy bool) (DockerCommand, error) { +func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool, cacheCopy bool, cacheRun bool) (DockerCommand, error) { switch c := cmd.(type) { case *instructions.RunCommand: if useNewRun { - return &RunMarkerCommand{cmd: c}, nil + return &RunMarkerCommand{cmd: c, shdCache: cacheRun}, nil } - return &RunCommand{cmd: c}, nil + return &RunCommand{cmd: c, shdCache: cacheRun}, nil case *instructions.CopyCommand: return &CopyCommand{cmd: c, fileContext: fileContext, shdCache: cacheCopy}, nil case *instructions.ExposeCommand: diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 3290ca019..f2f372b3d 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -35,7 +35,8 @@ import ( type RunCommand struct { BaseCommand - cmd *instructions.RunCommand + cmd *instructions.RunCommand + shdCache bool } // for testing @@ -187,7 +188,7 @@ func (r *RunCommand) RequiresUnpackedFS() bool { } func (r *RunCommand) ShouldCacheOutput() bool { - return true + return r.shdCache } type CachingRunCommand struct { diff --git a/pkg/commands/run_marker.go b/pkg/commands/run_marker.go index bc7482f08..ab33271c0 100644 --- a/pkg/commands/run_marker.go +++ b/pkg/commands/run_marker.go @@ -28,8 +28,9 @@ import ( type RunMarkerCommand struct { BaseCommand - cmd *instructions.RunCommand - Files []string + cmd *instructions.RunCommand + Files []string + shdCache bool } func (r *RunMarkerCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { @@ -77,7 +78,7 @@ func (r *RunMarkerCommand) RequiresUnpackedFS() bool { } func (r *RunMarkerCommand) ShouldCacheOutput() bool { - return true + return r.shdCache } func (r *RunMarkerCommand) ShouldDetectDeletedFiles() bool { diff --git a/pkg/config/options.go b/pkg/config/options.go index b8214ca97..51f5a7884 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -77,6 +77,7 @@ type KanikoOptions struct { SkipUnusedStages bool RunV2 bool CacheCopyLayers bool + CacheRunLayers bool ForceBuildMetadata bool } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index a033f5226..7e57896fa 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -133,7 +133,7 @@ func newStageBuilder(args *dockerfile.BuildArgs, opts *config.KanikoOptions, sta } for _, cmd := range s.stage.Commands { - command, err := commands.GetCommand(cmd, fileContext, opts.RunV2, opts.CacheCopyLayers) + command, err := commands.GetCommand(cmd, fileContext, opts.RunV2, opts.CacheCopyLayers, opts.CacheRunLayers) if err != nil { return nil, err } diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 678e747f8..99fd13497 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -913,7 +913,7 @@ COPY %s foo.txt expectedCacheKeys: []string{copyCommandCacheKey}, // CachingCopyCommand is not pushed to the cache pushedCacheKeys: []string{}, - commands: getCommands(util.FileContext{Root: dir}, cmds, true), + commands: getCommands(util.FileContext{Root: dir}, cmds, true, false), fileName: filename, } }(), @@ -970,7 +970,7 @@ COPY %s foo.txt rootDir: dir, expectedCacheKeys: []string{hash}, pushedCacheKeys: []string{hash}, - commands: getCommands(util.FileContext{Root: dir}, cmds, true), + commands: getCommands(util.FileContext{Root: dir}, cmds, true, false), fileName: filename, } }(), @@ -1034,7 +1034,7 @@ COPY %s bar.txt cmds := stage.Commands return testcase{ description: "cached run command followed by uncached copy command results in consistent read and write hashes", - opts: &config.KanikoOptions{Cache: true, CacheCopyLayers: true}, + opts: &config.KanikoOptions{Cache: true, CacheCopyLayers: true, CacheRunLayers: true}, rootDir: dir, config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, layerCache: &fakeLayerCache{ @@ -1045,7 +1045,7 @@ COPY %s bar.txt // hash1 is the read cachekey for the first layer expectedCacheKeys: []string{hash1, hash2}, pushedCacheKeys: []string{hash2}, - commands: getCommands(util.FileContext{Root: dir}, cmds, true), + commands: getCommands(util.FileContext{Root: dir}, cmds, true, true), } }(), func() testcase { @@ -1108,7 +1108,7 @@ RUN foobar cmds := stage.Commands return testcase{ description: "copy command followed by cached run command results in consistent read and write hashes", - opts: &config.KanikoOptions{Cache: true}, + opts: &config.KanikoOptions{Cache: true, CacheRunLayers: true}, rootDir: dir, config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, layerCache: &fakeLayerCache{ @@ -1118,7 +1118,7 @@ RUN foobar image: image, expectedCacheKeys: []string{runHash}, pushedCacheKeys: []string{}, - commands: getCommands(util.FileContext{Root: dir}, cmds, false), + commands: getCommands(util.FileContext{Root: dir}, cmds, false, true), } }(), func() testcase { @@ -1331,7 +1331,7 @@ func assertCacheKeys(t *testing.T, expectedCacheKeys, actualCacheKeys []string, } } -func getCommands(fileContext util.FileContext, cmds []instructions.Command, cacheCopy bool) []commands.DockerCommand { +func getCommands(fileContext util.FileContext, cmds []instructions.Command, cacheCopy, cacheRun bool) []commands.DockerCommand { outCommands := make([]commands.DockerCommand, 0) for _, c := range cmds { cmd, err := commands.GetCommand( @@ -1339,6 +1339,7 @@ func getCommands(fileContext util.FileContext, cmds []instructions.Command, cach fileContext, false, cacheCopy, + cacheRun, ) if err != nil { panic(err) @@ -1434,7 +1435,7 @@ func Test_stageBuild_populateCompositeKeyForCopyCommand(t *testing.T) { } fc := util.FileContext{Root: "workspace"} - copyCommand, err := commands.GetCommand(instructions[0], fc, false, true) + copyCommand, err := commands.GetCommand(instructions[0], fc, false, true, true) if err != nil { t.Fatal(err) }