public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

add caching copy layers back (#1518) 

* add caching copy layers back

* fix test

* lint

* fix test - 2

* Add integration test

* fix lint
This commit is contained in:
Tejal Desai 2020-12-11 00:25:03 -08:00 committed by GitHub
parent dde98a8e73
commit b04399eeac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 405 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -386,7 +386,7 @@ as a remote image destination:
### Caching
#### Caching Layers
kaniko can cache layers created by `RUN` commands in a remote repository.
kaniko can cache layers created by `RUN` and `Copy` (configured by flag `--cache-copy-layers`) commands in a remote repository.
Before executing a command, kaniko checks the cache for the layer.
If it exists, kaniko will pull and extract the cached layer instead of executing the command.
If not, kaniko will execute the command and then push the newly created layer to the cache.

1
cmd/executor/cmd/root.go
View File

@ -179,6 +179,7 @@ func addKanikoOptionsFlags() {
RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile")
RootCmd.PersistentFlags().BoolVarP(&opts.RunV2, "use-new-run", "", false, "Use the experimental run implementation for detecting changes without requiring file system snapshots.")
RootCmd.PersistentFlags().Var(&opts.Git, "git", "Branch to clone if build context is a git repository")
RootCmd.PersistentFlags().BoolVarP(&opts.CacheCopyLayers, "cache-copy-layers", "", false, "Caches copy layers")
}
// addHiddenFlags marks certain flags as hidden from the executor help text

5
integration/images.go
View File

@ -308,7 +308,7 @@ func populateVolumeCache() error {
}
// buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int) error {
func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int, args []string) error {
imageRepo, serviceAccount := config.imageRepo, config.serviceAccount
_, ex, _, _ := runtime.Caller(0)
cwd := filepath.Dir(ex)
@ -334,6 +334,9 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac
cacheFlag,
"--cache-repo", cacheRepo,
"--cache-dir", cacheDir)
for _, v := range args {
dockerRunFlags = append(dockerRunFlags, v)
}
kanikoCmd := exec.Command("docker", dockerRunFlags...)
_, err := RunCommandWithoutTest(kanikoCmd)

8
integration/integration_test.go
View File

@ -464,17 +464,21 @@ func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder
func TestCache(t *testing.T) {
populateVolumeCache()
for dockerfile := range imageBuilder.TestCacheDockerfiles {
args := []string{}
if dockerfile == "Dockerfile_test_cache_copy" {
args = append(args, "--cache-copy-layers=true")
}
t.Run("test_cache_"+dockerfile, func(t *testing.T) {
dockerfile := dockerfile
t.Parallel()
cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano()))
// Build the initial image which will cache layers
if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 0); err != nil {
if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 0, args); err != nil {
t.Fatalf("error building cached image for the first time: %v", err)
}
// Build the second image which should pull from the cache
if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 1); err != nil {
if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 1, args); err != nil {
t.Fatalf("error building cached image for the first time: %v", err)
}
// Make sure both images are the same

4
pkg/commands/commands.go
View File

@ -60,7 +60,7 @@ type DockerCommand interface {
ShouldDetectDeletedFiles() bool
}
func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool) (DockerCommand, error) {
func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool, cacheCopy bool) (DockerCommand, error) {
switch c := cmd.(type) {
case *instructions.RunCommand:
if useNewRun {
@ -68,7 +68,7 @@ func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRu
}
return &RunCommand{cmd: c}, nil
case *instructions.CopyCommand:
return &CopyCommand{cmd: c, fileContext: fileContext}, nil
return &CopyCommand{cmd: c, fileContext: fileContext, shdCache: cacheCopy}, nil
case *instructions.ExposeCommand:
return &ExposeCommand{cmd: c}, nil
case *instructions.EnvCommand:

81
pkg/commands/copy.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package commands
import (
"fmt"
"os"
"path/filepath"
"strings"
@ -41,6 +42,7 @@ type CopyCommand struct {
cmd *instructions.CopyCommand
fileContext util.FileContext
snapshotFiles []string
shdCache bool
}
func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
@ -145,6 +147,85 @@ func (c *CopyCommand) From() string {
return c.cmd.From
}
func (c *CopyCommand) ShouldCacheOutput() bool {
return c.shdCache
}
// CacheCommand returns true since this command should be cached
func (c *CopyCommand) CacheCommand(img v1.Image) DockerCommand {
return &CachingCopyCommand{
img: img,
cmd: c.cmd,
fileContext: c.fileContext,
extractFn: util.ExtractFile,
}
}
type CachingCopyCommand struct {
BaseCommand
caching
img v1.Image
extractedFiles []string
cmd *instructions.CopyCommand
fileContext util.FileContext
extractFn util.ExtractFunction
}
func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
logrus.Infof("Found cached layer, extracting to filesystem")
var err error
if cr.img == nil {
return errors.New(fmt.Sprintf("cached command image is nil %v", cr.String()))
}
layers, err := cr.img.Layers()
if err != nil {
return errors.Wrapf(err, "retrieve image layers")
}
if len(layers) != 1 {
return errors.New(fmt.Sprintf("expected %d layers but got %d", 1, len(layers)))
}
cr.layer = layers[0]
cr.extractedFiles, err = util.GetFSFromLayers(kConfig.RootDir, layers, util.ExtractFunc(cr.extractFn), util.IncludeWhiteout())
logrus.Debugf("extractedFiles: %s", cr.extractedFiles)
if err != nil {
return errors.Wrap(err, "extracting fs from image")
}
return nil
}
func (cr *CachingCopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
return copyCmdFilesUsedFromContext(config, buildArgs, cr.cmd, cr.fileContext)
}
func (cr *CachingCopyCommand) FilesToSnapshot() []string {
f := cr.extractedFiles
logrus.Debugf("%d files extracted by caching copy command", len(f))
logrus.Tracef("Extracted files: %s", f)
return f
}
func (cr *CachingCopyCommand) MetadataOnly() bool {
return false
}
func (cr *CachingCopyCommand) String() string {
if cr.cmd == nil {
return "nil command"
}
return cr.cmd.String()
}
func (cr *CachingCopyCommand) From() string {
return cr.cmd.From
}
func resolveIfSymlink(destPath string) (string, error) {
if !filepath.IsAbs(destPath) {
return "", errors.New("dest path must be abs")

155
pkg/commands/copy_test.go
View File

@ -16,6 +16,7 @@ limitations under the License.
package commands
import (
"archive/tar"
"fmt"
"io"
"io/ioutil"
@ -105,6 +106,160 @@ func setupTestTemp() string {
return tempDir
}
func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
tempDir := setupTestTemp()
tarContent, err := prepareTarFixture([]string{"foo.txt"})
if err != nil {
t.Errorf("couldn't prepare tar fixture %v", err)
}
config := &v1.Config{}
buildArgs := &dockerfile.BuildArgs{}
type testCase struct {
desctiption string
expectLayer bool
expectErr bool
count *int
expectedCount int
command *CachingCopyCommand
extractedFiles []string
contextFiles []string
}
testCases := []testCase{
func() testCase {
err = ioutil.WriteFile(filepath.Join(tempDir, "foo.txt"), []byte("meow"), 0644)
if err != nil {
t.Errorf("couldn't write tempfile %v", err)
t.FailNow()
}
c := &CachingCopyCommand{
img: fakeImage{
ImageLayers: []v1.Layer{
fakeLayer{TarContent: tarContent},
},
},
fileContext: util.FileContext{Root: tempDir},
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{
"foo.txt", "foo.txt",
},
},
}
count := 0
tc := testCase{
desctiption: "with valid image and valid layer",
count: &count,
expectedCount: 1,
expectLayer: true,
extractedFiles: []string{"/foo.txt"},
contextFiles: []string{"foo.txt"},
}
c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
*tc.count++
return nil
}
tc.command = c
return tc
}(),
func() testCase {
c := &CachingCopyCommand{}
tc := testCase{
desctiption: "with no image",
expectErr: true,
}
c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
return nil
}
tc.command = c
return tc
}(),
func() testCase {
c := &CachingCopyCommand{
img: fakeImage{},
}
c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
return nil
}
return testCase{
desctiption: "with image containing no layers",
expectErr: true,
command: c,
}
}(),
func() testCase {
c := &CachingCopyCommand{
img: fakeImage{
ImageLayers: []v1.Layer{
fakeLayer{},
},
},
}
c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
return nil
}
tc := testCase{
desctiption: "with image one layer which has no tar content",
expectErr: false, // this one probably should fail but doesn't because of how ExecuteCommand and util.GetFSFromLayers are implemented - cvgw- 2019-11-25
expectLayer: true,
}
tc.command = c
return tc
}(),
}
for _, tc := range testCases {
t.Run(tc.desctiption, func(t *testing.T) {
c := tc.command
err := c.ExecuteCommand(config, buildArgs)
if !tc.expectErr && err != nil {
t.Errorf("Expected err to be nil but was %v", err)
} else if tc.expectErr && err == nil {
t.Error("Expected err but was nil")
}
if tc.count != nil {
if *tc.count != tc.expectedCount {
t.Errorf("Expected extractFn to be called %v times but was called %v times", tc.expectedCount, *tc.count)
}
for _, file := range tc.extractedFiles {
match := false
cFiles := c.FilesToSnapshot()
for _, cFile := range cFiles {
if file == cFile {
match = true
break
}
}
if !match {
t.Errorf("Expected extracted files to include %v but did not %v", file, cFiles)
}
}
cmdFiles, err := c.FilesUsedFromContext(
config, buildArgs,
)
if err != nil {
t.Errorf("failed to get files used from context from command %v", err)
}
if len(cmdFiles) != len(tc.contextFiles) {
t.Errorf("expected files used from context to equal %v but was %v", tc.contextFiles, cmdFiles)
}
}
if c.layer == nil && tc.expectLayer {
t.Error("expected the command to have a layer set but instead was nil")
} else if c.layer != nil && !tc.expectLayer {
t.Error("expected the command to have no layer set but instead found a layer")
}
})
}
}
func TestCopyExecuteCmd(t *testing.T) {
tempDir := setupTestTemp()
defer os.RemoveAll(tempDir)

1
pkg/config/options.go
View File

@ -68,6 +68,7 @@ type KanikoOptions struct {
IgnoreVarRun bool
SkipUnusedStages bool
RunV2 bool
CacheCopyLayers bool
Git KanikoGitOptions
}

3
pkg/executor/build.go
View File

@ -127,7 +127,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross
}
for _, cmd := range s.stage.Commands {
command, err := commands.GetCommand(cmd, fileContext, opts.RunV2)
command, err := commands.GetCommand(cmd, fileContext, opts.RunV2, opts.CacheCopyLayers)
if err != nil {
return nil, err
}
@ -184,6 +184,7 @@ func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string
compositeKey.AddKey(resolvedCmd)
switch v := command.(type) {
case *commands.CopyCommand:
case *commands.CachingCopyCommand:
compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey)
}

160
pkg/executor/build_test.go
View File

@ -795,6 +795,135 @@ func Test_stageBuilder_build(t *testing.T) {
retrieve: true,
},
},
func() testcase {
dir, filenames := tempDirAndFile(t)
filename := filenames[0]
filepath := filepath.Join(dir, filename)
tarContent := generateTar(t, dir, filename)
ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename))
ch.AddPath(filepath, util.FileContext{})
hash, err := ch.Hash()
if err != nil {
t.Errorf("couldn't create hash %v", err)
}
copyCommandCacheKey := hash
dockerFile := fmt.Sprintf(`
FROM ubuntu:16.04
COPY %s foo.txt
`, filename)
f, _ := ioutil.TempFile("", "")
ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755)
opts := &config.KanikoOptions{
DockerfilePath: f.Name(),
Cache: true,
CacheCopyLayers: true,
}
testStages, metaArgs, err := dockerfile.ParseStages(opts)
if err != nil {
t.Errorf("Failed to parse test dockerfile to stages: %s", err)
}
kanikoStages, err := dockerfile.MakeKanikoStages(opts, testStages, metaArgs)
if err != nil {
t.Errorf("Failed to parse stages to Kaniko Stages: %s", err)
}
_ = ResolveCrossStageInstructions(kanikoStages)
stage := kanikoStages[0]
cmds := stage.Commands
return testcase{
description: "copy command cache enabled and key in cache",
opts: opts,
image: fakeImage{
ImageLayers: []v1.Layer{
fakeLayer{
TarContent: tarContent,
},
},
},
layerCache: &fakeLayerCache{
retrieve: true,
img: fakeImage{
ImageLayers: []v1.Layer{
fakeLayer{
TarContent: tarContent,
},
},
},
},
rootDir: dir,
expectedCacheKeys: []string{copyCommandCacheKey},
// CachingCopyCommand is not pushed to the cache
pushedCacheKeys: []string{},
commands: getCommands(util.FileContext{Root: dir}, cmds, true),
fileName: filename,
}
}(),
func() testcase {
dir, filenames := tempDirAndFile(t)
filename := filenames[0]
tarContent := []byte{}
destDir, err := ioutil.TempDir("", "baz")
if err != nil {
t.Errorf("could not create temp dir %v", err)
}
filePath := filepath.Join(dir, filename)
ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename))
ch.AddPath(filePath, util.FileContext{})
hash, err := ch.Hash()
if err != nil {
t.Errorf("couldn't create hash %v", err)
}
dockerFile := fmt.Sprintf(`
FROM ubuntu:16.04
COPY %s foo.txt
`, filename)
f, _ := ioutil.TempFile("", "")
ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755)
opts := &config.KanikoOptions{
DockerfilePath: f.Name(),
Cache: true,
CacheCopyLayers: true,
}
testStages, metaArgs, err := dockerfile.ParseStages(opts)
if err != nil {
t.Errorf("Failed to parse test dockerfile to stages: %s", err)
}
kanikoStages, err := dockerfile.MakeKanikoStages(opts, testStages, metaArgs)
if err != nil {
t.Errorf("Failed to parse stages to Kaniko Stages: %s", err)
}
_ = ResolveCrossStageInstructions(kanikoStages)
stage := kanikoStages[0]
cmds := stage.Commands
return testcase{
description: "copy command cache enabled and key is not in cache",
opts: opts,
config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}},
layerCache: &fakeLayerCache{},
image: fakeImage{
ImageLayers: []v1.Layer{
fakeLayer{
TarContent: tarContent,
},
},
},
rootDir: dir,
expectedCacheKeys: []string{hash},
pushedCacheKeys: []string{hash},
commands: getCommands(util.FileContext{Root: dir}, cmds, true),
fileName: filename,
}
}(),
func() testcase {
dir, filenames := tempDirAndFile(t)
filename := filenames[0]
@ -804,14 +933,26 @@ func Test_stageBuilder_build(t *testing.T) {
if err != nil {
t.Errorf("could not create temp dir %v", err)
}
filePath := filepath.Join(dir, filename)
ch := NewCompositeCache("", fmt.Sprintf("RUN foobar"))
ch := NewCompositeCache("", "RUN foobar")
hash1, err := ch.Hash()
if err != nil {
t.Errorf("couldn't create hash %v", err)
}
ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
ch.AddPath(filePath, util.FileContext{})
hash2, err := ch.Hash()
if err != nil {
t.Errorf("couldn't create hash %v", err)
}
ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename))
ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
ch.AddPath(filePath, util.FileContext{})
image := fakeImage{
ImageLayers: []v1.Layer{
fakeLayer{
@ -845,8 +986,8 @@ COPY %s bar.txt
cmds := stage.Commands
return testcase{
description: "cached run command followed by copy command results in consistent read and write hashes",
opts: &config.KanikoOptions{Cache: true},
description: "cached run command followed by uncached copy command results in consistent read and write hashes",
opts: &config.KanikoOptions{Cache: true, CacheCopyLayers: true},
rootDir: dir,
config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}},
layerCache: &fakeLayerCache{
@ -855,9 +996,9 @@ COPY %s bar.txt
},
image: image,
// hash1 is the read cachekey for the first layer
expectedCacheKeys: []string{hash1},
pushedCacheKeys: []string{},
commands: getCommands(util.FileContext{Root: dir}, cmds),
expectedCacheKeys: []string{hash1, hash2},
pushedCacheKeys: []string{hash2},
commands: getCommands(util.FileContext{Root: dir}, cmds, true),
}
}(),
func() testcase {
@ -933,7 +1074,7 @@ RUN foobar
image: image,
expectedCacheKeys: []string{runHash},
pushedCacheKeys: []string{},
commands: getCommands(util.FileContext{Root: dir}, cmds),
commands: getCommands(util.FileContext{Root: dir}, cmds, false),
}
}(),
func() testcase {
@ -1107,7 +1248,7 @@ RUN foobar
if err != nil {
t.Errorf("Expected error to be nil but was %v", err)
}
fmt.Println(lc.receivedKeys)
assertCacheKeys(t, tc.expectedCacheKeys, lc.receivedKeys, "receive")
assertCacheKeys(t, tc.pushedCacheKeys, keys, "push")
@ -1140,13 +1281,14 @@ func assertCacheKeys(t *testing.T, expectedCacheKeys, actualCacheKeys []string,
}
}
func getCommands(fileContext util.FileContext, cmds []instructions.Command) []commands.DockerCommand {
func getCommands(fileContext util.FileContext, cmds []instructions.Command, cacheCopy bool) []commands.DockerCommand {
outCommands := make([]commands.DockerCommand, 0)
for _, c := range cmds {
cmd, err := commands.GetCommand(
c,
fileContext,
false,
cacheCopy,
)
if err != nil {
panic(err)

1
pkg/executor/push.go
View File

@ -252,6 +252,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error {
}
}
timing.DefaultRun.Stop(t)
logrus.Infof("Pushed images to %d destinations", len(destRefs))
return writeImageOutputs(image, destRefs)
}