diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index a2ea7788d..cc58d51c7 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -17,6 +17,7 @@ limitations under the License. package commands import ( + "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" @@ -24,6 +25,12 @@ import ( "github.com/sirupsen/logrus" ) +var RootDir string + +func init() { + RootDir = constants.RootDir +} + type CurrentCacheKey func() (string, error) type DockerCommand interface { diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index ad75f6f3f..4071f957b 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -171,7 +171,7 @@ type CachingCopyCommand struct { func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { logrus.Infof("Found cached layer, extracting to filesystem") var err error - cr.extractedFiles, err = util.GetFSFromImage(constants.RootDir, cr.img) + cr.extractedFiles, err = util.GetFSFromImage(RootDir, cr.img) logrus.Infof("extractedFiles: %s", cr.extractedFiles) if err != nil { return errors.Wrap(err, "extracting fs from image") diff --git a/pkg/executor/build.go b/pkg/executor/build.go index c1d987120..e3c48b5e1 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -23,7 +23,7 @@ import ( "strconv" "time" - "github.com/otiai10/copy" + otiai10Cpy "github.com/otiai10/copy" "github.com/google/go-containerregistry/pkg/v1/partial" @@ -52,12 +52,21 @@ import ( // This is the size of an empty tar in Go const emptyTarSize = 1024 +type cachePusher func(*config.KanikoOptions, string, string, string) error +type snapShotter interface { + Init() error + TakeSnapshotFS() (string, error) + TakeSnapshot([]string) (string, error) +} + // stageBuilder contains all fields necessary to build one stage of a Dockerfile type stageBuilder struct { stage config.KanikoStage image v1.Image cf *v1.ConfigFile - snapshotter *snapshot.Snapshotter + snapshotter snapShotter + layerCache cache.LayerCache + pushCache cachePusher baseImageDigest string finalCacheKey string opts *config.KanikoOptions @@ -103,6 +112,10 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross opts: opts, crossStageDeps: crossStageDeps, digestToCacheKeyMap: dcm, + layerCache: &cache.RegistryCache{ + Opts: opts, + }, + pushCache: pushLayerToCache, } for _, cmd := range s.stage.Commands { @@ -138,9 +151,6 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro return nil } - layerCache := &cache.RegistryCache{ - Opts: s.opts, - } stopCache := false // Possibly replace commands with their cached implementations. // We walk through all the commands, running any commands that only operate on metadata. @@ -154,21 +164,21 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro // If the command uses files from the context, add them. files, err := command.FilesUsedFromContext(&cfg, s.args) if err != nil { - return err + return errors.Wrap(err, "failed to get files used from context") } for _, f := range files { if err := compositeKey.AddPath(f); err != nil { - return err + return errors.Wrap(err, "failed to add path to composite key") } } ck, err := compositeKey.Hash() if err != nil { - return err + return errors.Wrap(err, "failed to hash composite key") } s.finalCacheKey = ck if command.ShouldCacheOutput() && !stopCache { - img, err := layerCache.RetrieveLayer(ck) + img, err := s.layerCache.RetrieveLayer(ck) if err != nil { logrus.Debugf("Failed to retrieve layer: %s", err) logrus.Infof("No cached layer found for cmd %s", command.String()) @@ -205,7 +215,7 @@ func (s *stageBuilder) build() error { // Apply optimizations to the instructions. if err := s.optimize(*compositeKey, s.cf.Config); err != nil { - return err + return errors.Wrap(err, "failed to optimize instructions") } // Unpack file system to root if we need to. @@ -224,14 +234,14 @@ func (s *stageBuilder) build() error { if shouldUnpack { t := timing.Start("FS Unpacking") if _, err := util.GetFSFromImage(constants.RootDir, s.image); err != nil { - return err + return errors.Wrap(err, "failed to get filesystem from image") } timing.DefaultRun.Stop(t) } else { logrus.Info("Skipping unpacking as no commands require it.") } if err := util.DetectFilesystemWhitelist(constants.WhitelistPath); err != nil { - return err + return errors.Wrap(err, "failed to check filesystem whitelist") } // Take initial snapshot t := timing.Start("Initial FS snapshot") @@ -252,17 +262,17 @@ func (s *stageBuilder) build() error { // If the command uses files from the context, add them. files, err := command.FilesUsedFromContext(&s.cf.Config, s.args) if err != nil { - return err + return errors.Wrap(err, "failed to get files used from context") } for _, f := range files { if err := compositeKey.AddPath(f); err != nil { - return err + return errors.Wrap(err, fmt.Sprintf("failed to add path to composite key %v", f)) } } logrus.Info(command.String()) if err := command.ExecuteCommand(&s.cf.Config, s.args); err != nil { - return err + return errors.Wrap(err, "failed to execute command") } files = command.FilesToSnapshot() timing.DefaultRun.Stop(t) @@ -273,21 +283,21 @@ func (s *stageBuilder) build() error { tarPath, err := s.takeSnapshot(files) if err != nil { - return err + return errors.Wrap(err, "failed to take snapshot") } ck, err := compositeKey.Hash() if err != nil { - return err + return errors.Wrap(err, "failed to hash composite key") } // Push layer to cache (in parallel) now along with new config file if s.opts.Cache && command.ShouldCacheOutput() { cacheGroup.Go(func() error { - return pushLayerToCache(s.opts, ck, tarPath, command.String()) + return s.pushCache(s.opts, ck, tarPath, command.String()) }) } if err := s.saveSnapshotToImage(command.String(), tarPath); err != nil { - return err + return errors.Wrap(err, "failed to save snapshot to image") } } if err := cacheGroup.Wait(); err != nil { @@ -343,7 +353,7 @@ func (s *stageBuilder) saveSnapshotToImage(createdBy string, tarPath string) err } fi, err := os.Stat(tarPath) if err != nil { - return err + return errors.Wrap(err, "tar file path does not exist") } if fi.Size() <= emptyTarSize { logrus.Info("No files were changed, appending empty layer to config. No layer added to image.") @@ -505,7 +515,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { } for _, p := range filesToSave { logrus.Infof("Saving file %s for later use.", p) - copy.Copy(p, filepath.Join(dstDir, p)) + otiai10Cpy.Copy(p, filepath.Join(dstDir, p)) } // Delete the filesystem diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 44f6a1211..66c0ab5b5 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -17,6 +17,9 @@ limitations under the License. package executor import ( + "archive/tar" + "bytes" + "fmt" "io/ioutil" "os" "path/filepath" @@ -24,6 +27,7 @@ import ( "sort" "testing" + "github.com/GoogleContainerTools/kaniko/pkg/commands" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/testutil" @@ -32,6 +36,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/sirupsen/logrus" ) func Test_reviewConfig(t *testing.T) { @@ -462,3 +467,455 @@ func TestInitializeConfig(t *testing.T) { testutil.CheckDeepEqual(t, tt.expected, actual.Config) } } + +func Test_stageBuilder_optimize(t *testing.T) { + testCases := []struct { + opts *config.KanikoOptions + retrieve bool + name string + }{ + { + name: "cache enabled and layer not present in cache", + opts: &config.KanikoOptions{Cache: true}, + }, + { + name: "cache enabled and layer present in cache", + opts: &config.KanikoOptions{Cache: true}, + retrieve: true, + }, + { + name: "cache disabled and layer not present in cache", + opts: &config.KanikoOptions{Cache: false}, + }, + { + name: "cache disabled and layer present in cache", + opts: &config.KanikoOptions{Cache: false}, + retrieve: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cf := &v1.ConfigFile{} + snap := fakeSnapShotter{} + lc := &fakeLayerCache{retrieve: tc.retrieve} + sb := &stageBuilder{opts: tc.opts, cf: cf, snapshotter: snap, layerCache: lc} + ck := CompositeCache{} + file, err := ioutil.TempFile("", "foo") + if err != nil { + t.Error(err) + } + command := MockDockerCommand{ + contextFiles: []string{file.Name()}, + cacheCommand: MockCachedDockerCommand{}, + } + sb.cmds = []commands.DockerCommand{command} + err = sb.optimize(ck, cf.Config) + if err != nil { + t.Errorf("Expected error to be nil but was %v", err) + } + + }) + } +} + +func Test_stageBuilder_build(t *testing.T) { + type testcase struct { + description string + opts *config.KanikoOptions + layerCache *fakeLayerCache + expectedCacheKeys []string + pushedCacheKeys []string + commands []commands.DockerCommand + fileName string + rootDir string + image v1.Image + config *v1.ConfigFile + } + + testCases := []testcase{ + func() testcase { + dir, files := tempDirAndFile(t) + file := files[0] + filePath := filepath.Join(dir, file) + ch := NewCompositeCache("", "meow") + + ch.AddPath(filePath) + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + command := MockDockerCommand{ + contextFiles: []string{filePath}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{filePath}, + }, + } + + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + return testcase{ + description: "fake command cache enabled but key not in cache", + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + opts: &config.KanikoOptions{Cache: true}, + expectedCacheKeys: []string{hash}, + pushedCacheKeys: []string{hash}, + commands: []commands.DockerCommand{command}, + rootDir: dir, + } + }(), + func() testcase { + dir, files := tempDirAndFile(t) + file := files[0] + filePath := filepath.Join(dir, file) + ch := NewCompositeCache("", "meow") + + ch.AddPath(filePath) + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + command := MockDockerCommand{ + contextFiles: []string{filePath}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{filePath}, + }, + } + + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + return testcase{ + description: "fake command cache enabled and key in cache", + opts: &config.KanikoOptions{Cache: true}, + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + layerCache: &fakeLayerCache{ + retrieve: true, + }, + expectedCacheKeys: []string{hash}, + pushedCacheKeys: []string{}, + commands: []commands.DockerCommand{command}, + rootDir: dir, + } + }(), + { + description: "fake command cache disabled and key not in cache", + opts: &config.KanikoOptions{Cache: false}, + }, + { + description: "fake command cache disabled and key in cache", + opts: &config.KanikoOptions{Cache: false}, + layerCache: &fakeLayerCache{ + retrieve: true, + }, + }, + func() testcase { + dir, filenames := tempDirAndFile(t) + filename := filenames[0] + filepath := filepath.Join(dir, filename) + + tarContent := generateTar(t, dir, filename) + + ch := NewCompositeCache("", "") + ch.AddPath(filepath) + logrus.SetLevel(logrus.DebugLevel) + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + copyCommandCacheKey := hash + return testcase{ + description: "copy command cache enabled and key in cache", + opts: &config.KanikoOptions{Cache: true}, + layerCache: &fakeLayerCache{ + retrieve: true, + img: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + }, + }, + rootDir: dir, + expectedCacheKeys: []string{copyCommandCacheKey}, + // CachingCopyCommand is not pushed to the cache + pushedCacheKeys: []string{}, + commands: getCommands(dir, []instructions.Command{ + &instructions.CopyCommand{ + SourcesAndDest: []string{ + filename, "foo.txt", + }, + }, + }), + fileName: filename, + } + }(), + func() testcase { + dir, filenames := tempDirAndFile(t) + filename := filenames[0] + tarContent := []byte{} + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + filePath := filepath.Join(dir, filename) + ch := NewCompositeCache("", "") + ch.AddPath(filePath) + logrus.SetLevel(logrus.DebugLevel) + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + return testcase{ + description: "copy command cache enabled and key is not in cache", + opts: &config.KanikoOptions{Cache: true}, + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + layerCache: &fakeLayerCache{}, + image: fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + }, + rootDir: dir, + expectedCacheKeys: []string{hash}, + pushedCacheKeys: []string{hash}, + commands: getCommands(dir, []instructions.Command{ + &instructions.CopyCommand{ + SourcesAndDest: []string{ + filename, "foo.txt", + }, + }, + }), + fileName: filename, + } + }(), + func() testcase { + dir, filenames := tempDirAndFile(t) + filename := filenames[0] + tarContent := generateTar(t, filename) + destDir, err := ioutil.TempDir("", "baz") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + filePath := filepath.Join(dir, filename) + ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) + ch.AddPath(filePath) + logrus.SetLevel(logrus.DebugLevel) + logrus.Infof("test composite key %v", ch) + hash1, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) + ch.AddPath(filePath) + logrus.Infof("test composite key %v", ch) + hash2, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) + ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) + ch.AddPath(filePath) + logrus.Infof("test composite key %v", ch) + hash3, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + image := fakeImage{ + ImageLayers: []v1.Layer{ + fakeLayer{ + TarContent: tarContent, + }, + }, + } + + dockerFile := fmt.Sprintf(` +FROM ubuntu:16.04 +COPY %s foo.txt +COPY %s bar.txt +`, filename, filename) + f, _ := ioutil.TempFile("", "") + ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755) + opts := &config.KanikoOptions{ + DockerfilePath: f.Name(), + } + + stages, err := dockerfile.Stages(opts) + if err != nil { + t.Errorf("could not parse test dockerfile") + } + stage := stages[0] + cmds := stage.Commands + return testcase{ + description: "cached copy command followed by uncached copy command result in different read and write hashes", + opts: &config.KanikoOptions{Cache: true}, + rootDir: dir, + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}}, + layerCache: &fakeLayerCache{ + keySequence: []string{hash1}, + img: image, + }, + image: image, + // hash1 is the read cachekey for the first layer + // hash2 is the read cachekey for the second layer + expectedCacheKeys: []string{hash1, hash2}, + // Due to CachingCopyCommand and CopyCommand returning different values the write cache key for the second copy command will never match the read cache key + // hash3 is the cachekey used to write to the cache for layer 2 + pushedCacheKeys: []string{hash3}, + commands: getCommands(dir, cmds), + } + }(), + } + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + var fileName string + if tc.commands == nil { + file, err := ioutil.TempFile("", "foo") + if err != nil { + t.Error(err) + } + command := MockDockerCommand{ + contextFiles: []string{file.Name()}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{file.Name()}, + }, + } + tc.commands = []commands.DockerCommand{command} + fileName = file.Name() + } else { + fileName = tc.fileName + } + + cf := tc.config + if cf == nil { + cf = &v1.ConfigFile{ + Config: v1.Config{ + Env: make([]string, 0), + }, + } + } + + snap := fakeSnapShotter{file: fileName} + lc := tc.layerCache + if lc == nil { + lc = &fakeLayerCache{} + } + keys := []string{} + sb := &stageBuilder{ + args: &dockerfile.BuildArgs{}, //required or code will panic + image: tc.image, + opts: tc.opts, + cf: cf, + snapshotter: snap, + layerCache: lc, + pushCache: func(_ *config.KanikoOptions, cacheKey, _, _ string) error { + keys = append(keys, cacheKey) + return nil + }, + } + sb.cmds = tc.commands + tmp := commands.RootDir + if tc.rootDir != "" { + commands.RootDir = tc.rootDir + } + err := sb.build() + if err != nil { + t.Errorf("Expected error to be nil but was %v", err) + } + + assertCacheKeys(t, tc.expectedCacheKeys, lc.receivedKeys, "receive") + assertCacheKeys(t, tc.pushedCacheKeys, keys, "push") + + commands.RootDir = tmp + + }) + } +} + +func assertCacheKeys(t *testing.T, expectedCacheKeys, actualCacheKeys []string, description string) { + if len(expectedCacheKeys) != len(actualCacheKeys) { + t.Errorf("expected to %v %v keys but was %v", description, len(expectedCacheKeys), len(actualCacheKeys)) + } + + sort.Slice(expectedCacheKeys, func(x, y int) bool { + return expectedCacheKeys[x] > expectedCacheKeys[y] + }) + sort.Slice(actualCacheKeys, func(x, y int) bool { + return actualCacheKeys[x] > actualCacheKeys[y] + }) + for i, key := range expectedCacheKeys { + if key != actualCacheKeys[i] { + t.Errorf("expected to %v keys %d to be %v but was %v %v", description, i, key, actualCacheKeys[i], actualCacheKeys) + } + } +} + +func getCommands(dir string, cmds []instructions.Command) []commands.DockerCommand { + outCommands := make([]commands.DockerCommand, 0) + for _, c := range cmds { + cmd, err := commands.GetCommand( + c, + dir, + ) + if err != nil { + panic(err) + } + outCommands = append(outCommands, cmd) + } + return outCommands + +} + +func tempDirAndFile(t *testing.T) (string, []string) { + filenames := []string{"bar.txt"} + + dir, err := ioutil.TempDir("", "foo") + if err != nil { + t.Errorf("could not create temp dir %v", err) + } + for _, filename := range filenames { + filepath := filepath.Join(dir, filename) + err = ioutil.WriteFile(filepath, []byte(`meow`), 0777) + if err != nil { + t.Errorf("could not create temp file %v", err) + } + } + + return dir, filenames +} +func generateTar(t *testing.T, dir string, fileNames ...string) []byte { + buf := bytes.NewBuffer([]byte{}) + writer := tar.NewWriter(buf) + defer writer.Close() + + for _, filename := range fileNames { + filePath := filepath.Join(dir, filename) + info, err := os.Stat(filePath) + if err != nil { + t.Errorf("could not get file info for temp file %v", err) + } + hdr, err := tar.FileInfoHeader(info, filename) + if err != nil { + t.Errorf("could not get tar header for temp file %v", err) + } + + if err := writer.WriteHeader(hdr); err != nil { + t.Errorf("could not write tar header %v", err) + } + + content, err := ioutil.ReadFile(filePath) + if err != nil { + t.Errorf("could not read tempfile %v", err) + } + + if _, err := writer.Write(content); err != nil { + t.Errorf("could not write file contents to tar") + } + } + return buf.Bytes() +} diff --git a/pkg/executor/composite_cache_test.go b/pkg/executor/composite_cache_test.go new file mode 100644 index 000000000..a3c7f55af --- /dev/null +++ b/pkg/executor/composite_cache_test.go @@ -0,0 +1,137 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" +) + +func Test_NewCompositeCache(t *testing.T) { + r := NewCompositeCache() + if reflect.TypeOf(r).String() != "*executor.CompositeCache" { + t.Errorf("expected return to be *executor.CompositeCache but was %v", reflect.TypeOf(r).String()) + } +} + +func Test_CompositeCache_AddKey(t *testing.T) { + keys := []string{ + "meow", + "purr", + } + r := NewCompositeCache() + r.AddKey(keys...) + if len(r.keys) != 2 { + t.Errorf("expected keys to have length 2 but was %v", len(r.keys)) + } +} + +func Test_CompositeCache_Key(t *testing.T) { + r := NewCompositeCache("meow", "purr") + k := r.Key() + if k != "meow-purr" { + t.Errorf("expected result to equal meow-purr but was %v", k) + } +} + +func Test_CompositeCache_Hash(t *testing.T) { + r := NewCompositeCache("meow", "purr") + h, err := r.Hash() + if err != nil { + t.Errorf("expected error to be nil but was %v", err) + } + + expectedHash := "b4fd5a11af812a11a79d794007c842794cc668c8e7ebaba6d1e6d021b8e06c71" + if h != expectedHash { + t.Errorf("expected result to equal %v but was %v", expectedHash, h) + } +} + +func Test_CompositeCache_AddPath_dir(t *testing.T) { + tmpDir, err := ioutil.TempDir("/tmp", "foo") + if err != nil { + t.Errorf("got error setting up test %v", err) + } + + content := `meow meow meow` + if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo.txt"), []byte(content), 0777); err != nil { + t.Errorf("got error writing temp file %v", err) + } + + fn := func() string { + r := NewCompositeCache() + if err := r.AddPath(tmpDir); err != nil { + t.Errorf("expected error to be nil but was %v", err) + } + + if len(r.keys) != 1 { + t.Errorf("expected len of keys to be 1 but was %v", len(r.keys)) + } + hash, err := r.Hash() + if err != nil { + t.Errorf("couldnt generate hash from test cache") + } + return hash + } + + hash1 := fn() + hash2 := fn() + if hash1 != hash2 { + t.Errorf("expected hash %v to equal hash %v", hash1, hash2) + } +} +func Test_CompositeCache_AddPath_file(t *testing.T) { + tmpfile, err := ioutil.TempFile("/tmp", "foo.txt") + if err != nil { + t.Errorf("got error setting up test %v", err) + } + defer os.Remove(tmpfile.Name()) // clean up + + content := `meow meow meow` + if _, err := tmpfile.Write([]byte(content)); err != nil { + t.Errorf("got error writing temp file %v", err) + } + if err := tmpfile.Close(); err != nil { + t.Errorf("got error closing temp file %v", err) + } + + p := tmpfile.Name() + fn := func() string { + r := NewCompositeCache() + if err := r.AddPath(p); err != nil { + t.Errorf("expected error to be nil but was %v", err) + } + + if len(r.keys) != 1 { + t.Errorf("expected len of keys to be 1 but was %v", len(r.keys)) + } + hash, err := r.Hash() + if err != nil { + t.Errorf("couldnt generate hash from test cache") + } + return hash + } + + hash1 := fn() + hash2 := fn() + if hash1 != hash2 { + t.Errorf("expected hash %v to equal hash %v", hash1, hash2) + } +} diff --git a/pkg/executor/fakes.go b/pkg/executor/fakes.go new file mode 100644 index 000000000..e9cfdc694 --- /dev/null +++ b/pkg/executor/fakes.go @@ -0,0 +1,184 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// for use in tests +package executor + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + + "github.com/GoogleContainerTools/kaniko/pkg/commands" + "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type fakeSnapShotter struct { + file string + tarPath string +} + +func (f fakeSnapShotter) Init() error { return nil } +func (f fakeSnapShotter) TakeSnapshotFS() (string, error) { + return f.tarPath, nil +} +func (f fakeSnapShotter) TakeSnapshot(_ []string) (string, error) { + return f.tarPath, nil +} + +type MockDockerCommand struct { + contextFiles []string + cacheCommand commands.DockerCommand +} + +func (m MockDockerCommand) ExecuteCommand(c *v1.Config, args *dockerfile.BuildArgs) error { return nil } +func (m MockDockerCommand) String() string { + return "meow" +} +func (m MockDockerCommand) FilesToSnapshot() []string { + return []string{"meow-snapshot-no-cache"} +} +func (m MockDockerCommand) CacheCommand(image v1.Image) commands.DockerCommand { + return m.cacheCommand +} +func (m MockDockerCommand) FilesUsedFromContext(c *v1.Config, args *dockerfile.BuildArgs) ([]string, error) { + return m.contextFiles, nil +} +func (m MockDockerCommand) MetadataOnly() bool { + return false +} +func (m MockDockerCommand) RequiresUnpackedFS() bool { + return false +} +func (m MockDockerCommand) ShouldCacheOutput() bool { + return true +} + +type MockCachedDockerCommand struct { + contextFiles []string +} + +func (m MockCachedDockerCommand) ExecuteCommand(c *v1.Config, args *dockerfile.BuildArgs) error { + return nil +} +func (m MockCachedDockerCommand) String() string { + return "meow" +} +func (m MockCachedDockerCommand) FilesToSnapshot() []string { + return []string{"meow-snapshot"} +} +func (m MockCachedDockerCommand) CacheCommand(image v1.Image) commands.DockerCommand { + return nil +} +func (m MockCachedDockerCommand) FilesUsedFromContext(c *v1.Config, args *dockerfile.BuildArgs) ([]string, error) { + return m.contextFiles, nil +} +func (m MockCachedDockerCommand) MetadataOnly() bool { + return false +} +func (m MockCachedDockerCommand) RequiresUnpackedFS() bool { + return false +} +func (m MockCachedDockerCommand) ShouldCacheOutput() bool { + return false +} + +type fakeLayerCache struct { + retrieve bool + receivedKeys []string + img v1.Image + keySequence []string +} + +func (f *fakeLayerCache) RetrieveLayer(key string) (v1.Image, error) { + f.receivedKeys = append(f.receivedKeys, key) + if len(f.keySequence) > 0 { + if f.keySequence[0] == key { + f.keySequence = f.keySequence[1:] + return f.img, nil + } + return f.img, errors.New("could not find layer") + } + + if !f.retrieve { + return nil, errors.New("could not find layer") + } + return f.img, nil +} + +type fakeLayer struct { + TarContent []byte +} + +func (f fakeLayer) Digest() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeLayer) DiffID() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeLayer) Compressed() (io.ReadCloser, error) { + return nil, nil +} +func (f fakeLayer) Uncompressed() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(f.TarContent)), nil +} +func (f fakeLayer) Size() (int64, error) { + return 0, nil +} +func (f fakeLayer) MediaType() (types.MediaType, error) { + return "", nil +} + +type fakeImage struct { + ImageLayers []v1.Layer +} + +func (f fakeImage) Layers() ([]v1.Layer, error) { + return f.ImageLayers, nil +} +func (f fakeImage) MediaType() (types.MediaType, error) { + return "", nil +} +func (f fakeImage) Size() (int64, error) { + return 0, nil +} +func (f fakeImage) ConfigName() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeImage) ConfigFile() (*v1.ConfigFile, error) { + return &v1.ConfigFile{}, nil +} +func (f fakeImage) RawConfigFile() ([]byte, error) { + return []byte{}, nil +} +func (f fakeImage) Digest() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f fakeImage) Manifest() (*v1.Manifest, error) { + return &v1.Manifest{}, nil +} +func (f fakeImage) RawManifest() ([]byte, error) { + return []byte{}, nil +} +func (f fakeImage) LayerByDigest(v1.Hash) (v1.Layer, error) { + return fakeLayer{}, nil +} +func (f fakeImage) LayerByDiffID(v1.Hash) (v1.Layer, error) { + return fakeLayer{}, nil +} diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 5577da087..b54897d59 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -176,7 +176,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { // Only add changed files. fileChanged, err := s.l.CheckFileChange(path) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("could not check if file has changed %s %s", path, err) } if fileChanged { logrus.Debugf("Adding %s to layer, because it was changed.", path) diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go index c4b3437ea..103d22e0b 100644 --- a/pkg/util/command_util.go +++ b/pkg/util/command_util.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + "fmt" "net/http" "net/url" "os" @@ -25,7 +26,7 @@ import ( "strings" "github.com/GoogleContainerTools/kaniko/pkg/constants" - "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/frontend/dockerfile/shell" @@ -77,13 +78,16 @@ func ResolveEnvAndWildcards(sd instructions.SourcesAndDest, buildcontext string, // First, resolve any environment replacement resolvedEnvs, err := ResolveEnvironmentReplacementList(sd, envs, true) if err != nil { - return nil, "", err + return nil, "", errors.Wrap(err, "failed to resolve environment") + } + if len(resolvedEnvs) == 0 { + return nil, "", errors.New("resolved envs is empty") } dest := resolvedEnvs[len(resolvedEnvs)-1] // Resolve wildcards and get a list of resolved sources srcs, err := ResolveSources(resolvedEnvs[0:len(resolvedEnvs)-1], buildcontext) if err != nil { - return nil, "", err + return nil, "", errors.Wrap(err, "failed to resolve sources") } err = IsSrcsValid(sd, srcs, buildcontext) return srcs, dest, err @@ -219,9 +223,10 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri if IsSrcRemoteFileURL(resolvedSources[0]) { return nil } - fi, err := os.Lstat(filepath.Join(root, resolvedSources[0])) + path := filepath.Join(root, resolvedSources[0]) + fi, err := os.Lstat(path) if err != nil { - return err + return errors.Wrap(err, fmt.Sprintf("failed to get fileinfo for %v", path)) } if fi.IsDir() { return nil @@ -237,7 +242,7 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri src = filepath.Clean(src) files, err := RelativeFiles(src, root) if err != nil { - return err + return errors.Wrap(err, "failed to get relative files") } for _, file := range files { if excludeFile(file, root) {