From 876f24b0c985b25bc98815f26e2c33fdd7f91425 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Fri, 17 Jan 2020 14:16:42 -0800 Subject: [PATCH] update caching run and copy extracted files Update caching run and copy commands to use the new GetFSFromLayers method and include the whiteout option so that whiteout files are extracted and included in extractedFiles --- pkg/commands/copy.go | 13 +++++++++++-- pkg/commands/run.go | 18 ++++++++++++++++-- pkg/util/fs_util.go | 10 ++++++---- pkg/util/fs_util_test.go | 15 ++++++--------- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index fd4b18a04..bdbc65c4d 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -167,7 +167,13 @@ func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *docke if cr.img == nil { return errors.New(fmt.Sprintf("cached command image is nil %v", cr.String())) } - cr.extractedFiles, err = util.GetFSFromImage(RootDir, cr.img, cr.extractFn) + + layers, err := cr.img.Layers() + if err != nil { + return errors.Wrapf(err, "retrieve image layers") + } + + cr.extractedFiles, err = util.GetFSFromLayers(RootDir, layers, util.ExtractFunc(cr.extractFn), util.IncludeWhiteout()) logrus.Infof("extractedFiles: %s", cr.extractedFiles) if err != nil { @@ -182,7 +188,10 @@ func (cr *CachingCopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs } func (cr *CachingCopyCommand) FilesToSnapshot() []string { - return cr.extractedFiles + f := cr.extractedFiles + logrus.Debugf("files extracted by caching copy command %s", f) + + return f } func (cr *CachingCopyCommand) String() string { diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 0dcd4ed52..1687000c4 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -197,7 +197,18 @@ func (cr *CachingRunCommand) ExecuteCommand(config *v1.Config, buildArgs *docker if cr.img == nil { return errors.New(fmt.Sprintf("command image is nil %v", cr.String())) } - cr.extractedFiles, err = util.GetFSFromImage(constants.RootDir, cr.img, cr.extractFn) + + layers, err := cr.img.Layers() + if err != nil { + return errors.Wrap(err, "retrieving image layers") + } + + cr.extractedFiles, err = util.GetFSFromLayers( + constants.RootDir, + layers, + util.ExtractFunc(cr.extractFn), + util.IncludeWhiteout(), + ) if err != nil { return errors.Wrap(err, "extracting fs from image") } @@ -206,7 +217,10 @@ func (cr *CachingRunCommand) ExecuteCommand(config *v1.Config, buildArgs *docker } func (cr *CachingRunCommand) FilesToSnapshot() []string { - return cr.extractedFiles + f := cr.extractedFiles + logrus.Debugf("files extracted from caching run command %s", f) + + return f } func (cr *CachingRunCommand) String() string { diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 53a7ce6ff..5b704f33f 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -160,12 +160,14 @@ func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, e } if !cfg.includeWhiteout { + logrus.Debug("not including whiteout files") continue } - } else { - if err := cfg.extractFunc(root, hdr, tr); err != nil { - return nil, err - } + + } + + if err := cfg.extractFunc(root, hdr, tr); err != nil { + return nil, err } extractedFiles = append(extractedFiles, filepath.Join(root, filepath.Clean(hdr.Name))) diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index 65eb461a6..d2adaebaa 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -892,21 +892,20 @@ func Test_GetFSFromLayers_with_whiteouts_include_whiteout_enabled(t *testing.T) expectErr := false f := func(expectedFiles []string, tw *tar.Writer) { - body := "Hello World\n" for _, f := range expectedFiles { f := strings.TrimPrefix(strings.TrimPrefix(f, root), "/") hdr := &tar.Header{ Name: f, Mode: 0644, - Size: int64(len(body)), + Size: int64(len("Hello World\n")), } if err := tw.WriteHeader(hdr); err != nil { t.Fatal(err) } - if _, err := tw.Write([]byte(body)); err != nil { + if _, err := tw.Write([]byte("Hello World\n")); err != nil { t.Fatal(err) } } @@ -982,21 +981,20 @@ func Test_GetFSFromLayers_with_whiteouts_include_whiteout_disabled(t *testing.T) expectErr := false f := func(expectedFiles []string, tw *tar.Writer) { - body := "Hello World\n" for _, f := range expectedFiles { f := strings.TrimPrefix(strings.TrimPrefix(f, root), "/") hdr := &tar.Header{ Name: f, Mode: 0644, - Size: int64(len(body)), + Size: int64(len("Hello world\n")), } if err := tw.WriteHeader(hdr); err != nil { t.Fatal(err) } - if _, err := tw.Write([]byte(body)); err != nil { + if _, err := tw.Write([]byte("Hello world\n")); err != nil { t.Fatal(err) } } @@ -1075,21 +1073,20 @@ func Test_GetFSFromLayers(t *testing.T) { buf := new(bytes.Buffer) tw := tar.NewWriter(buf) - body := "Hello World\n" for _, f := range expectedFiles { f := strings.TrimPrefix(strings.TrimPrefix(f, root), "/") hdr := &tar.Header{ Name: f, Mode: 0644, - Size: int64(len(body)), + Size: int64(len("Hello world\n")), } if err := tw.WriteHeader(hdr); err != nil { t.Fatal(err) } - if _, err := tw.Write([]byte(body)); err != nil { + if _, err := tw.Write([]byte("Hello world\n")); err != nil { t.Fatal(err) } }