From 71aed35094e2e10fecb044b30e9064735264b779 Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Fri, 17 Jan 2020 13:36:23 -0800 Subject: [PATCH] GetFSFromLayers * add util.GetFSFromLayers * GetFSFromImage delegates to GetFSFromLayers * add FSOpts and FSConfig for GetFSFromLayers * add tests for GetFSFromLayers * add gomock for test support * add mock_v1 for layers --- go.mod | 3 +- go.sum | 6 + .../Dockerfile_test_deleted_file_cached | 5 + .../go-containerregistry/mock_v1/mocks.go | 126 ++++++++ pkg/util/fs_util.go | 67 ++++- pkg/util/fs_util_test.go | 282 ++++++++++++++++++ 6 files changed, 476 insertions(+), 13 deletions(-) create mode 100644 integration/dockerfiles/Dockerfile_test_deleted_file_cached create mode 100644 pkg/mocks/go-containerregistry/mock_v1/mocks.go diff --git a/go.mod b/go.mod index 47c583a23..6eaca49cb 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/gliderlabs/ssh v0.2.2 // indirect github.com/gogo/protobuf v1.1.1 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/mock v1.3.1 github.com/golang/protobuf v1.1.0 // indirect github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a // indirect github.com/google/go-cmp v0.2.0 @@ -94,7 +95,7 @@ require ( go.opencensus.io v0.14.0 // indirect golang.org/x/net v0.0.0-20190311183353-d8887717615a golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc - golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f + golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79 // indirect google.golang.org/appengine v1.1.0 // indirect diff --git a/go.sum b/go.sum index 32b35ffad..de01172fb 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -239,6 +241,8 @@ golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc h1:3ElrZeO6IBP+M8kgu5YFwR golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564 h1:o6ENHFwwr1TZ9CUPQcfo1HGvLP1OPsPOTB7xCIOPNmU= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -252,6 +256,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262 h1:qsl9y/CJx34tuA7QCPNp86JNJe4spst6Ff8MjvPUdPg= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79 h1:wCy2/9bhO1JeP2zZUALrj7ZdZuZoR4mRV57kTxjqRpo= google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= diff --git a/integration/dockerfiles/Dockerfile_test_deleted_file_cached b/integration/dockerfiles/Dockerfile_test_deleted_file_cached new file mode 100644 index 000000000..17864adaf --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_deleted_file_cached @@ -0,0 +1,5 @@ +FROM alpine + +RUN mkdir -p /some/dir/ && echo 'first' > /some/dir/first.txt + +RUN rm /some/dir/first.txt diff --git a/pkg/mocks/go-containerregistry/mock_v1/mocks.go b/pkg/mocks/go-containerregistry/mock_v1/mocks.go new file mode 100644 index 000000000..0415fe721 --- /dev/null +++ b/pkg/mocks/go-containerregistry/mock_v1/mocks.go @@ -0,0 +1,126 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/google/go-containerregistry/pkg/v1 (interfaces: Layer) + +// Package mock_v1 is a generated GoMock package. +package mock_v1 + +import ( + gomock "github.com/golang/mock/gomock" + v1 "github.com/google/go-containerregistry/pkg/v1" + types "github.com/google/go-containerregistry/pkg/v1/types" + io "io" + reflect "reflect" +) + +// MockLayer is a mock of Layer interface +type MockLayer struct { + ctrl *gomock.Controller + recorder *MockLayerMockRecorder +} + +// MockLayerMockRecorder is the mock recorder for MockLayer +type MockLayerMockRecorder struct { + mock *MockLayer +} + +// NewMockLayer creates a new mock instance +func NewMockLayer(ctrl *gomock.Controller) *MockLayer { + mock := &MockLayer{ctrl: ctrl} + mock.recorder = &MockLayerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockLayer) EXPECT() *MockLayerMockRecorder { + return m.recorder +} + +// Compressed mocks base method +func (m *MockLayer) Compressed() (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Compressed") + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Compressed indicates an expected call of Compressed +func (mr *MockLayerMockRecorder) Compressed() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compressed", reflect.TypeOf((*MockLayer)(nil).Compressed)) +} + +// DiffID mocks base method +func (m *MockLayer) DiffID() (v1.Hash, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiffID") + ret0, _ := ret[0].(v1.Hash) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DiffID indicates an expected call of DiffID +func (mr *MockLayerMockRecorder) DiffID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiffID", reflect.TypeOf((*MockLayer)(nil).DiffID)) +} + +// Digest mocks base method +func (m *MockLayer) Digest() (v1.Hash, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Digest") + ret0, _ := ret[0].(v1.Hash) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Digest indicates an expected call of Digest +func (mr *MockLayerMockRecorder) Digest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Digest", reflect.TypeOf((*MockLayer)(nil).Digest)) +} + +// MediaType mocks base method +func (m *MockLayer) MediaType() (types.MediaType, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MediaType") + ret0, _ := ret[0].(types.MediaType) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MediaType indicates an expected call of MediaType +func (mr *MockLayerMockRecorder) MediaType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MediaType", reflect.TypeOf((*MockLayer)(nil).MediaType)) +} + +// Size mocks base method +func (m *MockLayer) Size() (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Size") + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Size indicates an expected call of Size +func (mr *MockLayerMockRecorder) Size() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Size", reflect.TypeOf((*MockLayer)(nil).Size)) +} + +// Uncompressed mocks base method +func (m *MockLayer) Uncompressed() (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Uncompressed") + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Uncompressed indicates an expected call of Uncompressed +func (mr *MockLayerMockRecorder) Uncompressed() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Uncompressed", reflect.TypeOf((*MockLayer)(nil).Uncompressed)) +} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 0792e1d66..53a7ce6ff 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -71,25 +71,58 @@ var excluded []string type ExtractFunction func(string, *tar.Header, io.Reader) error +type FSConfig struct { + includeWhiteout bool + extractFunc ExtractFunction +} + +type FSOpt func(*FSConfig) + +func IncludeWhiteout() FSOpt { + return func(opts *FSConfig) { + opts.includeWhiteout = true + } +} + +func ExtractFunc(extractFunc ExtractFunction) FSOpt { + return func(opts *FSConfig) { + opts.extractFunc = extractFunc + } +} + // GetFSFromImage extracts the layers of img to root // It returns a list of all files extracted func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]string, error) { - if extract == nil { - return nil, errors.New("must supply an extract function") - } if img == nil { return nil, errors.New("image cannot be nil") } - if err := DetectFilesystemWhitelist(constants.WhitelistPath); err != nil { - return nil, err - } - logrus.Debugf("Mounted directories: %v", whitelist) + layers, err := img.Layers() if err != nil { return nil, err } - extractedFiles := []string{} + return GetFSFromLayers(root, layers, ExtractFunc(extract)) +} + +func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, error) { + cfg := new(FSConfig) + + for _, opt := range opts { + opt(cfg) + } + + if cfg.extractFunc == nil { + return nil, errors.New("must supply an extract function") + } + + if err := DetectFilesystemWhitelist(constants.WhitelistPath); err != nil { + return nil, err + } + + logrus.Debugf("Mounted directories: %v", whitelist) + + extractedFiles := []string{} for i, l := range layers { if mediaType, err := l.MediaType(); err == nil { logrus.Tracef("Extracting layer %d of media type %s", i, mediaType) @@ -102,29 +135,39 @@ func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]strin return nil, err } defer r.Close() + tr := tar.NewReader(r) for { hdr, err := tr.Next() if err == io.EOF { break } + if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("error reading tar %d", i)) } + path := filepath.Join(root, filepath.Clean(hdr.Name)) base := filepath.Base(path) dir := filepath.Dir(path) + if strings.HasPrefix(base, ".wh.") { logrus.Debugf("Whiting out %s", path) + name := strings.TrimPrefix(base, ".wh.") if err := os.RemoveAll(filepath.Join(dir, name)); err != nil { return nil, errors.Wrapf(err, "removing whiteout %s", hdr.Name) } - continue - } - if err := extract(root, hdr, tr); err != nil { - return nil, err + + if !cfg.includeWhiteout { + continue + } + } else { + if err := cfg.extractFunc(root, hdr, tr); err != nil { + return nil, err + } } + extractedFiles = append(extractedFiles, filepath.Join(root, filepath.Clean(hdr.Name))) } } diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index 1ea074f58..65eb461a6 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -20,6 +20,7 @@ import ( "archive/tar" "bytes" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -28,7 +29,11 @@ import ( "strings" "testing" + "github.com/GoogleContainerTools/kaniko/pkg/mocks/go-containerregistry/mock_v1" "github.com/GoogleContainerTools/kaniko/testutil" + "github.com/golang/mock/gomock" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" ) func Test_DetectFilesystemWhitelist(t *testing.T) { @@ -863,3 +868,280 @@ func Test_CopyFile_skips_self(t *testing.T) { t.Fatalf("expected file contents to be %q, but got %q", expected, actual) } } + +func fakeExtract(dest string, hdr *tar.Header, tr io.Reader) error { + return nil +} + +func Test_GetFSFromLayers_with_whiteouts_include_whiteout_enabled(t *testing.T) { + ctrl := gomock.NewController(t) + + root, err := ioutil.TempDir("", "layers-test") + if err != nil { + t.Fatal(err) + } + defer os.Remove(root) + + opts := []FSOpt{ + // I'd rather use the real func (util.ExtractFile) + // but you have to be root to chown + ExtractFunc(fakeExtract), + IncludeWhiteout(), + } + + expectErr := false + + f := func(expectedFiles []string, tw *tar.Writer) { + body := "Hello World\n" + for _, f := range expectedFiles { + f := strings.TrimPrefix(strings.TrimPrefix(f, root), "/") + + hdr := &tar.Header{ + Name: f, + Mode: 0644, + Size: int64(len(body)), + } + + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + } + + expectedFiles := []string{ + filepath.Join(root, "foobar"), + } + + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + f(expectedFiles, tw) + + mockLayer := mock_v1.NewMockLayer(ctrl) + mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil) + + rc := ioutil.NopCloser(buf) + mockLayer.EXPECT().Uncompressed().Return(rc, nil) + + secondLayerFiles := []string{ + filepath.Join(root, ".wh.foobar"), + } + + buf = new(bytes.Buffer) + tw = tar.NewWriter(buf) + + f(secondLayerFiles, tw) + + mockLayer2 := mock_v1.NewMockLayer(ctrl) + mockLayer2.EXPECT().MediaType().Return(types.OCILayer, nil) + + rc = ioutil.NopCloser(buf) + mockLayer2.EXPECT().Uncompressed().Return(rc, nil) + + layers := []v1.Layer{ + mockLayer, + mockLayer2, + } + + expectedFiles = append(expectedFiles, secondLayerFiles...) + + actualFiles, err := GetFSFromLayers(root, layers, opts...) + + assertGetFSFromLayers( + t, + actualFiles, + expectedFiles, + err, + expectErr, + ) +} + +func Test_GetFSFromLayers_with_whiteouts_include_whiteout_disabled(t *testing.T) { + ctrl := gomock.NewController(t) + + root, err := ioutil.TempDir("", "layers-test") + if err != nil { + t.Fatal(err) + } + defer os.Remove(root) + + opts := []FSOpt{ + // I'd rather use the real func (util.ExtractFile) + // but you have to be root to chown + ExtractFunc(fakeExtract), + } + + expectErr := false + + f := func(expectedFiles []string, tw *tar.Writer) { + body := "Hello World\n" + for _, f := range expectedFiles { + f := strings.TrimPrefix(strings.TrimPrefix(f, root), "/") + + hdr := &tar.Header{ + Name: f, + Mode: 0644, + Size: int64(len(body)), + } + + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + } + + expectedFiles := []string{ + filepath.Join(root, "foobar"), + } + + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + f(expectedFiles, tw) + + mockLayer := mock_v1.NewMockLayer(ctrl) + mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil) + + rc := ioutil.NopCloser(buf) + mockLayer.EXPECT().Uncompressed().Return(rc, nil) + + secondLayerFiles := []string{ + filepath.Join(root, ".wh.foobar"), + } + + buf = new(bytes.Buffer) + tw = tar.NewWriter(buf) + + f(secondLayerFiles, tw) + + mockLayer2 := mock_v1.NewMockLayer(ctrl) + mockLayer2.EXPECT().MediaType().Return(types.OCILayer, nil) + + rc = ioutil.NopCloser(buf) + mockLayer2.EXPECT().Uncompressed().Return(rc, nil) + + layers := []v1.Layer{ + mockLayer, + mockLayer2, + } + + actualFiles, err := GetFSFromLayers(root, layers, opts...) + + assertGetFSFromLayers( + t, + actualFiles, + expectedFiles, + err, + expectErr, + ) +} + +func Test_GetFSFromLayers(t *testing.T) { + ctrl := gomock.NewController(t) + + root, err := ioutil.TempDir("", "layers-test") + if err != nil { + t.Fatal(err) + } + defer os.Remove(root) + + opts := []FSOpt{ + // I'd rather use the real func (util.ExtractFile) + // but you have to be root to chown + ExtractFunc(fakeExtract), + } + + expectErr := false + expectedFiles := []string{ + filepath.Join(root, "foobar"), + } + + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + body := "Hello World\n" + for _, f := range expectedFiles { + f := strings.TrimPrefix(strings.TrimPrefix(f, root), "/") + + hdr := &tar.Header{ + Name: f, + Mode: 0644, + Size: int64(len(body)), + } + + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + mockLayer := mock_v1.NewMockLayer(ctrl) + mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil) + + rc := ioutil.NopCloser(buf) + mockLayer.EXPECT().Uncompressed().Return(rc, nil) + + layers := []v1.Layer{ + mockLayer, + } + + actualFiles, err := GetFSFromLayers(root, layers, opts...) + + assertGetFSFromLayers( + t, + actualFiles, + expectedFiles, + err, + expectErr, + ) +} + +func assertGetFSFromLayers( + t *testing.T, + actualFiles []string, + expectedFiles []string, + err error, + expectErr bool, +) { + if !expectErr && err != nil { + t.Error(err) + t.FailNow() + } else if expectErr && err == nil { + t.Error("expected err to not be nil") + t.FailNow() + } + + if len(actualFiles) != len(expectedFiles) { + t.Errorf("expected %s to equal %s", actualFiles, expectedFiles) + t.FailNow() + } + + for i := range expectedFiles { + if actualFiles[i] != expectedFiles[i] { + t.Errorf("expected %s to equal %s", actualFiles[i], expectedFiles[i]) + } + } +}