GetFSFromLayers

* add util.GetFSFromLayers
* GetFSFromImage delegates to GetFSFromLayers
* add FSOpts and FSConfig for GetFSFromLayers
* add tests for GetFSFromLayers
* add gomock for test support
* add mock_v1 for layers
This commit is contained in:
Cole Wippern 2020-01-17 13:36:23 -08:00
parent e19cc228ba
commit 71aed35094
6 changed files with 476 additions and 13 deletions

3
go.mod
View File

@ -36,6 +36,7 @@ require (
github.com/gliderlabs/ssh v0.2.2 // indirect
github.com/gogo/protobuf v1.1.1 // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
github.com/golang/mock v1.3.1
github.com/golang/protobuf v1.1.0 // indirect
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a // indirect
github.com/google/go-cmp v0.2.0
@ -94,7 +95,7 @@ require (
go.opencensus.io v0.14.0 // indirect
golang.org/x/net v0.0.0-20190311183353-d8887717615a
golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect
google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79 // indirect
google.golang.org/appengine v1.1.0 // indirect

6
go.sum
View File

@ -78,6 +78,8 @@ github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc=
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -239,6 +241,8 @@ golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc h1:3ElrZeO6IBP+M8kgu5YFwR
golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190130150945-aca44879d564 h1:o6ENHFwwr1TZ9CUPQcfo1HGvLP1OPsPOTB7xCIOPNmU=
golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -252,6 +256,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262 h1:qsl9y/CJx34tuA7QCPNp86JNJe4spst6Ff8MjvPUdPg=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79 h1:wCy2/9bhO1JeP2zZUALrj7ZdZuZoR4mRV57kTxjqRpo=
google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=

View File

@ -0,0 +1,5 @@
FROM alpine
RUN mkdir -p /some/dir/ && echo 'first' > /some/dir/first.txt
RUN rm /some/dir/first.txt

View File

@ -0,0 +1,126 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/google/go-containerregistry/pkg/v1 (interfaces: Layer)
// Package mock_v1 is a generated GoMock package.
package mock_v1
import (
gomock "github.com/golang/mock/gomock"
v1 "github.com/google/go-containerregistry/pkg/v1"
types "github.com/google/go-containerregistry/pkg/v1/types"
io "io"
reflect "reflect"
)
// MockLayer is a mock of Layer interface
type MockLayer struct {
ctrl *gomock.Controller
recorder *MockLayerMockRecorder
}
// MockLayerMockRecorder is the mock recorder for MockLayer
type MockLayerMockRecorder struct {
mock *MockLayer
}
// NewMockLayer creates a new mock instance
func NewMockLayer(ctrl *gomock.Controller) *MockLayer {
mock := &MockLayer{ctrl: ctrl}
mock.recorder = &MockLayerMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockLayer) EXPECT() *MockLayerMockRecorder {
return m.recorder
}
// Compressed mocks base method
func (m *MockLayer) Compressed() (io.ReadCloser, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Compressed")
ret0, _ := ret[0].(io.ReadCloser)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Compressed indicates an expected call of Compressed
func (mr *MockLayerMockRecorder) Compressed() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compressed", reflect.TypeOf((*MockLayer)(nil).Compressed))
}
// DiffID mocks base method
func (m *MockLayer) DiffID() (v1.Hash, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DiffID")
ret0, _ := ret[0].(v1.Hash)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// DiffID indicates an expected call of DiffID
func (mr *MockLayerMockRecorder) DiffID() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiffID", reflect.TypeOf((*MockLayer)(nil).DiffID))
}
// Digest mocks base method
func (m *MockLayer) Digest() (v1.Hash, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Digest")
ret0, _ := ret[0].(v1.Hash)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Digest indicates an expected call of Digest
func (mr *MockLayerMockRecorder) Digest() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Digest", reflect.TypeOf((*MockLayer)(nil).Digest))
}
// MediaType mocks base method
func (m *MockLayer) MediaType() (types.MediaType, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "MediaType")
ret0, _ := ret[0].(types.MediaType)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// MediaType indicates an expected call of MediaType
func (mr *MockLayerMockRecorder) MediaType() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MediaType", reflect.TypeOf((*MockLayer)(nil).MediaType))
}
// Size mocks base method
func (m *MockLayer) Size() (int64, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Size")
ret0, _ := ret[0].(int64)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Size indicates an expected call of Size
func (mr *MockLayerMockRecorder) Size() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Size", reflect.TypeOf((*MockLayer)(nil).Size))
}
// Uncompressed mocks base method
func (m *MockLayer) Uncompressed() (io.ReadCloser, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Uncompressed")
ret0, _ := ret[0].(io.ReadCloser)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Uncompressed indicates an expected call of Uncompressed
func (mr *MockLayerMockRecorder) Uncompressed() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Uncompressed", reflect.TypeOf((*MockLayer)(nil).Uncompressed))
}

View File

@ -71,25 +71,58 @@ var excluded []string
type ExtractFunction func(string, *tar.Header, io.Reader) error
type FSConfig struct {
includeWhiteout bool
extractFunc ExtractFunction
}
type FSOpt func(*FSConfig)
func IncludeWhiteout() FSOpt {
return func(opts *FSConfig) {
opts.includeWhiteout = true
}
}
func ExtractFunc(extractFunc ExtractFunction) FSOpt {
return func(opts *FSConfig) {
opts.extractFunc = extractFunc
}
}
// GetFSFromImage extracts the layers of img to root
// It returns a list of all files extracted
func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]string, error) {
if extract == nil {
return nil, errors.New("must supply an extract function")
}
if img == nil {
return nil, errors.New("image cannot be nil")
}
if err := DetectFilesystemWhitelist(constants.WhitelistPath); err != nil {
return nil, err
}
logrus.Debugf("Mounted directories: %v", whitelist)
layers, err := img.Layers()
if err != nil {
return nil, err
}
extractedFiles := []string{}
return GetFSFromLayers(root, layers, ExtractFunc(extract))
}
func GetFSFromLayers(root string, layers []v1.Layer, opts ...FSOpt) ([]string, error) {
cfg := new(FSConfig)
for _, opt := range opts {
opt(cfg)
}
if cfg.extractFunc == nil {
return nil, errors.New("must supply an extract function")
}
if err := DetectFilesystemWhitelist(constants.WhitelistPath); err != nil {
return nil, err
}
logrus.Debugf("Mounted directories: %v", whitelist)
extractedFiles := []string{}
for i, l := range layers {
if mediaType, err := l.MediaType(); err == nil {
logrus.Tracef("Extracting layer %d of media type %s", i, mediaType)
@ -102,29 +135,39 @@ func GetFSFromImage(root string, img v1.Image, extract ExtractFunction) ([]strin
return nil, err
}
defer r.Close()
tr := tar.NewReader(r)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("error reading tar %d", i))
}
path := filepath.Join(root, filepath.Clean(hdr.Name))
base := filepath.Base(path)
dir := filepath.Dir(path)
if strings.HasPrefix(base, ".wh.") {
logrus.Debugf("Whiting out %s", path)
name := strings.TrimPrefix(base, ".wh.")
if err := os.RemoveAll(filepath.Join(dir, name)); err != nil {
return nil, errors.Wrapf(err, "removing whiteout %s", hdr.Name)
}
continue
}
if err := extract(root, hdr, tr); err != nil {
return nil, err
if !cfg.includeWhiteout {
continue
}
} else {
if err := cfg.extractFunc(root, hdr, tr); err != nil {
return nil, err
}
}
extractedFiles = append(extractedFiles, filepath.Join(root, filepath.Clean(hdr.Name)))
}
}

View File

@ -20,6 +20,7 @@ import (
"archive/tar"
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -28,7 +29,11 @@ import (
"strings"
"testing"
"github.com/GoogleContainerTools/kaniko/pkg/mocks/go-containerregistry/mock_v1"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/golang/mock/gomock"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
)
func Test_DetectFilesystemWhitelist(t *testing.T) {
@ -863,3 +868,280 @@ func Test_CopyFile_skips_self(t *testing.T) {
t.Fatalf("expected file contents to be %q, but got %q", expected, actual)
}
}
func fakeExtract(dest string, hdr *tar.Header, tr io.Reader) error {
return nil
}
func Test_GetFSFromLayers_with_whiteouts_include_whiteout_enabled(t *testing.T) {
ctrl := gomock.NewController(t)
root, err := ioutil.TempDir("", "layers-test")
if err != nil {
t.Fatal(err)
}
defer os.Remove(root)
opts := []FSOpt{
// I'd rather use the real func (util.ExtractFile)
// but you have to be root to chown
ExtractFunc(fakeExtract),
IncludeWhiteout(),
}
expectErr := false
f := func(expectedFiles []string, tw *tar.Writer) {
body := "Hello World\n"
for _, f := range expectedFiles {
f := strings.TrimPrefix(strings.TrimPrefix(f, root), "/")
hdr := &tar.Header{
Name: f,
Mode: 0644,
Size: int64(len(body)),
}
if err := tw.WriteHeader(hdr); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
}
expectedFiles := []string{
filepath.Join(root, "foobar"),
}
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
f(expectedFiles, tw)
mockLayer := mock_v1.NewMockLayer(ctrl)
mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil)
rc := ioutil.NopCloser(buf)
mockLayer.EXPECT().Uncompressed().Return(rc, nil)
secondLayerFiles := []string{
filepath.Join(root, ".wh.foobar"),
}
buf = new(bytes.Buffer)
tw = tar.NewWriter(buf)
f(secondLayerFiles, tw)
mockLayer2 := mock_v1.NewMockLayer(ctrl)
mockLayer2.EXPECT().MediaType().Return(types.OCILayer, nil)
rc = ioutil.NopCloser(buf)
mockLayer2.EXPECT().Uncompressed().Return(rc, nil)
layers := []v1.Layer{
mockLayer,
mockLayer2,
}
expectedFiles = append(expectedFiles, secondLayerFiles...)
actualFiles, err := GetFSFromLayers(root, layers, opts...)
assertGetFSFromLayers(
t,
actualFiles,
expectedFiles,
err,
expectErr,
)
}
func Test_GetFSFromLayers_with_whiteouts_include_whiteout_disabled(t *testing.T) {
ctrl := gomock.NewController(t)
root, err := ioutil.TempDir("", "layers-test")
if err != nil {
t.Fatal(err)
}
defer os.Remove(root)
opts := []FSOpt{
// I'd rather use the real func (util.ExtractFile)
// but you have to be root to chown
ExtractFunc(fakeExtract),
}
expectErr := false
f := func(expectedFiles []string, tw *tar.Writer) {
body := "Hello World\n"
for _, f := range expectedFiles {
f := strings.TrimPrefix(strings.TrimPrefix(f, root), "/")
hdr := &tar.Header{
Name: f,
Mode: 0644,
Size: int64(len(body)),
}
if err := tw.WriteHeader(hdr); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
}
expectedFiles := []string{
filepath.Join(root, "foobar"),
}
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
f(expectedFiles, tw)
mockLayer := mock_v1.NewMockLayer(ctrl)
mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil)
rc := ioutil.NopCloser(buf)
mockLayer.EXPECT().Uncompressed().Return(rc, nil)
secondLayerFiles := []string{
filepath.Join(root, ".wh.foobar"),
}
buf = new(bytes.Buffer)
tw = tar.NewWriter(buf)
f(secondLayerFiles, tw)
mockLayer2 := mock_v1.NewMockLayer(ctrl)
mockLayer2.EXPECT().MediaType().Return(types.OCILayer, nil)
rc = ioutil.NopCloser(buf)
mockLayer2.EXPECT().Uncompressed().Return(rc, nil)
layers := []v1.Layer{
mockLayer,
mockLayer2,
}
actualFiles, err := GetFSFromLayers(root, layers, opts...)
assertGetFSFromLayers(
t,
actualFiles,
expectedFiles,
err,
expectErr,
)
}
func Test_GetFSFromLayers(t *testing.T) {
ctrl := gomock.NewController(t)
root, err := ioutil.TempDir("", "layers-test")
if err != nil {
t.Fatal(err)
}
defer os.Remove(root)
opts := []FSOpt{
// I'd rather use the real func (util.ExtractFile)
// but you have to be root to chown
ExtractFunc(fakeExtract),
}
expectErr := false
expectedFiles := []string{
filepath.Join(root, "foobar"),
}
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
body := "Hello World\n"
for _, f := range expectedFiles {
f := strings.TrimPrefix(strings.TrimPrefix(f, root), "/")
hdr := &tar.Header{
Name: f,
Mode: 0644,
Size: int64(len(body)),
}
if err := tw.WriteHeader(hdr); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
mockLayer := mock_v1.NewMockLayer(ctrl)
mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil)
rc := ioutil.NopCloser(buf)
mockLayer.EXPECT().Uncompressed().Return(rc, nil)
layers := []v1.Layer{
mockLayer,
}
actualFiles, err := GetFSFromLayers(root, layers, opts...)
assertGetFSFromLayers(
t,
actualFiles,
expectedFiles,
err,
expectErr,
)
}
func assertGetFSFromLayers(
t *testing.T,
actualFiles []string,
expectedFiles []string,
err error,
expectErr bool,
) {
if !expectErr && err != nil {
t.Error(err)
t.FailNow()
} else if expectErr && err == nil {
t.Error("expected err to not be nil")
t.FailNow()
}
if len(actualFiles) != len(expectedFiles) {
t.Errorf("expected %s to equal %s", actualFiles, expectedFiles)
t.FailNow()
}
for i := range expectedFiles {
if actualFiles[i] != expectedFiles[i] {
t.Errorf("expected %s to equal %s", actualFiles[i], expectedFiles[i])
}
}
}