Switch from containers/image to go-containerregistry (#140)

* Vendor changes for go-containerregistry switch.

* Manual changes for go-containerregistry switch.

The biggest change is refactoring the tarball unpacking.

* Pull more of container-diff out.

* More vendor removals.

* More unit tests.

pkg/util/fs_util.go

@@ -17,6 +17,7 @@ limitations under the License.
 package util
 
 import (
+	"archive/tar"
 	"bufio"
 	"io"
 	"net/http"
@@ -25,38 +26,163 @@ import (
 	"strings"
 	"time"
 
-	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
+	"github.com/google/go-containerregistry/v1"
+
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
-	"github.com/containers/image/docker"
 	"github.com/sirupsen/logrus"
 )
 
-var whitelist = []string{"/kaniko"}
+var whitelist = []string{
+	"/kaniko",
+	// /var/run is a special case. It's common to mount in /var/run/docker.sock or something similar
+	// which leads to a special mount on the /var/run/docker.sock file itself, but the directory to exist
+	// in the image with no way to tell if it came from the base image or not.
+	"/var/run",
+}
 var volumeWhitelist = []string{}
 
-// ExtractFileSystemFromImage pulls an image and unpacks it to a file system at root
-func ExtractFileSystemFromImage(img string) error {
+func GetFSFromImage(img v1.Image) error {
 	whitelist, err := fileSystemWhitelist(constants.WhitelistPath)
 	if err != nil {
 		return err
 	}
-	logrus.Infof("Whitelisted directories are %s", whitelist)
-	if img == constants.NoBaseImage {
-		logrus.Info("No base image, nothing to extract")
-		return nil
-	}
-	ref, err := docker.ParseReference("//" + img)
+	logrus.Infof("Mounted directories: %v", whitelist)
+	layers, err := img.Layers()
 	if err != nil {
 		return err
 	}
-	imgSrc, err := ref.NewImageSource(nil)
-	if err != nil {
-		return err
+
+	fs := map[string]struct{}{}
+	whiteouts := map[string]struct{}{}
+
+	for i := len(layers) - 1; i >= 0; i-- {
+		logrus.Infof("Unpacking layer: %d", i)
+		l := layers[i]
+		r, err := l.Uncompressed()
+		if err != nil {
+			return err
+		}
+		tr := tar.NewReader(r)
+		for {
+			hdr, err := tr.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			path := filepath.Join("/", filepath.Clean(hdr.Name))
+			base := filepath.Base(path)
+			dir := filepath.Dir(path)
+			if strings.HasPrefix(base, ".wh.") {
+				logrus.Infof("Whiting out %s", path)
+				name := strings.TrimPrefix(base, ".wh.")
+				whiteouts[filepath.Join(dir, name)] = struct{}{}
+				continue
+			}
+
+			if checkWhiteouts(path, whiteouts) {
+				logrus.Infof("Not adding %s because it is whited out", path)
+				continue
+			}
+			if _, ok := fs[path]; ok {
+				logrus.Infof("Not adding %s because it was added by a prior layer", path)
+				continue
+			}
+
+			if checkWhitelist(path, whitelist) {
+				logrus.Infof("Not adding %s because it is whitelisted", path)
+				continue
+			}
+			fs[path] = struct{}{}
+
+			if err := extractFile("/", hdr, tr); err != nil {
+				return err
+			}
+		}
 	}
-	return pkgutil.GetFileSystemFromReference(ref, imgSrc, constants.RootDir, whitelist)
+	return nil
+}
+
+func unTar(r io.Reader, dest string) error {
+	tr := tar.NewReader(r)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if err := extractFile(dest, hdr, tr); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func extractFile(dest string, hdr *tar.Header, tr io.Reader) error {
+	path := filepath.Join(dest, filepath.Clean(hdr.Name))
+	base := filepath.Base(path)
+	dir := filepath.Dir(path)
+	mode := hdr.FileInfo().Mode()
+	switch hdr.Typeflag {
+	case tar.TypeReg:
+		logrus.Debugf("creating file %s", path)
+		// It's possible a file is in the tar before it's directory.
+		if _, err := os.Stat(dir); os.IsNotExist(err) {
+			logrus.Debugf("base %s for file %s does not exist. Creating.", base, path)
+			if err := os.MkdirAll(dir, 0755); err != nil {
+				return err
+			}
+		}
+		currFile, err := os.Create(path)
+		if err != nil {
+			return err
+		}
+		// manually set permissions on file, since the default umask (022) will interfere
+		if err = os.Chmod(path, mode); err != nil {
+			return err
+		}
+		if _, err = io.Copy(currFile, tr); err != nil {
+			return err
+		}
+		currFile.Close()
+
+	case tar.TypeDir:
+		logrus.Debugf("creating dir %s", path)
+		if err := os.MkdirAll(path, mode); err != nil {
+			return err
+		}
+		// In some cases, MkdirAll doesn't change the permissions, so run Chmod
+		if err := os.Chmod(path, mode); err != nil {
+			return err
+		}
+
+	case tar.TypeLink:
+		logrus.Debugf("link from %s to %s", hdr.Linkname, path)
+		// The base directory for a link may not exist before it is created.
+		dir := filepath.Dir(path)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return err
+		}
+		if err := os.Symlink(filepath.Clean(filepath.Join("/", hdr.Linkname)), path); err != nil {
+			return err
+		}
+	case tar.TypeSymlink:
+		logrus.Debugf("symlink from %s to %s", hdr.Linkname, path)
+		// The base directory for a symlink may not exist before it is created.
+		dir := filepath.Dir(path)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return err
+		}
+		if err := os.Symlink(hdr.Linkname, path); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
-// PathInWhitelist returns true if the path is whitelisted
 func PathInWhitelist(path, directory string) bool {
 	for _, c := range constants.KanikoBuildFiles {
 		if path == c {
@@ -65,7 +191,30 @@ func PathInWhitelist(path, directory string) bool {
 	}
 	for _, d := range whitelist {
 		dirPath := filepath.Join(directory, d)
-		if pkgutil.HasFilepathPrefix(path, dirPath) {
+		if HasFilepathPrefix(path, dirPath) {
+			return true
+		}
+	}
+	return false
+}
+
+func checkWhiteouts(path string, whiteouts map[string]struct{}) bool {
+	// Don't add the file if it or it's directory are whited out.
+	if _, ok := whiteouts[path]; ok {
+		return true
+	}
+	for wd := range whiteouts {
+		if HasFilepathPrefix(path, wd) {
+			logrus.Infof("Not adding %s because it's directory is whited out", path)
+			return true
+		}
+	}
+	return false
+}
+
+func checkWhitelist(path string, whitelist []string) bool {
+	for _, wl := range whitelist {
+		if HasFilepathPrefix(path, wl) {
 			return true
 		}
 	}
@@ -268,3 +417,22 @@ func CopyFile(src, dest string) error {
 	defer srcFile.Close()
 	return CreateFile(dest, srcFile, fi.Mode())
 }
+
+// HasFilepathPrefix checks if the given file path begins with prefix
+func HasFilepathPrefix(path, prefix string) bool {
+	path = filepath.Clean(path)
+	prefix = filepath.Clean(prefix)
+	pathArray := strings.Split(path, "/")
+	prefixArray := strings.Split(prefix, "/")
+
+	if len(pathArray) < len(prefixArray) {
+		return false
+	}
+	for index := range prefixArray {
+		if prefixArray[index] == pathArray[index] {
+			continue
+		}
+		return false
+	}
+	return true
+}

pkg/util/fs_util_test.go

@@ -17,9 +17,12 @@ limitations under the License.
 package util
 
 import (
+	"archive/tar"
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"reflect"
 	"sort"
 	"testing"
 
@@ -47,7 +50,7 @@ func Test_fileSystemWhitelist(t *testing.T) {
 	}
 
 	actualWhitelist, err := fileSystemWhitelist(path)
-	expectedWhitelist := []string{"/kaniko", "/proc", "/dev", "/dev/pts", "/sys"}
+	expectedWhitelist := []string{"/kaniko", "/proc", "/dev", "/dev/pts", "/sys", "/var/run"}
 	sort.Strings(actualWhitelist)
 	sort.Strings(expectedWhitelist)
 	testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist)
@@ -131,3 +134,358 @@ func Test_RelativeFiles(t *testing.T) {
 		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles)
 	}
 }
+
+func Test_checkWhiteouts(t *testing.T) {
+	type args struct {
+		path      string
+		whiteouts map[string]struct{}
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "file whited out",
+			args: args{
+				path:      "/foo",
+				whiteouts: map[string]struct{}{"/foo": {}},
+			},
+			want: true,
+		},
+		{
+			name: "directory whited out",
+			args: args{
+				path:      "/foo/bar",
+				whiteouts: map[string]struct{}{"/foo": {}},
+			},
+			want: true,
+		},
+		{
+			name: "grandparent whited out",
+			args: args{
+				path:      "/foo/bar/baz",
+				whiteouts: map[string]struct{}{"/foo": {}},
+			},
+			want: true,
+		},
+		{
+			name: "sibling whited out",
+			args: args{
+				path:      "/foo/bar/baz",
+				whiteouts: map[string]struct{}{"/foo/bat": {}},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := checkWhiteouts(tt.args.path, tt.args.whiteouts); got != tt.want {
+				t.Errorf("checkWhiteouts() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_checkWhitelist(t *testing.T) {
+	type args struct {
+		path      string
+		whitelist []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "file whitelisted",
+			args: args{
+				path:      "/foo",
+				whitelist: []string{"/foo"},
+			},
+			want: true,
+		},
+		{
+			name: "directory whitelisted",
+			args: args{
+				path:      "/foo/bar",
+				whitelist: []string{"/foo"},
+			},
+			want: true,
+		},
+		{
+			name: "grandparent whitelisted",
+			args: args{
+				path:      "/foo/bar/baz",
+				whitelist: []string{"/foo"},
+			},
+			want: true,
+		},
+		{
+			name: "sibling whitelisted",
+			args: args{
+				path:      "/foo/bar/baz",
+				whitelist: []string{"/foo/bat"},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := checkWhitelist(tt.args.path, tt.args.whitelist); got != tt.want {
+				t.Errorf("checkWhitelist() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestHasFilepathPrefix(t *testing.T) {
+	type args struct {
+		path   string
+		prefix string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "parent",
+			args: args{
+				path:   "/foo/bar",
+				prefix: "/foo",
+			},
+			want: true,
+		},
+		{
+			name: "nested parent",
+			args: args{
+				path:   "/foo/bar/baz",
+				prefix: "/foo/bar",
+			},
+			want: true,
+		},
+		{
+			name: "sibling",
+			args: args{
+				path:   "/foo/bar",
+				prefix: "/bar",
+			},
+			want: false,
+		},
+		{
+			name: "nested sibling",
+			args: args{
+				path:   "/foo/bar/baz",
+				prefix: "/foo/bar",
+			},
+			want: true,
+		},
+		{
+			name: "name prefix",
+			args: args{
+				path:   "/foo2/bar",
+				prefix: "/foo",
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := HasFilepathPrefix(tt.args.path, tt.args.prefix); got != tt.want {
+				t.Errorf("HasFilepathPrefix() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+type checker func(root string, t *testing.T)
+
+func fileExists(p string) checker {
+	return func(root string, t *testing.T) {
+		_, err := os.Stat(filepath.Join(root, p))
+		if err != nil {
+			t.Fatalf("File does not exist")
+		}
+	}
+}
+
+func fileMatches(p string, c []byte) checker {
+	return func(root string, t *testing.T) {
+		actual, err := ioutil.ReadFile(filepath.Join(root, p))
+		if err != nil {
+			t.Fatalf("error reading file: %s", p)
+		}
+		if !reflect.DeepEqual(actual, c) {
+			t.Errorf("file contents do not match. %v!=%v", actual, c)
+		}
+	}
+}
+
+func permissionsMatch(p string, perms os.FileMode) checker {
+	return func(root string, t *testing.T) {
+		fi, err := os.Stat(filepath.Join(root, p))
+		if err != nil {
+			t.Fatalf("error statting file %s", p)
+		}
+		if fi.Mode() != perms {
+			t.Errorf("Permissions do not match. %s != %s", fi.Mode(), perms)
+		}
+	}
+}
+
+func linkPointsTo(src, dst string) checker {
+	return func(root string, t *testing.T) {
+		link := filepath.Join(root, src)
+		got, err := os.Readlink(link)
+		if err != nil {
+			t.Fatalf("error reading link %s: %s", link, err)
+		}
+		if got != dst {
+			t.Errorf("link destination does not match: %s != %s", got, dst)
+		}
+	}
+}
+
+func fileHeader(name string, contents string, mode int64) *tar.Header {
+	return &tar.Header{
+		Name:     name,
+		Size:     int64(len(contents)),
+		Mode:     mode,
+		Typeflag: tar.TypeReg,
+	}
+}
+
+func linkHeader(name, linkname string) *tar.Header {
+	return &tar.Header{
+		Name:     name,
+		Size:     0,
+		Typeflag: tar.TypeSymlink,
+		Linkname: linkname,
+	}
+}
+
+func hardlinkHeader(name, linkname string) *tar.Header {
+	return &tar.Header{
+		Name:     name,
+		Size:     0,
+		Typeflag: tar.TypeLink,
+		Linkname: linkname,
+	}
+}
+
+func dirHeader(name string, mode int64) *tar.Header {
+	return &tar.Header{
+		Name:     name,
+		Size:     0,
+		Typeflag: tar.TypeDir,
+		Mode:     mode,
+	}
+}
+
+func TestExtractFile(t *testing.T) {
+	type tc struct {
+		name     string
+		hdrs     []*tar.Header
+		contents []byte
+		checkers []checker
+	}
+
+	tcs := []tc{
+		{
+			name:     "normal file",
+			contents: []byte("helloworld"),
+			hdrs:     []*tar.Header{fileHeader("./bar", "helloworld", 0644)},
+			checkers: []checker{
+				fileExists("/bar"),
+				fileMatches("/bar", []byte("helloworld")),
+				permissionsMatch("/bar", 0644),
+			},
+		},
+		{
+			name:     "normal file, directory does not exist",
+			contents: []byte("helloworld"),
+			hdrs:     []*tar.Header{fileHeader("./foo/bar", "helloworld", 0644)},
+			checkers: []checker{
+				fileExists("/foo/bar"),
+				fileMatches("/foo/bar", []byte("helloworld")),
+				permissionsMatch("/foo/bar", 0644),
+				permissionsMatch("/foo", 0755|os.ModeDir),
+			},
+		},
+		{
+			name:     "normal file, directory is created after",
+			contents: []byte("helloworld"),
+			hdrs: []*tar.Header{
+				fileHeader("./foo/bar", "helloworld", 0644),
+				dirHeader("./foo", 0722),
+			},
+			checkers: []checker{
+				fileExists("/foo/bar"),
+				fileMatches("/foo/bar", []byte("helloworld")),
+				permissionsMatch("/foo/bar", 0644),
+				permissionsMatch("/foo", 0722|os.ModeDir),
+			},
+		},
+		{
+			name: "symlink",
+			hdrs: []*tar.Header{linkHeader("./bar", "bar/bat")},
+			checkers: []checker{
+				linkPointsTo("/bar", "bar/bat"),
+			},
+		},
+		{
+			name: "symlink relative path",
+			hdrs: []*tar.Header{linkHeader("./bar", "./foo/bar/baz")},
+			checkers: []checker{
+				linkPointsTo("/bar", "./foo/bar/baz"),
+			},
+		},
+		{
+			name: "symlink parent does not exist",
+			hdrs: []*tar.Header{linkHeader("./foo/bar/baz", "../../bat")},
+			checkers: []checker{
+				linkPointsTo("/foo/bar/baz", "../../bat"),
+			},
+		},
+		{
+			name: "symlink parent does not exist",
+			hdrs: []*tar.Header{linkHeader("./foo/bar/baz", "../../bat")},
+			checkers: []checker{
+				linkPointsTo("/foo/bar/baz", "../../bat"),
+				permissionsMatch("/foo", 0755|os.ModeDir),
+				permissionsMatch("/foo/bar", 0755|os.ModeDir),
+			},
+		},
+		{
+			name: "hardlink",
+			hdrs: []*tar.Header{
+				fileHeader("/bin/gzip", "gzip-binary", 0751),
+				hardlinkHeader("/bin/uncompress", "/bin/gzip"),
+			},
+			checkers: []checker{
+				linkPointsTo("/bin/uncompress", "/bin/gzip"),
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			tc := tc
+			t.Parallel()
+			r, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer os.RemoveAll(r)
+			for _, hdr := range tc.hdrs {
+				if err := extractFile(r, hdr, bytes.NewReader(tc.contents)); err != nil {
+					t.Fatal(err)
+				}
+			}
+			for _, checker := range tc.checkers {
+				checker(r, t)
+			}
+		})
+	}
+}

pkg/util/tar_util.go

@@ -26,7 +26,6 @@ import (
 	"path/filepath"
 	"syscall"
 
-	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -122,7 +121,7 @@ func UnpackLocalTarArchive(path, dest string) error {
 			return UnpackCompressedTar(path, dest)
 		} else if compressionLevel == archive.Bzip2 {
 			bzr := bzip2.NewReader(file)
-			return pkgutil.UnTar(bzr, dest, nil)
+			return unTar(bzr, dest)
 		}
 	}
 	if fileIsUncompressedTar(path) {
@@ -131,7 +130,7 @@ func UnpackLocalTarArchive(path, dest string) error {
 			return err
 		}
 		defer file.Close()
-		return pkgutil.UnTar(file, dest, nil)
+		return unTar(file, dest)
 	}
 	return errors.New("path does not lead to local tar archive")
 }
@@ -195,5 +194,5 @@ func UnpackCompressedTar(path, dir string) error {
 		return err
 	}
 	defer gzr.Close()
-	return pkgutil.UnTar(gzr, dir, nil)
+	return unTar(gzr, dir)
 }