Merge branch 'master' into opts

pkg/image/image_util.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
+	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/cache"
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
@@ -117,6 +118,11 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) {
 			return nil, err
 		}
 
+		ref, err = normalizeReference(ref, image)
+		if err != nil {
+			return nil, err
+		}
+
 		toSet = true
 	}
 
@@ -134,16 +140,31 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) {
 			tag.Repository.Registry = newReg
 			ref = tag
 		}
+
 		if digest, ok := ref.(name.Digest); ok {
 			digest.Repository.Registry = newReg
 			ref = digest
 		}
 	}
 
+	logrus.Infof("Retrieving image %s", ref)
+
 	rOpts := remoteOptions(registryName, opts)
 	return remote.Image(ref, rOpts...)
 }
 
+// normalizeReference adds the library/ prefix to images without it.
+//
+// It is mostly useful when using a registry mirror that is not able to perform
+// this fix automatically.
+func normalizeReference(ref name.Reference, image string) (name.Reference, error) {
+	if !strings.ContainsRune(image, '/') {
+		return name.ParseReference("library/"+image, name.WeakValidation)
+	}
+
+	return ref, nil
+}
+
 func remoteOptions(registryName string, opts *config.KanikoOptions) []remote.Option {
 	tr := util.MakeTransport(opts, registryName)
 

pkg/image/image_util_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
@@ -109,6 +110,25 @@ func Test_ScratchImageFromMirror(t *testing.T) {
 	testutil.CheckErrorAndDeepEqual(t, false, err, expected, actual)
 }
 
+func Test_normalizeReference(t *testing.T) {
+	image := "debian"
+	expected := "index.docker.io/library/debian:latest"
+
+	ref, err := name.ParseReference(image)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ref2, err := normalizeReference(ref, image)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if ref2.Name() != ref.Name() || ref2.Name() != expected {
+		t.Errorf("%s should have been normalized to %s, got %s", ref2.Name(), expected, ref.Name())
+	}
+}
+
 // parse parses the contents of a Dockerfile and returns a list of commands
 func parse(s string) ([]instructions.Stage, error) {
 	p, err := parser.Parse(bytes.NewReader([]byte(s)))

pkg/snapshot/snapshot.go

@@ -33,7 +33,7 @@ import (
 )
 
 // For testing
-var snapshotPathPrefix = config.KanikoDir
+var snapshotPathPrefix = ""
 
 // Snapshotter holds the root directory from which to take snapshots, and a list of snapshots taken
 type Snapshotter struct {
@@ -108,6 +108,9 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string,
 			}
 		}
 	}
+
+	sort.Strings(filesToWhiteout)
+
 	t := util.NewTar(f)
 	defer t.Close()
 	if err := writeToTar(t, filesToAdd, filesToWhiteout); err != nil {
@@ -119,7 +122,7 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string,
 // TakeSnapshotFS takes a snapshot of the filesystem, avoiding directories in the ignorelist, and creates
 // a tarball of the changed files.
 func (s *Snapshotter) TakeSnapshotFS() (string, error) {
-	f, err := ioutil.TempFile(snapshotPathPrefix, "")
+	f, err := ioutil.TempFile(s.getSnashotPathPrefix(), "")
 	if err != nil {
 		return "", err
 	}
@@ -138,6 +141,13 @@ func (s *Snapshotter) TakeSnapshotFS() (string, error) {
 	return f.Name(), nil
 }
 
+func (s *Snapshotter) getSnashotPathPrefix() string {
+	if snapshotPathPrefix == "" {
+		return config.KanikoDir
+	}
+	return snapshotPathPrefix
+}
+
 func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
 	logrus.Info("Taking snapshot of full filesystem...")
 
@@ -178,7 +188,10 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
 		}
 	}
 	timing.DefaultRun.Stop(timer)
+
 	sort.Strings(filesToAdd)
+	sort.Strings(filesToWhiteOut)
+
 	// Add files to the layered map
 	for _, file := range filesToAdd {
 		if err := s.l.Add(file); err != nil {

pkg/snapshot/snapshot_test.go

@@ -391,6 +391,88 @@ func TestSnasphotPreservesFileOrder(t *testing.T) {
 	}
 }
 
+func TestSnasphotPreservesWhiteoutOrder(t *testing.T) {
+	newFiles := map[string]string{
+		"foo":     "newbaz1",
+		"bar/bat": "baz",
+		"bar/qux": "quuz",
+		"qux":     "quuz",
+		"corge":   "grault",
+		"garply":  "waldo",
+		"fred":    "plugh",
+		"xyzzy":   "thud",
+	}
+
+	newFileNames := []string{}
+
+	for fileName := range newFiles {
+		newFileNames = append(newFileNames, fileName)
+	}
+
+	filesInTars := [][]string{}
+
+	for i := 0; i <= 2; i++ {
+		testDir, snapshotter, cleanup, err := setUpTest()
+		testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/")
+		defer cleanup()
+
+		if err != nil {
+			t.Fatal(err)
+		}
+		// Make some changes to the filesystem
+		if err := testutil.SetupFiles(testDir, newFiles); err != nil {
+			t.Fatalf("Error setting up fs: %s", err)
+		}
+
+		filesToSnapshot := []string{}
+		for _, file := range newFileNames {
+			filesToSnapshot = append(filesToSnapshot, filepath.Join(testDir, file))
+		}
+
+		// Take a snapshot
+		_, err = snapshotter.TakeSnapshot(filesToSnapshot, false)
+		if err != nil {
+			t.Fatalf("Error taking snapshot of fs: %s", err)
+		}
+
+		// Delete all files
+		for p := range newFiles {
+			err := os.Remove(filepath.Join(testDir, p))
+			if err != nil {
+				t.Fatalf("Error deleting file: %s", err)
+			}
+		}
+
+		// Take a snapshot again
+		tarPath, err := snapshotter.TakeSnapshot(filesToSnapshot, true)
+		if err != nil {
+			t.Fatalf("Error taking snapshot of fs: %s", err)
+		}
+
+		f, err := os.Open(tarPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+		tr := tar.NewReader(f)
+		filesInTars = append(filesInTars, []string{})
+		for {
+			hdr, err := tr.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			filesInTars[i] = append(filesInTars[i], strings.TrimPrefix(hdr.Name, testDirWithoutLeadingSlash))
+		}
+	}
+
+	// Check contents of all snapshots, make sure files appear in consistent order
+	for i := 1; i < len(filesInTars); i++ {
+		testutil.CheckErrorAndDeepEqual(t, false, nil, filesInTars[0], filesInTars[i])
+	}
+}
+
 func TestSnapshotOmitsUnameGname(t *testing.T) {
 	_, snapshotter, cleanup, err := setUpTest()
 

pkg/util/fs_util.go

@@ -545,6 +545,11 @@ func DownloadFileToDest(rawurl, dest string, uid, gid int64) error {
 		return err
 	}
 	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("invalid response status %d", resp.StatusCode)
+	}
+
 	if err := CreateFile(dest, resp.Body, 0600, uint32(uid), uint32(gid)); err != nil {
 		return err
 	}