Merge branch 'master' into opts

CHANGELOG.md

@@ -1,3 +1,77 @@
+# v0.24.0 Release 2020-07-01
+This is the 24th release of Kaniko!
+
+New features:
+* Add a new run command along with a new flag [#1300](https://github.com/GoogleContainerTools/kaniko/pull/1300)
+* Add redo snapshotter.  [#1301](https://github.com/GoogleContainerTools/kaniko/pull/1301)
+* Add pkg.dev to automagic config file population [#1328](https://github.com/GoogleContainerTools/kaniko/pull/1328)
+* kaniko now clone git repositories recursing submodules by default [#1320](https://github.com/GoogleContainerTools/kaniko/pull/1320)
+
+Bug fixes:
+* Fix README.md [#1323](https://github.com/GoogleContainerTools/kaniko/pull/1323)
+* Fix docker-credential-gcr owner and group id [#1307](https://github.com/GoogleContainerTools/kaniko/pull/1307)
+
+Refactors:
+* check file changed in loop [#1302](https://github.com/GoogleContainerTools/kaniko/pull/1302)
+* ADD GCB benchmark code [#1299](https://github.com/GoogleContainerTools/kaniko/pull/1299)
+* benchmark FileSystem snapshot project added [#1288](https://github.com/GoogleContainerTools/kaniko/pull/1288)
+* [Perf] Reduce loops over files when taking FS snapshot. [#1283](https://github.com/GoogleContainerTools/kaniko/pull/1283)
+* Fix README.md [#1323](https://github.com/GoogleContainerTools/kaniko/pull/1323)
+* Fix docker-credential-gcr owner and group id [#1307](https://github.com/GoogleContainerTools/kaniko/pull/1307)
+* benchmark FileSystem snapshot project added [#1288](https://github.com/GoogleContainerTools/kaniko/pull/1288)
+* [Perf] Reduce loops over files when taking FS snapshot. [#1283](https://github.com/GoogleContainerTools/kaniko/pull/1283)
+
+Huge thank you for this release towards our contributors:
+- Alexander Sharov
+- Alex Szakaly
+- Anthony Davies
+- Art Begolli
+- Batuhan Apaydın
+- Ben Einaudi
+- Carlos Alexandro Becker
+- Carlos Sanchez
+- Chris Sng
+- Cole Wippern
+- cvgw
+- Daniel Marks
+- Dani Raznikov
+- David Dooling
+- DracoBlue
+- Gábor Lipták
+- Gabriel Virga
+- Gilbert Gilb's
+- Giovan Isa Musthofa
+- James Ravn
+- Jon Henrik Bjørnstad
+- Jon Johnson
+- Jordan Goasdoué
+- Liubov Grinkevich
+- Logan.Price
+- Lukasz Jakimczuk
+- Mehdi Abaakouk
+- Michel Hollands
+- Mitchell Friedman
+- Moritz Wanzenböck
+- ohchang-kwon
+- Or Sela
+- PhoenixMage
+- priyawadhwa
+- Sam Stoelinga
+- Tejal Desai
+- Thomas Bonfort
+- Thomas Stromberg
+- Thomas Strömberg
+- tinkerborg
+- Tom Prince
+- Vincent Latombe
+- Wietse Muizelaar
+- xanonid
+- Yoan Blanc
+- Yoriyasu Yano
+- Yuheng Zhang
+- yw-liu
+
+
 # v0.23.0 Release 2020-06-04
 This is the 23rd release of Kaniko! 
 

Makefile

@@ -14,7 +14,7 @@
 
 # Bump these on release
 VERSION_MAJOR ?= 0
-VERSION_MINOR ?= 23
+VERSION_MINOR ?= 24
 VERSION_BUILD ?= 0
 
 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)

integration/dockerfiles/Dockerfile_test_add_404

@@ -0,0 +1,4 @@
+FROM debian:9.11
+
+# Testing that any HTTP failure is handled properly
+ADD https://httpstat.us/404 .

integration/images.go

@@ -167,6 +167,7 @@ type DockerFileBuilder struct {
 func NewDockerFileBuilder() *DockerFileBuilder {
 	d := DockerFileBuilder{filesBuilt: map[string]struct{}{}}
 	d.DockerfilesToIgnore = map[string]struct{}{
+		"Dockerfile_test_add_404": {},
 		// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
 		"Dockerfile_test_user_run": {},
 	}

integration/integration_test.go

@@ -386,6 +386,40 @@ func TestBuildWithLabels(t *testing.T) {
 	checkContainerDiffOutput(t, diff, expected)
 }
 
+func TestBuildWithHTTPError(t *testing.T) {
+	repo := getGitRepo()
+	dockerfile := fmt.Sprintf("%s/%s/Dockerfile_test_add_404", integrationPath, dockerfilesPath)
+
+	// Build with docker
+	dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_test_add_404")
+	dockerCmd := exec.Command("docker",
+		append([]string{"build",
+			"-t", dockerImage,
+			"-f", dockerfile,
+			repo})...)
+	out, err := RunCommandWithoutTest(dockerCmd)
+	if err == nil {
+		t.Errorf("an error was expected, got %s", string(out))
+	}
+
+	// Build with kaniko
+	kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_add_404")
+	dockerRunFlags := []string{"run", "--net=host"}
+	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
+		"-f", dockerfile,
+		"-d", kanikoImage,
+		"-c", fmt.Sprintf("git://%s", repo),
+	)
+
+	kanikoCmd := exec.Command("docker", dockerRunFlags...)
+
+	out, err = RunCommandWithoutTest(kanikoCmd)
+	if err == nil {
+		t.Errorf("an error was expected, got %s", string(out))
+	}
+}
+
 func TestLayers(t *testing.T) {
 	offset := map[string]int{
 		"Dockerfile_test_add":     12,

pkg/image/image_util.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
+	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/cache"
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
@@ -117,6 +118,11 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) {
 			return nil, err
 		}
 
+		ref, err = normalizeReference(ref, image)
+		if err != nil {
+			return nil, err
+		}
+
 		toSet = true
 	}
 
@@ -134,16 +140,31 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) {
 			tag.Repository.Registry = newReg
 			ref = tag
 		}
+
 		if digest, ok := ref.(name.Digest); ok {
 			digest.Repository.Registry = newReg
 			ref = digest
 		}
 	}
 
+	logrus.Infof("Retrieving image %s", ref)
+
 	rOpts := remoteOptions(registryName, opts)
 	return remote.Image(ref, rOpts...)
 }
 
+// normalizeReference adds the library/ prefix to images without it.
+//
+// It is mostly useful when using a registry mirror that is not able to perform
+// this fix automatically.
+func normalizeReference(ref name.Reference, image string) (name.Reference, error) {
+	if !strings.ContainsRune(image, '/') {
+		return name.ParseReference("library/"+image, name.WeakValidation)
+	}
+
+	return ref, nil
+}
+
 func remoteOptions(registryName string, opts *config.KanikoOptions) []remote.Option {
 	tr := util.MakeTransport(opts, registryName)
 

pkg/image/image_util_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
@@ -109,6 +110,25 @@ func Test_ScratchImageFromMirror(t *testing.T) {
 	testutil.CheckErrorAndDeepEqual(t, false, err, expected, actual)
 }
 
+func Test_normalizeReference(t *testing.T) {
+	image := "debian"
+	expected := "index.docker.io/library/debian:latest"
+
+	ref, err := name.ParseReference(image)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ref2, err := normalizeReference(ref, image)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if ref2.Name() != ref.Name() || ref2.Name() != expected {
+		t.Errorf("%s should have been normalized to %s, got %s", ref2.Name(), expected, ref.Name())
+	}
+}
+
 // parse parses the contents of a Dockerfile and returns a list of commands
 func parse(s string) ([]instructions.Stage, error) {
 	p, err := parser.Parse(bytes.NewReader([]byte(s)))

pkg/snapshot/snapshot.go

@@ -33,7 +33,7 @@ import (
 )
 
 // For testing
-var snapshotPathPrefix = config.KanikoDir
+var snapshotPathPrefix = ""
 
 // Snapshotter holds the root directory from which to take snapshots, and a list of snapshots taken
 type Snapshotter struct {
@@ -108,6 +108,9 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string,
 			}
 		}
 	}
+
+	sort.Strings(filesToWhiteout)
+
 	t := util.NewTar(f)
 	defer t.Close()
 	if err := writeToTar(t, filesToAdd, filesToWhiteout); err != nil {
@@ -119,7 +122,7 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string,
 // TakeSnapshotFS takes a snapshot of the filesystem, avoiding directories in the ignorelist, and creates
 // a tarball of the changed files.
 func (s *Snapshotter) TakeSnapshotFS() (string, error) {
-	f, err := ioutil.TempFile(snapshotPathPrefix, "")
+	f, err := ioutil.TempFile(s.getSnashotPathPrefix(), "")
 	if err != nil {
 		return "", err
 	}
@@ -138,6 +141,13 @@ func (s *Snapshotter) TakeSnapshotFS() (string, error) {
 	return f.Name(), nil
 }
 
+func (s *Snapshotter) getSnashotPathPrefix() string {
+	if snapshotPathPrefix == "" {
+		return config.KanikoDir
+	}
+	return snapshotPathPrefix
+}
+
 func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
 	logrus.Info("Taking snapshot of full filesystem...")
 
@@ -178,7 +188,10 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
 		}
 	}
 	timing.DefaultRun.Stop(timer)
+
 	sort.Strings(filesToAdd)
+	sort.Strings(filesToWhiteOut)
+
 	// Add files to the layered map
 	for _, file := range filesToAdd {
 		if err := s.l.Add(file); err != nil {

pkg/snapshot/snapshot_test.go

@@ -391,6 +391,88 @@ func TestSnasphotPreservesFileOrder(t *testing.T) {
 	}
 }
 
+func TestSnasphotPreservesWhiteoutOrder(t *testing.T) {
+	newFiles := map[string]string{
+		"foo":     "newbaz1",
+		"bar/bat": "baz",
+		"bar/qux": "quuz",
+		"qux":     "quuz",
+		"corge":   "grault",
+		"garply":  "waldo",
+		"fred":    "plugh",
+		"xyzzy":   "thud",
+	}
+
+	newFileNames := []string{}
+
+	for fileName := range newFiles {
+		newFileNames = append(newFileNames, fileName)
+	}
+
+	filesInTars := [][]string{}
+
+	for i := 0; i <= 2; i++ {
+		testDir, snapshotter, cleanup, err := setUpTest()
+		testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/")
+		defer cleanup()
+
+		if err != nil {
+			t.Fatal(err)
+		}
+		// Make some changes to the filesystem
+		if err := testutil.SetupFiles(testDir, newFiles); err != nil {
+			t.Fatalf("Error setting up fs: %s", err)
+		}
+
+		filesToSnapshot := []string{}
+		for _, file := range newFileNames {
+			filesToSnapshot = append(filesToSnapshot, filepath.Join(testDir, file))
+		}
+
+		// Take a snapshot
+		_, err = snapshotter.TakeSnapshot(filesToSnapshot, false)
+		if err != nil {
+			t.Fatalf("Error taking snapshot of fs: %s", err)
+		}
+
+		// Delete all files
+		for p := range newFiles {
+			err := os.Remove(filepath.Join(testDir, p))
+			if err != nil {
+				t.Fatalf("Error deleting file: %s", err)
+			}
+		}
+
+		// Take a snapshot again
+		tarPath, err := snapshotter.TakeSnapshot(filesToSnapshot, true)
+		if err != nil {
+			t.Fatalf("Error taking snapshot of fs: %s", err)
+		}
+
+		f, err := os.Open(tarPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+		tr := tar.NewReader(f)
+		filesInTars = append(filesInTars, []string{})
+		for {
+			hdr, err := tr.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			filesInTars[i] = append(filesInTars[i], strings.TrimPrefix(hdr.Name, testDirWithoutLeadingSlash))
+		}
+	}
+
+	// Check contents of all snapshots, make sure files appear in consistent order
+	for i := 1; i < len(filesInTars); i++ {
+		testutil.CheckErrorAndDeepEqual(t, false, nil, filesInTars[0], filesInTars[i])
+	}
+}
+
 func TestSnapshotOmitsUnameGname(t *testing.T) {
 	_, snapshotter, cleanup, err := setUpTest()
 

pkg/util/fs_util.go

@@ -545,6 +545,11 @@ func DownloadFileToDest(rawurl, dest string, uid, gid int64) error {
 		return err
 	}
 	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("invalid response status %d", resp.StatusCode)
+	}
+
 	if err := CreateFile(dest, resp.Body, 0600, uint32(uid), uint32(gid)); err != nil {
 		return err
 	}