diff --git a/CHANGELOG.md b/CHANGELOG.md index b97a00174..d738526d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,77 @@ +# v0.24.0 Release 2020-07-01 +This is the 24th release of Kaniko! + +New features: +* Add a new run command along with a new flag [#1300](https://github.com/GoogleContainerTools/kaniko/pull/1300) +* Add redo snapshotter. [#1301](https://github.com/GoogleContainerTools/kaniko/pull/1301) +* Add pkg.dev to automagic config file population [#1328](https://github.com/GoogleContainerTools/kaniko/pull/1328) +* kaniko now clone git repositories recursing submodules by default [#1320](https://github.com/GoogleContainerTools/kaniko/pull/1320) + +Bug fixes: +* Fix README.md [#1323](https://github.com/GoogleContainerTools/kaniko/pull/1323) +* Fix docker-credential-gcr owner and group id [#1307](https://github.com/GoogleContainerTools/kaniko/pull/1307) + +Refactors: +* check file changed in loop [#1302](https://github.com/GoogleContainerTools/kaniko/pull/1302) +* ADD GCB benchmark code [#1299](https://github.com/GoogleContainerTools/kaniko/pull/1299) +* benchmark FileSystem snapshot project added [#1288](https://github.com/GoogleContainerTools/kaniko/pull/1288) +* [Perf] Reduce loops over files when taking FS snapshot. [#1283](https://github.com/GoogleContainerTools/kaniko/pull/1283) +* Fix README.md [#1323](https://github.com/GoogleContainerTools/kaniko/pull/1323) +* Fix docker-credential-gcr owner and group id [#1307](https://github.com/GoogleContainerTools/kaniko/pull/1307) +* benchmark FileSystem snapshot project added [#1288](https://github.com/GoogleContainerTools/kaniko/pull/1288) +* [Perf] Reduce loops over files when taking FS snapshot. [#1283](https://github.com/GoogleContainerTools/kaniko/pull/1283) + +Huge thank you for this release towards our contributors: +- Alexander Sharov +- Alex Szakaly +- Anthony Davies +- Art Begolli +- Batuhan Apaydın +- Ben Einaudi +- Carlos Alexandro Becker +- Carlos Sanchez +- Chris Sng +- Cole Wippern +- cvgw +- Daniel Marks +- Dani Raznikov +- David Dooling +- DracoBlue +- Gábor Lipták +- Gabriel Virga +- Gilbert Gilb's +- Giovan Isa Musthofa +- James Ravn +- Jon Henrik Bjørnstad +- Jon Johnson +- Jordan Goasdoué +- Liubov Grinkevich +- Logan.Price +- Lukasz Jakimczuk +- Mehdi Abaakouk +- Michel Hollands +- Mitchell Friedman +- Moritz Wanzenböck +- ohchang-kwon +- Or Sela +- PhoenixMage +- priyawadhwa +- Sam Stoelinga +- Tejal Desai +- Thomas Bonfort +- Thomas Stromberg +- Thomas Strömberg +- tinkerborg +- Tom Prince +- Vincent Latombe +- Wietse Muizelaar +- xanonid +- Yoan Blanc +- Yoriyasu Yano +- Yuheng Zhang +- yw-liu + + # v0.23.0 Release 2020-06-04 This is the 23rd release of Kaniko! diff --git a/Makefile b/Makefile index 0966fe693..5c40d51fd 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ # Bump these on release VERSION_MAJOR ?= 0 -VERSION_MINOR ?= 23 +VERSION_MINOR ?= 24 VERSION_BUILD ?= 0 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD) diff --git a/integration/dockerfiles/Dockerfile_test_add_404 b/integration/dockerfiles/Dockerfile_test_add_404 new file mode 100644 index 000000000..21b31ae1f --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_add_404 @@ -0,0 +1,4 @@ +FROM debian:9.11 + +# Testing that any HTTP failure is handled properly +ADD https://httpstat.us/404 . diff --git a/integration/images.go b/integration/images.go index ea5d1f464..3c3f0ccd7 100644 --- a/integration/images.go +++ b/integration/images.go @@ -167,6 +167,7 @@ type DockerFileBuilder struct { func NewDockerFileBuilder() *DockerFileBuilder { d := DockerFileBuilder{filesBuilt: map[string]struct{}{}} d.DockerfilesToIgnore = map[string]struct{}{ + "Dockerfile_test_add_404": {}, // TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed "Dockerfile_test_user_run": {}, } diff --git a/integration/integration_test.go b/integration/integration_test.go index 117ba76d7..6f0b9b8a6 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -386,6 +386,40 @@ func TestBuildWithLabels(t *testing.T) { checkContainerDiffOutput(t, diff, expected) } +func TestBuildWithHTTPError(t *testing.T) { + repo := getGitRepo() + dockerfile := fmt.Sprintf("%s/%s/Dockerfile_test_add_404", integrationPath, dockerfilesPath) + + // Build with docker + dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_test_add_404") + dockerCmd := exec.Command("docker", + append([]string{"build", + "-t", dockerImage, + "-f", dockerfile, + repo})...) + out, err := RunCommandWithoutTest(dockerCmd) + if err == nil { + t.Errorf("an error was expected, got %s", string(out)) + } + + // Build with kaniko + kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_add_404") + dockerRunFlags := []string{"run", "--net=host"} + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount) + dockerRunFlags = append(dockerRunFlags, ExecutorImage, + "-f", dockerfile, + "-d", kanikoImage, + "-c", fmt.Sprintf("git://%s", repo), + ) + + kanikoCmd := exec.Command("docker", dockerRunFlags...) + + out, err = RunCommandWithoutTest(kanikoCmd) + if err == nil { + t.Errorf("an error was expected, got %s", string(out)) + } +} + func TestLayers(t *testing.T) { offset := map[string]int{ "Dockerfile_test_add": 12, diff --git a/pkg/image/image_util.go b/pkg/image/image_util.go index fb7a7ce89..44f2f399f 100644 --- a/pkg/image/image_util.go +++ b/pkg/image/image_util.go @@ -21,6 +21,7 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "github.com/GoogleContainerTools/kaniko/pkg/cache" "github.com/GoogleContainerTools/kaniko/pkg/config" @@ -117,6 +118,11 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { return nil, err } + ref, err = normalizeReference(ref, image) + if err != nil { + return nil, err + } + toSet = true } @@ -134,16 +140,31 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { tag.Repository.Registry = newReg ref = tag } + if digest, ok := ref.(name.Digest); ok { digest.Repository.Registry = newReg ref = digest } } + logrus.Infof("Retrieving image %s", ref) + rOpts := remoteOptions(registryName, opts) return remote.Image(ref, rOpts...) } +// normalizeReference adds the library/ prefix to images without it. +// +// It is mostly useful when using a registry mirror that is not able to perform +// this fix automatically. +func normalizeReference(ref name.Reference, image string) (name.Reference, error) { + if !strings.ContainsRune(image, '/') { + return name.ParseReference("library/"+image, name.WeakValidation) + } + + return ref, nil +} + func remoteOptions(registryName string, opts *config.KanikoOptions) []remote.Option { tr := util.MakeTransport(opts, registryName) diff --git a/pkg/image/image_util_test.go b/pkg/image/image_util_test.go index 3b9e039cf..4a99d4ef1 100644 --- a/pkg/image/image_util_test.go +++ b/pkg/image/image_util_test.go @@ -20,6 +20,7 @@ import ( "bytes" "testing" + "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/moby/buildkit/frontend/dockerfile/instructions" @@ -109,6 +110,25 @@ func Test_ScratchImageFromMirror(t *testing.T) { testutil.CheckErrorAndDeepEqual(t, false, err, expected, actual) } +func Test_normalizeReference(t *testing.T) { + image := "debian" + expected := "index.docker.io/library/debian:latest" + + ref, err := name.ParseReference(image) + if err != nil { + t.Fatal(err) + } + + ref2, err := normalizeReference(ref, image) + if err != nil { + t.Fatal(err) + } + + if ref2.Name() != ref.Name() || ref2.Name() != expected { + t.Errorf("%s should have been normalized to %s, got %s", ref2.Name(), expected, ref.Name()) + } +} + // parse parses the contents of a Dockerfile and returns a list of commands func parse(s string) ([]instructions.Stage, error) { p, err := parser.Parse(bytes.NewReader([]byte(s))) diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index e8523d4e3..81c8371f3 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -33,7 +33,7 @@ import ( ) // For testing -var snapshotPathPrefix = config.KanikoDir +var snapshotPathPrefix = "" // Snapshotter holds the root directory from which to take snapshots, and a list of snapshots taken type Snapshotter struct { @@ -108,6 +108,9 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string, } } } + + sort.Strings(filesToWhiteout) + t := util.NewTar(f) defer t.Close() if err := writeToTar(t, filesToAdd, filesToWhiteout); err != nil { @@ -119,7 +122,7 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string, // TakeSnapshotFS takes a snapshot of the filesystem, avoiding directories in the ignorelist, and creates // a tarball of the changed files. func (s *Snapshotter) TakeSnapshotFS() (string, error) { - f, err := ioutil.TempFile(snapshotPathPrefix, "") + f, err := ioutil.TempFile(s.getSnashotPathPrefix(), "") if err != nil { return "", err } @@ -138,6 +141,13 @@ func (s *Snapshotter) TakeSnapshotFS() (string, error) { return f.Name(), nil } +func (s *Snapshotter) getSnashotPathPrefix() string { + if snapshotPathPrefix == "" { + return config.KanikoDir + } + return snapshotPathPrefix +} + func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { logrus.Info("Taking snapshot of full filesystem...") @@ -178,7 +188,10 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { } } timing.DefaultRun.Stop(timer) + sort.Strings(filesToAdd) + sort.Strings(filesToWhiteOut) + // Add files to the layered map for _, file := range filesToAdd { if err := s.l.Add(file); err != nil { diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index 7150ae0c4..c10b68426 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -391,6 +391,88 @@ func TestSnasphotPreservesFileOrder(t *testing.T) { } } +func TestSnasphotPreservesWhiteoutOrder(t *testing.T) { + newFiles := map[string]string{ + "foo": "newbaz1", + "bar/bat": "baz", + "bar/qux": "quuz", + "qux": "quuz", + "corge": "grault", + "garply": "waldo", + "fred": "plugh", + "xyzzy": "thud", + } + + newFileNames := []string{} + + for fileName := range newFiles { + newFileNames = append(newFileNames, fileName) + } + + filesInTars := [][]string{} + + for i := 0; i <= 2; i++ { + testDir, snapshotter, cleanup, err := setUpTest() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") + defer cleanup() + + if err != nil { + t.Fatal(err) + } + // Make some changes to the filesystem + if err := testutil.SetupFiles(testDir, newFiles); err != nil { + t.Fatalf("Error setting up fs: %s", err) + } + + filesToSnapshot := []string{} + for _, file := range newFileNames { + filesToSnapshot = append(filesToSnapshot, filepath.Join(testDir, file)) + } + + // Take a snapshot + _, err = snapshotter.TakeSnapshot(filesToSnapshot, false) + if err != nil { + t.Fatalf("Error taking snapshot of fs: %s", err) + } + + // Delete all files + for p := range newFiles { + err := os.Remove(filepath.Join(testDir, p)) + if err != nil { + t.Fatalf("Error deleting file: %s", err) + } + } + + // Take a snapshot again + tarPath, err := snapshotter.TakeSnapshot(filesToSnapshot, true) + if err != nil { + t.Fatalf("Error taking snapshot of fs: %s", err) + } + + f, err := os.Open(tarPath) + if err != nil { + t.Fatal(err) + } + tr := tar.NewReader(f) + filesInTars = append(filesInTars, []string{}) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + filesInTars[i] = append(filesInTars[i], strings.TrimPrefix(hdr.Name, testDirWithoutLeadingSlash)) + } + } + + // Check contents of all snapshots, make sure files appear in consistent order + for i := 1; i < len(filesInTars); i++ { + testutil.CheckErrorAndDeepEqual(t, false, nil, filesInTars[0], filesInTars[i]) + } +} + func TestSnapshotOmitsUnameGname(t *testing.T) { _, snapshotter, cleanup, err := setUpTest() diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 9a34f2974..71657ac94 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -545,6 +545,11 @@ func DownloadFileToDest(rawurl, dest string, uid, gid int64) error { return err } defer resp.Body.Close() + + if resp.StatusCode >= 400 { + return fmt.Errorf("invalid response status %d", resp.StatusCode) + } + if err := CreateFile(dest, resp.Body, 0600, uint32(uid), uint32(gid)); err != nil { return err }