From a74dc78c65ba8d6ca59df280b1b80d725469df92 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Wed, 14 Nov 2018 11:20:16 -0800 Subject: [PATCH 01/45] create cache directory if it doesn't already exist --- cmd/warmer/cmd/root.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/warmer/cmd/root.go b/cmd/warmer/cmd/root.go index 0e4908d2b..345bd2aa5 100644 --- a/cmd/warmer/cmd/root.go +++ b/cmd/warmer/cmd/root.go @@ -51,6 +51,12 @@ var RootCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { + if _, err := os.Stat(opts.CacheDir); os.IsNotExist(err) { + err = os.MkdirAll(opts.CacheDir, 0755) + if err != nil { + exit(errors.Wrap(err, "Failed to create cache directory")) + } + } if err := cache.WarmCache(opts); err != nil { exit(errors.Wrap(err, "Failed warming cache")) } From 1e5286cbad85bad9cf79d24bf760d4227514acfd Mon Sep 17 00:00:00 2001 From: Niels Denissen Date: Mon, 17 Dec 2018 16:38:01 +0100 Subject: [PATCH 02/45] Add desc for `--skip-tls-verify-pull` to README Add a description for the `--skip-tls-verify-pull` option in the README. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 798dab056..4a5c711f3 100644 --- a/README.md +++ b/README.md @@ -379,6 +379,10 @@ This flag takes a single snapshot of the filesystem at the end of the build, so Set this flag to skip TLS certificate validation when connecting to a registry. It is supposed to be used for testing purposes only and should not be used in production! +#### --skip-tls-verify-pull + +Set this flag to skip TLS certificate validation when pulling from a registry. It is supposed to be used for testing purposes only and should not be used in production! + #### --snapshotMode You can set the `--snapshotMode=` flag to set how kaniko will snapshot the filesystem. From 319bfde932c2cbe948f4329388f65b10777f5cf3 Mon Sep 17 00:00:00 2001 From: Warren Seymour Date: Thu, 20 Dec 2018 12:00:25 +0000 Subject: [PATCH 03/45] Include warmer in debug image When using Kaniko with certain build systems, such as GitLab CI, it is necessary to use the 'debug' image and override the entrypoint, so that an arbitrary build script can be executed within the container. Unfortunately, the 'warmer' binary is not available in the 'debug' image, making it impossible to perform a base image cache warmup using such build systems. This patch addresses this by ensuring that the 'out/warmer' target is made in the initial stage, and that all 'out' files are copied to '/kaniko' in the final stage. --- deploy/Dockerfile_debug | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index 51cf02a17..1feb086b7 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -25,7 +25,7 @@ RUN docker-credential-gcr configure-docker RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 COPY . . -RUN make +RUN make && make out/warmer # Stage 1: Get the busybox shell FROM gcr.io/cloud-builders/bazel:latest @@ -35,7 +35,7 @@ RUN bazel build //experimental/busybox:busybox_tar RUN tar -C /distroless/bazel-genfiles/experimental/busybox/ -xf /distroless/bazel-genfiles/experimental/busybox/busybox.tar FROM scratch -COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/executor /kaniko/executor +COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/* /kaniko/ COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login COPY --from=1 /distroless/bazel-genfiles/experimental/busybox/busybox/ /busybox/ From 8f863213b81ce30acfd7afd0f9de3ef7acab6b2c Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Mon, 1 Apr 2019 11:13:10 +0200 Subject: [PATCH 04/45] Remove cruft --- pkg/executor/foo | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 pkg/executor/foo diff --git a/pkg/executor/foo b/pkg/executor/foo deleted file mode 100644 index e69de29bb..000000000 From 317d1b7017a42e4980577eee4262e2a0917c1759 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 May 2019 16:12:20 +0200 Subject: [PATCH 05/45] Improve git buildcontext integration test Build a dockerfile that will copy the LICENSE from the kaniko github repository into the image to make sure the git buildcontext works as expected. --- integration/dockerfiles/Dockerfile_git_buildcontext | 2 ++ integration/integration_test.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 integration/dockerfiles/Dockerfile_git_buildcontext diff --git a/integration/dockerfiles/Dockerfile_git_buildcontext b/integration/dockerfiles/Dockerfile_git_buildcontext new file mode 100644 index 000000000..9636fcd3d --- /dev/null +++ b/integration/dockerfiles/Dockerfile_git_buildcontext @@ -0,0 +1,2 @@ +FROM scratch +COPY LICENSE ./LICENSE diff --git a/integration/integration_test.go b/integration/integration_test.go index c11dd1378..3d046b53d 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -237,7 +237,7 @@ func TestRun(t *testing.T) { func TestGitBuildcontext(t *testing.T) { repo := "github.com/GoogleContainerTools/kaniko" - dockerfile := "integration/dockerfiles/Dockerfile_test_run_2" + dockerfile := "integration/dockerfiles/Dockerfile_git_buildcontext" // Build with docker dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_test_git") From 5c0603a9675b2399237b394f56130b38e12dd25d Mon Sep 17 00:00:00 2001 From: Taylor Barrella Date: Sat, 25 May 2019 15:47:26 -0700 Subject: [PATCH 06/45] Update go-containerregistry Resolves #607 * Deleted a duplicate Gopkg.lock block for github.com/otiai10/copy to prevent `dep ensure` from deleting it from vendor/ * Searched for breaking changes. Only found ones for remote.Delete/List/Write/WriteIndex. Searched for those and fixed * Noticed that NewInsecureRegistry was deprecated and replaced it --- Gopkg.lock | 28 +- Gopkg.toml | 2 +- pkg/cache/cache.go | 2 +- pkg/executor/push.go | 4 +- pkg/util/image_util.go | 2 +- .../cmd/ko/test/kodata/kenobi | 1 - .../go-containerregistry/pkg/name/check.go | 9 - .../go-containerregistry/pkg/name/digest.go | 9 +- .../go-containerregistry/pkg/name/doc.go | 42 +++ .../go-containerregistry/pkg/name/options.go | 49 ++++ .../go-containerregistry/pkg/name/ref.go | 6 +- .../go-containerregistry/pkg/name/registry.go | 20 +- .../pkg/name/repository.go | 7 +- .../go-containerregistry/pkg/name/tag.go | 7 +- .../go-containerregistry/pkg/v1/image.go | 1 + .../go-containerregistry/pkg/v1/index.go | 1 + .../go-containerregistry/pkg/v1/layer.go | 5 + .../pkg/v1/mutate/mutate.go | 11 +- .../pkg/v1/partial/compressed.go | 4 + .../pkg/v1/partial/uncompressed.go | 3 + .../pkg/v1/partial/with.go | 7 + .../pkg/v1/random/image.go | 8 + .../pkg/v1/remote/check.go | 5 +- .../pkg/v1/remote/delete.go | 9 +- .../pkg/v1/remote/descriptor.go | 255 ++++++++++++++++++ .../pkg/v1/remote/image.go | 206 +++----------- .../pkg/v1/remote/index.go | 131 ++++++--- .../pkg/v1/remote/list.go | 12 +- .../pkg/v1/remote/options.go | 86 ++++-- .../pkg/v1/remote/transport/bearer.go | 8 +- .../pkg/v1/remote/transport/error.go | 14 + .../pkg/v1/remote/write.go | 136 +++++----- .../pkg/v1/stream/layer.go | 8 + .../pkg/v1/tarball/image.go | 14 +- .../pkg/v1/tarball/layer.go | 18 ++ 35 files changed, 755 insertions(+), 375 deletions(-) delete mode 120000 vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi create mode 100644 vendor/github.com/google/go-containerregistry/pkg/name/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/name/options.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go diff --git a/Gopkg.lock b/Gopkg.lock index 7b09230e2..44926c25d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -445,7 +445,7 @@ version = "v0.2.0" [[projects]] - digest = "1:d40a26f0daf07f3b5c916356a3e10fabbf97d5166f77e57aa3983013ab57004c" + digest = "1:3ccc9b3dfd6b951b46e6e1c499af589fbc35c7e1172d6d840cbe836ae08d3536" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", @@ -465,7 +465,7 @@ "pkg/v1/v1util", ] pruneopts = "NUT" - revision = "8621d738a07bc74b2adeafd175a3c738423577a0" + revision = "bb17f50c1bc6808972811ed2894ecaaeb5de68ad" [[projects]] digest = "1:f4f203acd8b11b8747bdcd91696a01dbc95ccb9e2ca2db6abf81c3a4f5e950ce" @@ -719,6 +719,14 @@ revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" version = "v1.0.2" +[[projects]] + branch = "master" + digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c" + name = "github.com/otiai10/copy" + packages = ["."] + pruneopts = "NUT" + revision = "7e9a647135a142c2669943d4a4d29be015ce9392" + [[projects]] digest = "1:cf254277d898b713195cc6b4a3fac8bf738b9f1121625df27843b52b267eec6c" name = "github.com/pelletier/go-buffruneio" @@ -727,22 +735,6 @@ revision = "c37440a7cf42ac63b919c752ca73a85067e05992" version = "v0.2.0" -[[projects]] - branch = "master" - digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c" - name = "github.com/otiai10/copy" - packages = ["."] - pruneopts = "NUT" - revision = "7e9a647135a142c2669943d4a4d29be015ce9392" - -[[projects]] - branch = "master" - digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c" - name = "github.com/otiai10/copy" - packages = ["."] - pruneopts = "NUT" - revision = "7e9a647135a142c2669943d4a4d29be015ce9392" - [[projects]] branch = "master" digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" diff --git a/Gopkg.toml b/Gopkg.toml index 80db44131..163d821ad 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,7 +37,7 @@ required = [ [[constraint]] name = "github.com/google/go-containerregistry" - revision = "8621d738a07bc74b2adeafd175a3c738423577a0" + revision = "bb17f50c1bc6808972811ed2894ecaaeb5de68ad" [[override]] name = "k8s.io/apimachinery" diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index d11de89af..6ec9d0ce7 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -60,7 +60,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) { registryName := cacheRef.Repository.Registry.Name() if rc.Opts.InsecureRegistries.Contains(registryName) { - newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation) + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return nil, err } diff --git a/pkg/executor/push.go b/pkg/executor/push.go index a02d5b9b9..603b3a08a 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -114,7 +114,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { for _, destRef := range destRefs { registryName := destRef.Repository.Registry.Name() if opts.Insecure || opts.InsecureRegistries.Contains(registryName) { - newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation) + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return errors.Wrap(err, "getting new insecure registry") } @@ -135,7 +135,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { } rt := &withUserAgent{t: tr} - if err := remote.Write(destRef, image, pushAuth, rt); err != nil { + if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil { return errors.Wrap(err, fmt.Sprintf("failed to push to destination %s", destRef)) } } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 6d7ed8a8d..dcc09ada8 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -102,7 +102,7 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { registryName := ref.Context().RegistryStr() if opts.InsecurePull || opts.InsecureRegistries.Contains(registryName) { - newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation) + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi b/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi deleted file mode 120000 index 5d7eddc7f..000000000 --- a/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi +++ /dev/null @@ -1 +0,0 @@ -../kenobi \ No newline at end of file diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/check.go b/vendor/github.com/google/go-containerregistry/pkg/name/check.go index 01a25d554..01b03e562 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/check.go @@ -19,15 +19,6 @@ import ( "unicode/utf8" ) -// Strictness defines the level of strictness for name validation. -type Strictness int - -// Enums for CRUD operations. -const ( - StrictValidation Strictness = iota - WeakValidation -) - // stripRunesFn returns a function which returns -1 (i.e. a value which // signals deletion in strings.Map) for runes in 'runes', and the rune otherwise. func stripRunesFn(runes string) func(rune) rune { diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go index dc573ef1d..2dc0f7f37 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package name defines structured types for representing image references. package name import ( @@ -63,8 +62,8 @@ func checkDigest(name string) error { return checkElement("digest", name, digestChars, 7+64, 7+64) } -// NewDigest returns a new Digest representing the given name, according to the given strictness. -func NewDigest(name string, strict Strictness) (Digest, error) { +// NewDigest returns a new Digest representing the given name. +func NewDigest(name string, opts ...Option) (Digest, error) { // Split on "@" parts := strings.Split(name, digestDelim) if len(parts) != 2 { @@ -78,12 +77,12 @@ func NewDigest(name string, strict Strictness) (Digest, error) { return Digest{}, err } - tag, err := NewTag(base, strict) + tag, err := NewTag(base, opts...) if err == nil { base = tag.Repository.Name() } - repo, err := NewRepository(base, strict) + repo, err := NewRepository(base, opts...) if err != nil { return Digest{}, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/doc.go b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go new file mode 100644 index 000000000..b294794dc --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go @@ -0,0 +1,42 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package name defines structured types for representing image references. +// +// What's in a name? For image references, not nearly enough! +// +// Image references look a lot like URLs, but they differ in that they don't +// contain the scheme (http or https), they can end with a :tag or a @digest +// (the latter being validated), and they perform defaulting for missing +// components. +// +// Since image references don't contain the scheme, we do our best to infer +// if we use http or https from the given hostname. We allow http fallback for +// any host that looks like localhost (localhost, 127.0.0.1, ::1), ends in +// ".local", or is in the "private" address space per RFC 1918. For everything +// else, we assume https only. To override this heuristic, use the Insecure +// option. +// +// Image references with a digest signal to us that we should verify the content +// of the image matches the digest. E.g. when pulling a Digest reference, we'll +// calculate the sha256 of the manifest returned by the registry and error out +// if it doesn't match what we asked for. +// +// For defaulting, we interpret "ubuntu" as +// "index.docker.io/library/ubuntu:latest" because we add the missing repo +// "library", the missing registry "index.docker.io", and the missing tag +// "latest". To disable this defaulting, use the StrictValidation option. This +// is useful e.g. to only allow image references that explicitly set a tag or +// digest, so that you don't accidentally pull "latest". +package name diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/options.go b/vendor/github.com/google/go-containerregistry/pkg/name/options.go new file mode 100644 index 000000000..98beaae11 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/name/options.go @@ -0,0 +1,49 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +type options struct { + strict bool // weak by default + insecure bool // secure by default +} + +func makeOptions(opts ...Option) options { + opt := options{} + for _, o := range opts { + o(&opt) + } + return opt +} + +// Option is a functional option for name parsing. +type Option func(*options) + +// StrictValidation is an Option that requires image references to be fully +// specified; i.e. no defaulting for registry (dockerhub), repo (library), +// or tag (latest). +func StrictValidation(opts *options) { + opts.strict = true +} + +// WeakValidation is an Option that sets defaults when parsing names, see +// StrictValidation. +func WeakValidation(opts *options) { + opts.strict = false +} + +// Insecure is an Option that allows image references to be fetched without TLS. +func Insecure(opts *options) { + opts.insecure = true +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go index 58775daa3..cca303405 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go @@ -38,11 +38,11 @@ type Reference interface { } // ParseReference parses the string as a reference, either by tag or digest. -func ParseReference(s string, strict Strictness) (Reference, error) { - if t, err := NewTag(s, strict); err == nil { +func ParseReference(s string, opts ...Option) (Reference, error) { + if t, err := NewTag(s, opts...); err == nil { return t, nil } - if d, err := NewDigest(s, strict); err == nil { + if d, err := NewDigest(s, opts...); err == nil { return d, nil } // TODO: Combine above errors into something more useful? diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go index ab7419308..c12dd46c2 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go @@ -114,8 +114,9 @@ func checkRegistry(name string) error { // NewRegistry returns a Registry based on the given name. // Strict validation requires explicit, valid RFC 3986 URI authorities to be given. -func NewRegistry(name string, strict Strictness) (Registry, error) { - if strict == StrictValidation && len(name) == 0 { +func NewRegistry(name string, opts ...Option) (Registry, error) { + opt := makeOptions(opts...) + if opt.strict && len(name) == 0 { return Registry{}, NewErrBadName("strict validation requires the registry to be explicitly defined") } @@ -129,16 +130,13 @@ func NewRegistry(name string, strict Strictness) (Registry, error) { name = DefaultRegistry } - return Registry{registry: name}, nil + return Registry{registry: name, insecure: opt.insecure}, nil } // NewInsecureRegistry returns an Insecure Registry based on the given name. -// Strict validation requires explicit, valid RFC 3986 URI authorities to be given. -func NewInsecureRegistry(name string, strict Strictness) (Registry, error) { - reg, err := NewRegistry(name, strict) - if err != nil { - return Registry{}, err - } - reg.insecure = true - return reg, nil +// +// Deprecated: Use the Insecure Option with NewRegistry instead. +func NewInsecureRegistry(name string, opts ...Option) (Registry, error) { + opts = append(opts, Insecure) + return NewRegistry(name, opts...) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go index 43cc5b82b..f33377988 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go @@ -68,7 +68,8 @@ func checkRepository(repository string) error { } // NewRepository returns a new Repository representing the given name, according to the given strictness. -func NewRepository(name string, strict Strictness) (Repository, error) { +func NewRepository(name string, opts ...Option) (Repository, error) { + opt := makeOptions(opts...) if len(name) == 0 { return Repository{}, NewErrBadName("a repository name must be specified") } @@ -88,11 +89,11 @@ func NewRepository(name string, strict Strictness) (Repository, error) { return Repository{}, err } - reg, err := NewRegistry(registry, strict) + reg, err := NewRegistry(registry, opts...) if err != nil { return Repository{}, err } - if hasImplicitNamespace(repo, reg) && strict == StrictValidation { + if hasImplicitNamespace(repo, reg) && opt.strict { return Repository{}, NewErrBadName("strict validation requires the full repository path (missing 'library')") } return Repository{reg, repo}, nil diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/tag.go b/vendor/github.com/google/go-containerregistry/pkg/name/tag.go index b8375e1f9..e6cce34db 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/tag.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/tag.go @@ -71,7 +71,8 @@ func checkTag(name string) error { } // NewTag returns a new Tag representing the given name, according to the given strictness. -func NewTag(name string, strict Strictness) (Tag, error) { +func NewTag(name string, opts ...Option) (Tag, error) { + opt := makeOptions(opts...) base := name tag := "" @@ -87,13 +88,13 @@ func NewTag(name string, strict Strictness) (Tag, error) { // even when not being strict. // If we are being strict, we want to validate the tag regardless in case // it's empty. - if tag != "" || strict == StrictValidation { + if tag != "" || opt.strict { if err := checkTag(tag); err != nil { return Tag{}, err } } - repo, err := NewRepository(base, strict) + repo, err := NewRepository(base, opts...) if err != nil { return Tag{}, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/image.go index 17b9839a6..9ef026799 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/image.go @@ -19,6 +19,7 @@ import ( ) // Image defines the interface for interacting with an OCI v1 image. +//go:generate counterfeiter -o fake/image.go . Image type Image interface { // Layers returns the ordered collection of filesystem layers that comprise this image. // The order of the list is oldest/base layer first, and most-recent/top layer last. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/index.go index 604e6de36..b4e84e02a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/index.go @@ -19,6 +19,7 @@ import ( ) // ImageIndex defines the interface for interacting with an OCI image index. +//go:generate counterfeiter -o fake/index.go . ImageIndex type ImageIndex interface { // MediaType of this image's manifest. MediaType() (types.MediaType, error) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go index 8b5091e45..57447d263 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go @@ -16,6 +16,8 @@ package v1 import ( "io" + + "github.com/google/go-containerregistry/pkg/v1/types" ) // Layer is an interface for accessing the properties of a particular layer of a v1.Image @@ -34,4 +36,7 @@ type Layer interface { // Size returns the compressed size of the Layer. Size() (int64, error) + + // MediaType returns the media type of the Layer. + MediaType() (types.MediaType, error) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index a327e7594..11262f444 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -78,10 +78,11 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) { cf.Config = cfg - return configFile(base, cf) + return ConfigFile(base, cf) } -func configFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { +// ConfigFile mutates the provided v1.Image to have the provided v1.ConfigFile +func ConfigFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { m, err := base.Manifest() if err != nil { return nil, err @@ -106,7 +107,7 @@ func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) { cfg := cf.DeepCopy() cfg.Created = created - return configFile(base, cfg) + return ConfigFile(base, cfg) } type image struct { @@ -476,7 +477,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { h.Created = v1.Time{Time: t} } - return configFile(newImage, cfg) + return ConfigFile(newImage, cfg) } func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { @@ -555,5 +556,5 @@ func Canonical(img v1.Image) (v1.Image, error) { cfg.ContainerConfig.Hostname = "" cfg.DockerVersion = "" - return configFile(img, cfg) + return ConfigFile(img, cfg) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go index 497d1af0d..1c631b7aa 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go @@ -18,6 +18,7 @@ import ( "io" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -32,6 +33,9 @@ type CompressedLayer interface { // Size returns the compressed size of the Layer. Size() (int64, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) } // compressedLayerExtender implements v1.Image using the compressed base properties. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go index 9f75723ec..07658be42 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go @@ -32,6 +32,9 @@ type UncompressedLayer interface { // Uncompressed returns an io.ReadCloser for the uncompressed layer contents. Uncompressed() (io.ReadCloser, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) } // uncompressedLayerExtender implements v1.Image using the uncompressed base properties. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go index f724ec8ab..992672048 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go @@ -22,6 +22,7 @@ import ( "io/ioutil" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -80,6 +81,12 @@ func (cl *configLayer) Size() (int64, error) { return int64(len(cl.content)), nil } +func (cl *configLayer) MediaType() (types.MediaType, error) { + // Defaulting this to OCIConfigJSON as it should remain + // backwards compatible with DockerConfigJSON + return types.OCIConfigJSON, nil +} + var _ v1.Layer = (*configLayer)(nil) // ConfigLayer implements v1.Layer from the raw config bytes. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go index cc269d6b5..e09984cbf 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go @@ -45,6 +45,14 @@ func (ul *uncompressedLayer) Uncompressed() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewBuffer(ul.content)), nil } +// MediaType returns the media type of the layer +func (ul *uncompressedLayer) MediaType() (types.MediaType, error) { + // Technically the media type should be 'application/tar' but given that our + // v1.Layer doesn't force consumers to care about whether the layer is compressed + // we should be fine returning the DockerLayer media type + return types.DockerLayer, nil +} + var _ partial.UncompressedLayer = (*uncompressedLayer)(nil) // Image returns a pseudo-randomly generated Image. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go index aa574eb8b..da0fa24c3 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go @@ -1,6 +1,7 @@ package remote import ( + "fmt" "net/http" "github.com/google/go-containerregistry/pkg/authn" @@ -18,13 +19,13 @@ import ( func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { auth, err := kc.Resolve(ref.Context().Registry) if err != nil { - return err + return fmt.Errorf("resolving authorization for %v failed: %v", ref.Context().Registry, err) } scopes := []string{ref.Scope(transport.PushScope)} tr, err := transport.New(ref.Context().Registry, auth, t, scopes) if err != nil { - return err + return fmt.Errorf("creating push check transport for %v failed: %v", ref.Context().Registry, err) } // TODO(jasonhall): Against GCR, just doing the token handshake is // enough, but this doesn't extend to Dockerhub diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go index 2032e276e..c034435f9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go @@ -20,15 +20,18 @@ import ( "net/http" "net/url" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) // Delete removes the specified image reference from the remote registry. -func Delete(ref name.Reference, auth authn.Authenticator, t http.RoundTripper) error { +func Delete(ref name.Reference, options ...Option) error { + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } scopes := []string{ref.Scope(transport.DeleteScope)} - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go new file mode 100644 index 000000000..078de3a0b --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -0,0 +1,255 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var defaultPlatform = v1.Platform{ + Architecture: "amd64", + OS: "linux", +} + +// ErrSchema1 indicates that we received a schema1 manifest from the registry. +// This library doesn't have plans to support this legacy image format: +// https://github.com/google/go-containerregistry/issues/377 +var ErrSchema1 = errors.New("unsupported MediaType: https://github.com/google/go-containerregistry/issues/377") + +// Descriptor provides access to metadata about remote artifact and accessors +// for efficiently converting it into a v1.Image or v1.ImageIndex. +type Descriptor struct { + fetcher + v1.Descriptor + Manifest []byte + + // So we can share this implementation with Image.. + platform v1.Platform +} + +// Get returns a remote.Descriptor for the given reference. The response from +// the registry is left un-interpreted, for the most part. This is useful for +// querying what kind of artifact a reference represents. +func Get(ref name.Reference, options ...Option) (*Descriptor, error) { + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + types.DockerManifestList, + types.OCIImageIndex, + // Just to look at them. + types.DockerManifestSchema1, + types.DockerManifestSchema1Signed, + } + return get(ref, acceptable, options...) +} + +// Handle options and fetch the manifest with the acceptable MediaTypes in the +// Accept header. +func get(ref name.Reference, acceptable []types.MediaType, options ...Option) (*Descriptor, error) { + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return nil, err + } + + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, []string{ref.Scope(transport.PullScope)}) + if err != nil { + return nil, err + } + + f := fetcher{ + Ref: ref, + Client: &http.Client{Transport: tr}, + } + + b, desc, err := f.fetchManifest(ref, acceptable) + if err != nil { + return nil, err + } + + return &Descriptor{ + fetcher: f, + Manifest: b, + Descriptor: *desc, + platform: o.platform, + }, nil +} + +// Image converts the Descriptor into a v1.Image. +// +// If the fetched artifact is already an image, it will just return it. +// +// If the fetched artifact is an index, it will attempt to resolve the index to +// a child image with the appropriate platform. +// +// See WithPlatform to set the desired platform. +func (d *Descriptor) Image() (v1.Image, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, ErrSchema1 + case types.OCIImageIndex, types.DockerManifestList: + // We want an image but the registry has an index, resolve it to an image. + return d.remoteIndex().imageByPlatform(d.platform) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // These are expected. Enumerated here to allow a default case. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + + // Wrap the v1.Layers returned by this v1.Image in a hint for downstream + // remote.Write calls to facilitate cross-repo "mounting". + imgCore, err := partial.CompressedToImage(d.remoteImage()) + if err != nil { + return nil, err + } + return &mountableImage{ + Image: imgCore, + Reference: d.Ref, + }, nil +} + +// ImageIndex converts the Descriptor into a v1.ImageIndex. +func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, ErrSchema1 + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // We want an index but the registry has an image, nothing we can do. + return nil, fmt.Errorf("unexpected media type for ImageIndex(): %s; call Image() instead", d.MediaType) + case types.OCIImageIndex, types.DockerManifestList: + // These are expected. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + return d.remoteIndex(), nil +} + +func (d *Descriptor) remoteImage() *remoteImage { + return &remoteImage{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +func (d *Descriptor) remoteIndex() *remoteIndex { + return &remoteIndex{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +// fetcher implements methods for reading from a registry. +type fetcher struct { + Ref name.Reference + Client *http.Client +} + +// url returns a url.Url for the specified path in the context of this remote image reference. +func (f *fetcher) url(resource, identifier string) url.URL { + return url.URL{ + Scheme: f.Ref.Context().Registry.Scheme(), + Host: f.Ref.Context().RegistryStr(), + Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), + } +} + +func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { + u := f.url("manifests", ref.Identifier()) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, nil, err + } + accept := []string{} + for _, mt := range acceptable { + accept = append(accept, string(mt)) + } + req.Header.Set("Accept", strings.Join(accept, ",")) + + resp, err := f.Client.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, nil, err + } + + manifest, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + digest, size, err := v1.SHA256(bytes.NewReader(manifest)) + if err != nil { + return nil, nil, err + } + + mediaType := types.MediaType(resp.Header.Get("Content-Type")) + + // Validate the digest matches what we asked for, if pulling by digest. + if dgst, ok := ref.(name.Digest); ok { + if mediaType == types.DockerManifestSchema1Signed { + // Digests for this are stupid to calculate, ignore it. + } else if digest.String() != dgst.DigestStr() { + return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) + } + } else { + // Do nothing for tags; I give up. + // + // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, + // but so many registries implement this incorrectly that it's not worth checking. + // + // For reference: + // https://github.com/docker/distribution/issues/2395 + // https://github.com/GoogleContainerTools/kaniko/issues/298 + } + + // Return all this info since we have to calculate it anyway. + desc := v1.Descriptor{ + Digest: digest, + Size: size, + MediaType: mediaType, + } + + return manifest, &desc, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index 1be0ad2ea..752c71608 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -15,16 +15,12 @@ package remote import ( - "bytes" "fmt" "io" "io/ioutil" "net/http" - "net/url" - "strings" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" @@ -33,11 +29,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/v1util" ) -var defaultPlatform = v1.Platform{ - Architecture: "amd64", - OS: "linux", -} - // remoteImage accesses an image from a remote registry type remoteImage struct { fetcher @@ -46,135 +37,26 @@ type remoteImage struct { configLock sync.Mutex // Protects config config []byte mediaType types.MediaType - platform v1.Platform } -// ImageOption is a functional option for Image. -type ImageOption func(*imageOpener) error - var _ partial.CompressedImageCore = (*remoteImage)(nil) -type imageOpener struct { - auth authn.Authenticator - transport http.RoundTripper - ref name.Reference - client *http.Client - platform v1.Platform -} +// Image provides access to a remote image reference. +func Image(ref name.Reference, options ...Option) (v1.Image, error) { + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + // We resolve these to images later. + types.DockerManifestList, + types.OCIImageIndex, + } -func (i *imageOpener) Open() (v1.Image, error) { - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + desc, err := get(ref, acceptable, options...) if err != nil { return nil, err } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - platform: i.platform, - } - imgCore, err := partial.CompressedToImage(ri) - if err != nil { - return imgCore, err - } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: i.ref, - }, nil -} -// Image provides access to a remote image reference, applying functional options -// to the underlying imageOpener before resolving the reference into a v1.Image. -func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) { - img := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, - platform: defaultPlatform, - } - - for _, option := range options { - if err := option(img); err != nil { - return nil, err - } - } - return img.Open() -} - -// fetcher implements methods for reading from a remote image. -type fetcher struct { - Ref name.Reference - Client *http.Client -} - -// url returns a url.Url for the specified path in the context of this remote image reference. -func (f *fetcher) url(resource, identifier string) url.URL { - return url.URL{ - Scheme: f.Ref.Context().Registry.Scheme(), - Host: f.Ref.Context().RegistryStr(), - Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), - } -} - -func (f *fetcher) fetchManifest(acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { - u := f.url("manifests", f.Ref.Identifier()) - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return nil, nil, err - } - accept := []string{} - for _, mt := range acceptable { - accept = append(accept, string(mt)) - } - req.Header.Set("Accept", strings.Join(accept, ",")) - - resp, err := f.Client.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if err := transport.CheckError(resp, http.StatusOK); err != nil { - return nil, nil, err - } - - manifest, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } - - digest, size, err := v1.SHA256(bytes.NewReader(manifest)) - if err != nil { - return nil, nil, err - } - - // Validate the digest matches what we asked for, if pulling by digest. - if dgst, ok := f.Ref.(name.Digest); ok { - if digest.String() != dgst.DigestStr() { - return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) - } - } else { - // Do nothing for tags; I give up. - // - // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, - // but so many registries implement this incorrectly that it's not worth checking. - // - // For reference: - // https://github.com/docker/distribution/issues/2395 - // https://github.com/GoogleContainerTools/kaniko/issues/298 - } - - // Return all this info since we have to calculate it anyway. - desc := v1.Descriptor{ - Digest: digest, - Size: size, - MediaType: types.MediaType(resp.Header.Get("Content-Type")), - } - - return manifest, &desc, nil + return desc.Image() } func (r *remoteImage) MediaType() (types.MediaType, error) { @@ -184,7 +66,6 @@ func (r *remoteImage) MediaType() (types.MediaType, error) { return types.DockerManifestSchema2, nil } -// TODO(jonjohnsonjr): Handle manifest lists. func (r *remoteImage) RawManifest() ([]byte, error) { r.manifestLock.Lock() defer r.manifestLock.Unlock() @@ -192,26 +73,18 @@ func (r *remoteImage) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteImage. acceptable := []types.MediaType{ types.DockerManifestSchema2, types.OCIManifestSchema1, - // We'll resolve these to an image based on the platform. - types.DockerManifestList, - types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } - // We want an image but the registry has an index, resolve it to an image. - for desc.MediaType == types.DockerManifestList || desc.MediaType == types.OCIImageIndex { - manifest, desc, err = r.matchImage(manifest) - if err != nil { - return nil, err - } - } - r.mediaType = desc.MediaType r.manifest = manifest return r.manifest, nil @@ -278,6 +151,22 @@ func (rl *remoteLayer) Manifest() (*v1.Manifest, error) { return partial.Manifest(rl.ri) } +// MediaType implements v1.Layer +func (rl *remoteLayer) MediaType() (types.MediaType, error) { + m, err := rl.Manifest() + if err != nil { + return "", err + } + + for _, layer := range m.Layers { + if layer.Digest == rl.digest { + return layer.MediaType, nil + } + } + + return "", fmt.Errorf("unable to find layer with digest: %v", rl.digest) +} + // Size implements partial.CompressedLayer func (rl *remoteLayer) Size() (int64, error) { // Look up the size of this digest in the manifest to avoid a request. @@ -302,36 +191,3 @@ func (r *remoteImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) digest: h, }, nil } - -// This naively matches the first manifest with matching Architecture and OS. -// -// We should probably use this instead: -// github.com/containerd/containerd/platforms -// -// But first we'd need to migrate to: -// github.com/opencontainers/image-spec/specs-go/v1 -func (r *remoteImage) matchImage(rawIndex []byte) ([]byte, *v1.Descriptor, error) { - index, err := v1.ParseIndexManifest(bytes.NewReader(rawIndex)) - if err != nil { - return nil, nil, err - } - for _, childDesc := range index.Manifests { - // If platform is missing from child descriptor, assume it's amd64/linux. - p := defaultPlatform - if childDesc.Platform != nil { - p = *childDesc.Platform - } - if r.platform.Architecture == p.Architecture && r.platform.OS == p.OS { - childRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), childDesc.Digest), name.StrictValidation) - if err != nil { - return nil, nil, err - } - r.fetcher = fetcher{ - Client: r.Client, - Ref: childRef, - } - return r.fetchManifest([]types.MediaType{childDesc.MediaType}) - } - } - return nil, nil, fmt.Errorf("no matching image for %s/%s, index: %s", r.platform.Architecture, r.platform.OS, string(rawIndex)) -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index 03afc481a..1303dda9c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -17,14 +17,11 @@ package remote import ( "bytes" "fmt" - "net/http" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -36,30 +33,19 @@ type remoteIndex struct { mediaType types.MediaType } -// Index provides access to a remote index reference, applying functional options -// to the underlying imageOpener before resolving the reference into a v1.ImageIndex. -func Index(ref name.Reference, options ...ImageOption) (v1.ImageIndex, error) { - i := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, +// Index provides access to a remote index reference. +func Index(ref name.Reference, options ...Option) (v1.ImageIndex, error) { + acceptable := []types.MediaType{ + types.DockerManifestList, + types.OCIImageIndex, } - for _, option := range options { - if err := option(i); err != nil { - return nil, err - } - } - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + desc, err := get(ref, acceptable, options...) if err != nil { return nil, err } - return &remoteIndex{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - }, nil + + return desc.ImageIndex() } func (r *remoteIndex) MediaType() (types.MediaType, error) { @@ -80,11 +66,14 @@ func (r *remoteIndex) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteIndex. acceptable := []types.MediaType{ types.DockerManifestList, types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } @@ -103,37 +92,93 @@ func (r *remoteIndex) IndexManifest() (*v1.IndexManifest, error) { } func (r *remoteIndex) Image(h v1.Hash) (v1.Image, error) { - imgRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) + desc, err := r.childByHash(h) if err != nil { return nil, err } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: imgRef, - Client: r.Client, - }, - } - imgCore, err := partial.CompressedToImage(ri) - if err != nil { - return imgCore, err - } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: r.Ref, - }, nil + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() } func (r *remoteIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { - idxRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) + desc, err := r.childByHash(h) if err != nil { return nil, err } - return &remoteIndex{ + return desc.ImageIndex() +} + +func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) { + desc, err := r.childByPlatform(platform) + if err != nil { + return nil, err + } + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() +} + +// This naively matches the first manifest with matching Architecture and OS. +// +// We should probably use this instead: +// github.com/containerd/containerd/platforms +// +// But first we'd need to migrate to: +// github.com/opencontainers/image-spec/specs-go/v1 +func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + // If platform is missing from child descriptor, assume it's amd64/linux. + p := defaultPlatform + if childDesc.Platform != nil { + p = *childDesc.Platform + } + + if platform.Architecture == p.Architecture && platform.OS == p.OS { + return r.childDescriptor(childDesc, platform) + } + } + return nil, fmt.Errorf("no child with platform %s/%s in index %s", platform.Architecture, platform.OS, r.Ref) +} + +func (r *remoteIndex) childByHash(h v1.Hash) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + if h == childDesc.Digest { + return r.childDescriptor(childDesc, defaultPlatform) + } + } + return nil, fmt.Errorf("no child with digest %s in index %s", h, r.Ref) +} + +func (r *remoteIndex) childRef(h v1.Hash) (name.Reference, error) { + return name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) +} + +// Convert one of this index's child's v1.Descriptor into a remote.Descriptor, with the given platform option. +func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) (*Descriptor, error) { + ref, err := r.childRef(child.Digest) + if err != nil { + return nil, err + } + manifest, desc, err := r.fetchManifest(ref, []types.MediaType{child.MediaType}) + if err != nil { + return nil, err + } + return &Descriptor{ fetcher: fetcher{ - Ref: idxRef, + Ref: ref, Client: r.Client, }, + Manifest: manifest, + Descriptor: *desc, + platform: platform, }, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go index 1a36d0a4b..4cbdc4f19 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go @@ -20,7 +20,6 @@ import ( "net/http" "net/url" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) @@ -30,10 +29,15 @@ type tags struct { Tags []string `json:"tags"` } -// List calls /tags/list for the given repository. -func List(repo name.Repository, auth authn.Authenticator, t http.RoundTripper) ([]string, error) { +// List calls /tags/list for the given repository, returning the list of tags +// in the "tags" property. +func List(repo name.Repository, options ...Option) ([]string, error) { + o, err := makeOptions(repo.Registry, options...) + if err != nil { + return nil, err + } scopes := []string{repo.Scope(transport.PullScope)} - tr, err := transport.New(repo.Registry, auth, t, scopes) + tr, err := transport.New(repo.Registry, o.auth, o.transport, scopes) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 335e3fe5b..7edfcbabc 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -19,46 +19,88 @@ import ( "net/http" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" ) +// Option is a functional option for remote operations. +type Option func(*options) error + +type options struct { + auth authn.Authenticator + keychain authn.Keychain + transport http.RoundTripper + platform v1.Platform +} + +func makeOptions(reg name.Registry, opts ...Option) (*options, error) { + o := &options{ + auth: authn.Anonymous, + transport: http.DefaultTransport, + platform: defaultPlatform, + } + + for _, option := range opts { + if err := option(o); err != nil { + return nil, err + } + } + + if o.keychain != nil { + auth, err := o.keychain.Resolve(reg) + if err != nil { + return nil, err + } + if auth == authn.Anonymous { + log.Println("No matching credentials were found, falling back on anonymous") + } + o.auth = auth + } + + return o, nil +} + // WithTransport is a functional option for overriding the default transport -// on a remote image -func WithTransport(t http.RoundTripper) ImageOption { - return func(i *imageOpener) error { - i.transport = t +// for remote operations. +// +// The default transport its http.DefaultTransport. +func WithTransport(t http.RoundTripper) Option { + return func(o *options) error { + o.transport = t return nil } } // WithAuth is a functional option for overriding the default authenticator -// on a remote image -func WithAuth(auth authn.Authenticator) ImageOption { - return func(i *imageOpener) error { - i.auth = auth +// for remote operations. +// +// The default authenticator is authn.Anonymous. +func WithAuth(auth authn.Authenticator) Option { + return func(o *options) error { + o.auth = auth return nil } } // WithAuthFromKeychain is a functional option for overriding the default -// authenticator on a remote image using an authn.Keychain -func WithAuthFromKeychain(keys authn.Keychain) ImageOption { - return func(i *imageOpener) error { - auth, err := keys.Resolve(i.ref.Context().Registry) - if err != nil { - return err - } - if auth == authn.Anonymous { - log.Println("No matching credentials were found, falling back on anonymous") - } - i.auth = auth +// authenticator for remote operations, using an authn.Keychain to find +// credentials. +// +// The default authenticator is authn.Anonymous. +func WithAuthFromKeychain(keys authn.Keychain) Option { + return func(o *options) error { + o.keychain = keys return nil } } -func WithPlatform(p v1.Platform) ImageOption { - return func(i *imageOpener) error { - i.platform = p +// WithPlatform is a functional option for overriding the default platform +// that Image and Descriptor.Image use for resolving an index to an image. +// +// The default platform is amd64/linux. +func WithPlatform(p v1.Platform) Option { + return func(o *options) error { + o.platform = p return nil } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index f72ab276d..f9cd194ad 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -60,10 +60,14 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { // In case of redirect http.Client can use an empty Host, check URL too. if in.Host == bt.registry.RegistryStr() || in.URL.Host == bt.registry.RegistryStr() { in.Header.Set("Authorization", hdr) + + // When we ping() the registry, we determine whether to use http or https + // based on which scheme was successful. That is only valid for the + // registry server and not e.g. a separate token server or blob storage, + // so we should only override the scheme if the host is the registry. + in.URL.Scheme = bt.scheme } in.Header.Set("User-Agent", transportName) - - in.URL.Scheme = bt.scheme return bt.inner.RoundTrip(in) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 44885effa..5ca0b0881 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -48,6 +48,20 @@ func (e *Error) Error() string { } } +// ShouldRetry returns whether the request that preceded the error should be retried. +func (e *Error) ShouldRetry() bool { + if len(e.Errors) == 0 { + return false + } + for _, d := range e.Errors { + // TODO: Include other error types. + if d.Code != BlobUploadInvalidErrorCode { + return false + } + } + return true +} + // Diagnostic represents a single error returned by a Docker registry interaction. type Diagnostic struct { Code ErrorCode `json:"code"` diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 66f148155..557862aca 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -20,15 +20,15 @@ import ( "fmt" "io" "log" + "math" "net/http" "net/url" + "time" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" - "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "golang.org/x/sync/errgroup" ) @@ -40,14 +40,19 @@ type manifest interface { } // Write pushes the provided img to the specified image reference. -func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.RoundTripper) error { +func Write(ref name.Reference, img v1.Image, options ...Option) error { ls, err := img.Layers() if err != nil { return err } + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } + scopes := scopesForUploadingImage(ref, ls) - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } @@ -57,17 +62,17 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro } // Upload individual layers in goroutines and collect any errors. - // If we can dedupe by the layer digest, try to do so. If the layer is - // a stream.Layer, we can't dedupe and might re-upload. + // If we can dedupe by the layer digest, try to do so. If we can't determine + // the digest for whatever reason, we can't dedupe and might re-upload. var g errgroup.Group uploaded := map[v1.Hash]bool{} for _, l := range ls { l := l - if _, ok := l.(*stream.Layer); !ok { - h, err := l.Digest() - if err != nil { - return err - } + + // Streaming layers calculate their digests while uploading them. Assume + // an error here indicates we need to upload the layer. + h, err := l.Digest() + if err == nil { // If we can determine the layer's digest ahead of // time, use it to dedupe uploads. if uploaded[h] { @@ -81,14 +86,15 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro }) } - if l, err := partial.ConfigLayer(img); err == stream.ErrNotComputed { - // We can't read the ConfigLayer, because of streaming layers, since the - // config hasn't been calculated yet. + if l, err := partial.ConfigLayer(img); err != nil { + // We can't read the ConfigLayer, possibly because of streaming layers, + // since the layer DiffIDs haven't been calculated yet. Attempt to wait + // for the other layers to be uploaded, then try the config again. if err := g.Wait(); err != nil { return err } - // Now that all the layers are uploaded, upload the config file blob. + // Now that all the layers are uploaded, try to upload the config file blob. l, err := partial.ConfigLayer(img) if err != nil { return err @@ -96,9 +102,6 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro if err := w.uploadOne(l); err != nil { return err } - } else if err != nil { - // This is an actual error, not a streaming error, just return it. - return err } else { // We *can* read the ConfigLayer, so upload it concurrently with the layers. g.Go(func() error { @@ -285,19 +288,10 @@ func (w *writer) commitBlob(location, digest string) error { // uploadOne performs a complete upload of a single layer. func (w *writer) uploadOne(l v1.Layer) error { - var from, mount, digest string - if _, ok := l.(*stream.Layer); !ok { - // Layer isn't streamable, we should take advantage of that to - // skip uploading if possible. - // By sending ?digest= in the request, we'll also check that - // our computed digest matches the one computed by the - // registry. - h, err := l.Digest() - if err != nil { - return err - } - digest = h.String() - + var from, mount string + if h, err := l.Digest(); err == nil { + // If we know the digest, this isn't a streaming layer. Do an existence + // check so we can skip uploading the layer if possible. existing, err := w.checkExistingBlob(h) if err != nil { return err @@ -315,38 +309,56 @@ func (w *writer) uploadOne(l v1.Layer) error { } } - location, mounted, err := w.initiateUpload(from, mount) - if err != nil { - return err - } else if mounted { + tryUpload := func() error { + location, mounted, err := w.initiateUpload(from, mount) + if err != nil { + return err + } else if mounted { + h, err := l.Digest() + if err != nil { + return err + } + log.Printf("mounted blob: %s", h.String()) + return nil + } + + blob, err := l.Compressed() + if err != nil { + return err + } + location, err = w.streamBlob(blob, location) + if err != nil { + return err + } + h, err := l.Digest() if err != nil { return err } - log.Printf("mounted blob: %s", h.String()) + digest := h.String() + + if err := w.commitBlob(location, digest); err != nil { + return err + } + log.Printf("pushed blob: %s", digest) return nil } - - blob, err := l.Compressed() - if err != nil { - return err + const maxRetries = 2 + const backoffFactor = 0.5 + retries := 0 + for { + err := tryUpload() + if err == nil { + return nil + } + if te, ok := err.(*transport.Error); !(ok && te.ShouldRetry()) || retries >= maxRetries { + return err + } + log.Printf("retrying after error: %s", err) + retries++ + duration := time.Duration(backoffFactor*math.Pow(2, float64(retries))) * time.Second + time.Sleep(duration) } - location, err = w.streamBlob(blob, location) - if err != nil { - return err - } - - h, err := l.Digest() - if err != nil { - return err - } - digest = h.String() - - if err := w.commitBlob(location, digest); err != nil { - return err - } - log.Printf("pushed blob: %s", digest) - return nil } // commitImage does a PUT of the image's manifest. @@ -416,14 +428,18 @@ func scopesForUploadingImage(ref name.Reference, layers []v1.Layer) []string { // WriteIndex pushes the provided ImageIndex to the specified image reference. // WriteIndex will attempt to push all of the referenced manifests before // attempting to push the ImageIndex, to retain referential integrity. -func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, t http.RoundTripper) error { +func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { index, err := ii.IndexManifest() if err != nil { return err } + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } scopes := []string{ref.Scope(transport.PushScope)} - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } @@ -453,7 +469,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, return err } - if err := WriteIndex(ref, ii, auth, t); err != nil { + if err := WriteIndex(ref, ii, WithAuth(o.auth), WithTransport(o.transport)); err != nil { return err } case types.OCIManifestSchema1, types.DockerManifestSchema2: @@ -461,7 +477,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, if err != nil { return err } - if err := Write(ref, img, auth, t); err != nil { + if err := Write(ref, img, WithAuth(o.auth), WithTransport(o.transport)); err != nil { return err } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go index f8895a226..aa816359c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go @@ -24,6 +24,7 @@ import ( "sync" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" ) var ( @@ -81,6 +82,13 @@ func (l *Layer) Size() (int64, error) { return l.size, nil } +// MediaType implements v1.Layer +func (l *Layer) MediaType() (types.MediaType, error) { + // We return DockerLayer for now as uncompressed layers + // are unimplemented + return types.DockerLayer, nil +} + // Uncompressed implements v1.Layer. func (l *Layer) Uncompressed() (io.ReadCloser, error) { return nil, errors.New("NYI: stream.Layer.Uncompressed is not implemented") diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go index ced18735c..06ecd9a77 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go @@ -119,7 +119,7 @@ func (td tarDescriptor) findSpecifiedImageDescriptor(tag *name.Tag) (*singleImag } for _, img := range td { for _, tagStr := range img.RepoTags { - repoTag, err := name.NewTag(tagStr, name.WeakValidation) + repoTag, err := name.NewTag(tagStr) if err != nil { return nil, err } @@ -226,6 +226,13 @@ func (ulft *uncompressedLayerFromTarball) Uncompressed() (io.ReadCloser, error) return extractFileFromTar(ulft.opener, ulft.filePath) } +func (ulft *uncompressedLayerFromTarball) MediaType() (types.MediaType, error) { + // Technically the media type should be 'application/tar' but given that our + // v1.Layer doesn't force consumers to care about whether the layer is compressed + // we should be fine returning the DockerLayer media type + return types.DockerLayer, nil +} + func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) { cfg, err := partial.ConfigFile(i) if err != nil { @@ -310,6 +317,11 @@ func (clft *compressedLayerFromTarball) Compressed() (io.ReadCloser, error) { return extractFileFromTar(clft.opener, clft.filePath) } +// MediaType implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + // Size implements partial.CompressedLayer func (clft *compressedLayerFromTarball) Size() (int64, error) { r, err := clft.Compressed() diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index 00256e8f2..85c1b7838 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -15,12 +15,14 @@ package tarball import ( + "bytes" "compress/gzip" "io" "io/ioutil" "os" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -62,6 +64,10 @@ func (l *layer) Size() (int64, error) { return l.size, nil } +func (l *layer) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + // LayerFromFile returns a v1.Layer given a tarball func LayerFromFile(path string) (v1.Layer, error) { opener := func() (io.ReadCloser, error) { @@ -103,6 +109,18 @@ func LayerFromOpener(opener Opener) (v1.Layer, error) { }, nil } +// LayerFromReader returns a v1.Layer given a io.Reader. +func LayerFromReader(reader io.Reader) (v1.Layer, error) { + // Buffering due to Opener requiring multiple calls. + a, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + return LayerFromOpener(func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(a)), nil + }) +} + func computeDigest(opener Opener, compressed bool) (v1.Hash, int64, error) { rc, err := opener() if err != nil { From 35bb350a496327a3a1740df6c6c54b7c084916c9 Mon Sep 17 00:00:00 2001 From: Takeaki Matsumoto Date: Thu, 13 Jun 2019 11:17:03 +0900 Subject: [PATCH 07/45] Add support for S3 custom endpoint For S3-compatible object storage (like minio), this patch enable to use custom endpoint-url. Fix #531 --- pkg/buildcontext/s3.go | 17 +++++++++++++++-- pkg/constants/constants.go | 4 ++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pkg/buildcontext/s3.go b/pkg/buildcontext/s3.go index 5b77d9c4a..93d404c45 100644 --- a/pkg/buildcontext/s3.go +++ b/pkg/buildcontext/s3.go @@ -19,6 +19,7 @@ package buildcontext import ( "os" "path/filepath" + "strings" "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -36,9 +37,21 @@ type S3 struct { // UnpackTarFromBuildContext download and untar a file from s3 func (s *S3) UnpackTarFromBuildContext() (string, error) { bucket, item := util.GetBucketAndItem(s.context) - sess, err := session.NewSessionWithOptions(session.Options{ + option := session.Options{ SharedConfigState: session.SharedConfigEnable, - }) + } + endpoint := os.Getenv(constants.S3EndpointEnv) + forcePath := false + if strings.ToLower(os.Getenv(constants.S3ForcePathStyle)) == "true" { + forcePath = true + } + if endpoint != "" { + option.Config = aws.Config{ + Endpoint: aws.String(endpoint), + S3ForcePathStyle: aws.Bool(forcePath), + } + } + sess, err := session.NewSessionWithOptions(option) if err != nil { return bucket, err } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 0eb29cb9d..d0d84e3f3 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -70,6 +70,10 @@ const ( // Name of the .dockerignore file Dockerignore = ".dockerignore" + + // S3 Custom endpoint ENV name + S3EndpointEnv = "S3_ENDPOINT" + S3ForcePathStyle = "S3_FORCE_PATH_STYLE" ) // ScratchEnvVars are the default environment variables needed for a scratch image. From 7cc899b09e7fd7a2d5a546573c08dd0912df34e6 Mon Sep 17 00:00:00 2001 From: Andreas Bergmeier Date: Fri, 14 Jun 2019 21:34:55 +0200 Subject: [PATCH 08/45] Add SkipVerify support to CheckPushPermissions. (#663) Extract makeTransport, which allows using the current mechanism used for pushing. Fixes #628. --- pkg/executor/push.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/executor/push.go b/pkg/executor/push.go index a02d5b9b9..eea855710 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -64,7 +64,10 @@ func CheckPushPermissions(opts *config.KanikoOptions) error { if checked[destRef.Context().RepositoryStr()] { continue } - if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), http.DefaultTransport); err != nil { + + registryName := destRef.Repository.Registry.Name() + tr := makeTransport(opts, registryName) + if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), tr); err != nil { return errors.Wrapf(err, "checking push permission for %q", destRef) } checked[destRef.Context().RepositoryStr()] = true @@ -126,13 +129,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { return errors.Wrap(err, "resolving pushAuth") } - // Create a transport to set our user-agent. - tr := http.DefaultTransport - if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) { - tr.(*http.Transport).TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } + tr := makeTransport(opts, registryName) rt := &withUserAgent{t: tr} if err := remote.Write(destRef, image, pushAuth, rt); err != nil { @@ -143,6 +140,17 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { return nil } +func makeTransport(opts *config.KanikoOptions, registryName string) http.RoundTripper { + // Create a transport to set our user-agent. + tr := http.DefaultTransport + if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) { + tr.(*http.Transport).TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } + return tr +} + // pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache // if opts.Cache doesn't exist, infer the cache from the given destination func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error { From 7ed6fec424f96279eefca5e40abb9fbb3c2333ac Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 19 Jun 2019 10:41:35 -0700 Subject: [PATCH 09/45] Release v0.10.0 --- CHANGELOG.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Makefile | 2 +- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94dc5302e..5bc746dc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,75 @@ +# v0.10.0 Release - 2019-06-19 + +## Bug Fixes +* Fix kaniko caching [#639](https://github.com/GoogleContainerTools/kaniko/pull/639) +* chore: fix typo [#665](https://github.com/GoogleContainerTools/kaniko/pull/665) +* Fix file mode bug [#618](https://github.com/GoogleContainerTools/kaniko/pull/618) +* Fix arg handling for multi-stage images in COPY instructions. [#621](https://github.com/GoogleContainerTools/kaniko/pull/621) +* Fix parent directory permissions [#619](https://github.com/GoogleContainerTools/kaniko/pull/619) +* Environment variables should be replaced in URLs in ADD commands. [#580](https://github.com/GoogleContainerTools/kaniko/pull/580) +* Update the cache warmer to also save manifests. [#576](https://github.com/GoogleContainerTools/kaniko/pull/576) +* Fix typo in error message [#569](https://github.com/GoogleContainerTools/kaniko/pull/569) + +## New Features +* Add SkipVerify support to CheckPushPermissions. [#663](https://github.com/GoogleContainerTools/kaniko/pull/663) +* Creating github Build Context [#672](https://github.com/GoogleContainerTools/kaniko/pull/672) +* Add `--digest-file` flag to output built digest to file. [#655](https://github.com/GoogleContainerTools/kaniko/pull/655) +* README.md: update BuildKit/img comparison [#642](https://github.com/GoogleContainerTools/kaniko/pull/642) +* Add documentation for --verbosity flag [#634](https://github.com/GoogleContainerTools/kaniko/pull/634) +* Optimize file copying and stage saving between stages. [#605](https://github.com/GoogleContainerTools/kaniko/pull/605) +* Add an integration test for USER unpacking. [#600](https://github.com/GoogleContainerTools/kaniko/pull/600) +* Added missing documentation for --skip-tls-verify-pull arg [#593](https://github.com/GoogleContainerTools/kaniko/pull/593) +* README.me: update Buildah description [#586](https://github.com/GoogleContainerTools/kaniko/pull/586) +* Add missing tests for bucket util [#565](https://github.com/GoogleContainerTools/kaniko/pull/565) +* Look for manifests in the local cache next to the full images. [#570](https://github.com/GoogleContainerTools/kaniko/pull/570) +* Make the run_in_docker script support caching. [#564](https://github.com/GoogleContainerTools/kaniko/pull/564) +* Refactor snapshotting [#561](https://github.com/GoogleContainerTools/kaniko/pull/561) +* Stop storing a separate cache hash. [#560](https://github.com/GoogleContainerTools/kaniko/pull/560) +* Speed up workdir by always returning an empty filelist (rather than a… [#557](https://github.com/GoogleContainerTools/kaniko/pull/557) +* Refactor whitelist handling. [#559](https://github.com/GoogleContainerTools/kaniko/pull/559) +* Refactor the build loop to fetch stagebuilders earlier. [#558](https://github.com/GoogleContainerTools/kaniko/pull/558) + +## Additonal PRs +* Improve changelog dates [#657](https://github.com/GoogleContainerTools/kaniko/pull/657) +* Change verbose output from info to debug [#640](https://github.com/GoogleContainerTools/kaniko/pull/640) +* Check push permissions before building images [#622](https://github.com/GoogleContainerTools/kaniko/pull/622) +* Bump go-containerregistry to 8c1640add99804503b4126abc718931a4d93c31a [#609](https://github.com/GoogleContainerTools/kaniko/pull/609) +* Update go-containerregistry [#599](https://github.com/GoogleContainerTools/kaniko/pull/599) +* Log "Skipping paths under..." to debug [#571](https://github.com/GoogleContainerTools/kaniko/pull/571) + +Huge thank you for this release towards our contributors: +- Achilleas Pipinellis +- Adrian Duong +- Akihiro Suda +- Andreas Bergmeier +- Andrew Rynhard +- Anthony Weston +- Anurag Goel +- Balint Pato +- Christie Wilson +- Daisuke Taniwaki +- Dan Cecile +- Dirk Gustke +- dlorenc +- Fredrik Lönnegren +- Gijs +- Jake Shadle +- James Rawlings +- Jason Hall +- Johan Hernandez +- Johannes 'fish' Ziemke +- Kartik Verma +- linuxshokunin +- MMeent +- Myers Carpenter +- Nándor István Krácser +- Nao YONASHIRO +- Priya Wadhwa +- Sharif Elgamal +- Shuhei Kitagawa +- Valentin Rothberg +- Vincent Demeester + # v0.9.0 Release - 2019-02-08 ## Bug Fixes diff --git a/Makefile b/Makefile index 1e07ddac6..036f1e017 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ # Bump these on release VERSION_MAJOR ?= 0 -VERSION_MINOR ?= 9 +VERSION_MINOR ?= 10 VERSION_BUILD ?= 0 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD) From e2a4098b8ef9116ceff885637c7629724d76138a Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 19 Jun 2019 11:06:59 -0700 Subject: [PATCH 10/45] fixed dockerfile path --- integration/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index 3d046b53d..c11dd1378 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -237,7 +237,7 @@ func TestRun(t *testing.T) { func TestGitBuildcontext(t *testing.T) { repo := "github.com/GoogleContainerTools/kaniko" - dockerfile := "integration/dockerfiles/Dockerfile_git_buildcontext" + dockerfile := "integration/dockerfiles/Dockerfile_test_run_2" // Build with docker dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_test_git") From f0b9ad3a57f5e0ad16bb0b26097d6864a1e52006 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sat, 22 Jun 2019 11:45:07 -0300 Subject: [PATCH 11/45] feat: support specifying branch for cloning Signed-off-by: Carlos Alexandro Becker --- integration/integration_test.go | 43 +++++++++++++++++++++++++++++++++ pkg/buildcontext/git.go | 16 ++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index c11dd1378..b1bb4fd0d 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -278,6 +278,49 @@ func TestGitBuildcontext(t *testing.T) { checkContainerDiffOutput(t, diff, expected) } +func TestGitBuildContextWithBranch(t *testing.T) { + repo := "github.com/GoogleContainerTools/kaniko#v0.10.0" + dockerfile := "integration/dockerfiles/Dockerfile_test_run_2" + + // Build with docker + dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_test_git") + dockerCmd := exec.Command("docker", + append([]string{"build", + "-t", dockerImage, + "-f", dockerfile, + repo})...) + out, err := RunCommandWithoutTest(dockerCmd) + if err != nil { + t.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) + } + + // Build with kaniko + kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git") + kanikoCmd := exec.Command("docker", + append([]string{"run", + "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", + ExecutorImage, + "-f", dockerfile, + "-d", kanikoImage, + "-c", fmt.Sprintf("git://%s", repo)})...) + + out, err = RunCommandWithoutTest(kanikoCmd) + if err != nil { + t.Errorf("Failed to build image %s with kaniko command \"%s\": %v %s", dockerImage, kanikoCmd.Args, err, string(out)) + } + + // container-diff + daemonDockerImage := daemonPrefix + dockerImage + containerdiffCmd := exec.Command("container-diff", "diff", "--no-cache", + daemonDockerImage, kanikoImage, + "-q", "--type=file", "--type=metadata", "--json") + diff := RunCommand(containerdiffCmd, t) + t.Logf("diff = %s", string(diff)) + + expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) + checkContainerDiffOutput(t, diff, expected) +} + func TestLayers(t *testing.T) { offset := map[string]int{ "Dockerfile_test_add": 11, diff --git a/pkg/buildcontext/git.go b/pkg/buildcontext/git.go index 1aa8691be..7d61c6298 100644 --- a/pkg/buildcontext/git.go +++ b/pkg/buildcontext/git.go @@ -17,10 +17,13 @@ limitations under the License. package buildcontext import ( + "fmt" "os" + "strings" "github.com/GoogleContainerTools/kaniko/pkg/constants" git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" ) // Git unifies calls to download and unpack the build context. @@ -31,9 +34,18 @@ type Git struct { // UnpackTarFromBuildContext will provide the directory where Git Repository is Cloned func (g *Git) UnpackTarFromBuildContext() (string, error) { directory := constants.BuildContextDir + parts := strings.Split(g.context, "#") + url := "https://" + parts[0] + branch := "master" + if len(parts) > 1 { + branch = parts[1] + } + fmt.Println("will clone branch", branch) _, err := git.PlainClone(directory, false, &git.CloneOptions{ - URL: "https://" + g.context, - Progress: os.Stdout, + URL: url, + Progress: os.Stdout, + ReferenceName: plumbing.ReferenceName(branch), + SingleBranch: true, }) return directory, err } From c45e05f668f0077f266bf9e31657e2c344b4ad86 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sat, 22 Jun 2019 11:45:42 -0300 Subject: [PATCH 12/45] clean: remove debug msg Signed-off-by: Carlos Alexandro Becker --- pkg/buildcontext/git.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/buildcontext/git.go b/pkg/buildcontext/git.go index 7d61c6298..1909e8377 100644 --- a/pkg/buildcontext/git.go +++ b/pkg/buildcontext/git.go @@ -17,7 +17,6 @@ limitations under the License. package buildcontext import ( - "fmt" "os" "strings" @@ -40,7 +39,6 @@ func (g *Git) UnpackTarFromBuildContext() (string, error) { if len(parts) > 1 { branch = parts[1] } - fmt.Println("will clone branch", branch) _, err := git.PlainClone(directory, false, &git.CloneOptions{ URL: url, Progress: os.Stdout, From f578b09846bd5f26da89037c36c427bd2267caab Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sat, 22 Jun 2019 12:17:46 -0300 Subject: [PATCH 13/45] fix: remove single branch option Signed-off-by: Carlos Alexandro Becker --- pkg/buildcontext/git.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/pkg/buildcontext/git.go b/pkg/buildcontext/git.go index 1909e8377..9908350b1 100644 --- a/pkg/buildcontext/git.go +++ b/pkg/buildcontext/git.go @@ -34,16 +34,13 @@ type Git struct { func (g *Git) UnpackTarFromBuildContext() (string, error) { directory := constants.BuildContextDir parts := strings.Split(g.context, "#") - url := "https://" + parts[0] - branch := "master" - if len(parts) > 1 { - branch = parts[1] + options := git.CloneOptions{ + URL: "https://" + parts[0], + Progress: os.Stdout, } - _, err := git.PlainClone(directory, false, &git.CloneOptions{ - URL: url, - Progress: os.Stdout, - ReferenceName: plumbing.ReferenceName(branch), - SingleBranch: true, - }) + if len(parts) > 1 { + options.ReferenceName = plumbing.ReferenceName(parts[1]) + } + _, err := git.PlainClone(directory, false, &options) return directory, err } From 46a738f2b2661675588f804a55e051014cbd4455 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sat, 22 Jun 2019 12:18:42 -0300 Subject: [PATCH 14/45] fix: integration tests Signed-off-by: Carlos Alexandro Becker --- integration/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index b1bb4fd0d..fcd6246f9 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -279,7 +279,7 @@ func TestGitBuildcontext(t *testing.T) { } func TestGitBuildContextWithBranch(t *testing.T) { - repo := "github.com/GoogleContainerTools/kaniko#v0.10.0" + repo := "github.com/GoogleContainerTools/kaniko#refs/tags/v0.10.0" dockerfile := "integration/dockerfiles/Dockerfile_test_run_2" // Build with docker From 619fc5e59bda3428b2c573ad15166edc358832af Mon Sep 17 00:00:00 2001 From: Matthew Dawson Date: Thu, 11 Jul 2019 08:42:44 -0400 Subject: [PATCH 15/45] Make container layers captured using FS snapshots reproducible When a Dockerfile command requires using the TakeSnapshotFS function, the resulting layer has a random ordering of files. This causes the layer to have a non-deterministic hash defeating the reproducible flag. Issue #710 appears to document this issue as well. To fix, always sort the list of files to be added in scanFullFilesystem. This avoids trying to sort the file list during execution, and takes almost no time to complete. --- pkg/snapshot/snapshot.go | 3 +++ pkg/snapshot/snapshot_test.go | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 90638168d..2eb68ea2c 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "path/filepath" + "sort" "syscall" "github.com/GoogleContainerTools/kaniko/pkg/timing" @@ -186,6 +187,8 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { // Also add parent directories to keep the permission of them correctly. filesToAdd = filesWithParentDirs(filesToAdd) + sort.Strings(filesToAdd) + // Add files to the layered map for _, file := range filesToAdd { if err := s.l.Add(file); err != nil { diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index ea6f4bceb..bfa445f58 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -87,6 +87,45 @@ func TestSnapshotFSFileChange(t *testing.T) { } } +func TestSnapshotFSIsReproducible(t *testing.T) { + testDir, snapshotter, cleanup, err := setUpTestDir() + defer cleanup() + if err != nil { + t.Fatal(err) + } + // Make some changes to the filesystem + newFiles := map[string]string{ + "foo": "newbaz1", + "bar/bat": "baz", + } + if err := testutil.SetupFiles(testDir, newFiles); err != nil { + t.Fatalf("Error setting up fs: %s", err) + } + // Take another snapshot + tarPath, err := snapshotter.TakeSnapshotFS() + if err != nil { + t.Fatalf("Error taking snapshot of fs: %s", err) + } + + f, err := os.Open(tarPath) + if err != nil { + t.Fatal(err) + } + // Check contents of the snapshot, make sure contents are sorted by name + tr := tar.NewReader(f) + var filesInTar []string + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + filesInTar = append(filesInTar, hdr.Name) + } + if !sort.StringsAreSorted(filesInTar) { + t.Fatalf("Expected the file in the tar archive were sorted, actual list was not sorted: %v", filesInTar) + } +} + func TestSnapshotFSChangePermissions(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() defer cleanup() From 35bff80fdaf624831caceee5fe52690d556baffd Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sun, 14 Jul 2019 16:13:42 -0300 Subject: [PATCH 16/45] chore: empty commit Signed-off-by: Carlos Alexandro Becker From be0dac28237b153f4f58663e5ee5db7c06fdf2e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD=20=D0=9D=D0=B5=D0=B1=D0=B0?= =?UTF-8?q?=D0=BB=D1=83=D0=B5=D0=B2?= Date: Fri, 19 Jul 2019 18:36:29 +0500 Subject: [PATCH 17/45] fix unpacking archives via ADD --- pkg/commands/add.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/commands/add.go b/pkg/commands/add.go index 72f97653c..1256a840b 100644 --- a/pkg/commands/add.go +++ b/pkg/commands/add.go @@ -71,8 +71,12 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui } a.snapshotFiles = append(a.snapshotFiles, urlDest) } else if util.IsFileLocalTarArchive(fullPath) { - logrus.Infof("Unpacking local tar archive %s to %s", src, dest) - extractedFiles, err := util.UnpackLocalTarArchive(fullPath, dest) + tarDest, err := util.DestinationFilepath("", dest, config.WorkingDir) + if err != nil { + return err + } + logrus.Infof("Unpacking local tar archive %s to %s", src, tarDest) + extractedFiles, err := util.UnpackLocalTarArchive(fullPath, tarDest) if err != nil { return err } From 3422d5572af36c70c7875e4861703cb1c8588548 Mon Sep 17 00:00:00 2001 From: Taylor Barrella Date: Tue, 23 Jul 2019 18:10:22 -0400 Subject: [PATCH 18/45] Misc. small changes/refactoring (#712) --- cmd/executor/cmd/root.go | 2 +- pkg/commands/add.go | 2 +- pkg/commands/expose.go | 2 +- pkg/commands/volume.go | 2 +- pkg/dockerfile/dockerfile.go | 2 +- pkg/snapshot/layered_map.go | 4 ++-- pkg/snapshot/snapshot.go | 6 +++--- pkg/util/command_util.go | 26 ++++++++++++-------------- pkg/util/fs_util.go | 9 ++------- 9 files changed, 24 insertions(+), 31 deletions(-) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 1fc2672bc..0dd717f86 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -208,12 +208,12 @@ func resolveSourceContext() error { } if opts.Bucket != "" { if !strings.Contains(opts.Bucket, "://") { + // if no prefix use Google Cloud Storage as default for backwards compatibility opts.SrcContext = constants.GCSBuildContextPrefix + opts.Bucket } else { opts.SrcContext = opts.Bucket } } - // if no prefix use Google Cloud Storage as default for backwards compatibility contextExecutor, err := buildcontext.GetBuildContext(opts.SrcContext) if err != nil { return err diff --git a/pkg/commands/add.go b/pkg/commands/add.go index 72f97653c..27d666646 100644 --- a/pkg/commands/add.go +++ b/pkg/commands/add.go @@ -43,7 +43,7 @@ type AddCommand struct { // - If remote file has HTTP Last-Modified header, we set the mtime of the file to that timestamp // - If dest doesn't end with a slash, the filepath is inferred to be / // 2. If is a local tar archive: -// -If is a local tar archive, it is unpacked at the dest, as 'tar -x' would +// - it is unpacked at the dest, as 'tar -x' would func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { replacementEnvs := buildArgs.ReplacementEnvs(config.Env) diff --git a/pkg/commands/expose.go b/pkg/commands/expose.go index fa497f17d..9d56ee3ea 100644 --- a/pkg/commands/expose.go +++ b/pkg/commands/expose.go @@ -54,7 +54,7 @@ func (r *ExposeCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile. } protocol := strings.Split(p, "/")[1] if !validProtocol(protocol) { - return fmt.Errorf("Invalid protocol: %s", protocol) + return fmt.Errorf("invalid protocol: %s", protocol) } logrus.Infof("Adding exposed port: %s", p) existingPorts[p] = struct{}{} diff --git a/pkg/commands/volume.go b/pkg/commands/volume.go index 2ec0fcb2b..b6d92bf53 100644 --- a/pkg/commands/volume.go +++ b/pkg/commands/volume.go @@ -54,7 +54,7 @@ func (v *VolumeCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile. if _, err := os.Stat(volume); os.IsNotExist(err) { logrus.Infof("Creating directory %s", volume) if err := os.MkdirAll(volume, 0755); err != nil { - return fmt.Errorf("Could not create directory for volume %s: %s", volume, err) + return fmt.Errorf("could not create directory for volume %s: %s", volume, err) } } } diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 331219159..8865b17d0 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -111,7 +111,7 @@ func Parse(b []byte) ([]instructions.Stage, []instructions.ArgCommand, error) { if err != nil { return nil, nil, err } - return stages, metaArgs, err + return stages, metaArgs, nil } // targetStage returns the index of the target stage kaniko is trying to build diff --git a/pkg/snapshot/layered_map.go b/pkg/snapshot/layered_map.go index ad2bee4d2..56e8da4f0 100644 --- a/pkg/snapshot/layered_map.go +++ b/pkg/snapshot/layered_map.go @@ -103,13 +103,13 @@ func (l *LayeredMap) Add(s string) error { // Use hash function and add to layers newV, err := l.hasher(s) if err != nil { - return fmt.Errorf("Error creating hash for %s: %v", s, err) + return fmt.Errorf("error creating hash for %s: %v", s, err) } l.layers[len(l.layers)-1][s] = newV return nil } -// CheckFileChange checkes whether a given file changed +// CheckFileChange checks whether a given file changed // from the current layered map by its hashing function. // Returns true if the file is changed. func (l *LayeredMap) CheckFileChange(s string) (bool, error) { diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 90638168d..02f5ffcc7 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -80,7 +80,7 @@ func (s *Snapshotter) TakeSnapshot(files []string) (string, error) { // Add files to the layered map for _, file := range filesToAdd { if err := s.l.Add(file); err != nil { - return "", fmt.Errorf("Unable to add file %s to layered map: %s", file, err) + return "", fmt.Errorf("unable to add file %s to layered map: %s", file, err) } } @@ -183,13 +183,13 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { } } - // Also add parent directories to keep the permission of them correctly. + // Also add parent directories to keep their permissions correctly. filesToAdd = filesWithParentDirs(filesToAdd) // Add files to the layered map for _, file := range filesToAdd { if err := s.l.Add(file); err != nil { - return nil, nil, fmt.Errorf("Unable to add file %s to layered map: %s", file, err) + return nil, nil, fmt.Errorf("unable to add file %s to layered map: %s", file, err) } } diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go index a38972ced..6d9471ebd 100644 --- a/pkg/util/command_util.go +++ b/pkg/util/command_util.go @@ -55,7 +55,7 @@ func ResolveEnvironmentReplacementList(values, envs []string, isFilepath bool) ( // ResolveEnvironmentReplacement resolves replacing env variables in some text from envs // It takes in a string representation of the command, the value to be resolved, and a list of envs (config.Env) -// Ex: fp = $foo/newdir, envs = [foo=/foodir], then this should return /foodir/newdir +// Ex: value = $foo/newdir, envs = [foo=/foodir], then this should return /foodir/newdir // The dockerfile/shell package handles processing env values // It handles escape characters and supports expansion from the config.Env array // Shlex handles some of the following use cases (these and more are tested in integration tests) @@ -325,13 +325,12 @@ func GetUserFromUsername(userStr string, groupStr string) (string, string, error // Lookup by username userObj, err := user.Lookup(userStr) if err != nil { - if _, ok := err.(user.UnknownUserError); ok { - // Lookup by id - userObj, err = user.LookupId(userStr) - if err != nil { - return "", "", err - } - } else { + if _, ok := err.(user.UnknownUserError); !ok { + return "", "", err + } + // Lookup by id + userObj, err = user.LookupId(userStr) + if err != nil { return "", "", err } } @@ -341,12 +340,11 @@ func GetUserFromUsername(userStr string, groupStr string) (string, string, error if groupStr != "" { group, err = user.LookupGroup(groupStr) if err != nil { - if _, ok := err.(user.UnknownGroupError); ok { - group, err = user.LookupGroupId(groupStr) - if err != nil { - return "", "", err - } - } else { + if _, ok := err.(user.UnknownGroupError); !ok { + return "", "", err + } + group, err = user.LookupGroupId(groupStr) + if err != nil { return "", "", err } } diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 5f72c7784..d22dcb4aa 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -188,7 +188,7 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error { switch hdr.Typeflag { case tar.TypeReg: logrus.Debugf("creating file %s", path) - // It's possible a file is in the tar before it's directory. + // It's possible a file is in the tar before its directory. if _, err := os.Stat(dir); os.IsNotExist(err) { logrus.Debugf("base %s for file %s does not exist. Creating.", base, path) if err := os.MkdirAll(dir, 0755); err != nil { @@ -288,12 +288,7 @@ func checkWhitelistRoot(root string) bool { if root == constants.RootDir { return false } - for _, wl := range whitelist { - if HasFilepathPrefix(root, wl.Path, wl.PrefixMatchOnly) { - return true - } - } - return false + return CheckWhitelist(root) } // Get whitelist from roots of mounted files From 7750094ec172781d920945b92c05447b404ee9d9 Mon Sep 17 00:00:00 2001 From: "v.rul" Date: Wed, 24 Jul 2019 16:33:08 +0300 Subject: [PATCH 19/45] Add checking image presence in cache prior to downloading it This changes allow to use kaniko-warmer multiple times without unnecessary docker image downloads. To check image presence in cache directory I'm using existing cache function that is used by kaniko-executor. I've considered building separate function to only check image presence, but it will have pretty much the same code. Questionable decision is to embed CacheOptions type to KanikoOptions and WarmerOptions. Probably this should be resolved by creating interface providing needed options and implement it both mentioned structs. But I've struggled to get a meaningfull name to it. To replicate previous behaviour of downloading regardless of cache state I've added --force(-f) option. This changes provides crucial speed-up when downloading images from remote registry is slow. Closes #722 --- cmd/warmer/cmd/root.go | 3 +++ pkg/cache/cache.go | 2 +- pkg/cache/warm.go | 8 ++++++++ pkg/config/options.go | 14 ++++++++++---- pkg/util/image_util.go | 2 +- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/cmd/warmer/cmd/root.go b/cmd/warmer/cmd/root.go index 0e4908d2b..49850d432 100644 --- a/cmd/warmer/cmd/root.go +++ b/cmd/warmer/cmd/root.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "os" + "time" "github.com/GoogleContainerTools/kaniko/pkg/cache" "github.com/GoogleContainerTools/kaniko/pkg/config" @@ -61,6 +62,8 @@ var RootCmd = &cobra.Command{ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().VarP(&opts.Images, "image", "i", "Image to cache. Set it repeatedly for multiple images.") RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "c", "/cache", "Directory of the cache.") + RootCmd.PersistentFlags().BoolVarP(&opts.Force, "force", "f", false, "Force cache overwriting.") + RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.") } // addHiddenFlags marks certain flags as hidden from the executor help text diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 6ec9d0ce7..fe1d3276f 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -114,7 +114,7 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { } // LocalSource retieves a source image from a local cache given cacheKey -func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { +func LocalSource(opts *config.CacheOptions, cacheKey string) (v1.Image, error) { cache := opts.CacheDir if cache == "" { return nil, nil diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go index c03746e0b..ce1f55cb3 100644 --- a/pkg/cache/warm.go +++ b/pkg/cache/warm.go @@ -50,6 +50,14 @@ func WarmCache(opts *config.WarmerOptions) error { return errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image)) } cachePath := path.Join(cacheDir, digest.String()) + + if !opts.Force { + _, err := LocalSource(&opts.CacheOptions, digest.String()) + if err == nil { + continue + } + } + err = tarball.WriteToFile(cachePath, cacheRef, img) if err != nil { return errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image)) diff --git a/pkg/config/options.go b/pkg/config/options.go index ff4d60a13..627a51531 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -20,8 +20,15 @@ import ( "time" ) +// CacheOptions are base image cache options that are set by command line arguments +type CacheOptions struct { + CacheDir string + CacheTTL time.Duration +} + // KanikoOptions are options that are set by command line arguments type KanikoOptions struct { + CacheOptions DockerfilePath string SrcContext string SnapshotMode string @@ -29,7 +36,6 @@ type KanikoOptions struct { TarPath string Target string CacheRepo string - CacheDir string DigestFile string Destinations multiArg BuildArgs multiArg @@ -42,13 +48,13 @@ type KanikoOptions struct { NoPush bool Cache bool Cleanup bool - CacheTTL time.Duration InsecureRegistries multiArg SkipTLSVerifyRegistries multiArg } // WarmerOptions are options that are set by command line arguments to the cache warmer. type WarmerOptions struct { - Images multiArg - CacheDir string + CacheOptions + Images multiArg + Force bool } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index dcc09ada8..0978a6fef 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -149,5 +149,5 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { cacheKey = d.String() } - return cache.LocalSource(opts, cacheKey) + return cache.LocalSource(&opts.CacheOptions, cacheKey) } From 80421f2a73d49057ab0aea8170afbf867475855c Mon Sep 17 00:00:00 2001 From: Luke Wood Date: Wed, 24 Jul 2019 21:09:18 +0100 Subject: [PATCH 20/45] Update version of go-containerregistry. (#724) Brings in a change from upstream to resolve ports to well-known values when comparing Host values to decide whether or not to send the Bearer Authorization header when pushing an image. Upstream issue is https://github.com/google/go-containerregistry/issues/472. --- Gopkg.lock | 7 +- Gopkg.toml | 2 +- .../pkg/authn/keychain.go | 8 +- .../pkg/internal/retry/retry.go | 68 ++++++++++++++ .../go-containerregistry/pkg/logs/logs.go | 29 ++++++ .../pkg/v1/mutate/mutate.go | 4 +- .../go-containerregistry/pkg/v1/platform.go | 1 + .../pkg/v1/remote/descriptor.go | 10 ++- .../pkg/v1/remote/index.go | 50 ++++++++++- .../pkg/v1/remote/options.go | 8 +- .../pkg/v1/remote/transport/basic.go | 2 +- .../pkg/v1/remote/transport/bearer.go | 37 +++++++- .../pkg/v1/remote/transport/error.go | 4 +- .../pkg/v1/remote/transport/retry.go | 89 +++++++++++++++++++ .../pkg/v1/remote/write.go | 39 ++++---- 15 files changed, 317 insertions(+), 41 deletions(-) create mode 100644 vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/logs/logs.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go diff --git a/Gopkg.lock b/Gopkg.lock index 44926c25d..bd23f0578 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -445,11 +445,13 @@ version = "v0.2.0" [[projects]] - digest = "1:3ccc9b3dfd6b951b46e6e1c499af589fbc35c7e1172d6d840cbe836ae08d3536" + digest = "1:16c8837e951303ef6388132bc875337660a48ea2dedf1c941ca118ea92d2a3d2" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", "pkg/authn/k8schain", + "pkg/internal/retry", + "pkg/logs", "pkg/name", "pkg/v1", "pkg/v1/daemon", @@ -465,7 +467,7 @@ "pkg/v1/v1util", ] pruneopts = "NUT" - revision = "bb17f50c1bc6808972811ed2894ecaaeb5de68ad" + revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74" [[projects]] digest = "1:f4f203acd8b11b8747bdcd91696a01dbc95ccb9e2ca2db6abf81c3a4f5e950ce" @@ -1384,6 +1386,7 @@ "golang.org/x/oauth2", "golang.org/x/sync/errgroup", "gopkg.in/src-d/go-git.v4", + "gopkg.in/src-d/go-git.v4/plumbing", "k8s.io/client-go/discovery", ] solver-name = "gps-cdcl" diff --git a/Gopkg.toml b/Gopkg.toml index 163d821ad..64d3b5b7b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,7 +37,7 @@ required = [ [[constraint]] name = "github.com/google/go-containerregistry" - revision = "bb17f50c1bc6808972811ed2894ecaaeb5de68ad" + revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74" [[override]] name = "k8s.io/apimachinery" diff --git a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go index aee1fedb9..8b2ead5db 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go +++ b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go @@ -19,11 +19,11 @@ import ( "errors" "fmt" "io/ioutil" - "log" "os" "path/filepath" "runtime" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" ) @@ -100,19 +100,19 @@ var ( func (dk *defaultKeychain) Resolve(reg name.Registry) (Authenticator, error) { dir, err := configDir() if err != nil { - log.Printf("Unable to determine config dir: %v", err) + logs.Warn.Printf("Unable to determine config dir: %v", err) return Anonymous, nil } file := filepath.Join(dir, "config.json") content, err := ioutil.ReadFile(file) if err != nil { - log.Printf("Unable to read %q: %v", file, err) + logs.Warn.Printf("Unable to read %q: %v", file, err) return Anonymous, nil } var cf cfg if err := json.Unmarshal(content, &cf); err != nil { - log.Printf("Unable to parse %q: %v", file, err) + logs.Warn.Printf("Unable to parse %q: %v", file, err) return Anonymous, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go b/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go new file mode 100644 index 000000000..87f730955 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go @@ -0,0 +1,68 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package retry provides methods for retrying operations. It is a thin wrapper +// around k8s.io/apimachinery/pkg/util/wait to make certain operations easier. +package retry + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/util/wait" +) + +// This is implemented by several errors in the net package as well as our +// transport.Error. +type temporary interface { + Temporary() bool +} + +// IsTemporary returns true if err implements Temporary() and it returns true. +func IsTemporary(err error) bool { + if te, ok := err.(temporary); ok && te.Temporary() { + return true + } + return false +} + +// IsNotNil returns true if err is not nil. +func IsNotNil(err error) bool { + return err != nil +} + +// Predicate determines whether an error should be retried. +type Predicate func(error) (retry bool) + +// Retry retries a given function, f, until a predicate is satisfied, using +// exponential backoff. If the predicate is never satisfied, it will return the +// last error returned by f. +func Retry(f func() error, p Predicate, backoff wait.Backoff) (err error) { + if f == nil { + return fmt.Errorf("nil f passed to retry") + } + if p == nil { + return fmt.Errorf("nil p passed to retry") + } + + condition := func() (bool, error) { + err = f() + if p(err) { + return false, nil + } + return true, err + } + + wait.ExponentialBackoff(backoff, condition) + return +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go b/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go new file mode 100644 index 000000000..af3c1a3b7 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go @@ -0,0 +1,29 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logs exposes the loggers used by this library. +package logs + +import ( + "io/ioutil" + "log" +) + +var ( + // Warn is used to log non-fatal errors. + Warn = log.New(ioutil.Discard, "", log.LstdFlags) + + // Progress is used to log notable, successful events. + Progress = log.New(ioutil.Discard, "", log.LstdFlags) +) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index 11262f444..813205dad 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -77,6 +77,8 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) { } cf.Config = cfg + // Downstream tooling expects these to match. + cf.ContainerConfig = cfg return ConfigFile(base, cf) } @@ -468,7 +470,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { // Copy basic config over cfg.Config = ocf.Config - cfg.ContainerConfig = ocf.ContainerConfig + cfg.ContainerConfig = ocf.Config // Downstream tooling expects these to match. // Strip away timestamps from the config file cfg.Created = v1.Time{Time: t} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go b/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go index df9b2959e..bb9886433 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go @@ -21,4 +21,5 @@ type Platform struct { OSVersion string `json:"os.version,omitempty"` OSFeatures []string `json:"os.features,omitempty"` Variant string `json:"variant,omitempty"` + Features []string `json:"features,omitempty"` } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go index 078de3a0b..144b99ecc 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -225,12 +225,16 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType } mediaType := types.MediaType(resp.Header.Get("Content-Type")) + contentDigest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest")) + if err == nil && mediaType == types.DockerManifestSchema1Signed { + // If we can parse the digest from the header, and it's a signed schema 1 + // manifest, let's use that for the digest to appease older registries. + digest = contentDigest + } // Validate the digest matches what we asked for, if pulling by digest. if dgst, ok := ref.(name.Digest); ok { - if mediaType == types.DockerManifestSchema1Signed { - // Digests for this are stupid to calculate, ignore it. - } else if digest.String() != dgst.DigestStr() { + if digest.String() != dgst.DigestStr() { return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) } } else { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index 1303dda9c..043dc83b6 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -119,7 +119,7 @@ func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) { return desc.Image() } -// This naively matches the first manifest with matching Architecture and OS. +// This naively matches the first manifest with matching platform attributes. // // We should probably use this instead: // github.com/containerd/containerd/platforms @@ -138,7 +138,7 @@ func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) p = *childDesc.Platform } - if platform.Architecture == p.Architecture && platform.OS == p.OS { + if matchesPlatform(p, platform) { return r.childDescriptor(childDesc, platform) } } @@ -182,3 +182,49 @@ func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) platform: platform, }, nil } + +// matchesPlatform checks if the given platform matches the required platforms. +// The given platform matches the required platform if +// - architecture and OS are identical. +// - OS version and variant are identical if provided. +// - features and OS features of the required platform are subsets of those of the given platform. +func matchesPlatform(given, required v1.Platform) bool { + // Required fields that must be identical. + if given.Architecture != required.Architecture || given.OS != required.OS { + return false + } + + // Optional fields that may be empty, but must be identical if provided. + if required.OSVersion != "" && given.OSVersion != required.OSVersion { + return false + } + if required.Variant != "" && given.Variant != required.Variant { + return false + } + + // Verify required platform's features are a subset of given platform's features. + if !isSubset(given.OSFeatures, required.OSFeatures) { + return false + } + if !isSubset(given.Features, required.Features) { + return false + } + + return true +} + +// isSubset checks if the required array of strings is a subset of the given lst. +func isSubset(lst, required []string) bool { + set := make(map[string]bool) + for _, value := range lst { + set[value] = true + } + + for _, value := range required { + if _, ok := set[value]; !ok { + return false + } + } + + return true +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 7edfcbabc..c60344840 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -15,12 +15,13 @@ package remote import ( - "log" "net/http" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) // Option is a functional option for remote operations. @@ -52,11 +53,14 @@ func makeOptions(reg name.Registry, opts ...Option) (*options, error) { return nil, err } if auth == authn.Anonymous { - log.Println("No matching credentials were found, falling back on anonymous") + logs.Warn.Println("No matching credentials were found, falling back on anonymous") } o.auth = auth } + // Wrap the transport in something that can retry network flakes. + o.transport = transport.NewRetry(o.transport) + return o, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go index e77f47f69..b98b4a625 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go @@ -40,7 +40,7 @@ func (bt *basicTransport) RoundTrip(in *http.Request) (*http.Response, error) { // we are redirected, only set it when the authorization header matches // the host with which we are interacting. // In case of redirect http.Client can use an empty Host, check URL too. - if in.Host == bt.target || in.URL.Host == bt.target { + if hdr != "" && (in.Host == bt.target || in.URL.Host == bt.target) { in.Header.Set("Authorization", hdr) } in.Header.Set("User-Agent", transportName) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index f9cd194ad..326708b97 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -18,8 +18,10 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/url" + "strings" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" @@ -45,6 +47,11 @@ type bearerTransport struct { var _ http.RoundTripper = (*bearerTransport)(nil) +var portMap = map[string]string{ + "http": "80", + "https": "443", +} + // RoundTrip implements http.RoundTripper func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { sendRequest := func() (*http.Response, error) { @@ -58,7 +65,10 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { // we are redirected, only set it when the authorization header matches // the registry with which we are interacting. // In case of redirect http.Client can use an empty Host, check URL too. - if in.Host == bt.registry.RegistryStr() || in.URL.Host == bt.registry.RegistryStr() { + canonicalHeaderHost := bt.canonicalAddress(in.Host) + canonicalURLHost := bt.canonicalAddress(in.URL.Host) + canonicalRegistryHost := bt.canonicalAddress(bt.registry.RegistryStr()) + if canonicalHeaderHost == canonicalRegistryHost || canonicalURLHost == canonicalRegistryHost { in.Header.Set("Authorization", hdr) // When we ping() the registry, we determine whether to use http or https @@ -144,3 +154,28 @@ func (bt *bearerTransport) refresh() error { bt.bearer = &bearer return nil } + +func (bt *bearerTransport) canonicalAddress(host string) (address string) { + // The host may be any one of: + // - hostname + // - hostname:port + // - ipv4 + // - ipv4:port + // - ipv6 + // - [ipv6]:port + // As net.SplitHostPort returns an error if the host does not contain a port, we should only attempt + // to call it when we know that the address contains a port + if strings.Count(host, ":") == 1 || (strings.Count(host, ":") >= 2 && strings.Contains(host, "]:")) { + hostname, port, err := net.SplitHostPort(host) + if err != nil { + return host + } + if port == "" { + port = portMap[bt.scheme] + } + + return net.JoinHostPort(hostname, port) + } + + return net.JoinHostPort(host, portMap[bt.scheme]) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 5ca0b0881..3673a341b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -48,8 +48,8 @@ func (e *Error) Error() string { } } -// ShouldRetry returns whether the request that preceded the error should be retried. -func (e *Error) ShouldRetry() bool { +// Temporary returns whether the request that preceded the error is temporary. +func (e *Error) Temporary() bool { if len(e.Errors) == 0 { return false } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go new file mode 100644 index 000000000..b6b2181da --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go @@ -0,0 +1,89 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "net/http" + "time" + + "github.com/google/go-containerregistry/pkg/internal/retry" + "k8s.io/apimachinery/pkg/util/wait" +) + +// Sleep for 0.1, 0.3, 0.9, 2.7 seconds. This should cover networking blips. +var defaultBackoff = wait.Backoff{ + Duration: 100 * time.Millisecond, + Factor: 3.0, + Jitter: 0.1, + Steps: 5, +} + +var _ http.RoundTripper = (*retryTransport)(nil) + +// retryTransport wraps a RoundTripper and retries temporary network errors. +type retryTransport struct { + inner http.RoundTripper + backoff wait.Backoff + predicate retry.Predicate +} + +// Option is a functional option for retryTransport. +type Option func(*options) + +type options struct { + backoff wait.Backoff + predicate retry.Predicate +} + +// WithRetryBackoff sets the backoff for retry operations. +func WithRetryBackoff(backoff wait.Backoff) Option { + return func(o *options) { + o.backoff = backoff + } +} + +// WithRetryPredicate sets the predicate for retry operations. +func WithRetryPredicate(predicate func(error) bool) Option { + return func(o *options) { + o.predicate = predicate + } +} + +// NewRetry returns a transport that retries errors. +func NewRetry(inner http.RoundTripper, opts ...Option) http.RoundTripper { + o := &options{ + backoff: defaultBackoff, + predicate: retry.IsTemporary, + } + + for _, opt := range opts { + opt(o) + } + + return &retryTransport{ + inner: inner, + backoff: o.backoff, + predicate: o.predicate, + } +} + +func (t *retryTransport) RoundTrip(in *http.Request) (out *http.Response, err error) { + roundtrip := func() error { + out, err = t.inner.RoundTrip(in) + return err + } + retry.Retry(roundtrip, t.predicate, t.backoff) + return +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 557862aca..8806aa187 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -19,18 +19,19 @@ import ( "errors" "fmt" "io" - "log" - "math" "net/http" "net/url" "time" + "github.com/google/go-containerregistry/pkg/internal/retry" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/util/wait" ) type manifest interface { @@ -297,7 +298,7 @@ func (w *writer) uploadOne(l v1.Layer) error { return err } if existing { - log.Printf("existing blob: %v", h) + logs.Progress.Printf("existing blob: %v", h) return nil } @@ -318,7 +319,7 @@ func (w *writer) uploadOne(l v1.Layer) error { if err != nil { return err } - log.Printf("mounted blob: %s", h.String()) + logs.Progress.Printf("mounted blob: %s", h.String()) return nil } @@ -340,25 +341,19 @@ func (w *writer) uploadOne(l v1.Layer) error { if err := w.commitBlob(location, digest); err != nil { return err } - log.Printf("pushed blob: %s", digest) + logs.Progress.Printf("pushed blob: %s", digest) return nil } - const maxRetries = 2 - const backoffFactor = 0.5 - retries := 0 - for { - err := tryUpload() - if err == nil { - return nil - } - if te, ok := err.(*transport.Error); !(ok && te.ShouldRetry()) || retries >= maxRetries { - return err - } - log.Printf("retrying after error: %s", err) - retries++ - duration := time.Duration(backoffFactor*math.Pow(2, float64(retries))) * time.Second - time.Sleep(duration) + + // Try this three times, waiting 1s after first failure, 3s after second. + backoff := wait.Backoff{ + Duration: 1.0 * time.Second, + Factor: 3.0, + Jitter: 0.1, + Steps: 3, } + + return retry.Retry(tryUpload, retry.IsTemporary, backoff) } // commitImage does a PUT of the image's manifest. @@ -397,7 +392,7 @@ func (w *writer) commitImage(man manifest) error { } // The image was successfully pushed! - log.Printf("%v: digest: %v size: %d", w.ref, digest, len(raw)) + logs.Progress.Printf("%v: digest: %v size: %d", w.ref, digest, len(raw)) return nil } @@ -458,7 +453,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { return err } if exists { - log.Printf("existing manifest: %v", desc.Digest) + logs.Progress.Printf("existing manifest: %v", desc.Digest) continue } From 9454b5d28ba28c0718b9651dd6cd7867527da457 Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Wed, 31 Jul 2019 10:30:13 +0200 Subject: [PATCH 21/45] Document how to build from git reference --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 18d1d4e72..466dd4ab5 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Right now, kaniko supports these storage solutions: - GCS Bucket - S3 Bucket - Local Directory +- Git Repository _Note: the local directory option refers to a directory within the kaniko container. If you wish to use this option, you will need to mount in your build context into the container as a directory._ @@ -114,12 +115,12 @@ gsutil cp context.tar.gz gs:// When running kaniko, use the `--context` flag with the appropriate prefix to specify the location of your build context: -| Source | Prefix | -|---------|---------| -| Local Directory | dir://[path to a directory in the kaniko container] | -| GCS Bucket | gs://[bucket name]/[path to .tar.gz] | -| S3 Bucket | s3://[bucket name]/[path to .tar.gz] | -| Git Repository | git://[repository url] | +| Source | Prefix | Example | +|---------|---------|---------| +| Local Directory | dir://[path to a directory in the kaniko container] | `dir:///workspace` | +| GCS Bucket | gs://[bucket name]/[path to .tar.gz] | `gs://kaniko-bucket/path/to/context.tar.gz` | +| S3 Bucket | s3://[bucket name]/[path to .tar.gz] | `s3://kaniko-bucket/path/to/context.tar.gz` | +| Git Repository | git://[repository url][#reference] | `git://github.com/acme/myproject.git#refs/heads/mybranch` | If you don't specify a prefix, kaniko will assume a local directory. For example, to use a GCS bucket called `kaniko-bucket`, you would pass in `--context=gs://kaniko-bucket/path/to/context.tar.gz`. From 8a24115b6afa62db87827d6988328b9fa2ff415d Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 2 Aug 2019 13:25:20 -0700 Subject: [PATCH 22/45] Prevent panic on nil image --- pkg/cache/cache.go | 2 +- pkg/cache/warm.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 6ec9d0ce7..d7e17be2c 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -88,7 +88,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) { // Layer is stale, rebuild it. if expiry.Before(time.Now()) { logrus.Infof("Cache entry expired: %s", cache) - return nil, errors.New(fmt.Sprintf("Cache entry expired: %s", cache)) + return nil, fmt.Errorf("Cache entry expired: %s", cache) } // Force the manifest to be populated diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go index c03746e0b..3a78716a3 100644 --- a/pkg/cache/warm.go +++ b/pkg/cache/warm.go @@ -29,6 +29,7 @@ import ( "github.com/sirupsen/logrus" ) +// WarmCache populates the cache func WarmCache(opts *config.WarmerOptions) error { cacheDir := opts.CacheDir images := opts.Images @@ -41,7 +42,7 @@ func WarmCache(opts *config.WarmerOptions) error { return errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image)) } img, err := remote.Image(cacheRef) - if err != nil { + if err != nil || img == nil { return errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image)) } From 06638fa4c14574f0f85ce48a1a19467282ea87e3 Mon Sep 17 00:00:00 2001 From: Xueshan Feng Date: Thu, 8 Aug 2019 11:49:56 -0700 Subject: [PATCH 23/45] Bailout when there is not enough input arguments Currently the code carries on even missing required arguments. --- run_in_docker.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run_in_docker.sh b/run_in_docker.sh index b3bdd4988..9397f1a71 100755 --- a/run_in_docker.sh +++ b/run_in_docker.sh @@ -15,8 +15,9 @@ #!/bin/bash set -e -if [ $# -lt 3 ]; - then echo "Usage: run_in_docker.sh " +if [ $# -lt 3 ]; then + echo "Usage: run_in_docker.sh " + exit 1 fi dockerfile=$1 From c75749b840dffb83e56ce48a0a75c2aec2a80397 Mon Sep 17 00:00:00 2001 From: xanonid Date: Mon, 29 Jul 2019 20:07:31 +0200 Subject: [PATCH 24/45] Do not use leading slashes for paths in layer tarballs to be more compatible with docker --- pkg/util/tar_util.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go index bc1cc67a0..f358f48ad 100644 --- a/pkg/util/tar_util.go +++ b/pkg/util/tar_util.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "syscall" "github.com/docker/docker/pkg/archive" @@ -74,7 +75,14 @@ func (t *Tar) AddFileToTar(p string) error { if err != nil { return err } - hdr.Name = p + + if p != "/" { + // Docker uses no leading / in the tarball + hdr.Name = strings.TrimLeft(p, "/") + } else { + // allow entry for / to preserve permission changes etc. (currently ignored anyway by Docker runtime) + hdr.Name = p + } hardlink, linkDst := t.checkHardlink(p, i) if hardlink { From 1fa2527a7b0f73e00fa9ae25273c5fe87685b7a1 Mon Sep 17 00:00:00 2001 From: xanonid Date: Mon, 29 Jul 2019 20:08:34 +0200 Subject: [PATCH 25/45] Adapt tests such that expected paths do not contain a leading / --- pkg/snapshot/snapshot_test.go | 22 ++++++++++++++-------- pkg/util/fs_util.go | 18 ++++++++++++++++++ pkg/util/fs_util_test.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index ea6f4bceb..39b40415a 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "sort" + "strings" "testing" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -31,6 +32,7 @@ import ( func TestSnapshotFSFileChange(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { t.Fatal(err) @@ -55,16 +57,16 @@ func TestSnapshotFSFileChange(t *testing.T) { } // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles tr := tar.NewReader(f) - fooPath := filepath.Join(testDir, "foo") - batPath := filepath.Join(testDir, "bar/bat") + fooPath := filepath.Join(testDirWithoutLeadingSlash, "foo") + batPath := filepath.Join(testDirWithoutLeadingSlash, "bar/bat") snapshotFiles := map[string]string{ fooPath: "newbaz1", batPath: "baz", } - for _, dir := range util.ParentDirectories(fooPath) { + for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(fooPath) { snapshotFiles[dir] = "" } - for _, dir := range util.ParentDirectories(batPath) { + for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) { snapshotFiles[dir] = "" } numFiles := 0 @@ -89,12 +91,14 @@ func TestSnapshotFSFileChange(t *testing.T) { func TestSnapshotFSChangePermissions(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { t.Fatal(err) } // Change permissions on a file batPath := filepath.Join(testDir, "bar/bat") + batPathWithoutLeadingSlash := filepath.Join(testDirWithoutLeadingSlash, "bar/bat") if err := os.Chmod(batPath, 0600); err != nil { t.Fatalf("Error changing permissions on %s: %v", batPath, err) } @@ -110,9 +114,9 @@ func TestSnapshotFSChangePermissions(t *testing.T) { // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles tr := tar.NewReader(f) snapshotFiles := map[string]string{ - batPath: "baz2", + batPathWithoutLeadingSlash: "baz2", } - for _, dir := range util.ParentDirectories(batPath) { + for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) { snapshotFiles[dir] = "" } numFiles := 0 @@ -121,6 +125,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) { if err == io.EOF { break } + t.Logf("Info %s in tar", hdr.Name) numFiles++ if _, isFile := snapshotFiles[hdr.Name]; !isFile { t.Fatalf("File %s unexpectedly in tar", hdr.Name) @@ -137,6 +142,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) { func TestSnapshotFiles(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { t.Fatal(err) @@ -158,9 +164,9 @@ func TestSnapshotFiles(t *testing.T) { defer os.Remove(tarPath) expectedFiles := []string{ - filepath.Join(testDir, "foo"), + filepath.Join(testDirWithoutLeadingSlash, "foo"), } - expectedFiles = append(expectedFiles, util.ParentDirectories(filepath.Join(testDir, "foo"))...) + expectedFiles = append(expectedFiles, util.ParentDirectoriesWithoutLeadingSlash(filepath.Join(testDir, "foo"))...) f, err := os.Open(tarPath) if err != nil { diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index d22dcb4aa..4abfcd4a4 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -377,6 +377,24 @@ func ParentDirectories(path string) []string { return paths } +// ParentDirectoriesWithoutLeadingSlash returns a list of paths to all parent directories +// all subdirectories do not contain a leading / +// Ex. /some/temp/dir -> [/, some, some/temp, some/temp/dir] +func ParentDirectoriesWithoutLeadingSlash(path string) []string { + path = filepath.Clean(path) + dirs := strings.Split(path, "/") + dirPath := "" + paths := []string{constants.RootDir} + for index, dir := range dirs { + if dir == "" || index == (len(dirs)-1) { + continue + } + dirPath = filepath.Join(dirPath, dir) + paths = append(paths, dirPath) + } + return paths +} + // FilepathExists returns true if the path exists func FilepathExists(path string) bool { _, err := os.Lstat(path) diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index eff39d5d9..c44908056 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -177,6 +177,38 @@ func Test_ParentDirectories(t *testing.T) { } } +func Test_ParentDirectoriesWithoutLeadingSlash(t *testing.T) { + tests := []struct { + name string + path string + expected []string + }{ + { + name: "regular path", + path: "/path/to/dir", + expected: []string{ + "/", + "path", + "path/to", + }, + }, + { + name: "current directory", + path: ".", + expected: []string{ + "/", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := ParentDirectoriesWithoutLeadingSlash(tt.path) + testutil.CheckErrorAndDeepEqual(t, false, nil, tt.expected, actual) + }) + } +} + func Test_CheckWhitelist(t *testing.T) { type args struct { path string From 30db2c07d3a26fec70a4722b928bdb1ef336e6d3 Mon Sep 17 00:00:00 2001 From: xanonid Date: Fri, 9 Aug 2019 17:30:45 +0200 Subject: [PATCH 26/45] Also remove leading / from paths to file which are marked as deleted --- pkg/util/tar_util.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go index f358f48ad..213ef68e3 100644 --- a/pkg/util/tar_util.go +++ b/pkg/util/tar_util.go @@ -112,7 +112,8 @@ func (t *Tar) Whiteout(p string) error { name := ".wh." + filepath.Base(p) th := &tar.Header{ - Name: filepath.Join(dir, name), + // Docker uses no leading / in the tarball + Name: strings.TrimLeft(filepath.Join(dir, name), "/"), Size: 0, } if err := t.w.WriteHeader(th); err != nil { From c425f028660bcb75b0e207cf13bc6cafd22c7a4b Mon Sep 17 00:00:00 2001 From: Deniz Zoeteman Date: Fri, 16 Aug 2019 15:09:52 +0200 Subject: [PATCH 27/45] Reverted not including build args in cache key --- pkg/executor/build.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 6626577cf..13f9db92d 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -191,6 +191,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro func (s *stageBuilder) build() error { // Set the initial cache key to be the base image digest, the build args and the SrcContext. compositeKey := NewCompositeCache(s.baseImageDigest) + compositeKey.AddKey(s.opts.BuildArgs...) // Apply optimizations to the instructions. if err := s.optimize(*compositeKey, s.cf.Config); err != nil { From 6daffd8dd73bf349221e8c9b0e4054acd16c38aa Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 23 Aug 2019 11:18:46 -0700 Subject: [PATCH 28/45] add multiple user agents to kaniko if upstream_client_type value is set --- pkg/executor/push.go | 12 ++++++- pkg/executor/push_test.go | 71 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 pkg/executor/push_test.go diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 59d275d69..6508c7bd0 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -21,6 +21,8 @@ import ( "fmt" "io/ioutil" "net/http" + "os" + "strings" "time" "github.com/GoogleContainerTools/kaniko/pkg/cache" @@ -43,8 +45,16 @@ type withUserAgent struct { t http.RoundTripper } +const ( + UPSTREAM_CLIENT_UA_KEY = "UPSTREAM_CLIENT_TYPE" +) + func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) { - r.Header.Set("User-Agent", fmt.Sprintf("kaniko/%s", version.Version())) + ua := []string{fmt.Sprintf("kaniko/%s", version.Version())} + if upstream := os.Getenv(UPSTREAM_CLIENT_UA_KEY); upstream != "" { + ua = append(ua, upstream) + } + r.Header.Set("User-Agent", strings.Join(ua, ",")) return w.t.RoundTrip(r) } diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go new file mode 100644 index 000000000..2f9729960 --- /dev/null +++ b/pkg/executor/push_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "testing" + + "github.com/GoogleContainerTools/kaniko/testutil" +) + +func TestHeaderAdded(t *testing.T) { + tests := []struct { + name string + upstream string + expected string + }{{ + name: "upstream env variable set", + upstream: "skaffold-v0.25.45", + expected: "kaniko/unset,skaffold-v0.25.45", + }, { + name: "upstream env variable not set", + expected: "kaniko/unset", + }, + } + for _, test := range tests { + + t.Run(test.name, func(t *testing.T) { + rt := &withUserAgent{t: &mockRoundTripper{}} + if test.upstream != "" { + os.Setenv("UPSTREAM_CLIENT_TYPE", test.upstream) + defer func() { os.Unsetenv("UPSTREAM_CLIENT_TYPE") }() + } + req, err := http.NewRequest("GET", "dummy", nil) + if err != nil { + t.Fatalf("culd not create a req due to %s", err) + } + resp, err := rt.RoundTrip(req) + testutil.CheckError(t, false, err) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expected, string(body)) + }) + } + +} + +type mockRoundTripper struct { +} + +func (m *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + ua := r.UserAgent() + return &http.Response{Body: ioutil.NopCloser(bytes.NewBufferString(ua))}, nil +} From 96947b8ca4630e8c70b525f95ddf6eee1a47fa01 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 23 Aug 2019 13:13:14 -0700 Subject: [PATCH 29/45] fix lint --- pkg/executor/push.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 6508c7bd0..486bcd32a 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -46,12 +46,12 @@ type withUserAgent struct { } const ( - UPSTREAM_CLIENT_UA_KEY = "UPSTREAM_CLIENT_TYPE" + UpstreamClientUaKey = "UPSTREAM_CLIENT_TYPE" ) func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) { ua := []string{fmt.Sprintf("kaniko/%s", version.Version())} - if upstream := os.Getenv(UPSTREAM_CLIENT_UA_KEY); upstream != "" { + if upstream := os.Getenv(UpstreamClientUaKey); upstream != "" { ua = append(ua, upstream) } r.Header.Set("User-Agent", strings.Join(ua, ",")) From e2e16a0165ca6cdfa5d413ef7e4c5059576042fa Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 23 Aug 2019 15:01:51 -0700 Subject: [PATCH 30/45] Release v0.11.0 --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ Makefile | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bc746dc5..353250455 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,40 @@ +# v0.11.0 Release - 2019-08-23 + +## Bug Fixes +* fix unpacking archives via ADD [#717](https://github.com/GoogleContainerTools/kaniko/pull/717) +* Reverted not including build args in cache key [#739](https://github.com/GoogleContainerTools/kaniko/pull/739) +* Create cache directory if it doesn't already exist [#452](https://github.com/GoogleContainerTools/kaniko/pull/452) + +## New Features +* add multiple user agents to kaniko if upstream_client_type value is set [#750](https://github.com/GoogleContainerTools/kaniko/pull/750) +* Make container layers captured using FS snapshots reproducible [#714](https://github.com/GoogleContainerTools/kaniko/pull/714) +* Include warmer in debug image [#497](https://github.com/GoogleContainerTools/kaniko/pull/497) +* Bailout when there is not enough input arguments [#735](https://github.com/GoogleContainerTools/kaniko/pull/735) +* Add checking image presence in cache prior to downloading it [#723](https://github.com/GoogleContainerTools/kaniko/pull/723) + +## Additonal PRs +* Document how to build from git reference [#730](https://github.com/GoogleContainerTools/kaniko/pull/730) +* Misc. small changes/refactoring [#712](https://github.com/GoogleContainerTools/kaniko/pull/712) +* Update go-containerregistry [#680](https://github.com/GoogleContainerTools/kaniko/pull/680) +* Update version of go-containerregistry [#724](https://github.com/GoogleContainerTools/kaniko/pull/724) +* feat: support specifying branch for cloning [#703](https://github.com/GoogleContainerTools/kaniko/pull/703) + +Huge thank you for this release towards our contributors: +- Carlos Alexandro Becker +- Carlos Sanchez +- Deniz Zoeteman +- Luke Wood +- Matthew Dawson +- priyawadhwa +- sharifelgamal +- Sharif Elgamal +- Taylor Barrella +- Tejal Desai +- v.rul +- Warren Seymour +- Xueshan Feng +- Роман Небалуе + # v0.10.0 Release - 2019-06-19 ## Bug Fixes diff --git a/Makefile b/Makefile index 036f1e017..e5633bf20 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ # Bump these on release VERSION_MAJOR ?= 0 -VERSION_MINOR ?= 10 +VERSION_MINOR ?= 11 VERSION_BUILD ?= 0 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD) From 730b8b77c8ee999d3f3e1f6988e8340b57054367 Mon Sep 17 00:00:00 2001 From: chhsia0 Date: Tue, 20 Aug 2019 15:15:47 -0700 Subject: [PATCH 31/45] Added `--layout-path` flag to save image in OCI layout. Fixed #296. The output manifests may have `application/vnd.docker.distribution.manifest.v2+json` as their media types instead of `application/vnd.oci.image.manifest.v1+json`. --- Gopkg.lock | 6 +- Gopkg.toml | 2 +- cmd/executor/cmd/root.go | 1 + pkg/config/options.go | 1 + pkg/executor/push.go | 11 + .../pkg/v1/layout/blob.go | 38 +++ .../go-containerregistry/pkg/v1/layout/doc.go | 19 ++ .../pkg/v1/layout/image.go | 131 ++++++++ .../pkg/v1/layout/index.go | 146 +++++++++ .../pkg/v1/layout/layoutpath.go | 25 ++ .../pkg/v1/layout/options.go | 42 +++ .../pkg/v1/layout/read.go | 32 ++ .../pkg/v1/layout/write.go | 301 ++++++++++++++++++ .../pkg/v1/mutate/mutate.go | 10 +- .../pkg/v1/remote/descriptor.go | 20 +- .../pkg/v1/remote/transport/error.go | 20 +- 16 files changed, 787 insertions(+), 18 deletions(-) create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go diff --git a/Gopkg.lock b/Gopkg.lock index bd23f0578..57a55af22 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -445,7 +445,7 @@ version = "v0.2.0" [[projects]] - digest = "1:16c8837e951303ef6388132bc875337660a48ea2dedf1c941ca118ea92d2a3d2" + digest = "1:5924704ec96f00247784c512cc57f45a595030376a7ff2ff993bf356793a2cb0" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", @@ -456,6 +456,7 @@ "pkg/v1", "pkg/v1/daemon", "pkg/v1/empty", + "pkg/v1/layout", "pkg/v1/mutate", "pkg/v1/partial", "pkg/v1/random", @@ -467,7 +468,7 @@ "pkg/v1/v1util", ] pruneopts = "NUT" - revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74" + revision = "31e00cede111067bae48bfc2cbfc522b0b36207f" [[projects]] digest = "1:f4f203acd8b11b8747bdcd91696a01dbc95ccb9e2ca2db6abf81c3a4f5e950ce" @@ -1368,6 +1369,7 @@ "github.com/google/go-containerregistry/pkg/v1", "github.com/google/go-containerregistry/pkg/v1/daemon", "github.com/google/go-containerregistry/pkg/v1/empty", + "github.com/google/go-containerregistry/pkg/v1/layout", "github.com/google/go-containerregistry/pkg/v1/mutate", "github.com/google/go-containerregistry/pkg/v1/partial", "github.com/google/go-containerregistry/pkg/v1/remote", diff --git a/Gopkg.toml b/Gopkg.toml index 64d3b5b7b..e502f07ff 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,7 +37,7 @@ required = [ [[constraint]] name = "github.com/google/go-containerregistry" - revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74" + revision = "31e00cede111067bae48bfc2cbfc522b0b36207f" [[override]] name = "k8s.io/apimachinery" diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 0dd717f86..5179509ad 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -129,6 +129,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided") RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.") RootCmd.PersistentFlags().StringVarP(&opts.DigestFile, "digest-file", "", "", "Specify a file to save the digest of the built image to.") + RootCmd.PersistentFlags().StringVarP(&opts.LayoutPath, "layout-path", "", "", "Path to save the OCI image spec of the built image.") RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end") RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.") diff --git a/pkg/config/options.go b/pkg/config/options.go index 627a51531..d52641b75 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -37,6 +37,7 @@ type KanikoOptions struct { Target string CacheRepo string DigestFile string + LayoutPath string Destinations multiArg BuildArgs multiArg Insecure bool diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 486bcd32a..33cb8475c 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -34,6 +34,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" @@ -101,6 +102,16 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { } } + if opts.LayoutPath != "" { + path, err := layout.Write(opts.LayoutPath, empty.Index) + if err != nil { + return errors.Wrap(err, "writing empty layout") + } + if err := path.AppendImage(image); err != nil { + return errors.Wrap(err, "appending image") + } + } + destRefs := []name.Tag{} for _, destination := range opts.Destinations { destRef, err := name.NewTag(destination, name.WeakValidation) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go new file mode 100644 index 000000000..ba90d4cdb --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go @@ -0,0 +1,38 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" + "io/ioutil" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Blob returns a blob with the given hash from the Path. +func (l Path) Blob(h v1.Hash) (io.ReadCloser, error) { + return os.Open(l.blobPath(h)) +} + +// Bytes is a convenience function to return a blob from the Path as +// a byte slice. +func (l Path) Bytes(h v1.Hash) ([]byte, error) { + return ioutil.ReadFile(l.blobPath(h)) +} + +func (l Path) blobPath(h v1.Hash) string { + return l.path("blobs", h.Algorithm, h.Hex) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go new file mode 100644 index 000000000..d80d27363 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go @@ -0,0 +1,19 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout provides facilities for reading/writing artifacts from/to +// an OCI image layout on disk, see: +// +// https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go new file mode 100644 index 000000000..7c76a10cb --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go @@ -0,0 +1,131 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "fmt" + "io" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type layoutImage struct { + path Path + desc v1.Descriptor + manifestLock sync.Mutex // Protects rawManifest + rawManifest []byte +} + +var _ partial.CompressedImageCore = (*layoutImage)(nil) + +// Image reads a v1.Image with digest h from the Path. +func (l Path) Image(h v1.Hash) (v1.Image, error) { + ii, err := l.ImageIndex() + if err != nil { + return nil, err + } + + return ii.Image(h) +} + +func (li *layoutImage) MediaType() (types.MediaType, error) { + return li.desc.MediaType, nil +} + +// Implements WithManifest for partial.Blobset. +func (li *layoutImage) Manifest() (*v1.Manifest, error) { + return partial.Manifest(li) +} + +func (li *layoutImage) RawManifest() ([]byte, error) { + li.manifestLock.Lock() + defer li.manifestLock.Unlock() + if li.rawManifest != nil { + return li.rawManifest, nil + } + + b, err := li.path.Bytes(li.desc.Digest) + if err != nil { + return nil, err + } + + li.rawManifest = b + return li.rawManifest, nil +} + +func (li *layoutImage) RawConfigFile() ([]byte, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + return li.path.Bytes(manifest.Config.Digest) +} + +func (li *layoutImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + if h == manifest.Config.Digest { + return partial.CompressedLayer(&compressedBlob{ + path: li.path, + desc: manifest.Config, + }), nil + } + + for _, desc := range manifest.Layers { + if h == desc.Digest { + switch desc.MediaType { + case types.OCILayer, types.DockerLayer: + return partial.CompressedToLayer(&compressedBlob{ + path: li.path, + desc: desc, + }) + default: + // TODO: We assume everything is a compressed blob, but that might not be true. + // TODO: Handle foreign layers. + return nil, fmt.Errorf("unexpected media type: %v for layer: %v", desc.MediaType, desc.Digest) + } + } + } + + return nil, fmt.Errorf("could not find layer in image: %s", h) +} + +type compressedBlob struct { + path Path + desc v1.Descriptor +} + +func (b *compressedBlob) Digest() (v1.Hash, error) { + return b.desc.Digest, nil +} + +func (b *compressedBlob) Compressed() (io.ReadCloser, error) { + return b.path.Blob(b.desc.Digest) +} + +func (b *compressedBlob) Size() (int64, error) { + return b.desc.Size, nil +} + +func (b *compressedBlob) MediaType() (types.MediaType, error) { + return b.desc.MediaType, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go new file mode 100644 index 000000000..aba139d9d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go @@ -0,0 +1,146 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var _ v1.ImageIndex = (*layoutIndex)(nil) + +type layoutIndex struct { + path Path + rawIndex []byte +} + +// ImageIndexFromPath is a convenience function which constructs a Path and returns its v1.ImageIndex. +func ImageIndexFromPath(path string) (v1.ImageIndex, error) { + lp, err := FromPath(path) + if err != nil { + return nil, err + } + return lp.ImageIndex() +} + +// ImageIndex returns a v1.ImageIndex for the Path. +func (l Path) ImageIndex() (v1.ImageIndex, error) { + rawIndex, err := ioutil.ReadFile(l.path("index.json")) + if err != nil { + return nil, err + } + + idx := &layoutIndex{ + path: l, + rawIndex: rawIndex, + } + + return idx, nil +} + +func (i *layoutIndex) MediaType() (types.MediaType, error) { + return types.OCIImageIndex, nil +} + +func (i *layoutIndex) Digest() (v1.Hash, error) { + return partial.Digest(i) +} + +func (i *layoutIndex) IndexManifest() (*v1.IndexManifest, error) { + var index v1.IndexManifest + err := json.Unmarshal(i.rawIndex, &index) + return &index, err +} + +func (i *layoutIndex) RawManifest() ([]byte, error) { + return i.rawIndex, nil +} + +func (i *layoutIndex) Image(h v1.Hash) (v1.Image, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIManifestSchema1, types.DockerManifestSchema2) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + img := &layoutImage{ + path: i.path, + desc: *desc, + } + return partial.CompressedToImage(img) +} + +func (i *layoutIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIImageIndex, types.DockerManifestList) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + rawIndex, err := i.path.Bytes(h) + if err != nil { + return nil, err + } + + return &layoutIndex{ + path: i.path, + rawIndex: rawIndex, + }, nil +} + +func (i *layoutIndex) Blob(h v1.Hash) (io.ReadCloser, error) { + return i.path.Blob(h) +} + +func (i *layoutIndex) findDescriptor(h v1.Hash) (*v1.Descriptor, error) { + im, err := i.IndexManifest() + if err != nil { + return nil, err + } + + for _, desc := range im.Manifests { + if desc.Digest == h { + return &desc, nil + } + } + + return nil, fmt.Errorf("could not find descriptor in index: %s", h) +} + +// TODO: Pull this out into methods on types.MediaType? e.g. instead, have: +// * mt.IsIndex() +// * mt.IsImage() +func isExpectedMediaType(mt types.MediaType, expected ...types.MediaType) bool { + for _, allowed := range expected { + if mt == allowed { + return true + } + } + return false +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go new file mode 100644 index 000000000..a031ff5ae --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go @@ -0,0 +1,25 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import "path/filepath" + +// Path represents an OCI image layout rooted in a file system path +type Path string + +func (l Path) path(elem ...string) string { + complete := []string{string(l)} + return filepath.Join(append(complete, elem...)...) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go new file mode 100644 index 000000000..5569e51de --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go @@ -0,0 +1,42 @@ +package layout + +import v1 "github.com/google/go-containerregistry/pkg/v1" + +// Option is a functional option for Layout. +// +// TODO: We'll need to change this signature to support Sparse/Thin images. +// Or, alternatively, wrap it in a sparse.Image that returns an empty list for layers? +type Option func(*v1.Descriptor) error + +// WithAnnotations adds annotations to the artifact descriptor. +func WithAnnotations(annotations map[string]string) Option { + return func(desc *v1.Descriptor) error { + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + for k, v := range annotations { + desc.Annotations[k] = v + } + + return nil + } +} + +// WithURLs adds urls to the artifact descriptor. +func WithURLs(urls []string) Option { + return func(desc *v1.Descriptor) error { + if desc.URLs == nil { + desc.URLs = []string{} + } + desc.URLs = append(desc.URLs, urls...) + return nil + } +} + +// WithPlatform sets the platform of the artifact descriptor. +func WithPlatform(platform v1.Platform) Option { + return func(desc *v1.Descriptor) error { + desc.Platform = &platform + return nil + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go new file mode 100644 index 000000000..796abc7dd --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go @@ -0,0 +1,32 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "os" + "path/filepath" +) + +// FromPath reads an OCI image layout at path and constructs a layout.Path. +func FromPath(path string) (Path, error) { + // TODO: check oci-layout exists + + _, err := os.Stat(filepath.Join(path, "index.json")) + if err != nil { + return "", err + } + + return Path(path), nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go new file mode 100644 index 000000000..2abb9a586 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go @@ -0,0 +1,301 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "golang.org/x/sync/errgroup" +) + +var layoutFile = `{ + "imageLayoutVersion": "1.0.0" +}` + +// AppendImage writes a v1.Image to the Path and updates +// the index.json to reference it. +func (l Path) AppendImage(img v1.Image, options ...Option) error { + if err := l.writeImage(img); err != nil { + return err + } + + mt, err := img.MediaType() + if err != nil { + return err + } + + d, err := img.Digest() + if err != nil { + return err + } + + manifest, err := img.RawManifest() + if err != nil { + return err + } + + desc := v1.Descriptor{ + MediaType: mt, + Size: int64(len(manifest)), + Digest: d, + } + + for _, opt := range options { + if err := opt(&desc); err != nil { + return err + } + } + + return l.AppendDescriptor(desc) +} + +// AppendIndex writes a v1.ImageIndex to the Path and updates +// the index.json to reference it. +func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error { + if err := l.writeIndex(ii); err != nil { + return err + } + + mt, err := ii.MediaType() + if err != nil { + return err + } + + d, err := ii.Digest() + if err != nil { + return err + } + + manifest, err := ii.RawManifest() + if err != nil { + return err + } + + desc := v1.Descriptor{ + MediaType: mt, + Size: int64(len(manifest)), + Digest: d, + } + + for _, opt := range options { + if err := opt(&desc); err != nil { + return err + } + } + + return l.AppendDescriptor(desc) +} + +// AppendDescriptor adds a descriptor to the index.json of the Path. +func (l Path) AppendDescriptor(desc v1.Descriptor) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + index.Manifests = append(index.Manifests, desc) + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.writeFile("index.json", rawIndex) +} + +func (l Path) writeFile(name string, data []byte) error { + if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + return ioutil.WriteFile(l.path(name), data, os.ModePerm) + +} + +// WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at +// blobs/{hash.Algorithm}/{hash.Hex}. +func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error { + dir := l.path("blobs", hash.Algorithm) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + file := filepath.Join(dir, hash.Hex) + if _, err := os.Stat(file); err == nil { + // Blob already exists, that's fine. + return nil + } + w, err := os.Create(file) + if err != nil { + return err + } + defer w.Close() + + _, err = io.Copy(w, r) + return err +} + +// TODO: A streaming version of WriteBlob so we don't have to know the hash +// before we write it. + +// TODO: For streaming layers we should write to a tmp file then Rename to the +// final digest. +func (l Path) writeLayer(layer v1.Layer) error { + d, err := layer.Digest() + if err != nil { + return err + } + + r, err := layer.Compressed() + if err != nil { + return err + } + + return l.WriteBlob(d, r) +} + +func (l Path) writeImage(img v1.Image) error { + layers, err := img.Layers() + if err != nil { + return err + } + + // Write the layers concurrently. + var g errgroup.Group + for _, layer := range layers { + layer := layer + g.Go(func() error { + return l.writeLayer(layer) + }) + } + if err := g.Wait(); err != nil { + return err + } + + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return err + } + cfgBlob, err := img.RawConfigFile() + if err != nil { + return err + } + if err := l.WriteBlob(cfgName, ioutil.NopCloser(bytes.NewReader(cfgBlob))); err != nil { + return err + } + + // Write the img manifest. + d, err := img.Digest() + if err != nil { + return err + } + manifest, err := img.RawManifest() + if err != nil { + return err + } + + return l.WriteBlob(d, ioutil.NopCloser(bytes.NewReader(manifest))) +} + +func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error { + index, err := ii.IndexManifest() + if err != nil { + return err + } + + // Walk the descriptors and write any v1.Image or v1.ImageIndex that we find. + // If we come across something we don't expect, just write it as a blob. + for _, desc := range index.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + ii, err := ii.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := l.writeIndex(ii); err != nil { + return err + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := ii.Image(desc.Digest) + if err != nil { + return err + } + if err := l.writeImage(img); err != nil { + return err + } + default: + // TODO: The layout could reference arbitrary things, which we should + // probably just pass through. + } + } + + rawIndex, err := ii.RawManifest() + if err != nil { + return err + } + + return l.writeFile(indexFile, rawIndex) +} + +func (l Path) writeIndex(ii v1.ImageIndex) error { + // Always just write oci-layout file, since it's small. + if err := l.writeFile("oci-layout", []byte(layoutFile)); err != nil { + return err + } + + h, err := ii.Digest() + if err != nil { + return err + } + + indexFile := filepath.Join("blobs", h.Algorithm, h.Hex) + return l.writeIndexToFile(indexFile, ii) + +} + +// Write constructs a Path at path from an ImageIndex. +// +// The contents are written in the following format: +// At the top level, there is: +// One oci-layout file containing the version of this image-layout. +// One index.json file listing descriptors for the contained images. +// Under blobs/, there is, for each image: +// One file for each layer, named after the layer's SHA. +// One file for each config blob, named after its SHA. +// One file for each manifest blob, named after its SHA. +func Write(path string, ii v1.ImageIndex) (Path, error) { + lp := Path(path) + // Always just write oci-layout file, since it's small. + if err := lp.writeFile("oci-layout", []byte(layoutFile)); err != nil { + return "", err + } + + // TODO create blobs/ in case there is a blobs file which would prevent the directory from being created + + return lp, lp.writeIndexToFile("index.json", ii) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index 813205dad..eafd599b9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -165,11 +165,9 @@ func (i *image) compute() error { manifest := m.DeepCopy() manifestLayers := manifest.Layers for _, add := range i.adds { - d := v1.Descriptor{ - MediaType: types.DockerLayer, - } - + d := v1.Descriptor{} var err error + if d.Size, err = add.Layer.Size(); err != nil { return err } @@ -178,6 +176,10 @@ func (i *image) compute() error { return err } + if d.MediaType, err = add.Layer.MediaType(); err != nil { + return err + } + manifestLayers = append(manifestLayers, d) digestMap[d.Digest] = add.Layer } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go index 144b99ecc..6c3620740 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -16,7 +16,6 @@ package remote import ( "bytes" - "errors" "fmt" "io/ioutil" "net/http" @@ -38,7 +37,20 @@ var defaultPlatform = v1.Platform{ // ErrSchema1 indicates that we received a schema1 manifest from the registry. // This library doesn't have plans to support this legacy image format: // https://github.com/google/go-containerregistry/issues/377 -var ErrSchema1 = errors.New("unsupported MediaType: https://github.com/google/go-containerregistry/issues/377") +type ErrSchema1 struct { + schema string +} + +func NewErrSchema1(schema types.MediaType) error { + return &ErrSchema1{ + schema: string(schema), + } +} + +// Error implements error. +func (e *ErrSchema1) Error() string { + return fmt.Sprintf("unsupported MediaType: %q, see https://github.com/google/go-containerregistry/issues/377", e.schema) +} // Descriptor provides access to metadata about remote artifact and accessors // for efficiently converting it into a v1.Image or v1.ImageIndex. @@ -111,7 +123,7 @@ func (d *Descriptor) Image() (v1.Image, error) { case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: // We don't care to support schema 1 images: // https://github.com/google/go-containerregistry/issues/377 - return nil, ErrSchema1 + return nil, NewErrSchema1(d.MediaType) case types.OCIImageIndex, types.DockerManifestList: // We want an image but the registry has an index, resolve it to an image. return d.remoteIndex().imageByPlatform(d.platform) @@ -141,7 +153,7 @@ func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) { case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: // We don't care to support schema 1 images: // https://github.com/google/go-containerregistry/issues/377 - return nil, ErrSchema1 + return nil, NewErrSchema1(d.MediaType) case types.OCIManifestSchema1, types.DockerManifestSchema2: // We want an index but the registry has an image, nothing we can do. return nil, fmt.Errorf("unexpected media type for ImageIndex(): %s; call Image() instead", d.MediaType) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 3673a341b..35e5d798a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -26,6 +26,10 @@ import ( // https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors type Error struct { Errors []Diagnostic `json:"errors,omitempty"` + // The http status code returned. + StatusCode int + // The raw body if we couldn't understand it. + rawBody string } // Check that Error implements error @@ -35,7 +39,10 @@ var _ error = (*Error)(nil) func (e *Error) Error() string { switch len(e.Errors) { case 0: - return "" + if len(e.rawBody) == 0 { + return fmt.Sprintf("unsupported status code %d", e.StatusCode) + } + return fmt.Sprintf("unsupported status code %d; body: %s", e.StatusCode, e.rawBody) case 1: return e.Errors[0].String() default: @@ -115,11 +122,10 @@ func CheckError(resp *http.Response, codes ...int) error { } // https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors - var structuredError Error - if err := json.Unmarshal(b, &structuredError); err != nil { - // If the response isn't an unstructured error, then return some - // reasonable error response containing the response body. - return fmt.Errorf("unsupported status code %d; body: %s", resp.StatusCode, string(b)) + structuredError := &Error{} + if err := json.Unmarshal(b, structuredError); err != nil { + structuredError.rawBody = string(b) } - return &structuredError + structuredError.StatusCode = resp.StatusCode + return structuredError } From 7949d0de1d5898545d4d860b0a747c5f86683100 Mon Sep 17 00:00:00 2001 From: chhsia0 Date: Tue, 20 Aug 2019 15:35:14 -0700 Subject: [PATCH 32/45] Added a README for the flag. --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 466dd4ab5..0e134e9d0 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [--digest-file](#--digest-file) - [--insecure](#--insecure) - [--insecure-pull](#--insecure-pull) + - [--layout-path](#--layout-path) - [--no-push](#--no-push) - [--reproducible](#--reproducible) - [--single-snapshot](#--single-snapshot) @@ -374,6 +375,16 @@ will write the digest to that file, which is picked up by Kubernetes automatically as the `{{.state.terminated.message}}` of the container. +#### --layout-path + +Set this flag to specify a directory in the container where the OCI image +layout of a built image will be placed. This can be used to automatically +track the exact image built by Kaniko. + +For example, to surface the image digest built in a +[Tekton task](https://github.com/tektoncd/pipeline/blob/v0.6.0/docs/resources.md#surfacing-the-image-digest-built-in-a-task), +this flag should be set to match the image resource `outputImageDir`. + #### --insecure-registry Set this flag to use plain HTTP requests when accessing a registry. It is supposed to be used for testing purposes only and should not be used in production! From 11f3b791cdd891c8bd46bdd6cfcce267041fd17d Mon Sep 17 00:00:00 2001 From: chhsia0 Date: Wed, 21 Aug 2019 12:20:01 -0700 Subject: [PATCH 33/45] Renamed to `--oci-layout-path` and added a unit test. --- README.md | 7 +- cmd/executor/cmd/root.go | 2 +- pkg/config/options.go | 2 +- pkg/executor/push.go | 4 +- pkg/executor/push_test.go | 40 +++ .../pkg/v1/validate/doc.go | 16 + .../pkg/v1/validate/image.go | 297 ++++++++++++++++++ .../pkg/v1/validate/index.go | 123 ++++++++ 8 files changed, 485 insertions(+), 6 deletions(-) create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go diff --git a/README.md b/README.md index 0e134e9d0..38a4a63a3 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,8 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [--digest-file](#--digest-file) - [--insecure](#--insecure) - [--insecure-pull](#--insecure-pull) - - [--layout-path](#--layout-path) - [--no-push](#--no-push) + - [--oci-layout-path](#--oci-layout-path) - [--reproducible](#--reproducible) - [--single-snapshot](#--single-snapshot) - [--snapshotMode](#--snapshotmode) @@ -375,7 +375,7 @@ will write the digest to that file, which is picked up by Kubernetes automatically as the `{{.state.terminated.message}}` of the container. -#### --layout-path +#### --oci-layout-path Set this flag to specify a directory in the container where the OCI image layout of a built image will be placed. This can be used to automatically @@ -385,6 +385,9 @@ For example, to surface the image digest built in a [Tekton task](https://github.com/tektoncd/pipeline/blob/v0.6.0/docs/resources.md#surfacing-the-image-digest-built-in-a-task), this flag should be set to match the image resource `outputImageDir`. +_Note: Depending on the built image, the media type of the image manifest might be either +`application/vnd.oci.image.manifest.v1+json` or `application/vnd.docker.distribution.manifest.v2+json``._ + #### --insecure-registry Set this flag to use plain HTTP requests when accessing a registry. It is supposed to be used for testing purposes only and should not be used in production! diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 5179509ad..84fea6e8f 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -129,7 +129,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided") RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.") RootCmd.PersistentFlags().StringVarP(&opts.DigestFile, "digest-file", "", "", "Specify a file to save the digest of the built image to.") - RootCmd.PersistentFlags().StringVarP(&opts.LayoutPath, "layout-path", "", "", "Path to save the OCI image spec of the built image.") + RootCmd.PersistentFlags().StringVarP(&opts.OCILayoutPath, "oci-layout-path", "", "", "Path to save the OCI image layout of the built image.") RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end") RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.") diff --git a/pkg/config/options.go b/pkg/config/options.go index d52641b75..44af681ec 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -37,7 +37,7 @@ type KanikoOptions struct { Target string CacheRepo string DigestFile string - LayoutPath string + OCILayoutPath string Destinations multiArg BuildArgs multiArg Insecure bool diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 33cb8475c..e0ebf2111 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -102,8 +102,8 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { } } - if opts.LayoutPath != "" { - path, err := layout.Write(opts.LayoutPath, empty.Index) + if opts.OCILayoutPath != "" { + path, err := layout.Write(opts.OCILayoutPath, empty.Index) if err != nil { return errors.Wrap(err, "writing empty layout") } diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go index 2f9729960..220fc444f 100644 --- a/pkg/executor/push_test.go +++ b/pkg/executor/push_test.go @@ -23,7 +23,11 @@ import ( "os" "testing" + "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/testutil" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/validate" ) func TestHeaderAdded(t *testing.T) { @@ -69,3 +73,39 @@ func (m *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { ua := r.UserAgent() return &http.Response{Body: ioutil.NopCloser(bytes.NewBufferString(ua))}, nil } + +func Test_OCILayoutPath(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("could not create temp dir: %s", err) + } + defer os.RemoveAll(tmpDir) + + image, err := random.Image(1024, 4) + if err != nil { + t.Fatalf("could not create image: %s", err) + } + + digest, err := image.Digest() + if err != nil { + t.Fatalf("could not get image digest: %s", err) + } + + opts := config.KanikoOptions{ + NoPush: true, + OCILayoutPath: tmpDir, + } + + if err := DoPush(image, &opts); err != nil { + t.Fatalf("could not push image: %s", err) + } + + index, err := layout.ImageIndexFromPath(tmpDir) + if err != nil { + t.Fatalf("could not get index from layout: %s", err) + } + testutil.CheckError(t, false, validate.Index(index)) + + got, err := index.Image(digest) + testutil.CheckErrorAndDeepEqual(t, false, err, image, got) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go new file mode 100644 index 000000000..91ca87a5f --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package validate provides methods for validating image correctness. +package validate diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go new file mode 100644 index 000000000..c71d7d65e --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go @@ -0,0 +1,297 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "strings" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Image validates that img does not violate any invariants of the image format. +func Image(img v1.Image) error { + errs := []string{} + if err := validateLayers(img); err != nil { + errs = append(errs, fmt.Sprintf("validating layers: %v", err)) + } + + if err := validateConfig(img); err != nil { + errs = append(errs, fmt.Sprintf("validating config: %v", err)) + } + + if err := validateManifest(img); err != nil { + errs = append(errs, fmt.Sprintf("validating manifest: %v", err)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n\n")) + } + return nil +} + +func validateConfig(img v1.Image) error { + cn, err := img.ConfigName() + if err != nil { + return err + } + + rc, err := img.RawConfigFile() + if err != nil { + return err + } + + hash, size, err := v1.SHA256(bytes.NewReader(rc)) + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + cf, err := img.ConfigFile() + if err != nil { + return err + } + + pcf, err := v1.ParseConfigFile(bytes.NewReader(rc)) + if err != nil { + return err + } + + errs := []string{} + if cn != hash { + errs = append(errs, fmt.Sprintf("mismatched config digest: ConfigName()=%s, SHA256(RawConfigFile())=%s", cn, hash)) + } + + if want, got := m.Config.Size, size; want != got { + errs = append(errs, fmt.Sprintf("mismatched config size: Manifest.Config.Size()=%d, len(RawConfigFile())=%d", want, got)) + } + + if diff := cmp.Diff(pcf, cf); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched config content: (-ParseConfigFile(RawConfigFile()) +ConfigFile()) %s", diff)) + } + + if cf.RootFS.Type != "layers" { + errs = append(errs, fmt.Sprintf("invalid ConfigFile.RootFS.Type: %q != %q", cf.RootFS.Type, "layers")) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateLayers(img v1.Image) error { + layers, err := img.Layers() + if err != nil { + return err + } + + digests := []v1.Hash{} + diffids := []v1.Hash{} + sizes := []int64{} + for _, layer := range layers { + // TODO: Test layer.Uncompressed. + compressed, err := layer.Compressed() + if err != nil { + return err + } + + // Keep track of compressed digest. + digester := sha256.New() + // Everything read from compressed is written to digester to compute digest. + hashCompressed := io.TeeReader(compressed, digester) + + // Call io.Copy to write from the layer Reader through to the tarReader on + // the other side of the pipe. + pr, pw := io.Pipe() + var size int64 + go func() { + n, err := io.Copy(pw, hashCompressed) + if err != nil { + pw.CloseWithError(err) + return + } + size = n + + // Now close the compressed reader, to flush the gzip stream + // and calculate digest/diffID/size. This will cause pr to + // return EOF which will cause readers of the Compressed stream + // to finish reading. + pw.CloseWithError(compressed.Close()) + }() + + // Read the bytes through gzip.Reader to compute the DiffID. + uncompressed, err := gzip.NewReader(pr) + if err != nil { + return err + } + diffider := sha256.New() + hashUncompressed := io.TeeReader(uncompressed, diffider) + + // Ensure there aren't duplicate file paths. + tarReader := tar.NewReader(hashUncompressed) + files := make(map[string]struct{}) + for { + hdr, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if _, ok := files[hdr.Name]; ok { + return fmt.Errorf("duplicate file path: %s", hdr.Name) + } + files[hdr.Name] = struct{}{} + } + + // Discard any trailing padding that the tar.Reader doesn't consume. + if _, err := io.Copy(ioutil.Discard, hashUncompressed); err != nil { + return err + } + + if err := uncompressed.Close(); err != nil { + return err + } + + digest := v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(digester.Sum(make([]byte, 0, digester.Size()))), + } + + diffid := v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(diffider.Sum(make([]byte, 0, diffider.Size()))), + } + + // Compute all of these first before we call Config() and Manifest() to allow + // for lazy access e.g. for stream.Layer. + digests = append(digests, digest) + diffids = append(diffids, diffid) + sizes = append(sizes, size) + } + + cf, err := img.ConfigFile() + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + errs := []string{} + for i, layer := range layers { + digest, err := layer.Digest() + if err != nil { + return err + } + diffid, err := layer.DiffID() + if err != nil { + return err + } + size, err := layer.Size() + if err != nil { + return err + } + + if digest != digests[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Digest()=%s, SHA256(Compressed())=%s", i, digest, digests[i])) + } + + if m.Layers[i].Digest != digests[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Manifest.Layers[%d].Digest=%s, SHA256(Compressed())=%s", i, i, m.Layers[i].Digest, digests[i])) + } + + if diffid != diffids[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: DiffID()=%s, SHA256(Gunzip(Compressed()))=%s", i, diffid, diffids[i])) + } + + if cf.RootFS.DiffIDs[i] != diffids[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: ConfigFile.RootFS.DiffIDs[%d]=%s, SHA256(Gunzip(Compressed()))=%s", i, i, cf.RootFS.DiffIDs[i], diffids[i])) + } + + if size != sizes[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Size()=%d, len(Compressed())=%d", i, size, sizes[i])) + } + + if m.Layers[i].Size != sizes[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Manifest.Layers[%d].Size=%d, len(Compressed())=%d", i, i, m.Layers[i].Size, sizes[i])) + } + + } + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateManifest(img v1.Image) error { + digest, err := img.Digest() + if err != nil { + return err + } + + rm, err := img.RawManifest() + if err != nil { + return err + } + + hash, _, err := v1.SHA256(bytes.NewReader(rm)) + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + pm, err := v1.ParseManifest(bytes.NewReader(rm)) + if err != nil { + return err + } + + errs := []string{} + if digest != hash { + errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash)) + } + + if diff := cmp.Diff(pm, m); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseManifest(RawManifest()) +Manifest()) %s", diff)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go new file mode 100644 index 000000000..871e24153 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go @@ -0,0 +1,123 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Index validates that idx does not violate any invariants of the index format. +func Index(idx v1.ImageIndex) error { + errs := []string{} + + if err := validateChildren(idx); err != nil { + errs = append(errs, fmt.Sprintf("validating children: %v", err)) + } + + if err := validateIndexManifest(idx); err != nil { + errs = append(errs, fmt.Sprintf("validating index manifest: %v", err)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n\n")) + } + return nil +} + +func validateChildren(idx v1.ImageIndex) error { + manifest, err := idx.IndexManifest() + if err != nil { + return err + } + + errs := []string{} + for i, desc := range manifest.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + idx, err := idx.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := Index(idx); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate index Manifests[%d](%s): %v", i, desc.Digest, err)) + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := idx.Image(desc.Digest) + if err != nil { + return err + } + if err := Image(img); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate image Manifests[%d](%s): %v", i, desc.Digest, err)) + } + default: + return fmt.Errorf("todo: validate index Blob()") + } + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateIndexManifest(idx v1.ImageIndex) error { + digest, err := idx.Digest() + if err != nil { + return err + } + + rm, err := idx.RawManifest() + if err != nil { + return err + } + + hash, _, err := v1.SHA256(bytes.NewReader(rm)) + if err != nil { + return err + } + + m, err := idx.IndexManifest() + if err != nil { + return err + } + + pm, err := v1.ParseIndexManifest(bytes.NewReader(rm)) + if err != nil { + return err + } + + errs := []string{} + if digest != hash { + errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash)) + } + + if diff := cmp.Diff(pm, m); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseIndexManifest(RawManifest()) +Manifest()) %s", diff)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} From 2c44539151baca98a97f9787acba077114677fb4 Mon Sep 17 00:00:00 2001 From: Prashant Date: Fri, 30 Aug 2019 13:33:36 +0530 Subject: [PATCH 34/45] Setting PATH --- pkg/executor/build.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 13f9db92d..e584afe53 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -124,9 +124,7 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { return nil, err } - if img == empty.Image { - imageConfig.Config.Env = constants.ScratchEnvVars - } + imageConfig.Config.Env = constants.ScratchEnvVars return imageConfig, nil } From ea1a92712ca810286e9318e023d3945ba0f65131 Mon Sep 17 00:00:00 2001 From: chhsia0 Date: Sat, 24 Aug 2019 02:02:04 -0700 Subject: [PATCH 35/45] Avoid comparing uncompressed and compressed images in the unit test. --- pkg/executor/push_test.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go index 220fc444f..63857f4f0 100644 --- a/pkg/executor/push_test.go +++ b/pkg/executor/push_test.go @@ -74,7 +74,7 @@ func (m *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return &http.Response{Body: ioutil.NopCloser(bytes.NewBufferString(ua))}, nil } -func Test_OCILayoutPath(t *testing.T) { +func TestOCILayoutPath(t *testing.T) { tmpDir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("could not create temp dir: %s", err) @@ -91,6 +91,11 @@ func Test_OCILayoutPath(t *testing.T) { t.Fatalf("could not get image digest: %s", err) } + want, err := image.Manifest() + if err != nil { + t.Fatalf("could not get image manifest: %s", err) + } + opts := config.KanikoOptions{ NoPush: true, OCILayoutPath: tmpDir, @@ -100,12 +105,17 @@ func Test_OCILayoutPath(t *testing.T) { t.Fatalf("could not push image: %s", err) } - index, err := layout.ImageIndexFromPath(tmpDir) + layoutIndex, err := layout.ImageIndexFromPath(tmpDir) if err != nil { t.Fatalf("could not get index from layout: %s", err) } - testutil.CheckError(t, false, validate.Index(index)) + testutil.CheckError(t, false, validate.Index(layoutIndex)) - got, err := index.Image(digest) - testutil.CheckErrorAndDeepEqual(t, false, err, image, got) + layoutImage, err := layoutIndex.Image(digest) + if err != nil { + t.Fatalf("could not get image from layout: %s", err) + } + + got, err := layoutImage.Manifest() + testutil.CheckErrorAndDeepEqual(t, false, err, want, got) } From 17d1059ec4af33fc4b6194b0677e4b197f398c60 Mon Sep 17 00:00:00 2001 From: Prashant Date: Wed, 4 Sep 2019 16:44:55 +0530 Subject: [PATCH 36/45] Setting PATH to default PATH if PATH is missing --- pkg/executor/build.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index e584afe53..cdea08295 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -123,8 +123,9 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { if err != nil { return nil, err } - - imageConfig.Config.Env = constants.ScratchEnvVars + if imageConfig.Config.Env == nil { + imageConfig.Config.Env = constants.ScratchEnvVars + } return imageConfig, nil } From 0158cbf70c93674a75a3a62b73c7a2ec2cda267e Mon Sep 17 00:00:00 2001 From: Prashant Date: Wed, 4 Sep 2019 17:23:59 +0530 Subject: [PATCH 37/45] Setting PATH for empty image as well --- pkg/executor/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index cdea08295..0cf712fa5 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -123,7 +123,7 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { if err != nil { return nil, err } - if imageConfig.Config.Env == nil { + if imageConfig.Config.Env == nil || img == empty.Image { imageConfig.Config.Env = constants.ScratchEnvVars } return imageConfig, nil From a014c4a1e8cfec5fcd41212c904053bfb154bec0 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 13 Sep 2019 11:00:05 -0700 Subject: [PATCH 38/45] added unit tests --- pkg/executor/build_test.go | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index c48e4bb26..ddb01ec7e 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -29,6 +29,8 @@ import ( "github.com/GoogleContainerTools/kaniko/testutil" "github.com/google/go-cmp/cmp" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) @@ -405,3 +407,52 @@ func Test_filesToSave(t *testing.T) { }) } } + + +func TestInitializeConfig(t *testing.T) { + tests := []struct { + description string + config v1.Config + expected v1.Config + shouldErr bool + }{ + { + description: "env is empty in the image", + config: v1.Config{ + Image: "test", + }, + expected: v1.Config{ + Env: []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + }, + }, + { + description: "env is not empty in the image", + config: v1.Config{ + Env: []string{ + "PATH=/usr/local/something", + }, + }, + expected: v1.Config{ + Env: []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + }, + }, + { + description: "image is empty", + expected: v1.Config{ + Env: []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + }, + }, + } + for _, tt := range tests { + img := empty.Image + mutate.Config(img, tt.config) + actual, err :=initializeConfig(img) + testutil.CheckErrorAndDeepEqual(t,tt.shouldErr, err, tt.expected, actual.Config) + } +} \ No newline at end of file From f0e571839d45a310043ec664ac0abfe858354ef4 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 13 Sep 2019 11:21:43 -0700 Subject: [PATCH 39/45] add unit tests --- pkg/executor/build.go | 8 ++++---- pkg/executor/build_test.go | 39 ++++++++++++++++++++++---------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 0cf712fa5..e0e397131 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -123,7 +123,7 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { if err != nil { return nil, err } - if imageConfig.Config.Env == nil || img == empty.Image { + if imageConfig.Config.Env == nil { imageConfig.Config.Env = constants.ScratchEnvVars } return imageConfig, nil @@ -177,7 +177,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro } } - // Mutate the config for any commands that require it. + // Mutate the cfg for any commands that require it. if command.MetadataOnly() { if err := command.ExecuteCommand(&cfg, s.args); err != nil { return err @@ -269,7 +269,7 @@ func (s *stageBuilder) build() error { if err != nil { return err } - // Push layer to cache (in parallel) now along with new config file + // Push layer to cache (in parallel) now along with new cfg file if s.opts.Cache && command.ShouldCacheOutput() { cacheGroup.Go(func() error { return pushLayerToCache(s.opts, ck, tarPath, command.String()) @@ -335,7 +335,7 @@ func (s *stageBuilder) saveSnapshotToImage(createdBy string, tarPath string) err return err } if fi.Size() <= emptyTarSize { - logrus.Info("No files were changed, appending empty layer to config. No layer added to image.") + logrus.Info("No files were changed, appending empty layer to cfg. No layer added to image.") return nil } diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index ddb01ec7e..adc4f86d3 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -411,32 +411,36 @@ func Test_filesToSave(t *testing.T) { func TestInitializeConfig(t *testing.T) { tests := []struct { - description string - config v1.Config - expected v1.Config - shouldErr bool + description string + cfg v1.ConfigFile + expected v1.Config }{ { - description: "env is empty in the image", - config: v1.Config{ - Image: "test", + description: "env is not set in the image", + cfg: v1.ConfigFile{ + Config: v1.Config{ + Image: "test", + }, }, expected: v1.Config{ + Image: "test", Env: []string{ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", }, }, }, { - description: "env is not empty in the image", - config: v1.Config{ - Env: []string{ - "PATH=/usr/local/something", + description: "env is set in the image", + cfg: v1.ConfigFile{ + Config: v1.Config{ + Env: []string{ + "PATH=/usr/local/something", + }, }, }, expected: v1.Config{ Env: []string{ - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "PATH=/usr/local/something", }, }, }, @@ -450,9 +454,12 @@ func TestInitializeConfig(t *testing.T) { }, } for _, tt := range tests { - img := empty.Image - mutate.Config(img, tt.config) - actual, err :=initializeConfig(img) - testutil.CheckErrorAndDeepEqual(t,tt.shouldErr, err, tt.expected, actual.Config) + img, err := mutate.ConfigFile(empty.Image, &tt.cfg) + if err != nil { + t.Errorf("error seen when running test %s", err) + t.Fail() + } + actual, err := initializeConfig(img) + testutil.CheckDeepEqual(t, tt.expected, actual.Config) } } \ No newline at end of file From 469fdaa50dfbd6fa8932fd320c4e3dafe85e8716 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 13 Sep 2019 11:49:30 -0700 Subject: [PATCH 40/45] test --- pkg/executor/build.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index e0e397131..e134e52b5 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -123,6 +123,7 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { if err != nil { return nil, err } + if imageConfig.Config.Env == nil { imageConfig.Config.Env = constants.ScratchEnvVars } @@ -177,7 +178,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro } } - // Mutate the cfg for any commands that require it. + // Mutate the config for any commands that require it. if command.MetadataOnly() { if err := command.ExecuteCommand(&cfg, s.args); err != nil { return err @@ -269,7 +270,7 @@ func (s *stageBuilder) build() error { if err != nil { return err } - // Push layer to cache (in parallel) now along with new cfg file + // Push layer to cache (in parallel) now along with new config file if s.opts.Cache && command.ShouldCacheOutput() { cacheGroup.Go(func() error { return pushLayerToCache(s.opts, ck, tarPath, command.String()) @@ -335,7 +336,7 @@ func (s *stageBuilder) saveSnapshotToImage(createdBy string, tarPath string) err return err } if fi.Size() <= emptyTarSize { - logrus.Info("No files were changed, appending empty layer to cfg. No layer added to image.") + logrus.Info("No files were changed, appending empty layer to config. No layer added to image.") return nil } From 9b9fb815a72dd0af4527d3675152b6b6ef46f294 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 13 Sep 2019 11:56:57 -0700 Subject: [PATCH 41/45] fix format --- pkg/executor/build_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index adc4f86d3..83297e5f3 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -408,7 +408,6 @@ func Test_filesToSave(t *testing.T) { } } - func TestInitializeConfig(t *testing.T) { tests := []struct { description string @@ -460,6 +459,6 @@ func TestInitializeConfig(t *testing.T) { t.Fail() } actual, err := initializeConfig(img) - testutil.CheckDeepEqual(t, tt.expected, actual.Config) + testutil.CheckDeepEqual(t, tt.expected, actual.Config) } -} \ No newline at end of file +} From 30f1a7dae98441a0c98ba5d6a3e2d285c3d06ff9 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 13 Sep 2019 12:16:40 -0700 Subject: [PATCH 42/45] fix lint --- pkg/executor/build_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 83297e5f3..44f6a1211 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -458,7 +458,7 @@ func TestInitializeConfig(t *testing.T) { t.Errorf("error seen when running test %s", err) t.Fail() } - actual, err := initializeConfig(img) + actual, _ := initializeConfig(img) testutil.CheckDeepEqual(t, tt.expected, actual.Config) } } From 116d851c2fe741db933b0da85607a1a41698b015 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 13 Sep 2019 15:56:36 -0700 Subject: [PATCH 43/45] release kaniko v0.12.0 --- CHANGELOG.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 353250455..ef958ddba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,38 @@ +# v0.12.0 Release - 2019-09/13 + +## New Features +* Added `--oci-layout-path` flag to save image in OCI layout. [#744](https://github.com/GoogleContainerTools/kaniko/pull/744) + +## Bug Fixes +* Setting PATH [#760](https://github.com/GoogleContainerTools/kaniko/pull/760) +* Remove leading slash in layer tarball paths (Closes: #726) [#729](https://github.com/GoogleContainerTools/kaniko/pull/729) +* Add support for S3 custom endpoint [#698](https://github.com/GoogleContainerTools/kaniko/pull/698) + +## Updates and Refactors +* Remove cruft [#635](https://github.com/GoogleContainerTools/kaniko/pull/635) +* Add desc for `--skip-tls-verify-pull` to README [#493](https://github.com/GoogleContainerTools/kaniko/pull/493) + + +Huge thank you for this release towards our contributors: +- Carlos Alexandro Becker +- Carlos Sanchez +- chhsia0 +- Deniz Zoeteman +- Luke Wood +- Matthew Dawson +- Niels Denissen +- Priya Wadhwa +- Sharif Elgamal +- Takeaki Matsumoto +- Taylor Barrella +- Tejal Desai +- v.rul +- Warren Seymour +- xanonid +- Xueshan Feng +- Роман Небалуев + + # v0.11.0 Release - 2019-08-23 ## Bug Fixes From e04436f88de1f81711e9c8ab1b54364223d7a1d2 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 13 Sep 2019 16:16:15 -0700 Subject: [PATCH 44/45] move from bug to feature. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef958ddba..487cc1b30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,12 @@ ## New Features * Added `--oci-layout-path` flag to save image in OCI layout. [#744](https://github.com/GoogleContainerTools/kaniko/pull/744) +* Add support for S3 custom endpoint [#698](https://github.com/GoogleContainerTools/kaniko/pull/698) + ## Bug Fixes * Setting PATH [#760](https://github.com/GoogleContainerTools/kaniko/pull/760) * Remove leading slash in layer tarball paths (Closes: #726) [#729](https://github.com/GoogleContainerTools/kaniko/pull/729) -* Add support for S3 custom endpoint [#698](https://github.com/GoogleContainerTools/kaniko/pull/698) ## Updates and Refactors * Remove cruft [#635](https://github.com/GoogleContainerTools/kaniko/pull/635) From 7d23805adcc06bbe5e1e26339fb62a7e7ce56472 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 13 Sep 2019 16:16:50 -0700 Subject: [PATCH 45/45] Update CHANGELOG.md --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 487cc1b30..142baa25d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,6 @@ * Added `--oci-layout-path` flag to save image in OCI layout. [#744](https://github.com/GoogleContainerTools/kaniko/pull/744) * Add support for S3 custom endpoint [#698](https://github.com/GoogleContainerTools/kaniko/pull/698) - ## Bug Fixes * Setting PATH [#760](https://github.com/GoogleContainerTools/kaniko/pull/760) * Remove leading slash in layer tarball paths (Closes: #726) [#729](https://github.com/GoogleContainerTools/kaniko/pull/729) @@ -13,7 +12,6 @@ * Remove cruft [#635](https://github.com/GoogleContainerTools/kaniko/pull/635) * Add desc for `--skip-tls-verify-pull` to README [#493](https://github.com/GoogleContainerTools/kaniko/pull/493) - Huge thank you for this release towards our contributors: - Carlos Alexandro Becker - Carlos Sanchez