From 5c0603a9675b2399237b394f56130b38e12dd25d Mon Sep 17 00:00:00 2001 From: Taylor Barrella Date: Sat, 25 May 2019 15:47:26 -0700 Subject: [PATCH] Update go-containerregistry Resolves #607 * Deleted a duplicate Gopkg.lock block for github.com/otiai10/copy to prevent `dep ensure` from deleting it from vendor/ * Searched for breaking changes. Only found ones for remote.Delete/List/Write/WriteIndex. Searched for those and fixed * Noticed that NewInsecureRegistry was deprecated and replaced it --- Gopkg.lock | 28 +- Gopkg.toml | 2 +- pkg/cache/cache.go | 2 +- pkg/executor/push.go | 4 +- pkg/util/image_util.go | 2 +- .../cmd/ko/test/kodata/kenobi | 1 - .../go-containerregistry/pkg/name/check.go | 9 - .../go-containerregistry/pkg/name/digest.go | 9 +- .../go-containerregistry/pkg/name/doc.go | 42 +++ .../go-containerregistry/pkg/name/options.go | 49 ++++ .../go-containerregistry/pkg/name/ref.go | 6 +- .../go-containerregistry/pkg/name/registry.go | 20 +- .../pkg/name/repository.go | 7 +- .../go-containerregistry/pkg/name/tag.go | 7 +- .../go-containerregistry/pkg/v1/image.go | 1 + .../go-containerregistry/pkg/v1/index.go | 1 + .../go-containerregistry/pkg/v1/layer.go | 5 + .../pkg/v1/mutate/mutate.go | 11 +- .../pkg/v1/partial/compressed.go | 4 + .../pkg/v1/partial/uncompressed.go | 3 + .../pkg/v1/partial/with.go | 7 + .../pkg/v1/random/image.go | 8 + .../pkg/v1/remote/check.go | 5 +- .../pkg/v1/remote/delete.go | 9 +- .../pkg/v1/remote/descriptor.go | 255 ++++++++++++++++++ .../pkg/v1/remote/image.go | 206 +++----------- .../pkg/v1/remote/index.go | 131 ++++++--- .../pkg/v1/remote/list.go | 12 +- .../pkg/v1/remote/options.go | 86 ++++-- .../pkg/v1/remote/transport/bearer.go | 8 +- .../pkg/v1/remote/transport/error.go | 14 + .../pkg/v1/remote/write.go | 136 +++++----- .../pkg/v1/stream/layer.go | 8 + .../pkg/v1/tarball/image.go | 14 +- .../pkg/v1/tarball/layer.go | 18 ++ 35 files changed, 755 insertions(+), 375 deletions(-) delete mode 120000 vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi create mode 100644 vendor/github.com/google/go-containerregistry/pkg/name/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/name/options.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go diff --git a/Gopkg.lock b/Gopkg.lock index 7b09230e2..44926c25d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -445,7 +445,7 @@ version = "v0.2.0" [[projects]] - digest = "1:d40a26f0daf07f3b5c916356a3e10fabbf97d5166f77e57aa3983013ab57004c" + digest = "1:3ccc9b3dfd6b951b46e6e1c499af589fbc35c7e1172d6d840cbe836ae08d3536" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", @@ -465,7 +465,7 @@ "pkg/v1/v1util", ] pruneopts = "NUT" - revision = "8621d738a07bc74b2adeafd175a3c738423577a0" + revision = "bb17f50c1bc6808972811ed2894ecaaeb5de68ad" [[projects]] digest = "1:f4f203acd8b11b8747bdcd91696a01dbc95ccb9e2ca2db6abf81c3a4f5e950ce" @@ -719,6 +719,14 @@ revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" version = "v1.0.2" +[[projects]] + branch = "master" + digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c" + name = "github.com/otiai10/copy" + packages = ["."] + pruneopts = "NUT" + revision = "7e9a647135a142c2669943d4a4d29be015ce9392" + [[projects]] digest = "1:cf254277d898b713195cc6b4a3fac8bf738b9f1121625df27843b52b267eec6c" name = "github.com/pelletier/go-buffruneio" @@ -727,22 +735,6 @@ revision = "c37440a7cf42ac63b919c752ca73a85067e05992" version = "v0.2.0" -[[projects]] - branch = "master" - digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c" - name = "github.com/otiai10/copy" - packages = ["."] - pruneopts = "NUT" - revision = "7e9a647135a142c2669943d4a4d29be015ce9392" - -[[projects]] - branch = "master" - digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c" - name = "github.com/otiai10/copy" - packages = ["."] - pruneopts = "NUT" - revision = "7e9a647135a142c2669943d4a4d29be015ce9392" - [[projects]] branch = "master" digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" diff --git a/Gopkg.toml b/Gopkg.toml index 80db44131..163d821ad 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,7 +37,7 @@ required = [ [[constraint]] name = "github.com/google/go-containerregistry" - revision = "8621d738a07bc74b2adeafd175a3c738423577a0" + revision = "bb17f50c1bc6808972811ed2894ecaaeb5de68ad" [[override]] name = "k8s.io/apimachinery" diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index d11de89af..6ec9d0ce7 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -60,7 +60,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) { registryName := cacheRef.Repository.Registry.Name() if rc.Opts.InsecureRegistries.Contains(registryName) { - newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation) + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return nil, err } diff --git a/pkg/executor/push.go b/pkg/executor/push.go index a02d5b9b9..603b3a08a 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -114,7 +114,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { for _, destRef := range destRefs { registryName := destRef.Repository.Registry.Name() if opts.Insecure || opts.InsecureRegistries.Contains(registryName) { - newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation) + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return errors.Wrap(err, "getting new insecure registry") } @@ -135,7 +135,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { } rt := &withUserAgent{t: tr} - if err := remote.Write(destRef, image, pushAuth, rt); err != nil { + if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil { return errors.Wrap(err, fmt.Sprintf("failed to push to destination %s", destRef)) } } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 6d7ed8a8d..dcc09ada8 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -102,7 +102,7 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { registryName := ref.Context().RegistryStr() if opts.InsecurePull || opts.InsecureRegistries.Contains(registryName) { - newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation) + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi b/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi deleted file mode 120000 index 5d7eddc7f..000000000 --- a/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi +++ /dev/null @@ -1 +0,0 @@ -../kenobi \ No newline at end of file diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/check.go b/vendor/github.com/google/go-containerregistry/pkg/name/check.go index 01a25d554..01b03e562 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/check.go @@ -19,15 +19,6 @@ import ( "unicode/utf8" ) -// Strictness defines the level of strictness for name validation. -type Strictness int - -// Enums for CRUD operations. -const ( - StrictValidation Strictness = iota - WeakValidation -) - // stripRunesFn returns a function which returns -1 (i.e. a value which // signals deletion in strings.Map) for runes in 'runes', and the rune otherwise. func stripRunesFn(runes string) func(rune) rune { diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go index dc573ef1d..2dc0f7f37 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package name defines structured types for representing image references. package name import ( @@ -63,8 +62,8 @@ func checkDigest(name string) error { return checkElement("digest", name, digestChars, 7+64, 7+64) } -// NewDigest returns a new Digest representing the given name, according to the given strictness. -func NewDigest(name string, strict Strictness) (Digest, error) { +// NewDigest returns a new Digest representing the given name. +func NewDigest(name string, opts ...Option) (Digest, error) { // Split on "@" parts := strings.Split(name, digestDelim) if len(parts) != 2 { @@ -78,12 +77,12 @@ func NewDigest(name string, strict Strictness) (Digest, error) { return Digest{}, err } - tag, err := NewTag(base, strict) + tag, err := NewTag(base, opts...) if err == nil { base = tag.Repository.Name() } - repo, err := NewRepository(base, strict) + repo, err := NewRepository(base, opts...) if err != nil { return Digest{}, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/doc.go b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go new file mode 100644 index 000000000..b294794dc --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go @@ -0,0 +1,42 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package name defines structured types for representing image references. +// +// What's in a name? For image references, not nearly enough! +// +// Image references look a lot like URLs, but they differ in that they don't +// contain the scheme (http or https), they can end with a :tag or a @digest +// (the latter being validated), and they perform defaulting for missing +// components. +// +// Since image references don't contain the scheme, we do our best to infer +// if we use http or https from the given hostname. We allow http fallback for +// any host that looks like localhost (localhost, 127.0.0.1, ::1), ends in +// ".local", or is in the "private" address space per RFC 1918. For everything +// else, we assume https only. To override this heuristic, use the Insecure +// option. +// +// Image references with a digest signal to us that we should verify the content +// of the image matches the digest. E.g. when pulling a Digest reference, we'll +// calculate the sha256 of the manifest returned by the registry and error out +// if it doesn't match what we asked for. +// +// For defaulting, we interpret "ubuntu" as +// "index.docker.io/library/ubuntu:latest" because we add the missing repo +// "library", the missing registry "index.docker.io", and the missing tag +// "latest". To disable this defaulting, use the StrictValidation option. This +// is useful e.g. to only allow image references that explicitly set a tag or +// digest, so that you don't accidentally pull "latest". +package name diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/options.go b/vendor/github.com/google/go-containerregistry/pkg/name/options.go new file mode 100644 index 000000000..98beaae11 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/name/options.go @@ -0,0 +1,49 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +type options struct { + strict bool // weak by default + insecure bool // secure by default +} + +func makeOptions(opts ...Option) options { + opt := options{} + for _, o := range opts { + o(&opt) + } + return opt +} + +// Option is a functional option for name parsing. +type Option func(*options) + +// StrictValidation is an Option that requires image references to be fully +// specified; i.e. no defaulting for registry (dockerhub), repo (library), +// or tag (latest). +func StrictValidation(opts *options) { + opts.strict = true +} + +// WeakValidation is an Option that sets defaults when parsing names, see +// StrictValidation. +func WeakValidation(opts *options) { + opts.strict = false +} + +// Insecure is an Option that allows image references to be fetched without TLS. +func Insecure(opts *options) { + opts.insecure = true +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go index 58775daa3..cca303405 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go @@ -38,11 +38,11 @@ type Reference interface { } // ParseReference parses the string as a reference, either by tag or digest. -func ParseReference(s string, strict Strictness) (Reference, error) { - if t, err := NewTag(s, strict); err == nil { +func ParseReference(s string, opts ...Option) (Reference, error) { + if t, err := NewTag(s, opts...); err == nil { return t, nil } - if d, err := NewDigest(s, strict); err == nil { + if d, err := NewDigest(s, opts...); err == nil { return d, nil } // TODO: Combine above errors into something more useful? diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go index ab7419308..c12dd46c2 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go @@ -114,8 +114,9 @@ func checkRegistry(name string) error { // NewRegistry returns a Registry based on the given name. // Strict validation requires explicit, valid RFC 3986 URI authorities to be given. -func NewRegistry(name string, strict Strictness) (Registry, error) { - if strict == StrictValidation && len(name) == 0 { +func NewRegistry(name string, opts ...Option) (Registry, error) { + opt := makeOptions(opts...) + if opt.strict && len(name) == 0 { return Registry{}, NewErrBadName("strict validation requires the registry to be explicitly defined") } @@ -129,16 +130,13 @@ func NewRegistry(name string, strict Strictness) (Registry, error) { name = DefaultRegistry } - return Registry{registry: name}, nil + return Registry{registry: name, insecure: opt.insecure}, nil } // NewInsecureRegistry returns an Insecure Registry based on the given name. -// Strict validation requires explicit, valid RFC 3986 URI authorities to be given. -func NewInsecureRegistry(name string, strict Strictness) (Registry, error) { - reg, err := NewRegistry(name, strict) - if err != nil { - return Registry{}, err - } - reg.insecure = true - return reg, nil +// +// Deprecated: Use the Insecure Option with NewRegistry instead. +func NewInsecureRegistry(name string, opts ...Option) (Registry, error) { + opts = append(opts, Insecure) + return NewRegistry(name, opts...) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go index 43cc5b82b..f33377988 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go @@ -68,7 +68,8 @@ func checkRepository(repository string) error { } // NewRepository returns a new Repository representing the given name, according to the given strictness. -func NewRepository(name string, strict Strictness) (Repository, error) { +func NewRepository(name string, opts ...Option) (Repository, error) { + opt := makeOptions(opts...) if len(name) == 0 { return Repository{}, NewErrBadName("a repository name must be specified") } @@ -88,11 +89,11 @@ func NewRepository(name string, strict Strictness) (Repository, error) { return Repository{}, err } - reg, err := NewRegistry(registry, strict) + reg, err := NewRegistry(registry, opts...) if err != nil { return Repository{}, err } - if hasImplicitNamespace(repo, reg) && strict == StrictValidation { + if hasImplicitNamespace(repo, reg) && opt.strict { return Repository{}, NewErrBadName("strict validation requires the full repository path (missing 'library')") } return Repository{reg, repo}, nil diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/tag.go b/vendor/github.com/google/go-containerregistry/pkg/name/tag.go index b8375e1f9..e6cce34db 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/tag.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/tag.go @@ -71,7 +71,8 @@ func checkTag(name string) error { } // NewTag returns a new Tag representing the given name, according to the given strictness. -func NewTag(name string, strict Strictness) (Tag, error) { +func NewTag(name string, opts ...Option) (Tag, error) { + opt := makeOptions(opts...) base := name tag := "" @@ -87,13 +88,13 @@ func NewTag(name string, strict Strictness) (Tag, error) { // even when not being strict. // If we are being strict, we want to validate the tag regardless in case // it's empty. - if tag != "" || strict == StrictValidation { + if tag != "" || opt.strict { if err := checkTag(tag); err != nil { return Tag{}, err } } - repo, err := NewRepository(base, strict) + repo, err := NewRepository(base, opts...) if err != nil { return Tag{}, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/image.go index 17b9839a6..9ef026799 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/image.go @@ -19,6 +19,7 @@ import ( ) // Image defines the interface for interacting with an OCI v1 image. +//go:generate counterfeiter -o fake/image.go . Image type Image interface { // Layers returns the ordered collection of filesystem layers that comprise this image. // The order of the list is oldest/base layer first, and most-recent/top layer last. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/index.go index 604e6de36..b4e84e02a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/index.go @@ -19,6 +19,7 @@ import ( ) // ImageIndex defines the interface for interacting with an OCI image index. +//go:generate counterfeiter -o fake/index.go . ImageIndex type ImageIndex interface { // MediaType of this image's manifest. MediaType() (types.MediaType, error) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go index 8b5091e45..57447d263 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go @@ -16,6 +16,8 @@ package v1 import ( "io" + + "github.com/google/go-containerregistry/pkg/v1/types" ) // Layer is an interface for accessing the properties of a particular layer of a v1.Image @@ -34,4 +36,7 @@ type Layer interface { // Size returns the compressed size of the Layer. Size() (int64, error) + + // MediaType returns the media type of the Layer. + MediaType() (types.MediaType, error) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index a327e7594..11262f444 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -78,10 +78,11 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) { cf.Config = cfg - return configFile(base, cf) + return ConfigFile(base, cf) } -func configFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { +// ConfigFile mutates the provided v1.Image to have the provided v1.ConfigFile +func ConfigFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { m, err := base.Manifest() if err != nil { return nil, err @@ -106,7 +107,7 @@ func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) { cfg := cf.DeepCopy() cfg.Created = created - return configFile(base, cfg) + return ConfigFile(base, cfg) } type image struct { @@ -476,7 +477,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { h.Created = v1.Time{Time: t} } - return configFile(newImage, cfg) + return ConfigFile(newImage, cfg) } func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { @@ -555,5 +556,5 @@ func Canonical(img v1.Image) (v1.Image, error) { cfg.ContainerConfig.Hostname = "" cfg.DockerVersion = "" - return configFile(img, cfg) + return ConfigFile(img, cfg) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go index 497d1af0d..1c631b7aa 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go @@ -18,6 +18,7 @@ import ( "io" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -32,6 +33,9 @@ type CompressedLayer interface { // Size returns the compressed size of the Layer. Size() (int64, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) } // compressedLayerExtender implements v1.Image using the compressed base properties. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go index 9f75723ec..07658be42 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go @@ -32,6 +32,9 @@ type UncompressedLayer interface { // Uncompressed returns an io.ReadCloser for the uncompressed layer contents. Uncompressed() (io.ReadCloser, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) } // uncompressedLayerExtender implements v1.Image using the uncompressed base properties. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go index f724ec8ab..992672048 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go @@ -22,6 +22,7 @@ import ( "io/ioutil" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -80,6 +81,12 @@ func (cl *configLayer) Size() (int64, error) { return int64(len(cl.content)), nil } +func (cl *configLayer) MediaType() (types.MediaType, error) { + // Defaulting this to OCIConfigJSON as it should remain + // backwards compatible with DockerConfigJSON + return types.OCIConfigJSON, nil +} + var _ v1.Layer = (*configLayer)(nil) // ConfigLayer implements v1.Layer from the raw config bytes. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go index cc269d6b5..e09984cbf 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go @@ -45,6 +45,14 @@ func (ul *uncompressedLayer) Uncompressed() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewBuffer(ul.content)), nil } +// MediaType returns the media type of the layer +func (ul *uncompressedLayer) MediaType() (types.MediaType, error) { + // Technically the media type should be 'application/tar' but given that our + // v1.Layer doesn't force consumers to care about whether the layer is compressed + // we should be fine returning the DockerLayer media type + return types.DockerLayer, nil +} + var _ partial.UncompressedLayer = (*uncompressedLayer)(nil) // Image returns a pseudo-randomly generated Image. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go index aa574eb8b..da0fa24c3 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go @@ -1,6 +1,7 @@ package remote import ( + "fmt" "net/http" "github.com/google/go-containerregistry/pkg/authn" @@ -18,13 +19,13 @@ import ( func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { auth, err := kc.Resolve(ref.Context().Registry) if err != nil { - return err + return fmt.Errorf("resolving authorization for %v failed: %v", ref.Context().Registry, err) } scopes := []string{ref.Scope(transport.PushScope)} tr, err := transport.New(ref.Context().Registry, auth, t, scopes) if err != nil { - return err + return fmt.Errorf("creating push check transport for %v failed: %v", ref.Context().Registry, err) } // TODO(jasonhall): Against GCR, just doing the token handshake is // enough, but this doesn't extend to Dockerhub diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go index 2032e276e..c034435f9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go @@ -20,15 +20,18 @@ import ( "net/http" "net/url" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) // Delete removes the specified image reference from the remote registry. -func Delete(ref name.Reference, auth authn.Authenticator, t http.RoundTripper) error { +func Delete(ref name.Reference, options ...Option) error { + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } scopes := []string{ref.Scope(transport.DeleteScope)} - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go new file mode 100644 index 000000000..078de3a0b --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -0,0 +1,255 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var defaultPlatform = v1.Platform{ + Architecture: "amd64", + OS: "linux", +} + +// ErrSchema1 indicates that we received a schema1 manifest from the registry. +// This library doesn't have plans to support this legacy image format: +// https://github.com/google/go-containerregistry/issues/377 +var ErrSchema1 = errors.New("unsupported MediaType: https://github.com/google/go-containerregistry/issues/377") + +// Descriptor provides access to metadata about remote artifact and accessors +// for efficiently converting it into a v1.Image or v1.ImageIndex. +type Descriptor struct { + fetcher + v1.Descriptor + Manifest []byte + + // So we can share this implementation with Image.. + platform v1.Platform +} + +// Get returns a remote.Descriptor for the given reference. The response from +// the registry is left un-interpreted, for the most part. This is useful for +// querying what kind of artifact a reference represents. +func Get(ref name.Reference, options ...Option) (*Descriptor, error) { + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + types.DockerManifestList, + types.OCIImageIndex, + // Just to look at them. + types.DockerManifestSchema1, + types.DockerManifestSchema1Signed, + } + return get(ref, acceptable, options...) +} + +// Handle options and fetch the manifest with the acceptable MediaTypes in the +// Accept header. +func get(ref name.Reference, acceptable []types.MediaType, options ...Option) (*Descriptor, error) { + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return nil, err + } + + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, []string{ref.Scope(transport.PullScope)}) + if err != nil { + return nil, err + } + + f := fetcher{ + Ref: ref, + Client: &http.Client{Transport: tr}, + } + + b, desc, err := f.fetchManifest(ref, acceptable) + if err != nil { + return nil, err + } + + return &Descriptor{ + fetcher: f, + Manifest: b, + Descriptor: *desc, + platform: o.platform, + }, nil +} + +// Image converts the Descriptor into a v1.Image. +// +// If the fetched artifact is already an image, it will just return it. +// +// If the fetched artifact is an index, it will attempt to resolve the index to +// a child image with the appropriate platform. +// +// See WithPlatform to set the desired platform. +func (d *Descriptor) Image() (v1.Image, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, ErrSchema1 + case types.OCIImageIndex, types.DockerManifestList: + // We want an image but the registry has an index, resolve it to an image. + return d.remoteIndex().imageByPlatform(d.platform) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // These are expected. Enumerated here to allow a default case. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + + // Wrap the v1.Layers returned by this v1.Image in a hint for downstream + // remote.Write calls to facilitate cross-repo "mounting". + imgCore, err := partial.CompressedToImage(d.remoteImage()) + if err != nil { + return nil, err + } + return &mountableImage{ + Image: imgCore, + Reference: d.Ref, + }, nil +} + +// ImageIndex converts the Descriptor into a v1.ImageIndex. +func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, ErrSchema1 + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // We want an index but the registry has an image, nothing we can do. + return nil, fmt.Errorf("unexpected media type for ImageIndex(): %s; call Image() instead", d.MediaType) + case types.OCIImageIndex, types.DockerManifestList: + // These are expected. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + return d.remoteIndex(), nil +} + +func (d *Descriptor) remoteImage() *remoteImage { + return &remoteImage{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +func (d *Descriptor) remoteIndex() *remoteIndex { + return &remoteIndex{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +// fetcher implements methods for reading from a registry. +type fetcher struct { + Ref name.Reference + Client *http.Client +} + +// url returns a url.Url for the specified path in the context of this remote image reference. +func (f *fetcher) url(resource, identifier string) url.URL { + return url.URL{ + Scheme: f.Ref.Context().Registry.Scheme(), + Host: f.Ref.Context().RegistryStr(), + Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), + } +} + +func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { + u := f.url("manifests", ref.Identifier()) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, nil, err + } + accept := []string{} + for _, mt := range acceptable { + accept = append(accept, string(mt)) + } + req.Header.Set("Accept", strings.Join(accept, ",")) + + resp, err := f.Client.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, nil, err + } + + manifest, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + digest, size, err := v1.SHA256(bytes.NewReader(manifest)) + if err != nil { + return nil, nil, err + } + + mediaType := types.MediaType(resp.Header.Get("Content-Type")) + + // Validate the digest matches what we asked for, if pulling by digest. + if dgst, ok := ref.(name.Digest); ok { + if mediaType == types.DockerManifestSchema1Signed { + // Digests for this are stupid to calculate, ignore it. + } else if digest.String() != dgst.DigestStr() { + return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) + } + } else { + // Do nothing for tags; I give up. + // + // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, + // but so many registries implement this incorrectly that it's not worth checking. + // + // For reference: + // https://github.com/docker/distribution/issues/2395 + // https://github.com/GoogleContainerTools/kaniko/issues/298 + } + + // Return all this info since we have to calculate it anyway. + desc := v1.Descriptor{ + Digest: digest, + Size: size, + MediaType: mediaType, + } + + return manifest, &desc, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index 1be0ad2ea..752c71608 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -15,16 +15,12 @@ package remote import ( - "bytes" "fmt" "io" "io/ioutil" "net/http" - "net/url" - "strings" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" @@ -33,11 +29,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/v1util" ) -var defaultPlatform = v1.Platform{ - Architecture: "amd64", - OS: "linux", -} - // remoteImage accesses an image from a remote registry type remoteImage struct { fetcher @@ -46,135 +37,26 @@ type remoteImage struct { configLock sync.Mutex // Protects config config []byte mediaType types.MediaType - platform v1.Platform } -// ImageOption is a functional option for Image. -type ImageOption func(*imageOpener) error - var _ partial.CompressedImageCore = (*remoteImage)(nil) -type imageOpener struct { - auth authn.Authenticator - transport http.RoundTripper - ref name.Reference - client *http.Client - platform v1.Platform -} +// Image provides access to a remote image reference. +func Image(ref name.Reference, options ...Option) (v1.Image, error) { + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + // We resolve these to images later. + types.DockerManifestList, + types.OCIImageIndex, + } -func (i *imageOpener) Open() (v1.Image, error) { - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + desc, err := get(ref, acceptable, options...) if err != nil { return nil, err } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - platform: i.platform, - } - imgCore, err := partial.CompressedToImage(ri) - if err != nil { - return imgCore, err - } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: i.ref, - }, nil -} -// Image provides access to a remote image reference, applying functional options -// to the underlying imageOpener before resolving the reference into a v1.Image. -func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) { - img := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, - platform: defaultPlatform, - } - - for _, option := range options { - if err := option(img); err != nil { - return nil, err - } - } - return img.Open() -} - -// fetcher implements methods for reading from a remote image. -type fetcher struct { - Ref name.Reference - Client *http.Client -} - -// url returns a url.Url for the specified path in the context of this remote image reference. -func (f *fetcher) url(resource, identifier string) url.URL { - return url.URL{ - Scheme: f.Ref.Context().Registry.Scheme(), - Host: f.Ref.Context().RegistryStr(), - Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), - } -} - -func (f *fetcher) fetchManifest(acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { - u := f.url("manifests", f.Ref.Identifier()) - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return nil, nil, err - } - accept := []string{} - for _, mt := range acceptable { - accept = append(accept, string(mt)) - } - req.Header.Set("Accept", strings.Join(accept, ",")) - - resp, err := f.Client.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if err := transport.CheckError(resp, http.StatusOK); err != nil { - return nil, nil, err - } - - manifest, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } - - digest, size, err := v1.SHA256(bytes.NewReader(manifest)) - if err != nil { - return nil, nil, err - } - - // Validate the digest matches what we asked for, if pulling by digest. - if dgst, ok := f.Ref.(name.Digest); ok { - if digest.String() != dgst.DigestStr() { - return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) - } - } else { - // Do nothing for tags; I give up. - // - // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, - // but so many registries implement this incorrectly that it's not worth checking. - // - // For reference: - // https://github.com/docker/distribution/issues/2395 - // https://github.com/GoogleContainerTools/kaniko/issues/298 - } - - // Return all this info since we have to calculate it anyway. - desc := v1.Descriptor{ - Digest: digest, - Size: size, - MediaType: types.MediaType(resp.Header.Get("Content-Type")), - } - - return manifest, &desc, nil + return desc.Image() } func (r *remoteImage) MediaType() (types.MediaType, error) { @@ -184,7 +66,6 @@ func (r *remoteImage) MediaType() (types.MediaType, error) { return types.DockerManifestSchema2, nil } -// TODO(jonjohnsonjr): Handle manifest lists. func (r *remoteImage) RawManifest() ([]byte, error) { r.manifestLock.Lock() defer r.manifestLock.Unlock() @@ -192,26 +73,18 @@ func (r *remoteImage) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteImage. acceptable := []types.MediaType{ types.DockerManifestSchema2, types.OCIManifestSchema1, - // We'll resolve these to an image based on the platform. - types.DockerManifestList, - types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } - // We want an image but the registry has an index, resolve it to an image. - for desc.MediaType == types.DockerManifestList || desc.MediaType == types.OCIImageIndex { - manifest, desc, err = r.matchImage(manifest) - if err != nil { - return nil, err - } - } - r.mediaType = desc.MediaType r.manifest = manifest return r.manifest, nil @@ -278,6 +151,22 @@ func (rl *remoteLayer) Manifest() (*v1.Manifest, error) { return partial.Manifest(rl.ri) } +// MediaType implements v1.Layer +func (rl *remoteLayer) MediaType() (types.MediaType, error) { + m, err := rl.Manifest() + if err != nil { + return "", err + } + + for _, layer := range m.Layers { + if layer.Digest == rl.digest { + return layer.MediaType, nil + } + } + + return "", fmt.Errorf("unable to find layer with digest: %v", rl.digest) +} + // Size implements partial.CompressedLayer func (rl *remoteLayer) Size() (int64, error) { // Look up the size of this digest in the manifest to avoid a request. @@ -302,36 +191,3 @@ func (r *remoteImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) digest: h, }, nil } - -// This naively matches the first manifest with matching Architecture and OS. -// -// We should probably use this instead: -// github.com/containerd/containerd/platforms -// -// But first we'd need to migrate to: -// github.com/opencontainers/image-spec/specs-go/v1 -func (r *remoteImage) matchImage(rawIndex []byte) ([]byte, *v1.Descriptor, error) { - index, err := v1.ParseIndexManifest(bytes.NewReader(rawIndex)) - if err != nil { - return nil, nil, err - } - for _, childDesc := range index.Manifests { - // If platform is missing from child descriptor, assume it's amd64/linux. - p := defaultPlatform - if childDesc.Platform != nil { - p = *childDesc.Platform - } - if r.platform.Architecture == p.Architecture && r.platform.OS == p.OS { - childRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), childDesc.Digest), name.StrictValidation) - if err != nil { - return nil, nil, err - } - r.fetcher = fetcher{ - Client: r.Client, - Ref: childRef, - } - return r.fetchManifest([]types.MediaType{childDesc.MediaType}) - } - } - return nil, nil, fmt.Errorf("no matching image for %s/%s, index: %s", r.platform.Architecture, r.platform.OS, string(rawIndex)) -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index 03afc481a..1303dda9c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -17,14 +17,11 @@ package remote import ( "bytes" "fmt" - "net/http" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -36,30 +33,19 @@ type remoteIndex struct { mediaType types.MediaType } -// Index provides access to a remote index reference, applying functional options -// to the underlying imageOpener before resolving the reference into a v1.ImageIndex. -func Index(ref name.Reference, options ...ImageOption) (v1.ImageIndex, error) { - i := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, +// Index provides access to a remote index reference. +func Index(ref name.Reference, options ...Option) (v1.ImageIndex, error) { + acceptable := []types.MediaType{ + types.DockerManifestList, + types.OCIImageIndex, } - for _, option := range options { - if err := option(i); err != nil { - return nil, err - } - } - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + desc, err := get(ref, acceptable, options...) if err != nil { return nil, err } - return &remoteIndex{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - }, nil + + return desc.ImageIndex() } func (r *remoteIndex) MediaType() (types.MediaType, error) { @@ -80,11 +66,14 @@ func (r *remoteIndex) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteIndex. acceptable := []types.MediaType{ types.DockerManifestList, types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } @@ -103,37 +92,93 @@ func (r *remoteIndex) IndexManifest() (*v1.IndexManifest, error) { } func (r *remoteIndex) Image(h v1.Hash) (v1.Image, error) { - imgRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) + desc, err := r.childByHash(h) if err != nil { return nil, err } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: imgRef, - Client: r.Client, - }, - } - imgCore, err := partial.CompressedToImage(ri) - if err != nil { - return imgCore, err - } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: r.Ref, - }, nil + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() } func (r *remoteIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { - idxRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) + desc, err := r.childByHash(h) if err != nil { return nil, err } - return &remoteIndex{ + return desc.ImageIndex() +} + +func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) { + desc, err := r.childByPlatform(platform) + if err != nil { + return nil, err + } + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() +} + +// This naively matches the first manifest with matching Architecture and OS. +// +// We should probably use this instead: +// github.com/containerd/containerd/platforms +// +// But first we'd need to migrate to: +// github.com/opencontainers/image-spec/specs-go/v1 +func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + // If platform is missing from child descriptor, assume it's amd64/linux. + p := defaultPlatform + if childDesc.Platform != nil { + p = *childDesc.Platform + } + + if platform.Architecture == p.Architecture && platform.OS == p.OS { + return r.childDescriptor(childDesc, platform) + } + } + return nil, fmt.Errorf("no child with platform %s/%s in index %s", platform.Architecture, platform.OS, r.Ref) +} + +func (r *remoteIndex) childByHash(h v1.Hash) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + if h == childDesc.Digest { + return r.childDescriptor(childDesc, defaultPlatform) + } + } + return nil, fmt.Errorf("no child with digest %s in index %s", h, r.Ref) +} + +func (r *remoteIndex) childRef(h v1.Hash) (name.Reference, error) { + return name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) +} + +// Convert one of this index's child's v1.Descriptor into a remote.Descriptor, with the given platform option. +func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) (*Descriptor, error) { + ref, err := r.childRef(child.Digest) + if err != nil { + return nil, err + } + manifest, desc, err := r.fetchManifest(ref, []types.MediaType{child.MediaType}) + if err != nil { + return nil, err + } + return &Descriptor{ fetcher: fetcher{ - Ref: idxRef, + Ref: ref, Client: r.Client, }, + Manifest: manifest, + Descriptor: *desc, + platform: platform, }, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go index 1a36d0a4b..4cbdc4f19 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go @@ -20,7 +20,6 @@ import ( "net/http" "net/url" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) @@ -30,10 +29,15 @@ type tags struct { Tags []string `json:"tags"` } -// List calls /tags/list for the given repository. -func List(repo name.Repository, auth authn.Authenticator, t http.RoundTripper) ([]string, error) { +// List calls /tags/list for the given repository, returning the list of tags +// in the "tags" property. +func List(repo name.Repository, options ...Option) ([]string, error) { + o, err := makeOptions(repo.Registry, options...) + if err != nil { + return nil, err + } scopes := []string{repo.Scope(transport.PullScope)} - tr, err := transport.New(repo.Registry, auth, t, scopes) + tr, err := transport.New(repo.Registry, o.auth, o.transport, scopes) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 335e3fe5b..7edfcbabc 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -19,46 +19,88 @@ import ( "net/http" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" ) +// Option is a functional option for remote operations. +type Option func(*options) error + +type options struct { + auth authn.Authenticator + keychain authn.Keychain + transport http.RoundTripper + platform v1.Platform +} + +func makeOptions(reg name.Registry, opts ...Option) (*options, error) { + o := &options{ + auth: authn.Anonymous, + transport: http.DefaultTransport, + platform: defaultPlatform, + } + + for _, option := range opts { + if err := option(o); err != nil { + return nil, err + } + } + + if o.keychain != nil { + auth, err := o.keychain.Resolve(reg) + if err != nil { + return nil, err + } + if auth == authn.Anonymous { + log.Println("No matching credentials were found, falling back on anonymous") + } + o.auth = auth + } + + return o, nil +} + // WithTransport is a functional option for overriding the default transport -// on a remote image -func WithTransport(t http.RoundTripper) ImageOption { - return func(i *imageOpener) error { - i.transport = t +// for remote operations. +// +// The default transport its http.DefaultTransport. +func WithTransport(t http.RoundTripper) Option { + return func(o *options) error { + o.transport = t return nil } } // WithAuth is a functional option for overriding the default authenticator -// on a remote image -func WithAuth(auth authn.Authenticator) ImageOption { - return func(i *imageOpener) error { - i.auth = auth +// for remote operations. +// +// The default authenticator is authn.Anonymous. +func WithAuth(auth authn.Authenticator) Option { + return func(o *options) error { + o.auth = auth return nil } } // WithAuthFromKeychain is a functional option for overriding the default -// authenticator on a remote image using an authn.Keychain -func WithAuthFromKeychain(keys authn.Keychain) ImageOption { - return func(i *imageOpener) error { - auth, err := keys.Resolve(i.ref.Context().Registry) - if err != nil { - return err - } - if auth == authn.Anonymous { - log.Println("No matching credentials were found, falling back on anonymous") - } - i.auth = auth +// authenticator for remote operations, using an authn.Keychain to find +// credentials. +// +// The default authenticator is authn.Anonymous. +func WithAuthFromKeychain(keys authn.Keychain) Option { + return func(o *options) error { + o.keychain = keys return nil } } -func WithPlatform(p v1.Platform) ImageOption { - return func(i *imageOpener) error { - i.platform = p +// WithPlatform is a functional option for overriding the default platform +// that Image and Descriptor.Image use for resolving an index to an image. +// +// The default platform is amd64/linux. +func WithPlatform(p v1.Platform) Option { + return func(o *options) error { + o.platform = p return nil } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index f72ab276d..f9cd194ad 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -60,10 +60,14 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { // In case of redirect http.Client can use an empty Host, check URL too. if in.Host == bt.registry.RegistryStr() || in.URL.Host == bt.registry.RegistryStr() { in.Header.Set("Authorization", hdr) + + // When we ping() the registry, we determine whether to use http or https + // based on which scheme was successful. That is only valid for the + // registry server and not e.g. a separate token server or blob storage, + // so we should only override the scheme if the host is the registry. + in.URL.Scheme = bt.scheme } in.Header.Set("User-Agent", transportName) - - in.URL.Scheme = bt.scheme return bt.inner.RoundTrip(in) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 44885effa..5ca0b0881 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -48,6 +48,20 @@ func (e *Error) Error() string { } } +// ShouldRetry returns whether the request that preceded the error should be retried. +func (e *Error) ShouldRetry() bool { + if len(e.Errors) == 0 { + return false + } + for _, d := range e.Errors { + // TODO: Include other error types. + if d.Code != BlobUploadInvalidErrorCode { + return false + } + } + return true +} + // Diagnostic represents a single error returned by a Docker registry interaction. type Diagnostic struct { Code ErrorCode `json:"code"` diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 66f148155..557862aca 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -20,15 +20,15 @@ import ( "fmt" "io" "log" + "math" "net/http" "net/url" + "time" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" - "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "golang.org/x/sync/errgroup" ) @@ -40,14 +40,19 @@ type manifest interface { } // Write pushes the provided img to the specified image reference. -func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.RoundTripper) error { +func Write(ref name.Reference, img v1.Image, options ...Option) error { ls, err := img.Layers() if err != nil { return err } + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } + scopes := scopesForUploadingImage(ref, ls) - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } @@ -57,17 +62,17 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro } // Upload individual layers in goroutines and collect any errors. - // If we can dedupe by the layer digest, try to do so. If the layer is - // a stream.Layer, we can't dedupe and might re-upload. + // If we can dedupe by the layer digest, try to do so. If we can't determine + // the digest for whatever reason, we can't dedupe and might re-upload. var g errgroup.Group uploaded := map[v1.Hash]bool{} for _, l := range ls { l := l - if _, ok := l.(*stream.Layer); !ok { - h, err := l.Digest() - if err != nil { - return err - } + + // Streaming layers calculate their digests while uploading them. Assume + // an error here indicates we need to upload the layer. + h, err := l.Digest() + if err == nil { // If we can determine the layer's digest ahead of // time, use it to dedupe uploads. if uploaded[h] { @@ -81,14 +86,15 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro }) } - if l, err := partial.ConfigLayer(img); err == stream.ErrNotComputed { - // We can't read the ConfigLayer, because of streaming layers, since the - // config hasn't been calculated yet. + if l, err := partial.ConfigLayer(img); err != nil { + // We can't read the ConfigLayer, possibly because of streaming layers, + // since the layer DiffIDs haven't been calculated yet. Attempt to wait + // for the other layers to be uploaded, then try the config again. if err := g.Wait(); err != nil { return err } - // Now that all the layers are uploaded, upload the config file blob. + // Now that all the layers are uploaded, try to upload the config file blob. l, err := partial.ConfigLayer(img) if err != nil { return err @@ -96,9 +102,6 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro if err := w.uploadOne(l); err != nil { return err } - } else if err != nil { - // This is an actual error, not a streaming error, just return it. - return err } else { // We *can* read the ConfigLayer, so upload it concurrently with the layers. g.Go(func() error { @@ -285,19 +288,10 @@ func (w *writer) commitBlob(location, digest string) error { // uploadOne performs a complete upload of a single layer. func (w *writer) uploadOne(l v1.Layer) error { - var from, mount, digest string - if _, ok := l.(*stream.Layer); !ok { - // Layer isn't streamable, we should take advantage of that to - // skip uploading if possible. - // By sending ?digest= in the request, we'll also check that - // our computed digest matches the one computed by the - // registry. - h, err := l.Digest() - if err != nil { - return err - } - digest = h.String() - + var from, mount string + if h, err := l.Digest(); err == nil { + // If we know the digest, this isn't a streaming layer. Do an existence + // check so we can skip uploading the layer if possible. existing, err := w.checkExistingBlob(h) if err != nil { return err @@ -315,38 +309,56 @@ func (w *writer) uploadOne(l v1.Layer) error { } } - location, mounted, err := w.initiateUpload(from, mount) - if err != nil { - return err - } else if mounted { + tryUpload := func() error { + location, mounted, err := w.initiateUpload(from, mount) + if err != nil { + return err + } else if mounted { + h, err := l.Digest() + if err != nil { + return err + } + log.Printf("mounted blob: %s", h.String()) + return nil + } + + blob, err := l.Compressed() + if err != nil { + return err + } + location, err = w.streamBlob(blob, location) + if err != nil { + return err + } + h, err := l.Digest() if err != nil { return err } - log.Printf("mounted blob: %s", h.String()) + digest := h.String() + + if err := w.commitBlob(location, digest); err != nil { + return err + } + log.Printf("pushed blob: %s", digest) return nil } - - blob, err := l.Compressed() - if err != nil { - return err + const maxRetries = 2 + const backoffFactor = 0.5 + retries := 0 + for { + err := tryUpload() + if err == nil { + return nil + } + if te, ok := err.(*transport.Error); !(ok && te.ShouldRetry()) || retries >= maxRetries { + return err + } + log.Printf("retrying after error: %s", err) + retries++ + duration := time.Duration(backoffFactor*math.Pow(2, float64(retries))) * time.Second + time.Sleep(duration) } - location, err = w.streamBlob(blob, location) - if err != nil { - return err - } - - h, err := l.Digest() - if err != nil { - return err - } - digest = h.String() - - if err := w.commitBlob(location, digest); err != nil { - return err - } - log.Printf("pushed blob: %s", digest) - return nil } // commitImage does a PUT of the image's manifest. @@ -416,14 +428,18 @@ func scopesForUploadingImage(ref name.Reference, layers []v1.Layer) []string { // WriteIndex pushes the provided ImageIndex to the specified image reference. // WriteIndex will attempt to push all of the referenced manifests before // attempting to push the ImageIndex, to retain referential integrity. -func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, t http.RoundTripper) error { +func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { index, err := ii.IndexManifest() if err != nil { return err } + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } scopes := []string{ref.Scope(transport.PushScope)} - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } @@ -453,7 +469,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, return err } - if err := WriteIndex(ref, ii, auth, t); err != nil { + if err := WriteIndex(ref, ii, WithAuth(o.auth), WithTransport(o.transport)); err != nil { return err } case types.OCIManifestSchema1, types.DockerManifestSchema2: @@ -461,7 +477,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, if err != nil { return err } - if err := Write(ref, img, auth, t); err != nil { + if err := Write(ref, img, WithAuth(o.auth), WithTransport(o.transport)); err != nil { return err } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go index f8895a226..aa816359c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go @@ -24,6 +24,7 @@ import ( "sync" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" ) var ( @@ -81,6 +82,13 @@ func (l *Layer) Size() (int64, error) { return l.size, nil } +// MediaType implements v1.Layer +func (l *Layer) MediaType() (types.MediaType, error) { + // We return DockerLayer for now as uncompressed layers + // are unimplemented + return types.DockerLayer, nil +} + // Uncompressed implements v1.Layer. func (l *Layer) Uncompressed() (io.ReadCloser, error) { return nil, errors.New("NYI: stream.Layer.Uncompressed is not implemented") diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go index ced18735c..06ecd9a77 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go @@ -119,7 +119,7 @@ func (td tarDescriptor) findSpecifiedImageDescriptor(tag *name.Tag) (*singleImag } for _, img := range td { for _, tagStr := range img.RepoTags { - repoTag, err := name.NewTag(tagStr, name.WeakValidation) + repoTag, err := name.NewTag(tagStr) if err != nil { return nil, err } @@ -226,6 +226,13 @@ func (ulft *uncompressedLayerFromTarball) Uncompressed() (io.ReadCloser, error) return extractFileFromTar(ulft.opener, ulft.filePath) } +func (ulft *uncompressedLayerFromTarball) MediaType() (types.MediaType, error) { + // Technically the media type should be 'application/tar' but given that our + // v1.Layer doesn't force consumers to care about whether the layer is compressed + // we should be fine returning the DockerLayer media type + return types.DockerLayer, nil +} + func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) { cfg, err := partial.ConfigFile(i) if err != nil { @@ -310,6 +317,11 @@ func (clft *compressedLayerFromTarball) Compressed() (io.ReadCloser, error) { return extractFileFromTar(clft.opener, clft.filePath) } +// MediaType implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + // Size implements partial.CompressedLayer func (clft *compressedLayerFromTarball) Size() (int64, error) { r, err := clft.Compressed() diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index 00256e8f2..85c1b7838 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -15,12 +15,14 @@ package tarball import ( + "bytes" "compress/gzip" "io" "io/ioutil" "os" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -62,6 +64,10 @@ func (l *layer) Size() (int64, error) { return l.size, nil } +func (l *layer) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + // LayerFromFile returns a v1.Layer given a tarball func LayerFromFile(path string) (v1.Layer, error) { opener := func() (io.ReadCloser, error) { @@ -103,6 +109,18 @@ func LayerFromOpener(opener Opener) (v1.Layer, error) { }, nil } +// LayerFromReader returns a v1.Layer given a io.Reader. +func LayerFromReader(reader io.Reader) (v1.Layer, error) { + // Buffering due to Opener requiring multiple calls. + a, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + return LayerFromOpener(func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(a)), nil + }) +} + func computeDigest(opener Opener, compressed bool) (v1.Hash, int64, error) { rc, err := opener() if err != nil {