diff --git a/Gopkg.lock b/Gopkg.lock index 44926c25d..bd23f0578 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -445,11 +445,13 @@ version = "v0.2.0" [[projects]] - digest = "1:3ccc9b3dfd6b951b46e6e1c499af589fbc35c7e1172d6d840cbe836ae08d3536" + digest = "1:16c8837e951303ef6388132bc875337660a48ea2dedf1c941ca118ea92d2a3d2" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", "pkg/authn/k8schain", + "pkg/internal/retry", + "pkg/logs", "pkg/name", "pkg/v1", "pkg/v1/daemon", @@ -465,7 +467,7 @@ "pkg/v1/v1util", ] pruneopts = "NUT" - revision = "bb17f50c1bc6808972811ed2894ecaaeb5de68ad" + revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74" [[projects]] digest = "1:f4f203acd8b11b8747bdcd91696a01dbc95ccb9e2ca2db6abf81c3a4f5e950ce" @@ -1384,6 +1386,7 @@ "golang.org/x/oauth2", "golang.org/x/sync/errgroup", "gopkg.in/src-d/go-git.v4", + "gopkg.in/src-d/go-git.v4/plumbing", "k8s.io/client-go/discovery", ] solver-name = "gps-cdcl" diff --git a/Gopkg.toml b/Gopkg.toml index 163d821ad..64d3b5b7b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,7 +37,7 @@ required = [ [[constraint]] name = "github.com/google/go-containerregistry" - revision = "bb17f50c1bc6808972811ed2894ecaaeb5de68ad" + revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74" [[override]] name = "k8s.io/apimachinery" diff --git a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go index aee1fedb9..8b2ead5db 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go +++ b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go @@ -19,11 +19,11 @@ import ( "errors" "fmt" "io/ioutil" - "log" "os" "path/filepath" "runtime" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" ) @@ -100,19 +100,19 @@ var ( func (dk *defaultKeychain) Resolve(reg name.Registry) (Authenticator, error) { dir, err := configDir() if err != nil { - log.Printf("Unable to determine config dir: %v", err) + logs.Warn.Printf("Unable to determine config dir: %v", err) return Anonymous, nil } file := filepath.Join(dir, "config.json") content, err := ioutil.ReadFile(file) if err != nil { - log.Printf("Unable to read %q: %v", file, err) + logs.Warn.Printf("Unable to read %q: %v", file, err) return Anonymous, nil } var cf cfg if err := json.Unmarshal(content, &cf); err != nil { - log.Printf("Unable to parse %q: %v", file, err) + logs.Warn.Printf("Unable to parse %q: %v", file, err) return Anonymous, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go b/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go new file mode 100644 index 000000000..87f730955 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go @@ -0,0 +1,68 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package retry provides methods for retrying operations. It is a thin wrapper +// around k8s.io/apimachinery/pkg/util/wait to make certain operations easier. +package retry + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/util/wait" +) + +// This is implemented by several errors in the net package as well as our +// transport.Error. +type temporary interface { + Temporary() bool +} + +// IsTemporary returns true if err implements Temporary() and it returns true. +func IsTemporary(err error) bool { + if te, ok := err.(temporary); ok && te.Temporary() { + return true + } + return false +} + +// IsNotNil returns true if err is not nil. +func IsNotNil(err error) bool { + return err != nil +} + +// Predicate determines whether an error should be retried. +type Predicate func(error) (retry bool) + +// Retry retries a given function, f, until a predicate is satisfied, using +// exponential backoff. If the predicate is never satisfied, it will return the +// last error returned by f. +func Retry(f func() error, p Predicate, backoff wait.Backoff) (err error) { + if f == nil { + return fmt.Errorf("nil f passed to retry") + } + if p == nil { + return fmt.Errorf("nil p passed to retry") + } + + condition := func() (bool, error) { + err = f() + if p(err) { + return false, nil + } + return true, err + } + + wait.ExponentialBackoff(backoff, condition) + return +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go b/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go new file mode 100644 index 000000000..af3c1a3b7 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go @@ -0,0 +1,29 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logs exposes the loggers used by this library. +package logs + +import ( + "io/ioutil" + "log" +) + +var ( + // Warn is used to log non-fatal errors. + Warn = log.New(ioutil.Discard, "", log.LstdFlags) + + // Progress is used to log notable, successful events. + Progress = log.New(ioutil.Discard, "", log.LstdFlags) +) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index 11262f444..813205dad 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -77,6 +77,8 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) { } cf.Config = cfg + // Downstream tooling expects these to match. + cf.ContainerConfig = cfg return ConfigFile(base, cf) } @@ -468,7 +470,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { // Copy basic config over cfg.Config = ocf.Config - cfg.ContainerConfig = ocf.ContainerConfig + cfg.ContainerConfig = ocf.Config // Downstream tooling expects these to match. // Strip away timestamps from the config file cfg.Created = v1.Time{Time: t} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go b/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go index df9b2959e..bb9886433 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go @@ -21,4 +21,5 @@ type Platform struct { OSVersion string `json:"os.version,omitempty"` OSFeatures []string `json:"os.features,omitempty"` Variant string `json:"variant,omitempty"` + Features []string `json:"features,omitempty"` } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go index 078de3a0b..144b99ecc 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -225,12 +225,16 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType } mediaType := types.MediaType(resp.Header.Get("Content-Type")) + contentDigest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest")) + if err == nil && mediaType == types.DockerManifestSchema1Signed { + // If we can parse the digest from the header, and it's a signed schema 1 + // manifest, let's use that for the digest to appease older registries. + digest = contentDigest + } // Validate the digest matches what we asked for, if pulling by digest. if dgst, ok := ref.(name.Digest); ok { - if mediaType == types.DockerManifestSchema1Signed { - // Digests for this are stupid to calculate, ignore it. - } else if digest.String() != dgst.DigestStr() { + if digest.String() != dgst.DigestStr() { return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) } } else { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index 1303dda9c..043dc83b6 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -119,7 +119,7 @@ func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) { return desc.Image() } -// This naively matches the first manifest with matching Architecture and OS. +// This naively matches the first manifest with matching platform attributes. // // We should probably use this instead: // github.com/containerd/containerd/platforms @@ -138,7 +138,7 @@ func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) p = *childDesc.Platform } - if platform.Architecture == p.Architecture && platform.OS == p.OS { + if matchesPlatform(p, platform) { return r.childDescriptor(childDesc, platform) } } @@ -182,3 +182,49 @@ func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) platform: platform, }, nil } + +// matchesPlatform checks if the given platform matches the required platforms. +// The given platform matches the required platform if +// - architecture and OS are identical. +// - OS version and variant are identical if provided. +// - features and OS features of the required platform are subsets of those of the given platform. +func matchesPlatform(given, required v1.Platform) bool { + // Required fields that must be identical. + if given.Architecture != required.Architecture || given.OS != required.OS { + return false + } + + // Optional fields that may be empty, but must be identical if provided. + if required.OSVersion != "" && given.OSVersion != required.OSVersion { + return false + } + if required.Variant != "" && given.Variant != required.Variant { + return false + } + + // Verify required platform's features are a subset of given platform's features. + if !isSubset(given.OSFeatures, required.OSFeatures) { + return false + } + if !isSubset(given.Features, required.Features) { + return false + } + + return true +} + +// isSubset checks if the required array of strings is a subset of the given lst. +func isSubset(lst, required []string) bool { + set := make(map[string]bool) + for _, value := range lst { + set[value] = true + } + + for _, value := range required { + if _, ok := set[value]; !ok { + return false + } + } + + return true +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 7edfcbabc..c60344840 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -15,12 +15,13 @@ package remote import ( - "log" "net/http" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) // Option is a functional option for remote operations. @@ -52,11 +53,14 @@ func makeOptions(reg name.Registry, opts ...Option) (*options, error) { return nil, err } if auth == authn.Anonymous { - log.Println("No matching credentials were found, falling back on anonymous") + logs.Warn.Println("No matching credentials were found, falling back on anonymous") } o.auth = auth } + // Wrap the transport in something that can retry network flakes. + o.transport = transport.NewRetry(o.transport) + return o, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go index e77f47f69..b98b4a625 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go @@ -40,7 +40,7 @@ func (bt *basicTransport) RoundTrip(in *http.Request) (*http.Response, error) { // we are redirected, only set it when the authorization header matches // the host with which we are interacting. // In case of redirect http.Client can use an empty Host, check URL too. - if in.Host == bt.target || in.URL.Host == bt.target { + if hdr != "" && (in.Host == bt.target || in.URL.Host == bt.target) { in.Header.Set("Authorization", hdr) } in.Header.Set("User-Agent", transportName) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index f9cd194ad..326708b97 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -18,8 +18,10 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/url" + "strings" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" @@ -45,6 +47,11 @@ type bearerTransport struct { var _ http.RoundTripper = (*bearerTransport)(nil) +var portMap = map[string]string{ + "http": "80", + "https": "443", +} + // RoundTrip implements http.RoundTripper func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { sendRequest := func() (*http.Response, error) { @@ -58,7 +65,10 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { // we are redirected, only set it when the authorization header matches // the registry with which we are interacting. // In case of redirect http.Client can use an empty Host, check URL too. - if in.Host == bt.registry.RegistryStr() || in.URL.Host == bt.registry.RegistryStr() { + canonicalHeaderHost := bt.canonicalAddress(in.Host) + canonicalURLHost := bt.canonicalAddress(in.URL.Host) + canonicalRegistryHost := bt.canonicalAddress(bt.registry.RegistryStr()) + if canonicalHeaderHost == canonicalRegistryHost || canonicalURLHost == canonicalRegistryHost { in.Header.Set("Authorization", hdr) // When we ping() the registry, we determine whether to use http or https @@ -144,3 +154,28 @@ func (bt *bearerTransport) refresh() error { bt.bearer = &bearer return nil } + +func (bt *bearerTransport) canonicalAddress(host string) (address string) { + // The host may be any one of: + // - hostname + // - hostname:port + // - ipv4 + // - ipv4:port + // - ipv6 + // - [ipv6]:port + // As net.SplitHostPort returns an error if the host does not contain a port, we should only attempt + // to call it when we know that the address contains a port + if strings.Count(host, ":") == 1 || (strings.Count(host, ":") >= 2 && strings.Contains(host, "]:")) { + hostname, port, err := net.SplitHostPort(host) + if err != nil { + return host + } + if port == "" { + port = portMap[bt.scheme] + } + + return net.JoinHostPort(hostname, port) + } + + return net.JoinHostPort(host, portMap[bt.scheme]) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 5ca0b0881..3673a341b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -48,8 +48,8 @@ func (e *Error) Error() string { } } -// ShouldRetry returns whether the request that preceded the error should be retried. -func (e *Error) ShouldRetry() bool { +// Temporary returns whether the request that preceded the error is temporary. +func (e *Error) Temporary() bool { if len(e.Errors) == 0 { return false } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go new file mode 100644 index 000000000..b6b2181da --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go @@ -0,0 +1,89 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "net/http" + "time" + + "github.com/google/go-containerregistry/pkg/internal/retry" + "k8s.io/apimachinery/pkg/util/wait" +) + +// Sleep for 0.1, 0.3, 0.9, 2.7 seconds. This should cover networking blips. +var defaultBackoff = wait.Backoff{ + Duration: 100 * time.Millisecond, + Factor: 3.0, + Jitter: 0.1, + Steps: 5, +} + +var _ http.RoundTripper = (*retryTransport)(nil) + +// retryTransport wraps a RoundTripper and retries temporary network errors. +type retryTransport struct { + inner http.RoundTripper + backoff wait.Backoff + predicate retry.Predicate +} + +// Option is a functional option for retryTransport. +type Option func(*options) + +type options struct { + backoff wait.Backoff + predicate retry.Predicate +} + +// WithRetryBackoff sets the backoff for retry operations. +func WithRetryBackoff(backoff wait.Backoff) Option { + return func(o *options) { + o.backoff = backoff + } +} + +// WithRetryPredicate sets the predicate for retry operations. +func WithRetryPredicate(predicate func(error) bool) Option { + return func(o *options) { + o.predicate = predicate + } +} + +// NewRetry returns a transport that retries errors. +func NewRetry(inner http.RoundTripper, opts ...Option) http.RoundTripper { + o := &options{ + backoff: defaultBackoff, + predicate: retry.IsTemporary, + } + + for _, opt := range opts { + opt(o) + } + + return &retryTransport{ + inner: inner, + backoff: o.backoff, + predicate: o.predicate, + } +} + +func (t *retryTransport) RoundTrip(in *http.Request) (out *http.Response, err error) { + roundtrip := func() error { + out, err = t.inner.RoundTrip(in) + return err + } + retry.Retry(roundtrip, t.predicate, t.backoff) + return +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 557862aca..8806aa187 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -19,18 +19,19 @@ import ( "errors" "fmt" "io" - "log" - "math" "net/http" "net/url" "time" + "github.com/google/go-containerregistry/pkg/internal/retry" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/util/wait" ) type manifest interface { @@ -297,7 +298,7 @@ func (w *writer) uploadOne(l v1.Layer) error { return err } if existing { - log.Printf("existing blob: %v", h) + logs.Progress.Printf("existing blob: %v", h) return nil } @@ -318,7 +319,7 @@ func (w *writer) uploadOne(l v1.Layer) error { if err != nil { return err } - log.Printf("mounted blob: %s", h.String()) + logs.Progress.Printf("mounted blob: %s", h.String()) return nil } @@ -340,25 +341,19 @@ func (w *writer) uploadOne(l v1.Layer) error { if err := w.commitBlob(location, digest); err != nil { return err } - log.Printf("pushed blob: %s", digest) + logs.Progress.Printf("pushed blob: %s", digest) return nil } - const maxRetries = 2 - const backoffFactor = 0.5 - retries := 0 - for { - err := tryUpload() - if err == nil { - return nil - } - if te, ok := err.(*transport.Error); !(ok && te.ShouldRetry()) || retries >= maxRetries { - return err - } - log.Printf("retrying after error: %s", err) - retries++ - duration := time.Duration(backoffFactor*math.Pow(2, float64(retries))) * time.Second - time.Sleep(duration) + + // Try this three times, waiting 1s after first failure, 3s after second. + backoff := wait.Backoff{ + Duration: 1.0 * time.Second, + Factor: 3.0, + Jitter: 0.1, + Steps: 3, } + + return retry.Retry(tryUpload, retry.IsTemporary, backoff) } // commitImage does a PUT of the image's manifest. @@ -397,7 +392,7 @@ func (w *writer) commitImage(man manifest) error { } // The image was successfully pushed! - log.Printf("%v: digest: %v size: %d", w.ref, digest, len(raw)) + logs.Progress.Printf("%v: digest: %v size: %d", w.ref, digest, len(raw)) return nil } @@ -458,7 +453,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { return err } if exists { - log.Printf("existing manifest: %v", desc.Digest) + logs.Progress.Printf("existing manifest: %v", desc.Digest) continue }