Improve retry behavior for push operation (#1578)

* Update go-containerregistry to 0.4

* Implement --push-retry argument

vendor/github.com/google/go-containerregistry/pkg/v1/internal/gzip/zip.go

@@ -15,6 +15,7 @@
 package gzip
 
 import (
+	"bufio"
 	"bytes"
 	"compress/gzip"
 	"io"
@@ -38,11 +39,19 @@ func ReadCloser(r io.ReadCloser) io.ReadCloser {
 func ReadCloserLevel(r io.ReadCloser, level int) io.ReadCloser {
 	pr, pw := io.Pipe()
 
+	// For highly compressible layers, gzip.Writer will output a very small
+	// number of bytes per Write(). This is normally fine, but when pushing
+	// to a registry, we want to ensure that we're taking full advantage of
+	// the available bandwidth instead of sending tons of tiny writes over
+	// the wire.
+	// 64K ought to be small enough for anybody.
+	bw := bufio.NewWriterSize(pw, 2<<16)
+
 	// Returns err so we can pw.CloseWithError(err)
 	go func() error {
 		// TODO(go1.14): Just defer {pw,gw,r}.Close like you'd expect.
 		// Context: https://golang.org/issue/24283
-		gw, err := gzip.NewWriterLevel(pw, level)
+		gw, err := gzip.NewWriterLevel(bw, level)
 		if err != nil {
 			return pw.CloseWithError(err)
 		}
@@ -52,9 +61,20 @@ func ReadCloserLevel(r io.ReadCloser, level int) io.ReadCloser {
 			defer gw.Close()
 			return pw.CloseWithError(err)
 		}
+
+		// Close gzip writer to Flush it and write gzip trailers.
+		if err := gw.Close(); err != nil {
+			return pw.CloseWithError(err)
+		}
+
+		// Flush bufio writer to ensure we write out everything.
+		if err := bw.Flush(); err != nil {
+			return pw.CloseWithError(err)
+		}
+
+		// We dont' really care if these fail.
 		defer pw.Close()
 		defer r.Close()
-		defer gw.Close()
 
 		return nil
 	}()

vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go

@@ -23,6 +23,9 @@ import (
 	"path/filepath"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/match"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/partial"
 	"github.com/google/go-containerregistry/pkg/v1/types"
 	"golang.org/x/sync/errgroup"
 )
@@ -127,6 +130,85 @@ func (l Path) AppendDescriptor(desc v1.Descriptor) error {
 	return l.WriteFile("index.json", rawIndex, os.ModePerm)
 }
 
+// ReplaceImage writes a v1.Image to the Path and updates
+// the index.json to reference it, replacing any existing one that matches matcher, if found.
+func (l Path) ReplaceImage(img v1.Image, matcher match.Matcher, options ...Option) error {
+	if err := l.WriteImage(img); err != nil {
+		return err
+	}
+
+	return l.replaceDescriptor(img, matcher, options...)
+}
+
+// ReplaceIndex writes a v1.ImageIndex to the Path and updates
+// the index.json to reference it, replacing any existing one that matches matcher, if found.
+func (l Path) ReplaceIndex(ii v1.ImageIndex, matcher match.Matcher, options ...Option) error {
+	if err := l.WriteIndex(ii); err != nil {
+		return err
+	}
+
+	return l.replaceDescriptor(ii, matcher, options...)
+}
+
+// replaceDescriptor adds a descriptor to the index.json of the Path, replacing
+// any one matching matcher, if found.
+func (l Path) replaceDescriptor(append mutate.Appendable, matcher match.Matcher, options ...Option) error {
+	ii, err := l.ImageIndex()
+	if err != nil {
+		return err
+	}
+
+	desc, err := partial.Descriptor(append)
+	if err != nil {
+		return err
+	}
+
+	for _, opt := range options {
+		if err := opt(desc); err != nil {
+			return err
+		}
+	}
+
+	add := mutate.IndexAddendum{
+		Add:        append,
+		Descriptor: *desc,
+	}
+	ii = mutate.AppendManifests(mutate.RemoveManifests(ii, matcher), add)
+
+	index, err := ii.IndexManifest()
+	if err != nil {
+		return err
+	}
+
+	rawIndex, err := json.MarshalIndent(index, "", "   ")
+	if err != nil {
+		return err
+	}
+
+	return l.WriteFile("index.json", rawIndex, os.ModePerm)
+}
+
+// RemoveDescriptors removes any descriptors that match the match.Matcher from the index.json of the Path.
+func (l Path) RemoveDescriptors(matcher match.Matcher) error {
+	ii, err := l.ImageIndex()
+	if err != nil {
+		return err
+	}
+	ii = mutate.RemoveManifests(ii, matcher)
+
+	index, err := ii.IndexManifest()
+	if err != nil {
+		return err
+	}
+
+	rawIndex, err := json.MarshalIndent(index, "", "   ")
+	if err != nil {
+		return err
+	}
+
+	return l.WriteFile("index.json", rawIndex, os.ModePerm)
+}
+
 // WriteFile write a file with arbitrary data at an arbitrary location in a v1
 // layout. Used mostly internally to write files like "oci-layout" and
 // "index.json", also can be used to write other arbitrary files. Do *not* use

vendor/github.com/google/go-containerregistry/pkg/v1/mutate/README.md

@@ -46,11 +46,11 @@ e.g. to appease a registry with strict validation of images (_looking at you, GC
 
 Rebase has [its own README](/cmd/crane/rebase.md).
 
-This is the underlying implementation of [`crane rebase`](https://github.com/google/go-containerregistry/blob/master/cmd/crane/doc/crane_rebase.md).
+This is the underlying implementation of [`crane rebase`](https://github.com/google/go-containerregistry/blob/main/cmd/crane/doc/crane_rebase.md).
 
 ### `Extract`
 
 Extract will flatten an image filesystem into a single tar stream,
 respecting whiteout files.
 
-This is the underlying implementation of [`crane export`](https://github.com/google/go-containerregistry/blob/master/cmd/crane/doc/crane_export.md).
+This is the underlying implementation of [`crane export`](https://github.com/google/go-containerregistry/blob/main/cmd/crane/doc/crane_export.md).

vendor/github.com/google/go-containerregistry/pkg/v1/mutate/index.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/google/go-containerregistry/pkg/logs"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/match"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
 	"github.com/google/go-containerregistry/pkg/v1/types"
 )
@@ -56,6 +57,8 @@ func computeDescriptor(ia IndexAddendum) (*v1.Descriptor, error) {
 type index struct {
 	base v1.ImageIndex
 	adds []IndexAddendum
+	// remove is removed before adds
+	remove match.Matcher
 
 	computed  bool
 	manifest  *v1.IndexManifest
@@ -90,6 +93,17 @@ func (i *index) compute() error {
 	}
 	manifest := m.DeepCopy()
 	manifests := manifest.Manifests
+
+	if i.remove != nil {
+		var cleanedManifests []v1.Descriptor
+		for _, m := range manifests {
+			if !i.remove(m) {
+				cleanedManifests = append(cleanedManifests, m)
+			}
+		}
+		manifests = cleanedManifests
+	}
+
 	for _, add := range i.adds {
 		desc, err := computeDescriptor(add)
 		if err != nil {
@@ -105,6 +119,7 @@ func (i *index) compute() error {
 			logs.Warn.Printf("Unexpected index addendum: %T", add.Add)
 		}
 	}
+
 	manifest.Manifests = manifests
 
 	// With OCI media types, this should not be set, see discussion:

vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go

@@ -27,6 +27,7 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/internal/gzip"
+	"github.com/google/go-containerregistry/pkg/v1/match"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/google/go-containerregistry/pkg/v1/types"
 )
@@ -92,6 +93,14 @@ func AppendManifests(base v1.ImageIndex, adds ...IndexAddendum) v1.ImageIndex {
 	}
 }
 
+// RemoveManifests removes any descriptors that match the match.Matcher.
+func RemoveManifests(base v1.ImageIndex, matcher match.Matcher) v1.ImageIndex {
+	return &index{
+		base:   base,
+		remove: matcher,
+	}
+}
+
 // Config mutates the provided v1.Image to have the provided v1.Config
 func Config(base v1.Image, cfg v1.Config) (v1.Image, error) {
 	cf, err := base.ConfigFile()

vendor/github.com/google/go-containerregistry/pkg/v1/remote/README.md

@@ -76,7 +76,7 @@ which references an application/xml file from an image index.
 
 That could look something like this:
 
-![exotic image index anatomy](/images/index-anatomy-exotic.dot.svg)
+![strange image index anatomy](/images/index-anatomy-strange.dot.svg)
 
 Using a recursive index like this might not be possible with all registries,
 but this flexibility allows for some interesting applications, e.g. the
@@ -114,4 +114,4 @@ however, it's possible to do _something_ useful with them via [`remote.Get`](htt
 which doesn't try to interpret what is returned by the registry.
 
 [`crane.Copy`](https://godoc.org/github.com/google/go-containerregistry/pkg/crane#Copy) takes advantage of this to implement support for copying schema 1 images,
-see [here](https://github.com/google/go-containerregistry/blob/master/pkg/internal/legacy/copy.go).
+see [here](https://github.com/google/go-containerregistry/blob/main/pkg/internal/legacy/copy.go).

vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go

@@ -66,7 +66,7 @@ func makeOptions(target authn.Resource, opts ...Option) (*options, error) {
 			return nil, err
 		}
 		if auth == authn.Anonymous {
-			logs.Warn.Println("No matching credentials were found, falling back on anonymous")
+			logs.Warn.Printf("No matching credentials were found for %q, falling back on anonymous", target)
 		}
 		o.auth = auth
 	}

vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go

@@ -26,7 +26,7 @@ import (
 // The set of query string keys that we expect to send as part of the registry
 // protocol. Anything else is potentially dangerous to leak, as it's probably
 // from a redirect. These redirects often included tokens or signed URLs.
-var paramWhitelist = map[string]struct{}{
+var paramAllowlist = map[string]struct{}{
 	// Token exchange
 	"scope":   {},
 	"service": {},
@@ -89,7 +89,8 @@ func (e *Error) responseErr() string {
 // Temporary returns whether the request that preceded the error is temporary.
 func (e *Error) Temporary() bool {
 	if len(e.Errors) == 0 {
-		return false
+		_, ok := temporaryStatusCodes[e.StatusCode]
+		return ok
 	}
 	for _, d := range e.Errors {
 		if _, ok := temporaryErrorCodes[d.Code]; !ok {
@@ -104,8 +105,8 @@ func redactURL(original *url.URL) *url.URL {
 	qs := original.Query()
 	for k, v := range qs {
 		for i := range v {
-			if _, ok := paramWhitelist[k]; !ok {
-				// key is not in the whitelist
+			if _, ok := paramAllowlist[k]; !ok {
+				// key is not in the Allowlist
 				v[i] = "REDACTED"
 			}
 		}
@@ -161,6 +162,12 @@ var temporaryErrorCodes = map[ErrorCode]struct{}{
 	TooManyRequestsErrorCode:   {},
 }
 
+var temporaryStatusCodes = map[int]struct{}{
+	http.StatusInternalServerError: {},
+	http.StatusBadGateway:          {},
+	http.StatusServiceUnavailable:  {},
+}
+
 // CheckError returns a structured error if the response status is not in codes.
 func CheckError(resp *http.Response, codes ...int) error {
 	for _, code := range codes {

vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go

@@ -15,6 +15,7 @@
 package stream
 
 import (
+	"bufio"
 	"compress/gzip"
 	"crypto/sha256"
 	"encoding/hex"
@@ -130,6 +131,7 @@ type compressedReader struct {
 
 	h, zh hash.Hash // collects digests of compressed and uncompressed stream.
 	pr    io.Reader
+	bw    *bufio.Writer
 	count *countWriter
 
 	l *Layer // stream.Layer to update upon Close.
@@ -144,7 +146,14 @@ func newCompressedReader(l *Layer) (*compressedReader, error) {
 	// capture compressed digest, and a countWriter to capture compressed
 	// size.
 	pr, pw := io.Pipe()
-	zw, err := gzip.NewWriterLevel(io.MultiWriter(pw, zh, count), l.compression)
+
+	// Write compressed bytes to be read by the pipe.Reader, hashed by zh, and counted by count.
+	mw := io.MultiWriter(pw, zh, count)
+
+	// Buffer the output of the gzip writer so we don't have to wait on pr to keep writing.
+	// 64K ought to be small enough for anybody.
+	bw := bufio.NewWriterSize(mw, 2<<16)
+	zw, err := gzip.NewWriterLevel(bw, l.compression)
 	if err != nil {
 		return nil, err
 	}
@@ -152,6 +161,7 @@ func newCompressedReader(l *Layer) (*compressedReader, error) {
 	cr := &compressedReader{
 		closer: newMultiCloser(zw, l.blob),
 		pr:     pr,
+		bw:     bw,
 		h:      h,
 		zh:     zh,
 		count:  count,
@@ -183,6 +193,11 @@ func (cr *compressedReader) Close() error {
 		return err
 	}
 
+	// Flush the buffer.
+	if err := cr.bw.Flush(); err != nil {
+		return err
+	}
+
 	diffID, err := v1.NewHash("sha256:" + hex.EncodeToString(cr.h.Sum(nil)))
 	if err != nil {
 		return err

vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go

@@ -28,7 +28,6 @@ import (
 	gestargz "github.com/google/go-containerregistry/pkg/v1/internal/estargz"
 	ggzip "github.com/google/go-containerregistry/pkg/v1/internal/gzip"
 	"github.com/google/go-containerregistry/pkg/v1/types"
-	"github.com/google/go-containerregistry/pkg/v1/v1util"
 )
 
 type layer struct {
@@ -133,6 +132,45 @@ func WithEstargzOptions(opts ...estargz.Option) LayerOption {
 	}
 }
 
+// WithEstargz is a functional option that explicitly enables estargz support.
+func WithEstargz(l *layer) {
+	oguncompressed := l.uncompressedopener
+	estargz := func() (io.ReadCloser, error) {
+		crc, err := oguncompressed()
+		if err != nil {
+			return nil, err
+		}
+		eopts := append(l.estgzopts, estargz.WithCompressionLevel(l.compression))
+		rc, h, err := gestargz.ReadCloser(crc, eopts...)
+		if err != nil {
+			return nil, err
+		}
+		l.annotations[estargz.TOCJSONDigestAnnotation] = h.String()
+		return &and.ReadCloser{
+			Reader: rc,
+			CloseFunc: func() error {
+				err := rc.Close()
+				if err != nil {
+					return err
+				}
+				// As an optimization, leverage the DiffID exposed by the estargz ReadCloser
+				l.diffID, err = v1.NewHash(rc.DiffID().String())
+				return err
+			},
+		}, nil
+	}
+	uncompressed := func() (io.ReadCloser, error) {
+		urc, err := estargz()
+		if err != nil {
+			return nil, err
+		}
+		return ggzip.UnzipReadCloser(urc)
+	}
+
+	l.compressedopener = estargz
+	l.uncompressedopener = uncompressed
+}
+
 // LayerFromFile returns a v1.Layer given a tarball
 func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) {
 	opener := func() (io.ReadCloser, error) {
@@ -168,6 +206,10 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 		annotations: make(map[string]string, 1),
 	}
 
+	if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" {
+		opts = append([]LayerOption{WithEstargz}, opts...)
+	}
+
 	if compressed {
 		layer.compressedopener = opener
 		layer.uncompressedopener = func() (io.ReadCloser, error) {
@@ -177,38 +219,6 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 			}
 			return ggzip.UnzipReadCloser(urc)
 		}
-	} else if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" {
-		layer.compressedopener = func() (io.ReadCloser, error) {
-			crc, err := opener()
-			if err != nil {
-				return nil, err
-			}
-			eopts := append(layer.estgzopts, estargz.WithCompressionLevel(layer.compression))
-			rc, h, err := gestargz.ReadCloser(crc, eopts...)
-			if err != nil {
-				return nil, err
-			}
-			layer.annotations[estargz.TOCJSONDigestAnnotation] = h.String()
-			return &and.ReadCloser{
-				Reader: rc,
-				CloseFunc: func() error {
-					err := rc.Close()
-					if err != nil {
-						return err
-					}
-					// As an optimization, leverage the DiffID exposed by the estargz ReadCloser
-					layer.diffID, err = v1.NewHash(rc.DiffID().String())
-					return err
-				},
-			}, nil
-		}
-		layer.uncompressedopener = func() (io.ReadCloser, error) {
-			urc, err := layer.compressedopener()
-			if err != nil {
-				return nil, err
-			}
-			return v1util.GunzipReadCloser(urc)
-		}
 	} else {
 		layer.uncompressedopener = opener
 		layer.compressedopener = func() (io.ReadCloser, error) {

vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go

@@ -222,6 +222,10 @@ func writeImagesToTar(refToImage map[name.Reference]v1.Image, m []byte, size int
 func calculateManifest(refToImage map[name.Reference]v1.Image) (m Manifest, err error) {
 	imageToTags := dedupRefToImage(refToImage)
 
+	if len(imageToTags) == 0 {
+		return nil, errors.New("set of images is empty")
+	}
+
 	for img, tags := range imageToTags {
 		cfgName, err := img.ConfigName()
 		if err != nil {
@@ -334,11 +338,22 @@ func dedupRefToImage(refToImage map[name.Reference]v1.Image) map[v1.Image][]stri
 
 	for ref, img := range refToImage {
 		if tag, ok := ref.(name.Tag); ok {
-			if tags, ok := imageToTags[img]; ok && tags != nil {
-				imageToTags[img] = append(tags, tag.String())
-			} else {
-				imageToTags[img] = []string{tag.String()}
+			if tags, ok := imageToTags[img]; !ok || tags == nil {
+				imageToTags[img] = []string{}
 			}
+			// Docker cannot load tarballs without an explicit tag:
+			// https://github.com/google/go-containerregistry/issues/890
+			//
+			// We can't use the fully qualified tag.Name() because of rules_docker:
+			// https://github.com/google/go-containerregistry/issues/527
+			//
+			// If the tag is "latest", but tag.String() doesn't end in ":latest",
+			// just append it. Kind of gross, but should work for now.
+			ts := tag.String()
+			if tag.Identifier() == name.DefaultTag && !strings.HasSuffix(ts, ":"+name.DefaultTag) {
+				ts = fmt.Sprintf("%s:%s", ts, name.DefaultTag)
+			}
+			imageToTags[img] = append(imageToTags[img], ts)
 		} else {
 			if _, ok := imageToTags[img]; !ok {
 				imageToTags[img] = nil

vendor/github.com/google/go-containerregistry/pkg/v1/v1util/verify.go

@@ -1,23 +0,0 @@
-// Copyright 2018 Google LLC All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package v1util
-
-import "github.com/google/go-containerregistry/pkg/v1/internal/verify"
-
-// VerifyReadCloser wraps the given io.ReadCloser to verify that its contents match
-// the provided v1.Hash before io.EOF is returned.
-// TODO(#873): Remove this package.
-// Deprecated: please use v1.VerifyReadCloser
-var VerifyReadCloser = verify.ReadCloser

vendor/github.com/google/go-containerregistry/pkg/v1/v1util/zip.go

@@ -1,43 +0,0 @@
-// Copyright 2018 Google LLC All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package v1util
-
-import "github.com/google/go-containerregistry/pkg/v1/internal/gzip"
-
-// GzipReadCloser reads uncompressed input data from the io.ReadCloser and
-// returns an io.ReadCloser from which compressed data may be read.
-// This uses gzip.BestSpeed for the compression level.
-// TODO(#873): Remove this package.
-// Deprecated: please move off of this API
-var GzipReadCloser = gzip.ReadCloser
-
-// GzipReadCloserLevel reads uncompressed input data from the io.ReadCloser and
-// returns an io.ReadCloser from which compressed data may be read.
-// Refer to compress/gzip for the level:
-// https://golang.org/pkg/compress/gzip/#pkg-constants
-// TODO(#873): Remove this package.
-// Deprecated: please move off of this API
-var GzipReadCloserLevel = gzip.ReadCloserLevel
-
-// GunzipReadCloser reads compressed input data from the io.ReadCloser and
-// returns an io.ReadCloser from which uncompessed data may be read.
-// TODO(#873): Remove this package.
-// Deprecated: please move off of this API
-var GunzipReadCloser = gzip.UnzipReadCloser
-
-// IsGzipped detects whether the input stream is compressed.
-// TODO(#873): Remove this package.
-// Deprecated: please move off of this API
-var IsGzipped = gzip.Is