Improve retry behavior for push operation (#1578)

* Update go-containerregistry to 0.4

* Implement --push-retry argument

pkg/config/options.go

@@ -40,6 +40,7 @@ type RegistryOptions struct {
 	SkipTLSVerify           bool
 	InsecurePull            bool
 	SkipTLSVerifyPull       bool
+	PushRetry               int
 }
 
 // KanikoOptions are options that are set by command line arguments

pkg/executor/push.go

@@ -263,12 +263,18 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error {
 		tr := newRetry(util.MakeTransport(opts.RegistryOptions, registryName))
 		rt := &withUserAgent{t: tr}
 
-		if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil {
+		logrus.Infof("Pushing image to %s", destRef.String())
+
+		retryFunc := func() error {
+			return remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt))
+		}
+
+		if err := util.Retry(retryFunc, opts.PushRetry, 1000); err != nil {
 			return errors.Wrap(err, fmt.Sprintf("failed to push to destination %s", destRef))
 		}
 	}
 	timing.DefaultRun.Stop(t)
-	logrus.Infof("Pushed images to %d destinations", len(destRefs))
+	logrus.Infof("Pushed image to %d destinations", len(destRefs))
 	return writeImageOutputs(image, destRefs)
 }
 

pkg/util/util.go

@@ -22,12 +22,15 @@ import (
 	"encoding/hex"
 	"io"
 	"io/ioutil"
+	"math"
 	"os"
 	"strconv"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/minio/highwayhash"
+	"github.com/sirupsen/logrus"
 )
 
 // Hasher returns a hash function, used in snapshotting to determine if a file has changed
@@ -154,3 +157,18 @@ func GetInputFrom(r io.Reader) ([]byte, error) {
 	}
 	return output, nil
 }
+
+type retryFunc func() error
+
+// Retry retries an operation
+func Retry(operation retryFunc, retryCount int, initialDelayMilliseconds int) error {
+	err := operation()
+	for i := 0; err != nil && i < retryCount; i++ {
+		sleepDuration := time.Millisecond * time.Duration(int(math.Pow(2, float64(i)))*initialDelayMilliseconds)
+		logrus.Warnf("Retrying operation after %s due to %v", sleepDuration, err)
+		time.Sleep(sleepDuration)
+		err = operation()
+	}
+
+	return err
+}

pkg/util/util_test.go

@@ -19,6 +19,7 @@ package util
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"testing"
 
 	"github.com/GoogleContainerTools/kaniko/testutil"
@@ -30,3 +31,36 @@ func TestGetInputFrom(t *testing.T) {
 	validValue, err := GetInputFrom(validReader)
 	testutil.CheckErrorAndDeepEqual(t, false, err, validInput, validValue)
 }
+
+func makeRetryFunc(numFailures int) retryFunc {
+	i := -1
+
+	return func() error {
+		i++
+		if i < numFailures {
+			return fmt.Errorf("Failing with i=%v", i)
+		}
+		return nil
+	}
+}
+
+func TestRetry(t *testing.T) {
+	// test with a function that does not return an error
+	if err := Retry(makeRetryFunc(0), 0, 10); err != nil {
+		t.Fatalf("Not expecting error: %v", err)
+	}
+	if err := Retry(makeRetryFunc(0), 3, 10); err != nil {
+		t.Fatalf("Not expecting error: %v", err)
+	}
+
+	// test with a function that returns an error twice
+	if err := Retry(makeRetryFunc(2), 0, 10); err == nil {
+		t.Fatal("Expecting error", err)
+	}
+	if err := Retry(makeRetryFunc(2), 1, 10); err == nil {
+		t.Fatal("Expecting error", err)
+	}
+	if err := Retry(makeRetryFunc(2), 2, 10); err != nil {
+		t.Fatalf("Not expecting error: %v", err)
+	}
+}