Improve retry behavior for push operation (#1578)

* Update go-containerregistry to 0.4

* Implement --push-retry argument

pkg/util/util.go

@@ -22,12 +22,15 @@ import (
 	"encoding/hex"
 	"io"
 	"io/ioutil"
+	"math"
 	"os"
 	"strconv"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/minio/highwayhash"
+	"github.com/sirupsen/logrus"
 )
 
 // Hasher returns a hash function, used in snapshotting to determine if a file has changed
@@ -154,3 +157,18 @@ func GetInputFrom(r io.Reader) ([]byte, error) {
 	}
 	return output, nil
 }
+
+type retryFunc func() error
+
+// Retry retries an operation
+func Retry(operation retryFunc, retryCount int, initialDelayMilliseconds int) error {
+	err := operation()
+	for i := 0; err != nil && i < retryCount; i++ {
+		sleepDuration := time.Millisecond * time.Duration(int(math.Pow(2, float64(i)))*initialDelayMilliseconds)
+		logrus.Warnf("Retrying operation after %s due to %v", sleepDuration, err)
+		time.Sleep(sleepDuration)
+		err = operation()
+	}
+
+	return err
+}

pkg/util/util_test.go

@@ -19,6 +19,7 @@ package util
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"testing"
 
 	"github.com/GoogleContainerTools/kaniko/testutil"
@@ -30,3 +31,36 @@ func TestGetInputFrom(t *testing.T) {
 	validValue, err := GetInputFrom(validReader)
 	testutil.CheckErrorAndDeepEqual(t, false, err, validInput, validValue)
 }
+
+func makeRetryFunc(numFailures int) retryFunc {
+	i := -1
+
+	return func() error {
+		i++
+		if i < numFailures {
+			return fmt.Errorf("Failing with i=%v", i)
+		}
+		return nil
+	}
+}
+
+func TestRetry(t *testing.T) {
+	// test with a function that does not return an error
+	if err := Retry(makeRetryFunc(0), 0, 10); err != nil {
+		t.Fatalf("Not expecting error: %v", err)
+	}
+	if err := Retry(makeRetryFunc(0), 3, 10); err != nil {
+		t.Fatalf("Not expecting error: %v", err)
+	}
+
+	// test with a function that returns an error twice
+	if err := Retry(makeRetryFunc(2), 0, 10); err == nil {
+		t.Fatal("Expecting error", err)
+	}
+	if err := Retry(makeRetryFunc(2), 1, 10); err == nil {
+		t.Fatal("Expecting error", err)
+	}
+	if err := Retry(makeRetryFunc(2), 2, 10); err != nil {
+		t.Fatalf("Not expecting error: %v", err)
+	}
+}