feat: support multiple registry mirrors with fallback (#1498)

Fixes #1473

The initial implementation of the registry mirror only allowed a single mirror, and if pulling from the mirror failed, the build would fail.

This change introduces:
- multiple registry mirrors instead of a single one
- fallback if an image can't be pulled from a registry

This is the same behavior as the docker daemon and will allow using a registry mirror such as `mirror.gcr.io` which is incomplete and doesn't have all the content that the default registry on docker.io has.

Note that there are no changes in the CLI flags, the `--registry-mirror` flag is still valid. But now it can be used multiple times to set up more than one registry mirror.

Co-authored-by: Tejal Desai <tejaldesai@google.com>

README.md

@@ -660,7 +660,10 @@ Expected format is `my.registry.url=/path/to/the/certificate.cert`
 
 #### --registry-mirror
 
-Set this flag if you want to use a registry mirror instead of default `index.docker.io`.
+Set this flag if you want to use a registry mirror instead of the default `index.docker.io`. You can use this flag more than once, if you want to set multiple mirrors. If an image is not found on the first mirror, Kaniko will try the next mirror(s), and at the end fallback on the default registry.
+
+Expected format is `mirror.gcr.io` for example.
+
 
 Note that you can't specify a URL with scheme for this flag. Some valid options are:
 
@@ -668,6 +671,7 @@ Note that you can't specify a URL with scheme for this flag. Some valid options
 * `127.0.0.1`
 * `192.168.0.1:5000`
 
+
 #### --reproducible
 
 Set this flag to strip timestamps out of the built image and make it reproducible.

cmd/executor/cmd/root.go

@@ -173,7 +173,7 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().VarP(&opts.SkipTLSVerifyRegistries, "skip-tls-verify-registry", "", "Insecure registry ignoring TLS verify to push and pull. Set it repeatedly for multiple registries.")
 	opts.RegistriesCertificates = make(map[string]string)
 	RootCmd.PersistentFlags().VarP(&opts.RegistriesCertificates, "registry-certificate", "", "Use the provided certificate for TLS communication with the given registry. Expected format is 'my.registry.url=/path/to/the/server/certificate'.")
-	RootCmd.PersistentFlags().StringVarP(&opts.RegistryMirror, "registry-mirror", "", "", "Registry mirror to use has pull-through cache instead of docker.io.")
+	RootCmd.PersistentFlags().VarP(&opts.RegistryMirrors, "registry-mirror", "", "Registry mirror to use as pull-through cache instead of docker.io. Set it repeatedly for multiple mirrors.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.IgnoreVarRun, "whitelist-var-run", "", true, "Ignore /var/run directory when taking image snapshot. Set it to false to preserve /var/run/ in destination image. (Default true).")
 	RootCmd.PersistentFlags().VarP(&opts.Labels, "label", "", "Set metadata for an image. Set it repeatedly for multiple labels.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile")

integration/integration_test.go

@@ -304,7 +304,7 @@ func TestGitBuildcontextSubPath(t *testing.T) {
 	checkContainerDiffOutput(t, diff, expected)
 }
 
-func TestBuildViaRegistryMirror(t *testing.T) {
+func TestBuildViaRegistryMirrors(t *testing.T) {
 	repo := getGitRepo()
 	dockerfile := fmt.Sprintf("%s/%s/Dockerfile_registry_mirror", integrationPath, dockerfilesPath)
 
@@ -327,6 +327,7 @@ func TestBuildViaRegistryMirror(t *testing.T) {
 	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
 		"-f", dockerfile,
 		"-d", kanikoImage,
+		"--registry-mirror", "doesnotexist.example.com",
 		"--registry-mirror", "us-mirror.gcr.io",
 		"-c", fmt.Sprintf("git://%s", repo))
 

pkg/config/options.go

@@ -44,7 +44,7 @@ type KanikoOptions struct {
 	DigestFile              string
 	ImageNameDigestFile     string
 	OCILayoutPath           string
-	RegistryMirror          string
+	RegistryMirrors         multiArg
 	Destinations            multiArg
 	BuildArgs               multiArg
 	InsecureRegistries      multiArg

pkg/image/image_util.go

@@ -106,51 +106,45 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) {
 		return nil, err
 	}
 
-	registryName := ref.Context().RegistryStr()
+	if ref.Context().RegistryStr() == name.DefaultRegistry {
+		ref, err := normalizeReference(ref, image)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, registryMirror := range opts.RegistryMirrors {
 			var newReg name.Registry
-	toSet := false
+			if opts.InsecurePull || opts.InsecureRegistries.Contains(registryMirror) {
-
+				newReg, err = name.NewRegistry(registryMirror, name.WeakValidation, name.Insecure)
-	if opts.RegistryMirror != "" && registryName == name.DefaultRegistry {
+			} else {
-		registryName = opts.RegistryMirror
+				newReg, err = name.NewRegistry(registryMirror, name.StrictValidation)
-
+			}
-		newReg, err = name.NewRegistry(opts.RegistryMirror, name.StrictValidation)
 			if err != nil {
 				return nil, err
 			}
+			ref := setNewRegistry(ref, newReg)
 
-		ref, err = normalizeReference(ref, image)
+			logrus.Infof("Retrieving image %s from registry mirror %s", ref, registryMirror)
+			remoteImage, err := remote.Image(ref, remoteOptions(registryMirror, opts)...)
 			if err != nil {
-			return nil, err
+				logrus.Warnf("Failed to retrieve image %s from registry mirror %s: %s. Will try with the next mirror, or fallback to the default registry.", ref, registryMirror, err)
-		}
+				continue
-
+			}
-		toSet = true
+			return remoteImage, nil
+		}
 	}
 
+	registryName := ref.Context().RegistryStr()
 	if opts.InsecurePull || opts.InsecureRegistries.Contains(registryName) {
-		newReg, err = name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
+		newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
 		if err != nil {
 			return nil, err
 		}
-
+		ref = setNewRegistry(ref, newReg)
-		toSet = true
 	}
 
-	if toSet {
+	logrus.Infof("Retrieving image %s from registry %s", ref, registryName)
-		if tag, ok := ref.(name.Tag); ok {
+	return remote.Image(ref, remoteOptions(registryName, opts)...)
-			tag.Repository.Registry = newReg
-			ref = tag
-		}
-
-		if digest, ok := ref.(name.Digest); ok {
-			digest.Repository.Registry = newReg
-			ref = digest
-		}
-	}
-
-	logrus.Infof("Retrieving image %s", ref)
-
-	rOpts := remoteOptions(registryName, opts)
-	return remote.Image(ref, rOpts...)
 }
 
 // normalizeReference adds the library/ prefix to images without it.
@@ -165,6 +159,19 @@ func normalizeReference(ref name.Reference, image string) (name.Reference, error
 	return ref, nil
 }
 
+func setNewRegistry(ref name.Reference, newReg name.Registry) name.Reference {
+	switch r := ref.(type) {
+	case name.Tag:
+		r.Repository.Registry = newReg
+		return r
+	case name.Digest:
+		r.Repository.Registry = newReg
+		return r
+	default:
+		return ref
+	}
+}
+
 func remoteOptions(registryName string, opts *config.KanikoOptions) []remote.Option {
 	tr := util.MakeTransport(opts, registryName)
 

pkg/image/image_util_test.go

@@ -104,7 +104,7 @@ func Test_ScratchImageFromMirror(t *testing.T) {
 	actual, err := RetrieveSourceImage(config.KanikoStage{
 		Stage: stages[1],
 	}, &config.KanikoOptions{
-		RegistryMirror: "mirror.gcr.io",
+		RegistryMirrors: []string{"mirror.gcr.io"},
 	})
 	expected := empty.Image
 	testutil.CheckErrorAndDeepEqual(t, false, err, expected, actual)