Add flag to remap registries for any registry mirror (#2935)

* Add flag to remap registries for any registry mirror

The purpose of this PR is to add an option to remap registries, a kind of generalized `--registry-mirror`.
This is helpful for air-gapped environments and/or when local registry mirrors are available (not limited to docker.io).
This allows user to reference any images without having to change their location.
It also permit to separate infra related configuration (the mirrors) from CI/CD pipeline definition by using an environment variable for example (the reason behind the early return if flag provided but empty).
Therefore you can have a pipeline calling kaniko with `--registry-map=$REGISTRY_MAP` and have the `REGISTRY_MAP` populated via the runner's env by another team, and the absence of env wouldn't trigger a failure, it makes the pipeline env independent.

I've also considered the option of environment variables directly but it doesn't seems to be in kaniko's philosophy.

This makes quite some duplicated code :/ One option to keep the mirror flag and behavior would be to use only one codebase and convert `--registry-mirror=VALUE` to `--registry-map=index.docker.io=VALUE` internally. Suggestions welcome!

* Configure logging config sooner to be able to use it in flag parsing

* Replace registry mirrors by maps logic and use env var

* Add env vars to README.md

* Fix test

README.md

@@ -95,12 +95,13 @@ _If you are interested in contributing to kaniko, see
       - [Flag `--push-retry`](#flag---push-retry)
       - [Flag `--registry-certificate`](#flag---registry-certificate)
       - [Flag `--registry-client-cert`](#flag---registry-client-cert)
+      - [Flag `--registry-map`](#flag---registry-map)
       - [Flag `--registry-mirror`](#flag---registry-mirror)
       - [Flag `--skip-default-registry-fallback`](#flag---skip-default-registry-fallback)
       - [Flag `--reproducible`](#flag---reproducible)
       - [Flag `--single-snapshot`](#flag---single-snapshot)
-      - [Flag `--skip-tls-verify`](#flag---skip-tls-verify)
       - [Flag `--skip-push-permission-check`](#flag---skip-push-permission-check)
+      - [Flag `--skip-tls-verify`](#flag---skip-tls-verify)
       - [Flag `--skip-tls-verify-pull`](#flag---skip-tls-verify-pull)
       - [Flag `--skip-tls-verify-registry`](#flag---skip-tls-verify-registry)
       - [Flag `--skip-unused-stages`](#flag---skip-unused-stages)
@@ -112,8 +113,8 @@ _If you are interested in contributing to kaniko, see
       - [Flag `--ignore-var-run`](#flag---ignore-var-run)
       - [Flag `--ignore-path`](#flag---ignore-path)
       - [Flag `--image-fs-extract-retry`](#flag---image-fs-extract-retry)
-      - [Flag `--image-download-retry`](#flag---image-download-retry)    
-  - [Debug Image](#debug-image)
+      - [Flag `--image-download-retry`](#flag---image-download-retry)
+    - [Debug Image](#debug-image)
   - [Security](#security)
     - [Verifying Signed Kaniko Images](#verifying-signed-kaniko-images)
   - [Kaniko Builds - Profiling](#kaniko-builds---profiling)
@@ -981,6 +982,27 @@ for authentication.
 
 Expected format is `my.registry.url=/path/to/client/cert.crt,/path/to/client/key.key`
 
+#### Flag `--registry-map`
+
+Set this flag if you want to remap registries references. Usefull for air gap environement for example.
+You can use this flag more than once, if you want to set multiple mirrors for a given registry.
+You can mention several remap in a single flag too, separated by semi-colon.
+If an image is not found on the first mirror, Kaniko will try
+the next mirror(s), and at the end fallback on the original registry.
+
+Registry maps can also be defined through `KANIKO_REGISTRY_MAP` environment variable.
+
+Expected format is `original-registry=remapped-registry[;another-reg=another-remap[;...]]` for example.
+
+Note that you can't specify a URL with scheme for this flag. Some valid options
+are:
+
+- `index.docker.io=mirror.gcr.io`
+- `gcr.io=127.0.0.1`
+- `quay.io=192.168.0.1:5000`
+- `index.docker.io=docker-io.mirrors.corp.net;index.docker.io=mirror.gcr.io;gcr.io=127.0.0.1`
+   will try `docker-io.mirrors.corp.net` then `mirror.gcr.io` for `index.docker.io` and `127.0.0.1` for `gcr.io`
+
 #### Flag `--registry-mirror`
 
 Set this flag if you want to use a registry mirror instead of the default
@@ -988,6 +1010,8 @@ Set this flag if you want to use a registry mirror instead of the default
 multiple mirrors. If an image is not found on the first mirror, Kaniko will try
 the next mirror(s), and at the end fallback on the default registry.
 
+Mirror can also be defined through `KANIKO_REGISTRY_MIRROR` environment variable.
+
 Expected format is `mirror.gcr.io` for example.
 
 Note that you can't specify a URL with scheme for this flag. Some valid options

cmd/executor/cmd/root.go

@@ -34,6 +34,7 @@ import (
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/GoogleContainerTools/kaniko/pkg/util/proc"
 	"github.com/containerd/containerd/platforms"
+	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -70,6 +71,21 @@ func validateFlags() {
 		opts.RegistryMirrors.Set(val)
 	}
 
+	// Allow setting --registry-maps using an environment variable.
+	if val, ok := os.LookupEnv("KANIKO_REGISTRY_MAP"); ok {
+		opts.RegistryMaps.Set(val)
+	}
+
+	for _, target := range opts.RegistryMirrors {
+		opts.RegistryMaps.Set(fmt.Sprintf("%s=%s", name.DefaultRegistry, target))
+	}
+
+	if len(opts.RegistryMaps) > 0 {
+		for src, dsts := range opts.RegistryMaps {
+			logrus.Debugf("registry-map remaps %s to %s.", src, strings.Join(dsts, ", "))
+		}
+	}
+
 	// Default the custom platform flag to our current platform, and validate it.
 	if opts.CustomPlatform == "" {
 		opts.CustomPlatform = platforms.Format(platforms.Normalize(platforms.DefaultSpec()))
@@ -85,6 +101,10 @@ var RootCmd = &cobra.Command{
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 		if cmd.Use == "executor" {
 
+			if err := logging.Configure(logLevel, logFormat, logTimestamp); err != nil {
+				return err
+			}
+
 			validateFlags()
 
 			// Command line flag takes precedence over the KANIKO_DIR environment variable.
@@ -99,10 +119,6 @@ var RootCmd = &cobra.Command{
 
 			resolveEnvironmentBuildArgs(opts.BuildArgs, os.Getenv)
 
-			if err := logging.Configure(logLevel, logFormat, logTimestamp); err != nil {
-				return err
-			}
-
 			if !opts.NoPush && len(opts.Destinations) == 0 {
 				return errors.New("you must provide --destination, or use --no-push")
 			}
@@ -238,6 +254,8 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().VarP(&opts.RegistriesCertificates, "registry-certificate", "", "Use the provided certificate for TLS communication with the given registry. Expected format is 'my.registry.url=/path/to/the/server/certificate'.")
 	opts.RegistriesClientCertificates = make(map[string]string)
 	RootCmd.PersistentFlags().VarP(&opts.RegistriesClientCertificates, "registry-client-cert", "", "Use the provided client certificate for mutual TLS (mTLS) communication with the given registry. Expected format is 'my.registry.url=/path/to/client/cert,/path/to/client/key'.")
+	opts.RegistryMaps = make(map[string][]string)
+	RootCmd.PersistentFlags().VarP(&opts.RegistryMaps, "registry-map", "", "Registry map of mirror to use as pull-through cache instead. Expected format is 'orignal.registry=new.registry;other-original.registry=other-remap.registry'")
 	RootCmd.PersistentFlags().VarP(&opts.RegistryMirrors, "registry-mirror", "", "Registry mirror to use as pull-through cache instead of docker.io. Set it repeatedly for multiple mirrors.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.SkipDefaultRegistryFallback, "skip-default-registry-fallback", "", false, "If an image is not found on any mirrors (defined with registry-mirror) do not fallback to the default registry. If registry-mirror is not defined, this flag is ignored.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.IgnoreVarRun, "ignore-var-run", "", true, "Ignore /var/run directory when taking image snapshot. Set it to false to preserve /var/run/ in destination image.")

integration/integration_test.go

@@ -396,6 +396,46 @@ func TestBuildViaRegistryMirrors(t *testing.T) {
 	checkContainerDiffOutput(t, diff, expected)
 }
 
+func TestBuildViaRegistryMap(t *testing.T) {
+	repo := getGitRepo(false)
+	dockerfile := fmt.Sprintf("%s/%s/Dockerfile_registry_mirror", integrationPath, dockerfilesPath)
+
+	// Build with docker
+	dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_registry_mirror")
+	dockerCmd := exec.Command("docker",
+		append([]string{"build",
+			"-t", dockerImage,
+			"-f", dockerfile,
+			repo})...)
+	out, err := RunCommandWithoutTest(dockerCmd)
+	if err != nil {
+		t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out))
+	}
+
+	// Build with kaniko
+	kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_registry_mirror")
+	dockerRunFlags := []string{"run", "--net=host"}
+	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
+		"-f", dockerfile,
+		"-d", kanikoImage,
+		"--registry-map", "index.docker.io=doesnotexist.example.com",
+		"--registry-map", "index.docker.io=us-mirror.gcr.io",
+		"-c", fmt.Sprintf("git://%s", repo))
+
+	kanikoCmd := exec.Command("docker", dockerRunFlags...)
+
+	out, err = RunCommandWithoutTest(kanikoCmd)
+	if err != nil {
+		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
+	}
+
+	diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache")
+
+	expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage)
+	checkContainerDiffOutput(t, diff, expected)
+}
+
 func TestBuildSkipFallback(t *testing.T) {
 	repo := getGitRepo(false)
 	dockerfile := fmt.Sprintf("%s/%s/Dockerfile_registry_mirror", integrationPath, dockerfilesPath)

pkg/config/args.go

@@ -82,3 +82,46 @@ func (a *keyValueArg) Set(value string) error {
 func (a *keyValueArg) Type() string {
 	return "key-value-arg type"
 }
+
+type multiKeyMultiValueArg map[string][]string
+
+func (c *multiKeyMultiValueArg) parseKV(value string) error {
+	valueSplit := strings.SplitN(value, "=", 2)
+	if len(valueSplit) < 2 {
+		return fmt.Errorf("invalid argument value. expect key=value, got %s", value)
+	}
+	(*c)[valueSplit[0]] = append((*c)[valueSplit[0]], valueSplit[1])
+	return nil
+}
+
+func (c *multiKeyMultiValueArg) String() string {
+	var result []string
+	for key := range *c {
+		for _, val := range (*c)[key] {
+			result = append(result, fmt.Sprintf("%s=%s", key, val))
+		}
+	}
+	return strings.Join(result, ";")
+
+}
+
+func (c *multiKeyMultiValueArg) Set(value string) error {
+	if value == "" {
+		return nil
+	}
+	if strings.Contains(value, ";") {
+		kvpairs := strings.Split(value, ";")
+		for _, kv := range kvpairs {
+			err := c.parseKV(kv)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	return c.parseKV(value)
+}
+
+func (c *multiKeyMultiValueArg) Type() string {
+	return "key-multi-value-arg type"
+}

pkg/config/args_test.go

@@ -45,3 +45,36 @@ func Test_KeyValueArg_Set_shouldAcceptEqualAsValue(t *testing.T) {
 		t.Error("Invalid split. key=value=something should be split to key=>value=something")
 	}
 }
+
+func Test_multiKeyMultiValueArg_Set_shouldSplitArgumentLikeKVA(t *testing.T) {
+	arg := make(multiKeyMultiValueArg)
+	arg.Set("key=value")
+	if arg["key"][0] != "value" {
+		t.Error("Invalid split. key=value should be split to key=>value")
+	}
+}
+
+func Test_multiKeyMultiValueArg_Set_ShouldAppendIfRepeated(t *testing.T) {
+	arg := make(multiKeyMultiValueArg)
+	arg.Set("key=v1")
+	arg.Set("key=v2")
+	if arg["key"][0] != "v1" || arg["key"][1] != "v2" {
+		t.Error("Invalid repeat behavior. Repeated keys should append values")
+	}
+}
+
+func Test_multiKeyMultiValueArg_Set_Composed(t *testing.T) {
+	arg := make(multiKeyMultiValueArg)
+	arg.Set("key1=value1;key2=value2")
+	if arg["key1"][0] != "value1" || arg["key2"][0] != "value2" {
+		t.Error("Invalid composed value parsing. key=value;key2=value2 should generate 2 keys")
+	}
+}
+
+func Test_multiKeyMultiValueArg_Set_WithEmptyValueShouldWork(t *testing.T) {
+	arg := make(multiKeyMultiValueArg)
+	err := arg.Set("")
+	if len(arg) != 0 || err != nil {
+		t.Error("multiKeyMultiValueArg must handle empty value")
+	}
+}

pkg/config/options.go

@@ -32,6 +32,7 @@ type CacheOptions struct {
 
 // RegistryOptions are all the options related to the registries, set by command line arguments.
 type RegistryOptions struct {
+	RegistryMaps                 multiKeyMultiValueArg
 	RegistryMirrors              multiArg
 	InsecureRegistries           multiArg
 	SkipTLSVerifyRegistries      multiArg

pkg/image/remote/remote.go

@@ -17,7 +17,7 @@ limitations under the License.
 package remote
 
 import (
-	"errors"
+	"fmt"
 	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
@@ -51,33 +51,32 @@ func RetrieveRemoteImage(image string, opts config.RegistryOptions, customPlatfo
 		return nil, err
 	}
 
-	if ref.Context().RegistryStr() == name.DefaultRegistry {
+	if newRegURLs, found := opts.RegistryMaps[ref.Context().RegistryStr()]; found {
 		ref, err := normalizeReference(ref, image)
 		if err != nil {
 			return nil, err
 		}
 
-		for _, registryMirror := range opts.RegistryMirrors {
+		for _, regToMapTo := range newRegURLs {
 			var newReg name.Registry
-			if opts.InsecurePull || opts.InsecureRegistries.Contains(registryMirror) {
-				newReg, err = name.NewRegistry(registryMirror, name.WeakValidation, name.Insecure)
+			if opts.InsecurePull || opts.InsecureRegistries.Contains(regToMapTo) {
+				newReg, err = name.NewRegistry(regToMapTo, name.WeakValidation, name.Insecure)
 			} else {
-				newReg, err = name.NewRegistry(registryMirror, name.StrictValidation)
+				newReg, err = name.NewRegistry(regToMapTo, name.StrictValidation)
 			}
 			if err != nil {
 				return nil, err
 			}
 			ref := setNewRegistry(ref, newReg)
-
-			logrus.Infof("Retrieving image %s from registry mirror %s", ref, registryMirror)
+			logrus.Infof("Retrieving image %s from mapped registry %s", ref, regToMapTo)
 			retryFunc := func() (v1.Image, error) {
-				return remoteImageFunc(ref, remoteOptions(registryMirror, opts, customPlatform)...)
+				return remoteImageFunc(ref, remoteOptions(regToMapTo, opts, customPlatform)...)
 			}
 
 			var remoteImage v1.Image
 			var err error
 			if remoteImage, err = util.RetryWithResult(retryFunc, opts.ImageDownloadRetry, 1000); err != nil {
-				logrus.Warnf("Failed to retrieve image %s from registry mirror %s: %s. Will try with the next mirror, or fallback to the default registry.", ref, registryMirror, err)
+				logrus.Warnf("Failed to retrieve image %s from remapped registry %s: %s. Will try with the next registry, or fallback to the original registry.", ref, regToMapTo, err)
 				continue
 			}
 
@@ -86,8 +85,8 @@ func RetrieveRemoteImage(image string, opts config.RegistryOptions, customPlatfo
 			return remoteImage, nil
 		}
 
-		if len(opts.RegistryMirrors) > 0 && opts.SkipDefaultRegistryFallback {
-			return nil, errors.New("image not found on any configured mirror(s)")
+		if len(newRegURLs) > 0 && opts.SkipDefaultRegistryFallback {
+			return nil, fmt.Errorf("image not found on any configured mapped registries for %s", ref)
 		}
 	}
 

pkg/image/remote/remote_test.go

@@ -117,7 +117,7 @@ func Test_RetrieveRemoteImage_skipFallback(t *testing.T) {
 	registryMirror := "some-registry"
 
 	opts := config.RegistryOptions{
-		RegistryMirrors:             []string{registryMirror},
+		RegistryMaps:                map[string][]string{name.DefaultRegistry: {registryMirror}},
 		SkipDefaultRegistryFallback: false,
 	}