Merge remote-tracking branch 'origin/master' into registry-mirror

integration/config.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import "strings"
+
+type integrationTestConfig struct {
+	gcsBucket          string
+	imageRepo          string
+	onbuildBaseImage   string
+	hardlinkBaseImage  string
+	serviceAccount     string
+	dockerMajorVersion int
+}
+
+const gcrRepoPrefix string = "gcr.io/"
+
+func (config *integrationTestConfig) isGcrRepository() bool {
+	return strings.HasPrefix(config.imageRepo, gcrRepoPrefix)
+}

integration/dockerfiles/Dockerfile_test_arg_secret

@@ -0,0 +1,10 @@
+FROM debian:9.11
+
+ARG SSH_PRIVATE_KEY
+ARG SSH_PUBLIC_KEY
+
+RUN mkdir .ssh                                 && \
+    chmod 700 .ssh                             && \
+    echo "$SSH_PRIVATE_KEY" > .ssh/id_rsa      && \
+    echo "$SSH_PUBLIC_KEY"  > .ssh/id_rsa.pub  && \
+    chmod 600 .ssh/id_rsa .ssh/id_rsa.pub

integration/dockerfiles/Dockerfile_test_copy_symlink

@@ -2,5 +2,13 @@ FROM busybox as t
 RUN echo "hello" > /tmp/target
 RUN ln -s /tmp/target /tmp/link
 
+## Relative link
+RUN cd tmp && ln -s target relative_link
+
+## Relative link with paths
+RUN mkdir workdir && cd workdir && ln -s ../tmp/target relative_dir_link
+
 FROM scratch
-COPY --from=t /tmp/link /tmp
+COPY --from=t /tmp/link /tmp/
+COPY --from=t /tmp/relative_link /tmp/
+COPY --from=t /workdir/relative_dir_link /workdir/

integration/images.go

@@ -17,6 +17,7 @@ limitations under the License.
 package integration
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -46,10 +47,11 @@ const (
 
 // Arguments to build Dockerfiles with, used for both docker and kaniko builds
 var argsMap = map[string][]string{
-	"Dockerfile_test_run":     {"file=/file"},
-	"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
-	"Dockerfile_test_add":     {"file=context/foo"},
-	"Dockerfile_test_onbuild": {"file=/tmp/onbuild"},
+	"Dockerfile_test_run":        {"file=/file"},
+	"Dockerfile_test_workdir":    {"workdir=/arg/workdir"},
+	"Dockerfile_test_add":        {"file=context/foo"},
+	"Dockerfile_test_arg_secret": {"SSH_PRIVATE_KEY", "SSH_PUBLIC_KEY=Pµbl1cK€Y"},
+	"Dockerfile_test_onbuild":    {"file=/tmp/onbuild"},
 	"Dockerfile_test_scratch": {
 		"image=scratch",
 		"hello=hello-value",
@@ -59,6 +61,11 @@ var argsMap = map[string][]string{
 	"Dockerfile_test_multistage": {"file=/foo2"},
 }
 
+// Environment to build Dockerfiles with, used for both docker and kaniko builds
+var envsMap = map[string][]string{
+	"Dockerfile_test_arg_secret": {"SSH_PRIVATE_KEY=ThEPriv4t3Key"},
+}
+
 // Arguments to build Dockerfiles with when building with docker
 var additionalDockerFlagsMap = map[string][]string{
 	"Dockerfile_test_target": {"--target=second"},
@@ -71,6 +78,36 @@ var additionalKanikoFlagsMap = map[string][]string{
 	"Dockerfile_test_target":  {"--target=second"},
 }
 
+// output check to do when building with kaniko
+var outputChecks = map[string]func(string, []byte) error{
+	"Dockerfile_test_arg_secret": checkArgsNotPrinted,
+}
+
+// Checks if argument are not printed in output.
+// Argument may be passed through --build-arg key=value manner or --build-arg key with key in environment
+func checkArgsNotPrinted(dockerfile string, out []byte) error {
+	for _, arg := range argsMap[dockerfile] {
+		argSplitted := strings.Split(arg, "=")
+		if len(argSplitted) == 2 {
+			if idx := bytes.Index(out, []byte(argSplitted[1])); idx >= 0 {
+				return fmt.Errorf("Argument value %s for argument %s displayed in output", argSplitted[1], argSplitted[0])
+			}
+		} else if len(argSplitted) == 1 {
+			if envs, ok := envsMap[dockerfile]; ok {
+				for _, env := range envs {
+					envSplitted := strings.Split(env, "=")
+					if len(envSplitted) == 2 {
+						if idx := bytes.Index(out, []byte(envSplitted[1])); idx >= 0 {
+							return fmt.Errorf("Argument value %s for argument %s displayed in output", envSplitted[1], argSplitted[0])
+						}
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
 var bucketContextTests = []string{"Dockerfile_test_copy_bucket"}
 var reproducibleTests = []string{"Dockerfile_test_reproducible"}
 
@@ -154,7 +191,7 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string {
 // BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
 // The resulting image will be tagged with imageRepo. If the dockerfile will be built with
 // context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
-func (d *DockerFileBuilder) BuildImage(config *gcpConfig, dockerfilesPath, dockerfile string) error {
+func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
 	gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo
 	_, ex, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(ex)
@@ -164,8 +201,7 @@ func (d *DockerFileBuilder) BuildImage(config *gcpConfig, dockerfilesPath, docke
 	var buildArgs []string
 	buildArgFlag := "--build-arg"
 	for _, arg := range argsMap[dockerfile] {
-		buildArgs = append(buildArgs, buildArgFlag)
-		buildArgs = append(buildArgs, arg)
+		buildArgs = append(buildArgs, buildArgFlag, arg)
 	}
 	// build docker image
 	additionalFlags := append(buildArgs, additionalDockerFlagsMap[dockerfile]...)
@@ -177,6 +213,9 @@ func (d *DockerFileBuilder) BuildImage(config *gcpConfig, dockerfilesPath, docke
 			"."},
 			additionalFlags...)...,
 	)
+	if env, ok := envsMap[dockerfile]; ok {
+		dockerCmd.Env = append(dockerCmd.Env, env...)
+	}
 
 	timer := timing.Start(dockerfile + "_docker")
 	out, err := RunCommandWithoutTest(dockerCmd)
@@ -225,7 +264,11 @@ func (d *DockerFileBuilder) BuildImage(config *gcpConfig, dockerfilesPath, docke
 		"-v", cwd + ":/workspace",
 		"-v", benchmarkDir + ":/kaniko/benchmarks",
 	}
-
+	if env, ok := envsMap[dockerfile]; ok {
+		for _, envVariable := range env {
+			dockerRunFlags = append(dockerRunFlags, "-e", envVariable)
+		}
+	}
 	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
 
 	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
@@ -242,6 +285,11 @@ func (d *DockerFileBuilder) BuildImage(config *gcpConfig, dockerfilesPath, docke
 	if err != nil {
 		return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out))
 	}
+	if outputCheck := outputChecks[dockerfile]; outputCheck != nil {
+		if err := outputCheck(dockerfile, out); err != nil {
+			return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out))
+		}
+	}
 
 	d.FilesBuilt[dockerfile] = true
 	return nil
@@ -269,7 +317,7 @@ func populateVolumeCache() error {
 }
 
 // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
-func (d *DockerFileBuilder) buildCachedImages(config *gcpConfig, cacheRepo, dockerfilesPath string, version int) error {
+func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int) error {
 	imageRepo, serviceAccount := config.imageRepo, config.serviceAccount
 	_, ex, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(ex)

integration/integration_test.go

@@ -33,12 +33,14 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
 
+	"github.com/pkg/errors"
+
 	"github.com/GoogleContainerTools/kaniko/pkg/timing"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/GoogleContainerTools/kaniko/testutil"
 )
 
-var config *gcpConfig
+var config *integrationTestConfig
 var imageBuilder *DockerFileBuilder
 
 const (
@@ -80,37 +82,65 @@ func getDockerMajorVersion() int {
 	}
 	return ver
 }
+func launchTests(m *testing.M, dockerfiles []string) (int, error) {
+
+	if config.isGcrRepository() {
+		contextFile, err := CreateIntegrationTarball()
+		if err != nil {
+			return 1, errors.Wrap(err, "Failed to create tarball of integration files for build context")
+		}
+
+		fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile, contextFile)
+		if err != nil {
+			return 1, errors.Wrap(err, "Failed to upload build context")
+		}
+
+		if err = os.Remove(contextFile); err != nil {
+			return 1, errors.Wrap(err, fmt.Sprintf("Failed to remove tarball at %s", contextFile))
+		}
+
+		RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) })
+		defer DeleteFromBucket(fileInBucket)
+	} else {
+		var err error
+		var migratedFiles []string
+		if migratedFiles, err = MigrateGCRRegistry(dockerfilesPath, dockerfiles, config.imageRepo); err != nil {
+			RollbackMigratedFiles(dockerfilesPath, migratedFiles)
+			return 1, errors.Wrap(err, "Fail to migrate dockerfiles from gcs")
+		}
+		RunOnInterrupt(func() { RollbackMigratedFiles(dockerfilesPath, migratedFiles) })
+		defer RollbackMigratedFiles(dockerfilesPath, migratedFiles)
+	}
+	if err := buildRequiredImages(); err != nil {
+		return 1, errors.Wrap(err, "Error while building images")
+	}
+
+	imageBuilder = NewDockerFileBuilder(dockerfiles)
+
+	return m.Run(), nil
+}
 
 func TestMain(m *testing.M) {
 	if !meetsRequirements() {
 		fmt.Println("Missing required tools")
 		os.Exit(1)
 	}
-	config = initGCPConfig()
 
-	if config.uploadToGCS {
-		contextFile, err := CreateIntegrationTarball()
+	if dockerfiles, err := FindDockerFiles(dockerfilesPath); err != nil {
+		fmt.Println("Coudn't create map of dockerfiles", err)
+		os.Exit(1)
+	} else {
+		config = initIntegrationTestConfig()
+		exitCode, err := launchTests(m, dockerfiles)
 		if err != nil {
-			fmt.Println("Failed to create tarball of integration files for build context", err)
-			os.Exit(1)
+			fmt.Println(err)
 		}
-
-		fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile, contextFile)
-		if err != nil {
-			fmt.Println("Failed to upload build context", err)
-			os.Exit(1)
-		}
-
-		err = os.Remove(contextFile)
-		if err != nil {
-			fmt.Printf("Failed to remove tarball at %s: %s\n", contextFile, err)
-			os.Exit(1)
-		}
-
-		RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) })
-		defer DeleteFromBucket(fileInBucket)
+		os.Exit(exitCode)
 	}
 
+}
+
+func buildRequiredImages() error {
 	setupCommands := []struct {
 		name    string
 		command []string
@@ -145,20 +175,10 @@ func TestMain(m *testing.M) {
 		fmt.Println(setupCmd.name)
 		cmd := exec.Command(setupCmd.command[0], setupCmd.command[1:]...)
 		if out, err := RunCommandWithoutTest(cmd); err != nil {
-			fmt.Printf("%s failed: %s", setupCmd.name, err)
-			fmt.Println(string(out))
-			os.Exit(1)
+			return errors.Wrap(err, fmt.Sprintf("%s failed: %s", setupCmd.name, string(out)))
 		}
 	}
-
-	dockerfiles, err := FindDockerFiles(dockerfilesPath)
-	if err != nil {
-		fmt.Printf("Coudn't create map of dockerfiles: %s", err)
-		os.Exit(1)
-	}
-	imageBuilder = NewDockerFileBuilder(dockerfiles)
-
-	os.Exit(m.Run())
+	return nil
 }
 
 func TestRun(t *testing.T) {
@@ -580,16 +600,6 @@ func logBenchmarks(benchmark string) error {
 	return nil
 }
 
-type gcpConfig struct {
-	gcsBucket          string
-	imageRepo          string
-	onbuildBaseImage   string
-	hardlinkBaseImage  string
-	serviceAccount     string
-	dockerMajorVersion int
-	uploadToGCS        bool
-}
-
 type imageDetails struct {
 	name      string
 	numLayers int
@@ -600,12 +610,11 @@ func (i imageDetails) String() string {
 	return fmt.Sprintf("Image: [%s] Digest: [%s] Number of Layers: [%d]", i.name, i.digest, i.numLayers)
 }
 
-func initGCPConfig() *gcpConfig {
-	var c gcpConfig
+func initIntegrationTestConfig() *integrationTestConfig {
+	var c integrationTestConfig
 	flag.StringVar(&c.gcsBucket, "bucket", "gs://kaniko-test-bucket", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.")
 	flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo or serviceAccount must be set.")
 	flag.StringVar(&c.serviceAccount, "serviceAccount", "", "The path to the service account push images to GCR and upload/download files to GCS.")
-	flag.BoolVar(&c.uploadToGCS, "uploadToGCS", true, "Upload the tar-ed contents of `integration` dir to GCS bucket. Default is true. Set this to false to prevent uploading.")
 	flag.Parse()
 
 	if len(c.serviceAccount) > 0 {
@@ -620,8 +629,12 @@ func initGCPConfig() *gcpConfig {
 		os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", absPath)
 	}
 
-	if c.gcsBucket == "" || c.imageRepo == "" {
-		log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo)
+	if c.imageRepo == "" {
+		log.Fatal("You must provide a image repository")
+	}
+
+	if c.isGcrRepository() && c.gcsBucket == "" {
+		log.Fatalf("You must provide a gcs bucket when using a Google Container Registry (\"%s\" was provided)", c.imageRepo)
 	}
 	if !strings.HasSuffix(c.imageRepo, "/") {
 		c.imageRepo = c.imageRepo + "/"

integration/migrate_gcr.go

@@ -0,0 +1,155 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"strings"
+)
+
+// This function crawl through dockerfiles and replace all reference to "gcr.io/" in FROM instruction and replace it targetRepo
+// The result is the array containing all modified file
+// Each of this file is saved
+func MigrateGCRRegistry(dockerfilesPath string, dockerfiles []string, targetRepo string) ([]string, error) {
+	var savedFiles []string
+	importedImages := map[string]interface{}{}
+
+	for _, dockerfile := range dockerfiles {
+		if referencedImages, savedFile, err := migrateFile(dockerfilesPath, dockerfile, targetRepo); err != nil {
+			if savedFile {
+				savedFiles = append(savedFiles, dockerfile)
+			}
+			return savedFiles, err
+		} else if savedFile {
+			savedFiles = append(savedFiles, dockerfile)
+			for _, referencedImage := range referencedImages {
+				importedImages[referencedImage] = nil
+			}
+		}
+	}
+	for image := range importedImages {
+		if err := importImage(image, targetRepo); err != nil {
+			return savedFiles, err
+		}
+	}
+	return savedFiles, nil
+}
+
+// This function rollback all previously modified files
+func RollbackMigratedFiles(dockerfilesPath string, dockerfiles []string) []error {
+	var result []error
+	for _, dockerfile := range dockerfiles {
+		fmt.Printf("Rolling back %s\n", dockerfile)
+		if err := recoverDockerfile(dockerfilesPath, dockerfile); err != nil {
+			result = append(result, err)
+		}
+	}
+	return result
+}
+
+// Import the gcr.io image such as gcr.io/my-image to targetRepo
+func importImage(image string, targetRepo string) error {
+	fmt.Printf("Importing %s to %s\n", image, targetRepo)
+	targetImage := strings.ReplaceAll(image, "gcr.io/", targetRepo)
+	pullCmd := exec.Command("docker", "pull", image)
+	if out, err := RunCommandWithoutTest(pullCmd); err != nil {
+		return fmt.Errorf("Failed to pull image %s with docker command \"%s\": %s %s", image, pullCmd.Args, err, string(out))
+	}
+
+	tagCmd := exec.Command("docker", "tag", image, targetImage)
+	if out, err := RunCommandWithoutTest(tagCmd); err != nil {
+		return fmt.Errorf("Failed to tag image %s to %s with docker command \"%s\": %s %s", image, targetImage, tagCmd.Args, err, string(out))
+	}
+
+	pushCmd := exec.Command("docker", "push", targetImage)
+	if out, err := RunCommandWithoutTest(pushCmd); err != nil {
+		return fmt.Errorf("Failed to push image %s with docker command \"%s\": %s %s", targetImage, pushCmd.Args, err, string(out))
+	}
+	return nil
+}
+
+// takes a dockerfile and replace each gcr.io/ occurrence in FROM instruction and replace it with imageRepo
+// return true if the file was saved
+// if so, the array is non nil and contains each gcr image name
+func migrateFile(dockerfilesPath string, dockerfile string, imageRepo string) ([]string, bool, error) {
+	var input *os.File
+	var output *os.File
+	var err error
+	var referencedImages []string
+
+	if input, err = os.Open(path.Join(dockerfilesPath, dockerfile)); err != nil {
+		return nil, false, err
+	}
+	defer input.Close()
+
+	var lines []string
+	scanner := bufio.NewScanner(input)
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if isFromGcrBaseImageInstruction(line) {
+			referencedImages = append(referencedImages, strings.Trim(strings.Split(line, " ")[1], " "))
+			lines = append(lines, strings.ReplaceAll(line, gcrRepoPrefix, imageRepo))
+		} else {
+			lines = append(lines, line)
+		}
+	}
+	rawOutput := []byte(strings.Join(append(lines, ""), "\n"))
+
+	if len(referencedImages) == 0 {
+		return nil, false, nil
+	}
+
+	if err = saveDockerfile(dockerfilesPath, dockerfile); err != nil {
+		return nil, false, err
+	}
+
+	if output, err = os.Create(path.Join(dockerfilesPath, dockerfile)); err != nil {
+		return nil, true, err
+	}
+	defer output.Close()
+
+	if written, err := output.Write(rawOutput); err != nil {
+		return nil, true, err
+	} else if written != len(rawOutput) {
+		return nil, true, fmt.Errorf("invalid number of byte written. Got %d, expected %d", written, len(rawOutput))
+	}
+	return referencedImages, true, nil
+
+}
+
+func isFromGcrBaseImageInstruction(line string) bool {
+	result, _ := regexp.MatchString(fmt.Sprintf("FROM +%s", gcrRepoPrefix), line)
+	return result
+}
+
+func saveDockerfile(dockerfilesPath string, dockerfile string) error {
+	return os.Rename(path.Join(dockerfilesPath, dockerfile), path.Join(dockerfilesPath, saveName(dockerfile)))
+}
+
+func recoverDockerfile(dockerfilesPath string, dockerfile string) error {
+	return os.Rename(path.Join(dockerfilesPath, saveName(dockerfile)), path.Join(dockerfilesPath, dockerfile))
+}
+
+func saveName(dockerfile string) string {
+	return fmt.Sprintf("%s_save_%d", dockerfile, os.Getpid())
+}

integration/replace-gcr-with-local-registry.sh

@@ -1,43 +0,0 @@
-#!/bin/bash
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# This script is needed due to the following bug:
-# https://github.com/GoogleContainerTools/kaniko/issues/966
-
-
-if [ "$#" -ne 1 ]; then
-    echo "Please specify path to dockerfiles as first argument."
-    echo "Usage: `basename $0` integration/dockerfiles"
-	exit 2
-fi
-
-dir_with_docker_files=$1
-
-for dockerfile in $dir_with_docker_files/*; do
-    cat $dockerfile | grep '^FROM' | grep "gcr" | while read -r line; do
-        gcr_repo=$(echo "$line" | awk '{ print $2 }')
-        local_repo=$(echo "$gcr_repo" | sed -e "s/^.*gcr.io\(\/.*\)$/localhost:5000\1/")
-        remove_digest=$(echo "$local_repo" | cut -f1 -d"@")
-        echo "Running docker pull $gcr_repo"
-        docker pull "$gcr_repo"
-        echo "Running docker tag $gcr_repo $remove_digest"
-        docker tag "$gcr_repo" "$remove_digest"
-        echo "Running docker push $remove_digest"
-        docker push "$remove_digest"
-        echo "Updating dockerfile $dockerfile to use local repo $local_repo"
-        sed -i -e "s/^\(FROM \).*gcr.io\(.*\)$/\1localhost:5000\2/" $dockerfile
-    done
-done