Allow contributors to launch integration tests against local registry

This change allows user to launch integration tests with a local registry

Fixes #1012

.travis.yml

@@ -13,9 +13,7 @@ before_install:
   - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
   - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && sudo mv container-diff-linux-amd64 /usr/local/bin/container-diff
   - docker run -d -p 5000:5000 --restart always --name registry registry:2
-  - ./integration/replace-gcr-with-local-registry.sh integration/dockerfiles
-
 script:
   - make test
-  - ./integration-test.sh --uploadToGCS=false
+  - make integration-test
   - make images

DEVELOPMENT.md

@@ -67,18 +67,39 @@ _These tests will not run correctly unless you have [checked out your fork into
 
 ### Integration tests
 
-The integration tests live in [`integration`](./integration) and can be run with:
+Currently the integration tests that live in [`integration`](./integration) can be run against your own gcloud space or a local registry.
+
+In either case, you will need the following tools:
+
+* [`container-diff`](https://github.com/GoogleContainerTools/container-diff#installation)
+
+#### GCloud
+
+To run integration tests with your GCloud Storage, you will also need the following tools:
+
+* [`gcloud`](https://cloud.google.com/sdk/install)
+* [`gsutil`](https://cloud.google.com/storage/docs/gsutil_install)
+* A bucket in [GCS](https://cloud.google.com/storage/) which you have write access to via
+  the user currently logged into `gcloud`
+* An image repo which you have write access to via the user currently logged into `gcloud`
+
+Once this step done, you must override the project using environment variables:
+
+* `GCS_BUCKET` - The name of your GCS bucket
+* `IMAGE_REPO` - The path to your docker image repo
+
+This can be done as follows:
 
 ```shell
 export GCS_BUCKET="gs://<your bucket>"
 export IMAGE_REPO="gcr.io/somerepo"
-make integration-test
 ```
 
-If you want to run `make integration-test`, you must override the project using environment variables:
+Then you can launch integration tests as follows:
 
-* `GCS_BUCKET` - The name of your GCS bucket
+```shell
-* `IMAGE_REPO` - The path to your docker image repo
+make integration-test
+```
 
 You can also run tests with `go test`, for example to run tests individually:
 
@@ -86,16 +107,37 @@ You can also run tests with `go test`, for example to run tests individually:
 go test ./integration -v --bucket $GCS_BUCKET --repo $IMAGE_REPO -run TestLayers/test_layer_Dockerfile_test_copy_bucket
 ```
 
-Requirements:
+These tests will be kicked off by [reviewers](#reviews) for submitted PRs by the kokoro task.
+
+#### Local repository
+
+To run integration tests locally against a local registry, install a local docker registry
+
+```shell
+docker run --rm  -d -p 5000:5000 --name registry registry:2
+```
+
+Then export the `IMAGE_REPO` variable with  the `localhost:5000`value
+
+```shell
+export IMAGE_REPO=localhost:5000
+```
+
+And run the integration tests
+
+```shell
+make integration-test
+```
+
+You can also run tests with `go test`, for example to run tests individually:
+
+```shell
+go test ./integration -v --repo localhost:5000 -run TestLayers/test_layer_Dockerfile_test_copy_bucket
+```
+
+These tests will be kicked off by [reviewers](#reviews) for submitted PRs by the travis task.
 
-* [`gcloud`](https://cloud.google.com/sdk/install)
-* [`gsutil`](https://cloud.google.com/storage/docs/gsutil_install)
-* [`container-diff`](https://github.com/GoogleContainerTools/container-diff#installation)
-* A bucket in [GCS](https://cloud.google.com/storage/) which you have write access to via
-  the user currently logged into `gcloud`
-* An image repo which you have write access to via the user currently logged into `gcloud`
 
-These tests will be kicked off by [reviewers](#reviews) for submitted PRs.
 
 ### Benchmarking
 

integration/config.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import "strings"
+
+type integrationTestConfig struct {
+	gcsBucket          string
+	imageRepo          string
+	onbuildBaseImage   string
+	hardlinkBaseImage  string
+	serviceAccount     string
+	dockerMajorVersion int
+}
+
+const gcrRepoPrefix string = "gcr.io/"
+
+func (config *integrationTestConfig) isGcrRepository() bool {
+	return strings.HasPrefix(config.imageRepo, gcrRepoPrefix)
+}

integration/images.go

@@ -191,7 +191,7 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string {
 // BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
 // The resulting image will be tagged with imageRepo. If the dockerfile will be built with
 // context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
-func (d *DockerFileBuilder) BuildImage(config *gcpConfig, dockerfilesPath, dockerfile string) error {
+func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
 	gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo
 	_, ex, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(ex)
@@ -317,7 +317,7 @@ func populateVolumeCache() error {
 }
 
 // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
-func (d *DockerFileBuilder) buildCachedImages(config *gcpConfig, cacheRepo, dockerfilesPath string, version int) error {
+func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int) error {
 	imageRepo, serviceAccount := config.imageRepo, config.serviceAccount
 	_, ex, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(ex)

integration/integration_test.go

@@ -33,12 +33,14 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
 
+	"github.com/pkg/errors"
+
 	"github.com/GoogleContainerTools/kaniko/pkg/timing"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/GoogleContainerTools/kaniko/testutil"
 )
 
-var config *gcpConfig
+var config *integrationTestConfig
 var imageBuilder *DockerFileBuilder
 
 const (
@@ -80,37 +82,65 @@ func getDockerMajorVersion() int {
 	}
 	return ver
 }
+func launchTests(m *testing.M, dockerfiles []string) (int, error) {
+
+	if config.isGcrRepository() {
+		contextFile, err := CreateIntegrationTarball()
+		if err != nil {
+			return 1, errors.Wrap(err, "Failed to create tarball of integration files for build context")
+		}
+
+		fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile, contextFile)
+		if err != nil {
+			return 1, errors.Wrap(err, "Failed to upload build context")
+		}
+
+		if err = os.Remove(contextFile); err != nil {
+			return 1, errors.Wrap(err, fmt.Sprintf("Failed to remove tarball at %s", contextFile))
+		}
+
+		RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) })
+		defer DeleteFromBucket(fileInBucket)
+	} else {
+		var err error
+		var migratedFiles []string
+		if migratedFiles, err = MigrateGCRRegistry(dockerfilesPath, dockerfiles, config.imageRepo); err != nil {
+			RollbackMigratedFiles(dockerfilesPath, migratedFiles)
+			return 1, errors.Wrap(err, "Fail to migrate dockerfiles from gcs")
+		}
+		RunOnInterrupt(func() { RollbackMigratedFiles(dockerfilesPath, migratedFiles) })
+		defer RollbackMigratedFiles(dockerfilesPath, migratedFiles)
+	}
+	if err := buildRequiredImages(); err != nil {
+		return 1, errors.Wrap(err, "Error while building images")
+	}
+
+	imageBuilder = NewDockerFileBuilder(dockerfiles)
+
+	return m.Run(), nil
+}
 
 func TestMain(m *testing.M) {
 	if !meetsRequirements() {
 		fmt.Println("Missing required tools")
 		os.Exit(1)
 	}
-	config = initGCPConfig()
 
-	if config.uploadToGCS {
+	if dockerfiles, err := FindDockerFiles(dockerfilesPath); err != nil {
-		contextFile, err := CreateIntegrationTarball()
+		fmt.Println("Coudn't create map of dockerfiles", err)
+		os.Exit(1)
+	} else {
+		config = initIntegrationTestConfig()
+		exitCode, err := launchTests(m, dockerfiles)
 		if err != nil {
-			fmt.Println("Failed to create tarball of integration files for build context", err)
+			fmt.Println(err)
-			os.Exit(1)
 		}
-
+		os.Exit(exitCode)
-		fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile, contextFile)
-		if err != nil {
-			fmt.Println("Failed to upload build context", err)
-			os.Exit(1)
-		}
-
-		err = os.Remove(contextFile)
-		if err != nil {
-			fmt.Printf("Failed to remove tarball at %s: %s\n", contextFile, err)
-			os.Exit(1)
-		}
-
-		RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) })
-		defer DeleteFromBucket(fileInBucket)
 	}
 
+}
+
+func buildRequiredImages() error {
 	setupCommands := []struct {
 		name    string
 		command []string
@@ -145,20 +175,10 @@ func TestMain(m *testing.M) {
 		fmt.Println(setupCmd.name)
 		cmd := exec.Command(setupCmd.command[0], setupCmd.command[1:]...)
 		if out, err := RunCommandWithoutTest(cmd); err != nil {
-			fmt.Printf("%s failed: %s", setupCmd.name, err)
+			return errors.Wrap(err, fmt.Sprintf("%s failed: %s", setupCmd.name, string(out)))
-			fmt.Println(string(out))
-			os.Exit(1)
 		}
 	}
-
+	return nil
-	dockerfiles, err := FindDockerFiles(dockerfilesPath)
-	if err != nil {
-		fmt.Printf("Coudn't create map of dockerfiles: %s", err)
-		os.Exit(1)
-	}
-	imageBuilder = NewDockerFileBuilder(dockerfiles)
-
-	os.Exit(m.Run())
 }
 
 func TestRun(t *testing.T) {
@@ -535,16 +555,6 @@ func logBenchmarks(benchmark string) error {
 	return nil
 }
 
-type gcpConfig struct {
-	gcsBucket          string
-	imageRepo          string
-	onbuildBaseImage   string
-	hardlinkBaseImage  string
-	serviceAccount     string
-	dockerMajorVersion int
-	uploadToGCS        bool
-}
-
 type imageDetails struct {
 	name      string
 	numLayers int
@@ -555,12 +565,11 @@ func (i imageDetails) String() string {
 	return fmt.Sprintf("Image: [%s] Digest: [%s] Number of Layers: [%d]", i.name, i.digest, i.numLayers)
 }
 
-func initGCPConfig() *gcpConfig {
+func initIntegrationTestConfig() *integrationTestConfig {
-	var c gcpConfig
+	var c integrationTestConfig
 	flag.StringVar(&c.gcsBucket, "bucket", "gs://kaniko-test-bucket", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.")
 	flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo or serviceAccount must be set.")
 	flag.StringVar(&c.serviceAccount, "serviceAccount", "", "The path to the service account push images to GCR and upload/download files to GCS.")
-	flag.BoolVar(&c.uploadToGCS, "uploadToGCS", true, "Upload the tar-ed contents of `integration` dir to GCS bucket. Default is true. Set this to false to prevent uploading.")
 	flag.Parse()
 
 	if len(c.serviceAccount) > 0 {
@@ -575,8 +584,12 @@ func initGCPConfig() *gcpConfig {
 		os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", absPath)
 	}
 
-	if c.gcsBucket == "" || c.imageRepo == "" {
+	if c.imageRepo == "" {
-		log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo)
+		log.Fatal("You must provide a image repository")
+	}
+
+	if c.isGcrRepository() && c.gcsBucket == "" {
+		log.Fatalf("You must provide a gcs bucket when using a Google Container Registry (\"%s\" was provided)", c.imageRepo)
 	}
 	if !strings.HasSuffix(c.imageRepo, "/") {
 		c.imageRepo = c.imageRepo + "/"

integration/migrate_gcr.go

@@ -0,0 +1,155 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"strings"
+)
+
+// This function crawl through dockerfiles and replace all reference to "gcr.io/" in FROM instruction and replace it targetRepo
+// The result is the array containing all modified file
+// Each of this file is saved
+func MigrateGCRRegistry(dockerfilesPath string, dockerfiles []string, targetRepo string) ([]string, error) {
+	var savedFiles []string
+	importedImages := map[string]interface{}{}
+
+	for _, dockerfile := range dockerfiles {
+		if referencedImages, savedFile, err := migrateFile(dockerfilesPath, dockerfile, targetRepo); err != nil {
+			if savedFile {
+				savedFiles = append(savedFiles, dockerfile)
+			}
+			return savedFiles, err
+		} else if savedFile {
+			savedFiles = append(savedFiles, dockerfile)
+			for _, referencedImage := range referencedImages {
+				importedImages[referencedImage] = nil
+			}
+		}
+	}
+	for image := range importedImages {
+		if err := importImage(image, targetRepo); err != nil {
+			return savedFiles, err
+		}
+	}
+	return savedFiles, nil
+}
+
+// This function rollback all previously modified files
+func RollbackMigratedFiles(dockerfilesPath string, dockerfiles []string) []error {
+	var result []error
+	for _, dockerfile := range dockerfiles {
+		fmt.Printf("Rolling back %s\n", dockerfile)
+		if err := recoverDockerfile(dockerfilesPath, dockerfile); err != nil {
+			result = append(result, err)
+		}
+	}
+	return result
+}
+
+// Import the gcr.io image such as gcr.io/my-image to targetRepo
+func importImage(image string, targetRepo string) error {
+	fmt.Printf("Importing %s to %s\n", image, targetRepo)
+	targetImage := strings.ReplaceAll(image, "gcr.io/", targetRepo)
+	pullCmd := exec.Command("docker", "pull", image)
+	if out, err := RunCommandWithoutTest(pullCmd); err != nil {
+		return fmt.Errorf("Failed to pull image %s with docker command \"%s\": %s %s", image, pullCmd.Args, err, string(out))
+	}
+
+	tagCmd := exec.Command("docker", "tag", image, targetImage)
+	if out, err := RunCommandWithoutTest(tagCmd); err != nil {
+		return fmt.Errorf("Failed to tag image %s to %s with docker command \"%s\": %s %s", image, targetImage, tagCmd.Args, err, string(out))
+	}
+
+	pushCmd := exec.Command("docker", "push", targetImage)
+	if out, err := RunCommandWithoutTest(pushCmd); err != nil {
+		return fmt.Errorf("Failed to push image %s with docker command \"%s\": %s %s", targetImage, pushCmd.Args, err, string(out))
+	}
+	return nil
+}
+
+// takes a dockerfile and replace each gcr.io/ occurrence in FROM instruction and replace it with imageRepo
+// return true if the file was saved
+// if so, the array is non nil and contains each gcr image name
+func migrateFile(dockerfilesPath string, dockerfile string, imageRepo string) ([]string, bool, error) {
+	var input *os.File
+	var output *os.File
+	var err error
+	var referencedImages []string
+
+	if input, err = os.Open(path.Join(dockerfilesPath, dockerfile)); err != nil {
+		return nil, false, err
+	}
+	defer input.Close()
+
+	var lines []string
+	scanner := bufio.NewScanner(input)
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if isFromGcrBaseImageInstruction(line) {
+			referencedImages = append(referencedImages, strings.Trim(strings.Split(line, " ")[1], " "))
+			lines = append(lines, strings.ReplaceAll(line, gcrRepoPrefix, imageRepo))
+		} else {
+			lines = append(lines, line)
+		}
+	}
+	rawOutput := []byte(strings.Join(append(lines, ""), "\n"))
+
+	if len(referencedImages) == 0 {
+		return nil, false, nil
+	}
+
+	if err = saveDockerfile(dockerfilesPath, dockerfile); err != nil {
+		return nil, false, err
+	}
+
+	if output, err = os.Create(path.Join(dockerfilesPath, dockerfile)); err != nil {
+		return nil, true, err
+	}
+	defer output.Close()
+
+	if written, err := output.Write(rawOutput); err != nil {
+		return nil, true, err
+	} else if written != len(rawOutput) {
+		return nil, true, fmt.Errorf("invalid number of byte written. Got %d, expected %d", written, len(rawOutput))
+	}
+	return referencedImages, true, nil
+
+}
+
+func isFromGcrBaseImageInstruction(line string) bool {
+	result, _ := regexp.MatchString(fmt.Sprintf("FROM +%s", gcrRepoPrefix), line)
+	return result
+}
+
+func saveDockerfile(dockerfilesPath string, dockerfile string) error {
+	return os.Rename(path.Join(dockerfilesPath, dockerfile), path.Join(dockerfilesPath, saveName(dockerfile)))
+}
+
+func recoverDockerfile(dockerfilesPath string, dockerfile string) error {
+	return os.Rename(path.Join(dockerfilesPath, saveName(dockerfile)), path.Join(dockerfilesPath, dockerfile))
+}
+
+func saveName(dockerfile string) string {
+	return fmt.Sprintf("%s_save_%d", dockerfile, os.Getpid())
+}

integration/replace-gcr-with-local-registry.sh

@@ -1,43 +0,0 @@
-#!/bin/bash
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# This script is needed due to the following bug:
-# https://github.com/GoogleContainerTools/kaniko/issues/966
-
-
-if [ "$#" -ne 1 ]; then
-    echo "Please specify path to dockerfiles as first argument."
-    echo "Usage: `basename $0` integration/dockerfiles"
-	exit 2
-fi
-
-dir_with_docker_files=$1
-
-for dockerfile in $dir_with_docker_files/*; do
-    cat $dockerfile | grep '^FROM' | grep "gcr" | while read -r line; do
-        gcr_repo=$(echo "$line" | awk '{ print $2 }')
-        local_repo=$(echo "$gcr_repo" | sed -e "s/^.*gcr.io\(\/.*\)$/localhost:5000\1/")
-        remove_digest=$(echo "$local_repo" | cut -f1 -d"@")
-        echo "Running docker pull $gcr_repo"
-        docker pull "$gcr_repo"
-        echo "Running docker tag $gcr_repo $remove_digest"
-        docker tag "$gcr_repo" "$remove_digest"
-        echo "Running docker push $remove_digest"
-        docker push "$remove_digest"
-        echo "Updating dockerfile $dockerfile to use local repo $local_repo"
-        sed -i -e "s/^\(FROM \).*gcr.io\(.*\)$/\1localhost:5000\2/" $dockerfile
-    done
-done