Pass bucket and repo as args to tests

To allow contributors to run the integration tests with their own GCS buckets and image repos (since not all contributors will have accesss to the projects used by the kaniko maintainers) this updates the integration tests so that these can be provided on the command line. This allows tests to be run individually, without using `make integration-test`. Previously, part of the test setup was done in the shell script (creating the context tarball that is required for the tests that build images with context). Instead it will be done in the test iself, so we can use `go test` to run tests individually if we want to. If we are running only one individual test, we don't want to build all of the images, so this commit creates a builder which tracks which images it has built and can be used by a tests to check if it should build an image before running, or it will use the images that have already been built by a previous test. The name of the context tarball has also been made unique (it includes the unix timestamp) to avoid potential test flakes if two tests using the same GCS bucket run simultaneously.
2018-07-24 15:36:31 -07:00 · 2018-07-24 15:36:31 -07:00 · b5a4d7636f
parent e43968f02f
commit b5a4d7636f
7 changed files with 465 additions and 156 deletions
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@ -70,12 +70,32 @@ _These tests will not run correctly unless you have [checked out your fork into
 The integration tests live in [`integration`](./integration) and can be run with:

 ```shell
+export GCS_BUCKET="gs://<your bucket>"
+export IMAGE_REPO="gcr.io/somerepo"
 make integration-test
 ```

-_These tests require push access to a project in GCP, and so can only be run
-by maintainers who have access. These tests will be kicked off by [reviewers](#reviews)
-for submitted PRs._
+If you want to run `make integration-test`, you must override the project using environment variables:
+
+* `GCS_BUCKET` - The name of your GCS bucket
+* `IMAGE_REPO` - The path to your docker image repo
+
+You can also run tests with `go test`, for example to run tests individually:
+
+```shell
+go test -v --bucket $GCS_BUCKET --repo $IMAGE_REPO -run TestLayers/test_layer_Dockerfile_test_copy_bucket
+```
+
+Requirements:
+
+* [`gcloud`](https://cloud.google.com/sdk/install)
+* [`gsutil`](https://cloud.google.com/storage/docs/gsutil_install)
+* [`container-diff`](https://github.com/GoogleContainerTools/container-diff#installation)
+* A bucket in [GCS](https://cloud.google.com/storage/) which you have write access to via
+  the user currently logged into `gcloud`
+* An image repo which you have write access to via the user currently logged into `gcloud`
+
+These tests will be kicked off by [reviewers](#reviews) for submitted PRs.

 ## Creating a PR

--- a/integration-test.sh
+++ b/integration-test.sh
@ -15,6 +15,10 @@
 #!/bin/bash
 set -ex

+GCS_BUCKET="${GCS_BUCKET:-gs://kaniko-test-bucket}"
+IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}"
+
+# Sets up a kokoro (Google internal integration testing tool) environment
 if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then
    echo "Installing dependencies..."
    source "$KOKORO_GFILE_DIR/common.sh"
@ -28,12 +32,7 @@ if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then
    cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json
 fi

-echo "Creating build context tarball..."
-tar  -C ./integration -zcvf context.tar.gz .
-gsutil cp context.tar.gz gs://kaniko-test-bucket
-rm context.tar.gz
-
 echo "Running integration tests..."
 make out/executor
 pushd integration
-go test
+go test -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}"
--- a/integration/cleanup.go
+++ b/integration/cleanup.go
@ -0,0 +1,37 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"log"
+	"os"
+	"os/signal"
+)
+
+// RunOnInterrupt will execute the function f if execution is interrupted with the
+// interrupt signal.
+func RunOnInterrupt(f func()) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		for range c {
+			log.Println("Interrupted, cleaning up.")
+			f()
+			os.Exit(1)
+		}
+	}()
+}
--- a/integration/cmd.go
+++ b/integration/cmd.go
@ -0,0 +1,48 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"os/exec"
+	"testing"
+)
+
+// RunCommandWithoutTest will run cmd and if it fails will output relevant info
+// for debugging before returning an error. It can be run outside the context of a test.
+func RunCommandWithoutTest(cmd *exec.Cmd) ([]byte, error) {
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		fmt.Println(cmd.Args)
+		fmt.Println(string(output))
+	}
+	return output, nil
+}
+
+// RunCommand will run cmd and if it fails will output relevant info for debugging
+// before it fails. It must be run within the context of a test t and if the command
+// fails, it will the test. Returns the output from the command.
+func RunCommand(cmd *exec.Cmd, t *testing.T) []byte {
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Log(cmd.Args)
+		t.Log(string(output))
+		t.Error(err)
+		t.FailNow()
+	}
+	return output
+}
--- a/integration/gcs.go
+++ b/integration/gcs.go
@ -0,0 +1,73 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+)
+
+// CreateIntegrationTarball will take the contents of the integration directory and write
+// them to a tarball in a temmporary dir. It will return a path to the tarball.
+func CreateIntegrationTarball() (string, error) {
+	log.Println("Creating tarball of integration test files to use as build context")
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", fmt.Errorf("Failed find path to integration dir: %s", err)
+	}
+	tempDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return "", fmt.Errorf("Failed to create temporary directoy to hold tarball: %s", err)
+	}
+	contextFile := fmt.Sprintf("%s/context_%d.tar.gz", tempDir, time.Now().UnixNano())
+	cmd := exec.Command("tar", "-C", dir, "-zcvf", contextFile, ".")
+	_, err = RunCommandWithoutTest(cmd)
+	if err != nil {
+		return "", fmt.Errorf("Failed to create build context tarball from integration dir: %s", err)
+	}
+	return contextFile, err
+}
+
+// UploadFileToBucket will upload the at filePath to gcsBucket. It will return the path
+// of the file in gcsBucket.
+func UploadFileToBucket(gcsBucket string, filePath string) (string, error) {
+	log.Printf("Uploading file at %s to GCS bucket at %s\n", filePath, gcsBucket)
+
+	cmd := exec.Command("gsutil", "cp", filePath, gcsBucket)
+	_, err := RunCommandWithoutTest(cmd)
+	if err != nil {
+		return "", fmt.Errorf("Failed to copy tarball to GCS bucket %s: %s", gcsBucket, err)
+	}
+
+	return filepath.Join(gcsBucket, filePath), err
+}
+
+// DeleteFromBucket will remove the content at path. path should be the full path
+// to a file in GCS.
+func DeleteFromBucket(path string) error {
+	cmd := exec.Command("gsutil", "rm", path)
+	_, err := RunCommandWithoutTest(cmd)
+	if err != nil {
+		return fmt.Errorf("Failed to delete file %s from GCS: %s", path, err)
+	}
+	return err
+}
--- a/integration/images.go
+++ b/integration/images.go
@ -0,0 +1,188 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+const (
+	// ExecutorImage is the name of the kaniko executor image
+	ExecutorImage = "executor-image"
+
+	dockerPrefix     = "docker-"
+	kanikoPrefix     = "kaniko-"
+	buildContextPath = "/workspace"
+)
+
+// Arguments to build Dockerfiles with, used for both docker and kaniko builds
+var argsMap = map[string][]string{
+	"Dockerfile_test_run":     {"file=/file"},
+	"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
+	"Dockerfile_test_add":     {"file=context/foo"},
+	"Dockerfile_test_onbuild": {"file=/tmp/onbuild"},
+	"Dockerfile_test_scratch": {
+		"image=scratch",
+		"hello=hello-value",
+		"file=context/foo",
+		"file3=context/b*",
+	},
+	"Dockerfile_test_multistage": {"file=/foo2"},
+}
+
+// Arguments to build Dockerfiles with when building with docker
+var additionalDockerFlagsMap = map[string][]string{
+	"Dockerfile_test_target": {"--target=second"},
+}
+
+// Arguments to build Dockerfiles with when building with kaniko
+var additionalKanikoFlagsMap = map[string][]string{
+	"Dockerfile_test_add":     {"--single-snapshot"},
+	"Dockerfile_test_scratch": {"--single-snapshot"},
+	"Dockerfile_test_target":  {"--target=second"},
+}
+
+var bucketContextTests = []string{"Dockerfile_test_copy_bucket"}
+var reproducibleTests = []string{"Dockerfile_test_env"}
+
+// GetDockerImage constructs the name of the docker image that would be built with
+// dockerfile if it was tagged with imageRepo.
+func GetDockerImage(imageRepo, dockerfile string) string {
+	return strings.ToLower(imageRepo + dockerPrefix + dockerfile)
+}
+
+// GetKanikoImage constructs the name of the kaniko image that would be built with
+// dockerfile if it was tagged with imageRepo.
+func GetKanikoImage(imageRepo, dockerfile string) string {
+	return strings.ToLower(imageRepo + kanikoPrefix + dockerfile)
+}
+
+// FindDockerFiles will look for test docker files in the directory dockerfilesPath.
+// These files must start with `Dockerfile_test`. If the file is one we are intentionally
+// skipping, it will not be included in the returned list.
+func FindDockerFiles(dockerfilesPath string) ([]string, error) {
+	// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
+	testsToIgnore := map[string]bool{"Dockerfile_test_user_run": true}
+	allDockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*"))
+	if err != nil {
+		return []string{}, fmt.Errorf("Failed to find docker files at %s: %s", dockerfilesPath, err)
+	}
+
+	var dockerfiles []string
+	for _, dockerfile := range allDockerfiles {
+		// Remove the leading directory from the path
+		dockerfile = dockerfile[len("dockerfiles/"):]
+		if !testsToIgnore[dockerfile] {
+			dockerfiles = append(dockerfiles, dockerfile)
+		}
+	}
+	return dockerfiles, err
+}
+
+// DockerFileBuilder knows how to build docker files using both Kaniko and Docker and
+// keeps track of which files have been built.
+type DockerFileBuilder struct {
+	// Holds all available docker files and whether or not they've been built
+	FilesBuilt map[string]bool
+}
+
+// NewDockerFileBuilder will create a DockerFileBuilder initialized with dockerfiles, which
+// it will assume are all as yet unbuilt.
+func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder {
+	d := DockerFileBuilder{FilesBuilt: map[string]bool{}}
+	for _, f := range dockerfiles {
+		d.FilesBuilt[f] = false
+	}
+	return &d
+}
+
+// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
+// The resulting image will be tagged with imageRepo. If the dockerfile will be built with
+// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
+func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, dockerfile string) error {
+	_, ex, _, _ := runtime.Caller(0)
+	cwd := filepath.Dir(ex)
+
+	fmt.Printf("Building images for Dockerfile %s\n", dockerfile)
+
+	var buildArgs []string
+	buildArgFlag := "--build-arg"
+	for _, arg := range argsMap[dockerfile] {
+		buildArgs = append(buildArgs, buildArgFlag)
+		buildArgs = append(buildArgs, arg)
+	}
+	// build docker image
+	additionalFlags := append(buildArgs, additionalDockerFlagsMap[dockerfile]...)
+	dockerImage := strings.ToLower(imageRepo + dockerPrefix + dockerfile)
+	dockerCmd := exec.Command("docker",
+		append([]string{"build",
+			"-t", dockerImage,
+			"-f", path.Join(dockerfilesPath, dockerfile),
+			"."},
+			additionalFlags...)...,
+	)
+	_, err := RunCommandWithoutTest(dockerCmd)
+	if err != nil {
+		return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s", dockerImage, dockerCmd.Args, err)
+	}
+
+	contextFlag := "-c"
+	contextPath := buildContextPath
+	for _, d := range bucketContextTests {
+		if d == dockerfile {
+			contextFlag = "-b"
+			contextPath = gcsBucket
+			break
+		}
+	}
+
+	reproducibleFlag := ""
+	for _, d := range reproducibleTests {
+		if d == dockerfile {
+			reproducibleFlag = "--reproducible"
+			break
+		}
+	}
+
+	// build kaniko image
+	additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
+	kanikoImage := GetKanikoImage(imageRepo, dockerfile)
+	kanikoCmd := exec.Command("docker",
+		append([]string{"run",
+			"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
+			"-v", cwd + ":/workspace",
+			ExecutorImage,
+			"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
+			"-d", kanikoImage, reproducibleFlag,
+			contextFlag, contextPath},
+			additionalFlags...)...,
+	)
+
+	_, err = RunCommandWithoutTest(kanikoCmd)
+	if err != nil {
+		return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s", dockerImage, kanikoCmd.Args, err)
+	}
+
+	d.FilesBuilt[dockerfile] = true
+	return nil
+}
--- a/integration/integration_test.go
+++ b/integration/integration_test.go
@ -18,35 +18,50 @@ package integration

 import (
 	"encoding/json"
+	"flag"
 	"fmt"
+	"log"
 	"math"
 	"os"
 	"os/exec"
-	"path"
-	"path/filepath"
-	"runtime"
 	"strings"
 	"testing"

-	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"

 	"github.com/GoogleContainerTools/kaniko/testutil"
 )

+var config = initGCPConfig()
+var imageBuilder *DockerFileBuilder
+
+type gcpConfig struct {
+	gcsBucket        string
+	imageRepo        string
+	onbuildBaseImage string
+}
+
+func initGCPConfig() *gcpConfig {
+	var c gcpConfig
+	flag.StringVar(&c.gcsBucket, "bucket", "", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.")
+	flag.StringVar(&c.imageRepo, "repo", "", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo.")
+	flag.Parse()
+
+	if c.gcsBucket == "" || c.imageRepo == "" {
+		log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo)
+	}
+	if !strings.HasSuffix(c.imageRepo, "/") {
+		c.imageRepo = c.imageRepo + "/"
+	}
+	c.onbuildBaseImage = c.imageRepo + "onbuild-base:latest"
+	return &c
+}
+
 const (
-	executorImage      = "executor-image"
-	dockerImage        = "gcr.io/cloud-builders/docker"
 	ubuntuImage        = "ubuntu"
-	testRepo           = "gcr.io/kaniko-test/"
-	dockerPrefix       = "docker-"
-	kanikoPrefix       = "kaniko-"
 	daemonPrefix       = "daemon://"
-	kanikoTestBucket   = "kaniko-test-bucket"
 	dockerfilesPath    = "dockerfiles"
-	onbuildBaseImage   = testRepo + "onbuild-base:latest"
-	buildContextPath   = "/workspace"
 	emptyContainerDiff = `[
     {
       "Image1": "%s",
@ -70,130 +85,73 @@ const (
   ]`
 )

-// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
-var testsToIgnore = []string{"Dockerfile_test_user_run"}
+func meetsRequirements() bool {
+	requiredTools := []string{"container-diff", "gsutil"}
+	hasRequirements := true
+	for _, tool := range requiredTools {
+		_, err := exec.LookPath(tool)
+		if err != nil {
+			fmt.Printf("You must have %s installed and on your PATH\n", tool)
+			hasRequirements = false
+		}
+	}
+	return hasRequirements
+}

 func TestMain(m *testing.M) {
-	buildKaniko := exec.Command("docker", "build", "-t", executorImage, "-f", "../deploy/Dockerfile", "..")
-	err := buildKaniko.Run()
+	if !meetsRequirements() {
+		fmt.Println("Missing required tools")
+		os.Exit(1)
+	}
+	contextFile, err := CreateIntegrationTarball()
+	if err != nil {
+		fmt.Println("Failed to create tarball of integration files for build context", err)
+		os.Exit(1)
+	}
+
+	fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile)
+	if err != nil {
+		fmt.Println("Failed to upload build context", err)
+		os.Exit(1)
+	}
+
+	err = os.Remove(contextFile)
+	if err != nil {
+		fmt.Printf("Failed to remove tarball at %s: %s\n", contextFile, err)
+		os.Exit(1)
+	}
+
+	RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) })
+	defer DeleteFromBucket(fileInBucket)
+
+	fmt.Println("Building kaniko image")
+	buildKaniko := exec.Command("docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", "..")
+	err = buildKaniko.Run()
 	if err != nil {
 		fmt.Print(err)
 		fmt.Print("Building kaniko failed.")
 		os.Exit(1)
 	}

-	// Make sure container-diff is on user's PATH
-	_, err = exec.LookPath("container-diff")
+	dockerfiles, err := FindDockerFiles(dockerfilesPath)
 	if err != nil {
-		fmt.Print("Make sure you have container-diff installed and on your PATH")
+		fmt.Printf("Coudn't create map of dockerfiles: %s", err)
 		os.Exit(1)
 	}
-
+	imageBuilder = NewDockerFileBuilder(dockerfiles)
 	os.Exit(m.Run())
 }
-
 func TestRun(t *testing.T) {
-	dockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*"))
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-
-	// Map for test Dockerfile to expected ARGs
-	argsMap := map[string][]string{
-		"Dockerfile_test_run":     {"file=/file"},
-		"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
-		"Dockerfile_test_add":     {"file=context/foo"},
-		"Dockerfile_test_onbuild": {"file=/tmp/onbuild"},
-		"Dockerfile_test_scratch": {
-			"image=scratch",
-			"hello=hello-value",
-			"file=context/foo",
-			"file3=context/b*",
-		},
-		"Dockerfile_test_multistage": {"file=/foo2"},
-	}
-
-	// Map for additional docker flags
-	additionalDockerFlagsMap := map[string][]string{
-		"Dockerfile_test_target": {"--target=second"},
-	}
-	// Map for additional kaniko flags
-	additionalKanikoFlagsMap := map[string][]string{
-		"Dockerfile_test_add":     {"--single-snapshot"},
-		"Dockerfile_test_scratch": {"--single-snapshot"},
-		"Dockerfile_test_target":  {"--target=second"},
-	}
-
-	// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
-	testsToIgnore := []string{"Dockerfile_test_user_run"}
-	bucketContextTests := []string{"Dockerfile_test_copy_bucket"}
-	reproducibleTests := []string{"Dockerfile_test_env"}
-
-	_, ex, _, _ := runtime.Caller(0)
-	cwd := filepath.Dir(ex)
-
-	for _, dockerfile := range dockerfiles {
+	for dockerfile, built := range imageBuilder.FilesBuilt {
 		t.Run("test_"+dockerfile, func(t *testing.T) {
-			dockerfile = dockerfile[len("dockerfile/")+1:]
-			for _, d := range testsToIgnore {
-				if dockerfile == d {
-					t.SkipNow()
+			if !built {
+				err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
+				if err != nil {
+					t.Fatalf("Failed to build kaniko and docker images for %s: %s", dockerfile, err)
 				}
 			}
-			t.Logf("%s\n", dockerfile)
-
-			var buildArgs []string
-			buildArgFlag := "--build-arg"
-			for _, arg := range argsMap[dockerfile] {
-				buildArgs = append(buildArgs, buildArgFlag)
-				buildArgs = append(buildArgs, arg)
-			}
-			// build docker image
-			additionalFlags := append(buildArgs, additionalDockerFlagsMap[dockerfile]...)
-			dockerImage := strings.ToLower(testRepo + dockerPrefix + dockerfile)
-			dockerCmd := exec.Command("docker",
-				append([]string{"build",
-					"-t", dockerImage,
-					"-f", path.Join(dockerfilesPath, dockerfile),
-					"."},
-					additionalFlags...)...,
-			)
-			RunCommand(dockerCmd, t)
-
-			contextFlag := "-c"
-			contextPath := buildContextPath
-			for _, d := range bucketContextTests {
-				if d == dockerfile {
-					contextFlag = "-b"
-					contextPath = constants.GCSBuildContextPrefix + kanikoTestBucket
-					break
-				}
-			}
-
-			reproducibleFlag := ""
-			for _, d := range reproducibleTests {
-				if d == dockerfile {
-					reproducibleFlag = "--reproducible"
-					break
-				}
-			}
-
-			// build kaniko image
-			additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
-			kanikoImage := strings.ToLower(testRepo + kanikoPrefix + dockerfile)
-			kanikoCmd := exec.Command("docker",
-				append([]string{"run",
-					"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
-					"-v", cwd + ":/workspace",
-					executorImage,
-					"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
-					"-d", kanikoImage, reproducibleFlag,
-					contextFlag, contextPath},
-					additionalFlags...)...,
-			)
-
-			RunCommand(kanikoCmd, t)
+			dockerImage := GetDockerImage(config.imageRepo, dockerfile)
+			kanikoImage := GetKanikoImage(config.imageRepo, dockerfile)

 			// container-diff
 			daemonDockerImage := daemonPrefix + dockerImage
@ -210,7 +168,7 @@ func TestRun(t *testing.T) {
 			var diffInt interface{}
 			var expectedInt interface{}

-			err = json.Unmarshal(diff, &diffInt)
+			err := json.Unmarshal(diff, &diffInt)
 			if err != nil {
 				t.Error(err)
 				t.Fail()
@ -228,11 +186,6 @@ func TestRun(t *testing.T) {
 }

 func TestLayers(t *testing.T) {
-	dockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*"))
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
 	offset := map[string]int{
 		"Dockerfile_test_add":     9,
 		"Dockerfile_test_scratch": 3,
@ -240,17 +193,17 @@ func TestLayers(t *testing.T) {
 		// which is why this offset exists
 		"Dockerfile_test_volume": 1,
 	}
-	for _, dockerfile := range dockerfiles {
+	for dockerfile, built := range imageBuilder.FilesBuilt {
 		t.Run("test_layer_"+dockerfile, func(t *testing.T) {
-			dockerfile = dockerfile[len("dockerfile/")+1:]
-			for _, ignore := range testsToIgnore {
-				if dockerfile == ignore {
-					t.SkipNow()
+			if !built {
+				err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
+				if err != nil {
+					t.Fatalf("Failed to build kaniko and docker images for %s: %s", dockerfile, err)
 				}
 			}
 			// Pull the kaniko image
-			dockerImage := strings.ToLower(testRepo + dockerPrefix + dockerfile)
-			kanikoImage := strings.ToLower(testRepo + kanikoPrefix + dockerfile)
+			dockerImage := GetDockerImage(config.imageRepo, dockerfile)
+			kanikoImage := GetKanikoImage(config.imageRepo, dockerfile)
 			pullCmd := exec.Command("docker", "pull", kanikoImage)
 			RunCommand(pullCmd, t)
 			if err := checkLayers(dockerImage, kanikoImage, offset[dockerfile]); err != nil {
@ -264,11 +217,11 @@ func TestLayers(t *testing.T) {
 func checkLayers(image1, image2 string, offset int) error {
 	lenImage1, err := numLayers(image1)
 	if err != nil {
-		return err
+		return fmt.Errorf("Couldn't get number of layers for image1 (%s): %s", image1, err)
 	}
 	lenImage2, err := numLayers(image2)
 	if err != nil {
-		return err
+		return fmt.Errorf("Couldn't get number of layers for image2 (%s): %s", image2, err)
 	}
 	actualOffset := int(math.Abs(float64(lenImage1 - lenImage2)))
 	if actualOffset != offset {
@ -280,24 +233,15 @@ func checkLayers(image1, image2 string, offset int) error {
 func numLayers(image string) (int, error) {
 	ref, err := name.ParseReference(image, name.WeakValidation)
 	if err != nil {
-		return 0, err
+		return 0, fmt.Errorf("Couldn't parse referance to image %s: %s", image, err)
 	}
 	img, err := daemon.Image(ref)
 	if err != nil {
-		return 0, err
+		return 0, fmt.Errorf("Couldn't get reference to image %s from daemon: %s", image, err)
 	}
 	layers, err := img.Layers()
-	return len(layers), err
-}
-
-func RunCommand(cmd *exec.Cmd, t *testing.T) []byte {
-	output, err := cmd.CombinedOutput()
 	if err != nil {
-		t.Log(cmd.Args)
-		t.Log(string(output))
-		t.Error(err)
-		t.FailNow()
+		return 0, fmt.Errorf("Error getting layers for image %s: %s", image, err)
 	}
-
-	return output
+	return len(layers), nil
 }