Merge pull request #256 from bobcatfish/local_integration_tests

Local integration tests

DEVELOPMENT.md

@@ -70,12 +70,32 @@ _These tests will not run correctly unless you have [checked out your fork into
 The integration tests live in [`integration`](./integration) and can be run with:
 
 ```shell
+export GCS_BUCKET="gs://<your bucket>"
+export IMAGE_REPO="gcr.io/somerepo"
 make integration-test
 ```
 
-_These tests require push access to a project in GCP, and so can only be run
+If you want to run `make integration-test`, you must override the project using environment variables:
-by maintainers who have access. These tests will be kicked off by [reviewers](#reviews)
+
-for submitted PRs._
+* `GCS_BUCKET` - The name of your GCS bucket
+* `IMAGE_REPO` - The path to your docker image repo
+
+You can also run tests with `go test`, for example to run tests individually:
+
+```shell
+go test -v --bucket $GCS_BUCKET --repo $IMAGE_REPO -run TestLayers/test_layer_Dockerfile_test_copy_bucket
+```
+
+Requirements:
+
+* [`gcloud`](https://cloud.google.com/sdk/install)
+* [`gsutil`](https://cloud.google.com/storage/docs/gsutil_install)
+* [`container-diff`](https://github.com/GoogleContainerTools/container-diff#installation)
+* A bucket in [GCS](https://cloud.google.com/storage/) which you have write access to via
+  the user currently logged into `gcloud`
+* An image repo which you have write access to via the user currently logged into `gcloud`
+
+These tests will be kicked off by [reviewers](#reviews) for submitted PRs.
 
 ## Creating a PR
 

integration-test.sh

@@ -15,6 +15,10 @@
 #!/bin/bash
 set -ex
 
+GCS_BUCKET="${GCS_BUCKET:-gs://kaniko-test-bucket}"
+IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}"
+
+# Sets up a kokoro (Google internal integration testing tool) environment
 if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then
     echo "Installing dependencies..."
     source "$KOKORO_GFILE_DIR/common.sh"
@@ -28,12 +32,7 @@ if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then
     cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json
 fi
 
-echo "Creating build context tarball..."
-tar  -C ./integration -zcvf context.tar.gz .
-gsutil cp context.tar.gz gs://kaniko-test-bucket
-rm context.tar.gz
-
 echo "Running integration tests..."
 make out/executor
 pushd integration
-go test
+go test -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}"

integration/cleanup.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"log"
+	"os"
+	"os/signal"
+)
+
+// RunOnInterrupt will execute the function f if execution is interrupted with the
+// interrupt signal.
+func RunOnInterrupt(f func()) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		for range c {
+			log.Println("Interrupted, cleaning up.")
+			f()
+			os.Exit(1)
+		}
+	}()
+}

integration/cmd.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"os/exec"
+	"testing"
+)
+
+// RunCommandWithoutTest will run cmd and if it fails will output relevant info
+// for debugging before returning an error. It can be run outside the context of a test.
+func RunCommandWithoutTest(cmd *exec.Cmd) ([]byte, error) {
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		fmt.Println(cmd.Args)
+		fmt.Println(string(output))
+	}
+	return output, nil
+}
+
+// RunCommand will run cmd and if it fails will output relevant info for debugging
+// before it fails. It must be run within the context of a test t and if the command
+// fails, it will the test. Returns the output from the command.
+func RunCommand(cmd *exec.Cmd, t *testing.T) []byte {
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Log(cmd.Args)
+		t.Log(string(output))
+		t.Error(err)
+		t.FailNow()
+	}
+	return output
+}

integration/gcs.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+)
+
+// CreateIntegrationTarball will take the contents of the integration directory and write
+// them to a tarball in a temmporary dir. It will return a path to the tarball.
+func CreateIntegrationTarball() (string, error) {
+	log.Println("Creating tarball of integration test files to use as build context")
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", fmt.Errorf("Failed find path to integration dir: %s", err)
+	}
+	tempDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return "", fmt.Errorf("Failed to create temporary directoy to hold tarball: %s", err)
+	}
+	contextFile := fmt.Sprintf("%s/context_%d.tar.gz", tempDir, time.Now().UnixNano())
+	cmd := exec.Command("tar", "-C", dir, "-zcvf", contextFile, ".")
+	_, err = RunCommandWithoutTest(cmd)
+	if err != nil {
+		return "", fmt.Errorf("Failed to create build context tarball from integration dir: %s", err)
+	}
+	return contextFile, err
+}
+
+// UploadFileToBucket will upload the at filePath to gcsBucket. It will return the path
+// of the file in gcsBucket.
+func UploadFileToBucket(gcsBucket string, filePath string) (string, error) {
+	log.Printf("Uploading file at %s to GCS bucket at %s\n", filePath, gcsBucket)
+
+	cmd := exec.Command("gsutil", "cp", filePath, gcsBucket)
+	_, err := RunCommandWithoutTest(cmd)
+	if err != nil {
+		return "", fmt.Errorf("Failed to copy tarball to GCS bucket %s: %s", gcsBucket, err)
+	}
+
+	return filepath.Join(gcsBucket, filePath), err
+}
+
+// DeleteFromBucket will remove the content at path. path should be the full path
+// to a file in GCS.
+func DeleteFromBucket(path string) error {
+	cmd := exec.Command("gsutil", "rm", path)
+	_, err := RunCommandWithoutTest(cmd)
+	if err != nil {
+		return fmt.Errorf("Failed to delete file %s from GCS: %s", path, err)
+	}
+	return err
+}

integration/images.go

@@ -0,0 +1,188 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+const (
+	// ExecutorImage is the name of the kaniko executor image
+	ExecutorImage = "executor-image"
+
+	dockerPrefix     = "docker-"
+	kanikoPrefix     = "kaniko-"
+	buildContextPath = "/workspace"
+)
+
+// Arguments to build Dockerfiles with, used for both docker and kaniko builds
+var argsMap = map[string][]string{
+	"Dockerfile_test_run":     {"file=/file"},
+	"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
+	"Dockerfile_test_add":     {"file=context/foo"},
+	"Dockerfile_test_onbuild": {"file=/tmp/onbuild"},
+	"Dockerfile_test_scratch": {
+		"image=scratch",
+		"hello=hello-value",
+		"file=context/foo",
+		"file3=context/b*",
+	},
+	"Dockerfile_test_multistage": {"file=/foo2"},
+}
+
+// Arguments to build Dockerfiles with when building with docker
+var additionalDockerFlagsMap = map[string][]string{
+	"Dockerfile_test_target": {"--target=second"},
+}
+
+// Arguments to build Dockerfiles with when building with kaniko
+var additionalKanikoFlagsMap = map[string][]string{
+	"Dockerfile_test_add":     {"--single-snapshot"},
+	"Dockerfile_test_scratch": {"--single-snapshot"},
+	"Dockerfile_test_target":  {"--target=second"},
+}
+
+var bucketContextTests = []string{"Dockerfile_test_copy_bucket"}
+var reproducibleTests = []string{"Dockerfile_test_env"}
+
+// GetDockerImage constructs the name of the docker image that would be built with
+// dockerfile if it was tagged with imageRepo.
+func GetDockerImage(imageRepo, dockerfile string) string {
+	return strings.ToLower(imageRepo + dockerPrefix + dockerfile)
+}
+
+// GetKanikoImage constructs the name of the kaniko image that would be built with
+// dockerfile if it was tagged with imageRepo.
+func GetKanikoImage(imageRepo, dockerfile string) string {
+	return strings.ToLower(imageRepo + kanikoPrefix + dockerfile)
+}
+
+// FindDockerFiles will look for test docker files in the directory dockerfilesPath.
+// These files must start with `Dockerfile_test`. If the file is one we are intentionally
+// skipping, it will not be included in the returned list.
+func FindDockerFiles(dockerfilesPath string) ([]string, error) {
+	// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
+	testsToIgnore := map[string]bool{"Dockerfile_test_user_run": true}
+	allDockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*"))
+	if err != nil {
+		return []string{}, fmt.Errorf("Failed to find docker files at %s: %s", dockerfilesPath, err)
+	}
+
+	var dockerfiles []string
+	for _, dockerfile := range allDockerfiles {
+		// Remove the leading directory from the path
+		dockerfile = dockerfile[len("dockerfiles/"):]
+		if !testsToIgnore[dockerfile] {
+			dockerfiles = append(dockerfiles, dockerfile)
+		}
+	}
+	return dockerfiles, err
+}
+
+// DockerFileBuilder knows how to build docker files using both Kaniko and Docker and
+// keeps track of which files have been built.
+type DockerFileBuilder struct {
+	// Holds all available docker files and whether or not they've been built
+	FilesBuilt map[string]bool
+}
+
+// NewDockerFileBuilder will create a DockerFileBuilder initialized with dockerfiles, which
+// it will assume are all as yet unbuilt.
+func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder {
+	d := DockerFileBuilder{FilesBuilt: map[string]bool{}}
+	for _, f := range dockerfiles {
+		d.FilesBuilt[f] = false
+	}
+	return &d
+}
+
+// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
+// The resulting image will be tagged with imageRepo. If the dockerfile will be built with
+// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
+func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, dockerfile string) error {
+	_, ex, _, _ := runtime.Caller(0)
+	cwd := filepath.Dir(ex)
+
+	fmt.Printf("Building images for Dockerfile %s\n", dockerfile)
+
+	var buildArgs []string
+	buildArgFlag := "--build-arg"
+	for _, arg := range argsMap[dockerfile] {
+		buildArgs = append(buildArgs, buildArgFlag)
+		buildArgs = append(buildArgs, arg)
+	}
+	// build docker image
+	additionalFlags := append(buildArgs, additionalDockerFlagsMap[dockerfile]...)
+	dockerImage := strings.ToLower(imageRepo + dockerPrefix + dockerfile)
+	dockerCmd := exec.Command("docker",
+		append([]string{"build",
+			"-t", dockerImage,
+			"-f", path.Join(dockerfilesPath, dockerfile),
+			"."},
+			additionalFlags...)...,
+	)
+	_, err := RunCommandWithoutTest(dockerCmd)
+	if err != nil {
+		return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s", dockerImage, dockerCmd.Args, err)
+	}
+
+	contextFlag := "-c"
+	contextPath := buildContextPath
+	for _, d := range bucketContextTests {
+		if d == dockerfile {
+			contextFlag = "-b"
+			contextPath = gcsBucket
+			break
+		}
+	}
+
+	reproducibleFlag := ""
+	for _, d := range reproducibleTests {
+		if d == dockerfile {
+			reproducibleFlag = "--reproducible"
+			break
+		}
+	}
+
+	// build kaniko image
+	additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
+	kanikoImage := GetKanikoImage(imageRepo, dockerfile)
+	kanikoCmd := exec.Command("docker",
+		append([]string{"run",
+			"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
+			"-v", cwd + ":/workspace",
+			ExecutorImage,
+			"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
+			"-d", kanikoImage, reproducibleFlag,
+			contextFlag, contextPath},
+			additionalFlags...)...,
+	)
+
+	_, err = RunCommandWithoutTest(kanikoCmd)
+	if err != nil {
+		return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s", dockerImage, kanikoCmd.Args, err)
+	}
+
+	d.FilesBuilt[dockerfile] = true
+	return nil
+}

integration/integration_test.go

@@ -18,35 +18,50 @@ package integration
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
+	"log"
 	"math"
 	"os"
 	"os/exec"
-	"path"
-	"path/filepath"
-	"runtime"
 	"strings"
 	"testing"
 
-	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
 
 	"github.com/GoogleContainerTools/kaniko/testutil"
 )
 
+var config = initGCPConfig()
+var imageBuilder *DockerFileBuilder
+
+type gcpConfig struct {
+	gcsBucket        string
+	imageRepo        string
+	onbuildBaseImage string
+}
+
+func initGCPConfig() *gcpConfig {
+	var c gcpConfig
+	flag.StringVar(&c.gcsBucket, "bucket", "", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.")
+	flag.StringVar(&c.imageRepo, "repo", "", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo.")
+	flag.Parse()
+
+	if c.gcsBucket == "" || c.imageRepo == "" {
+		log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo)
+	}
+	if !strings.HasSuffix(c.imageRepo, "/") {
+		c.imageRepo = c.imageRepo + "/"
+	}
+	c.onbuildBaseImage = c.imageRepo + "onbuild-base:latest"
+	return &c
+}
+
 const (
-	executorImage      = "executor-image"
-	dockerImage        = "gcr.io/cloud-builders/docker"
 	ubuntuImage        = "ubuntu"
-	testRepo           = "gcr.io/kaniko-test/"
-	dockerPrefix       = "docker-"
-	kanikoPrefix       = "kaniko-"
 	daemonPrefix       = "daemon://"
-	kanikoTestBucket   = "kaniko-test-bucket"
 	dockerfilesPath    = "dockerfiles"
-	onbuildBaseImage   = testRepo + "onbuild-base:latest"
-	buildContextPath   = "/workspace"
 	emptyContainerDiff = `[
      {
        "Image1": "%s",
@@ -70,130 +85,73 @@ const (
    ]`
 )
 
-// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
+func meetsRequirements() bool {
-var testsToIgnore = []string{"Dockerfile_test_user_run"}
+	requiredTools := []string{"container-diff", "gsutil"}
+	hasRequirements := true
+	for _, tool := range requiredTools {
+		_, err := exec.LookPath(tool)
+		if err != nil {
+			fmt.Printf("You must have %s installed and on your PATH\n", tool)
+			hasRequirements = false
+		}
+	}
+	return hasRequirements
+}
 
 func TestMain(m *testing.M) {
-	buildKaniko := exec.Command("docker", "build", "-t", executorImage, "-f", "../deploy/Dockerfile", "..")
+	if !meetsRequirements() {
-	err := buildKaniko.Run()
+		fmt.Println("Missing required tools")
+		os.Exit(1)
+	}
+	contextFile, err := CreateIntegrationTarball()
+	if err != nil {
+		fmt.Println("Failed to create tarball of integration files for build context", err)
+		os.Exit(1)
+	}
+
+	fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile)
+	if err != nil {
+		fmt.Println("Failed to upload build context", err)
+		os.Exit(1)
+	}
+
+	err = os.Remove(contextFile)
+	if err != nil {
+		fmt.Printf("Failed to remove tarball at %s: %s\n", contextFile, err)
+		os.Exit(1)
+	}
+
+	RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) })
+	defer DeleteFromBucket(fileInBucket)
+
+	fmt.Println("Building kaniko image")
+	buildKaniko := exec.Command("docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", "..")
+	err = buildKaniko.Run()
 	if err != nil {
 		fmt.Print(err)
 		fmt.Print("Building kaniko failed.")
 		os.Exit(1)
 	}
 
-	// Make sure container-diff is on user's PATH
+	dockerfiles, err := FindDockerFiles(dockerfilesPath)
-	_, err = exec.LookPath("container-diff")
 	if err != nil {
-		fmt.Print("Make sure you have container-diff installed and on your PATH")
+		fmt.Printf("Coudn't create map of dockerfiles: %s", err)
 		os.Exit(1)
 	}
-
+	imageBuilder = NewDockerFileBuilder(dockerfiles)
 	os.Exit(m.Run())
 }
-
 func TestRun(t *testing.T) {
-	dockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*"))
+	for dockerfile, built := range imageBuilder.FilesBuilt {
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
-
-	// Map for test Dockerfile to expected ARGs
-	argsMap := map[string][]string{
-		"Dockerfile_test_run":     {"file=/file"},
-		"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
-		"Dockerfile_test_add":     {"file=context/foo"},
-		"Dockerfile_test_onbuild": {"file=/tmp/onbuild"},
-		"Dockerfile_test_scratch": {
-			"image=scratch",
-			"hello=hello-value",
-			"file=context/foo",
-			"file3=context/b*",
-		},
-		"Dockerfile_test_multistage": {"file=/foo2"},
-	}
-
-	// Map for additional docker flags
-	additionalDockerFlagsMap := map[string][]string{
-		"Dockerfile_test_target": {"--target=second"},
-	}
-	// Map for additional kaniko flags
-	additionalKanikoFlagsMap := map[string][]string{
-		"Dockerfile_test_add":     {"--single-snapshot"},
-		"Dockerfile_test_scratch": {"--single-snapshot"},
-		"Dockerfile_test_target":  {"--target=second"},
-	}
-
-	// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
-	testsToIgnore := []string{"Dockerfile_test_user_run"}
-	bucketContextTests := []string{"Dockerfile_test_copy_bucket"}
-	reproducibleTests := []string{"Dockerfile_test_env"}
-
-	_, ex, _, _ := runtime.Caller(0)
-	cwd := filepath.Dir(ex)
-
-	for _, dockerfile := range dockerfiles {
 		t.Run("test_"+dockerfile, func(t *testing.T) {
-			dockerfile = dockerfile[len("dockerfile/")+1:]
+			if !built {
-			for _, d := range testsToIgnore {
+				err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
-				if dockerfile == d {
+				if err != nil {
-					t.SkipNow()
+					t.Fatalf("Failed to build kaniko and docker images for %s: %s", dockerfile, err)
 				}
 			}
-			t.Logf("%s\n", dockerfile)
+			dockerImage := GetDockerImage(config.imageRepo, dockerfile)
-
+			kanikoImage := GetKanikoImage(config.imageRepo, dockerfile)
-			var buildArgs []string
-			buildArgFlag := "--build-arg"
-			for _, arg := range argsMap[dockerfile] {
-				buildArgs = append(buildArgs, buildArgFlag)
-				buildArgs = append(buildArgs, arg)
-			}
-			// build docker image
-			additionalFlags := append(buildArgs, additionalDockerFlagsMap[dockerfile]...)
-			dockerImage := strings.ToLower(testRepo + dockerPrefix + dockerfile)
-			dockerCmd := exec.Command("docker",
-				append([]string{"build",
-					"-t", dockerImage,
-					"-f", path.Join(dockerfilesPath, dockerfile),
-					"."},
-					additionalFlags...)...,
-			)
-			RunCommand(dockerCmd, t)
-
-			contextFlag := "-c"
-			contextPath := buildContextPath
-			for _, d := range bucketContextTests {
-				if d == dockerfile {
-					contextFlag = "-b"
-					contextPath = constants.GCSBuildContextPrefix + kanikoTestBucket
-					break
-				}
-			}
-
-			reproducibleFlag := ""
-			for _, d := range reproducibleTests {
-				if d == dockerfile {
-					reproducibleFlag = "--reproducible"
-					break
-				}
-			}
-
-			// build kaniko image
-			additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
-			kanikoImage := strings.ToLower(testRepo + kanikoPrefix + dockerfile)
-			kanikoCmd := exec.Command("docker",
-				append([]string{"run",
-					"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
-					"-v", cwd + ":/workspace",
-					executorImage,
-					"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
-					"-d", kanikoImage, reproducibleFlag,
-					contextFlag, contextPath},
-					additionalFlags...)...,
-			)
-
-			RunCommand(kanikoCmd, t)
 
 			// container-diff
 			daemonDockerImage := daemonPrefix + dockerImage
@@ -210,7 +168,7 @@ func TestRun(t *testing.T) {
 			var diffInt interface{}
 			var expectedInt interface{}
 
-			err = json.Unmarshal(diff, &diffInt)
+			err := json.Unmarshal(diff, &diffInt)
 			if err != nil {
 				t.Error(err)
 				t.Fail()
@@ -228,11 +186,6 @@ func TestRun(t *testing.T) {
 }
 
 func TestLayers(t *testing.T) {
-	dockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*"))
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
 	offset := map[string]int{
 		"Dockerfile_test_add":     9,
 		"Dockerfile_test_scratch": 3,
@@ -240,17 +193,17 @@ func TestLayers(t *testing.T) {
 		// which is why this offset exists
 		"Dockerfile_test_volume": 1,
 	}
-	for _, dockerfile := range dockerfiles {
+	for dockerfile, built := range imageBuilder.FilesBuilt {
 		t.Run("test_layer_"+dockerfile, func(t *testing.T) {
-			dockerfile = dockerfile[len("dockerfile/")+1:]
+			if !built {
-			for _, ignore := range testsToIgnore {
+				err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
-				if dockerfile == ignore {
+				if err != nil {
-					t.SkipNow()
+					t.Fatalf("Failed to build kaniko and docker images for %s: %s", dockerfile, err)
 				}
 			}
 			// Pull the kaniko image
-			dockerImage := strings.ToLower(testRepo + dockerPrefix + dockerfile)
+			dockerImage := GetDockerImage(config.imageRepo, dockerfile)
-			kanikoImage := strings.ToLower(testRepo + kanikoPrefix + dockerfile)
+			kanikoImage := GetKanikoImage(config.imageRepo, dockerfile)
 			pullCmd := exec.Command("docker", "pull", kanikoImage)
 			RunCommand(pullCmd, t)
 			if err := checkLayers(dockerImage, kanikoImage, offset[dockerfile]); err != nil {
@@ -264,11 +217,11 @@ func TestLayers(t *testing.T) {
 func checkLayers(image1, image2 string, offset int) error {
 	lenImage1, err := numLayers(image1)
 	if err != nil {
-		return err
+		return fmt.Errorf("Couldn't get number of layers for image1 (%s): %s", image1, err)
 	}
 	lenImage2, err := numLayers(image2)
 	if err != nil {
-		return err
+		return fmt.Errorf("Couldn't get number of layers for image2 (%s): %s", image2, err)
 	}
 	actualOffset := int(math.Abs(float64(lenImage1 - lenImage2)))
 	if actualOffset != offset {
@@ -280,24 +233,15 @@ func checkLayers(image1, image2 string, offset int) error {
 func numLayers(image string) (int, error) {
 	ref, err := name.ParseReference(image, name.WeakValidation)
 	if err != nil {
-		return 0, err
+		return 0, fmt.Errorf("Couldn't parse referance to image %s: %s", image, err)
 	}
 	img, err := daemon.Image(ref)
 	if err != nil {
-		return 0, err
+		return 0, fmt.Errorf("Couldn't get reference to image %s from daemon: %s", image, err)
 	}
 	layers, err := img.Layers()
-	return len(layers), err
-}
-
-func RunCommand(cmd *exec.Cmd, t *testing.T) []byte {
-	output, err := cmd.CombinedOutput()
 	if err != nil {
-		t.Log(cmd.Args)
+		return 0, fmt.Errorf("Error getting layers for image %s: %s", image, err)
-		t.Log(string(output))
-		t.Error(err)
-		t.FailNow()
 	}
-
+	return len(layers), nil
-	return output
 }