more changes

integration/benchmark_test.go

@@ -17,35 +17,85 @@ limitations under the License.
 package integration
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
+	"time"
 )
 
+type result struct {
+	totalBuildTime float64
+	resolvingFiles float64
+	walkingFiles   float64
+}
+
 func TestSnapshotBenchmark(t *testing.T) {
 	cwd, err := os.Getwd()
 	if err != nil {
 		t.Fatal(err)
 	}
-	contextDir := filepath.Join(cwd, "benchmark")
+	contextDir := filepath.Join(cwd, "benchmark_fs")
 
 	nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000, 800000}
 
+	var timeMap sync.Map
+	var wg sync.WaitGroup
 	for _, num := range nums {
-
-		t.Run("test_benchmark"+string(num), func(t *testing.T) {
-			t.Parallel()
-			dockerfile := "Dockerfile_FS_benchmark"
-			kanikoImage := GetKanikoImage(config.imageRepo, dockerfile)
-			buildArgs := []string{"--build-arg", fmt.Sprintf("NUM=%d", num)}
-			if _, err := buildKanikoImage("", dockerfile,
-				buildArgs, []string{}, kanikoImage, contextDir, config.gcsBucket, config.serviceAccount); err != nil {
-				t.Errorf("could not run benchmark results for num %d", num)
-			}
+		t.Run(fmt.Sprintf("test_benchmark_%d", num), func(t *testing.T) {
+			wg.Add(1)
+			go func(num int) {
+				dockerfile := "Dockerfile_fs_benchmark"
+				kanikoImage := fmt.Sprintf("%s_%d", GetKanikoImage(config.imageRepo, dockerfile), num)
+				buildArgs := []string{"--build-arg", fmt.Sprintf("NUM=%d", num)}
+				benchmarkDir, err := buildKanikoImage("", dockerfile,
+					buildArgs, []string{}, kanikoImage, contextDir, config.gcsBucket,
+					config.serviceAccount, false)
+				if err != nil {
+					t.Errorf("could not run benchmark results for num %d", num)
+				}
+				r := newResult(t, filepath.Join(benchmarkDir, dockerfile))
+				timeMap.Store(num, r)
+				wg.Done()
+				defer os.Remove(benchmarkDir)
+			}(num)
 		})
 	}
-	if err := logBenchmarks("benchmark"); err != nil {
-		t.Logf("Failed to create benchmark file: %v", err)
-	}
+	wg.Wait()
+
+	fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
+	timeMap.Range(func(key interface{}, value interface{}) bool {
+		d, _ := key.(int)
+		v, _ := value.(result)
+		fmt.Println(fmt.Sprintf("%d,%f,%f,%f", d, v.totalBuildTime, v.walkingFiles, v.resolvingFiles))
+		return true
+	})
+
+}
+
+func newResult(t *testing.T, f string) result {
+	var current map[string]time.Duration
+	jsonFile, err := os.Open(f)
+	defer jsonFile.Close()
+	if err != nil {
+		t.Errorf("could not read benchmark file %s", f)
+	}
+	byteValue, _ := ioutil.ReadAll(jsonFile)
+	if err := json.Unmarshal(byteValue, &current); err != nil {
+		t.Errorf("could not unmarshal benchmark file")
+	}
+	r := result{}
+	if c, ok := current["Resolving Paths"]; ok {
+		r.resolvingFiles = c.Seconds()
+	}
+	if c, ok := current["Walking filesystem"]; ok {
+		r.walkingFiles = c.Seconds()
+	}
+	if c, ok := current["Total Build Time"]; ok {
+		r.totalBuildTime = c.Seconds()
+	}
+	return r
 }

integration/images.go

@@ -269,17 +269,12 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
 	}
 
 	kanikoImage := GetKanikoImage(imageRepo, dockerfile)
-	out, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
-		contextDir, gcsBucket, serviceAccount)
-	if err != nil {
+	timer = timing.Start(dockerfile + "_kaniko")
+	if _, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
+		contextDir, gcsBucket, serviceAccount, true); err != nil {
 		return err
 	}
-
-	if outputCheck := outputChecks[dockerfile]; outputCheck != nil {
-		if err := outputCheck(dockerfile, out); err != nil {
-			return fmt.Errorf("Output check failed for image %s with kaniko command : %s %s", kanikoImage, err, string(out))
-		}
-	}
+	timing.DefaultRun.Stop(timer)
 
 	d.filesBuilt[dockerfile] = struct{}{}
 
@@ -336,9 +331,7 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac
 			"--cache-dir", cacheDir)
 		kanikoCmd := exec.Command("docker", dockerRunFlags...)
 
-		timer := timing.Start(dockerfile + "_kaniko_cached_" + strconv.Itoa(version))
 		_, err := RunCommandWithoutTest(kanikoCmd)
-		timing.DefaultRun.Stop(timer)
 		if err != nil {
 			return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %s", kanikoImage, kanikoCmd.Args, err)
 		}
@@ -391,19 +384,30 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi
 	return nil
 }
 
-func buildKanikoImage(dockerfilesPath string, dockerfile string, buildArgs []string, kanikoArgs []string, kanikoImage string,
-	contextDir string, gcsBucket string, serviceAccount string) ([]byte, error) {
+func buildKanikoImage(
+	dockerfilesPath string,
+	dockerfile string,
+	buildArgs []string,
+	kanikoArgs []string,
+	kanikoImage string,
+	contextDir string,
+	gcsBucket string,
+	serviceAccount string,
+	shdUpload bool,
+) (string, error) {
 	benchmarkEnv := "BENCHMARK_FILE=false"
 	benchmarkDir, err := ioutil.TempDir("", "")
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 	if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b {
 		benchmarkEnv = "BENCHMARK_FILE=/kaniko/benchmarks/" + dockerfile
-		benchmarkFile := path.Join(benchmarkDir, dockerfile)
-		fileName := fmt.Sprintf("run_%s_%s", time.Now().Format("2006-01-02-15:04"), dockerfile)
-		dst := path.Join("benchmarks", fileName)
-		defer UploadFileToBucket(gcsBucket, benchmarkFile, dst)
+		if shdUpload {
+			benchmarkFile := path.Join(benchmarkDir, dockerfile)
+			fileName := fmt.Sprintf("run_%s_%s", time.Now().Format("2006-01-02-15:04"), dockerfile)
+			dst := path.Join("benchmarks", fileName)
+			defer UploadFileToBucket(gcsBucket, benchmarkFile, dst)
+		}
 	}
 
 	// build kaniko image
@@ -426,7 +430,7 @@ func buildKanikoImage(dockerfilesPath string, dockerfile string, buildArgs []str
 
 	kanikoDockerfilePath := path.Join(buildContextPath, dockerfilesPath, dockerfile)
 	if dockerfilesPath == "" {
-		kanikoDockerfilePath = path.Join(buildContextPath, "Dockerfile")
+		kanikoDockerfilePath = path.Join(buildContextPath, dockerfile)
 	}
 
 	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
@@ -439,9 +443,13 @@ func buildKanikoImage(dockerfilesPath string, dockerfile string, buildArgs []str
 	timer := timing.Start(dockerfile + "_kaniko")
 	out, err := RunCommandWithoutTest(kanikoCmd)
 	timing.DefaultRun.Stop(timer)
-
 	if err != nil {
-		return nil, fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out))
+		return "", fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out))
 	}
-	return out, nil
+	if outputCheck := outputChecks[dockerfile]; outputCheck != nil {
+		if err := outputCheck(dockerfile, out); err != nil {
+			return "", fmt.Errorf("Output check failed for image %s with kaniko command : %s %s", kanikoImage, err, string(out))
+		}
+	}
+	return benchmarkDir, nil
 }