Merge pull request #1288 from tejal29/benchmark-snapshot

benchmark FileSystem snapshot project added

.gitignore

@@ -1,2 +1,3 @@
 out/
 .idea
+*.iml

integration/benchmark_fs/Dockerfile_fs_benchmark

@@ -0,0 +1,8 @@
+FROM bash:4.4
+
+ARG NUM
+COPY context.txt .
+COPY make.sh .
+SHELL ["/usr/local/bin/bash", "-c"]
+RUN ./make.sh $NUM
+RUN ls -al /workdir | wc

integration/benchmark_fs/context.txt

@@ -0,0 +1 @@
+hello world

integration/benchmark_fs/make.sh

@@ -0,0 +1,24 @@
+#!/usr/local/bin/bash
+
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir /workdir
+
+i=1
+while [ $i -le $1 ]
+do
+  cat context.txt >  /workdir/somefile$i
+  i=$(( $i + 1 ))
+done

integration/benchmark_test.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+)
+
+type result struct {
+	totalBuildTime float64
+	resolvingFiles float64
+	walkingFiles   float64
+}
+
+func TestSnapshotBenchmark(t *testing.T) {
+	if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err != nil || !b {
+		t.SkipNow()
+	}
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	contextDir := filepath.Join(cwd, "benchmark_fs")
+
+	nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000, 800000}
+
+	var timeMap sync.Map
+	var wg sync.WaitGroup
+	for _, num := range nums {
+		t.Run(fmt.Sprintf("test_benchmark_%d", num), func(t *testing.T) {
+			wg.Add(1)
+			var err error
+			go func(num int, err *error) {
+				dockerfile := "Dockerfile_fs_benchmark"
+				kanikoImage := fmt.Sprintf("%s_%d", GetKanikoImage(config.imageRepo, dockerfile), num)
+				buildArgs := []string{"--build-arg", fmt.Sprintf("NUM=%d", num)}
+				var benchmarkDir string
+				benchmarkDir, *err = buildKanikoImage("", dockerfile,
+					buildArgs, []string{}, kanikoImage, contextDir, config.gcsBucket,
+					config.serviceAccount, false)
+				if *err != nil {
+					return
+				}
+				r := newResult(t, filepath.Join(benchmarkDir, dockerfile))
+				timeMap.Store(num, r)
+				wg.Done()
+				defer os.Remove(benchmarkDir)
+			}(num, &err)
+			if err != nil {
+				t.Errorf("could not run benchmark results for num %d due to %s", num, err)
+			}
+		})
+	}
+	wg.Wait()
+
+	fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
+	timeMap.Range(func(key interface{}, value interface{}) bool {
+		d, _ := key.(int)
+		v, _ := value.(result)
+		fmt.Println(fmt.Sprintf("%d,%f,%f,%f", d, v.totalBuildTime, v.walkingFiles, v.resolvingFiles))
+		return true
+	})
+
+}
+
+func newResult(t *testing.T, f string) result {
+	var current map[string]time.Duration
+	jsonFile, err := os.Open(f)
+	defer jsonFile.Close()
+	if err != nil {
+		t.Errorf("could not read benchmark file %s", f)
+	}
+	byteValue, _ := ioutil.ReadAll(jsonFile)
+	if err := json.Unmarshal(byteValue, &current); err != nil {
+		t.Errorf("could not unmarshal benchmark file")
+	}
+	r := result{}
+	if c, ok := current["Resolving Paths"]; ok {
+		r.resolvingFiles = c.Seconds()
+	}
+	if c, ok := current["Walking filesystem"]; ok {
+		r.walkingFiles = c.Seconds()
+	}
+	if c, ok := current["Total Build Time"]; ok {
+		r.totalBuildTime = c.Seconds()
+	}
+	return r
+}

integration/images.go

@@ -259,73 +259,23 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
 		}
 	}
 
-	reproducibleFlag := ""
+	additionalKanikoFlags := additionalKanikoFlagsMap[dockerfile]
+	additionalKanikoFlags = append(additionalKanikoFlags, contextFlag, contextPath)
 	for _, d := range reproducibleTests {
 		if d == dockerfile {
-			reproducibleFlag = "--reproducible"
+			additionalKanikoFlags = append(additionalKanikoFlags, "--reproducible")
 			break
 		}
 	}
 
-	benchmarkEnv := "BENCHMARK_FILE=false"
+	kanikoImage := GetKanikoImage(imageRepo, dockerfile)
-	benchmarkDir, err := ioutil.TempDir("", "")
+	timer = timing.Start(dockerfile + "_kaniko")
-	if err != nil {
+	if _, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
+		contextDir, gcsBucket, serviceAccount, true); err != nil {
 		return err
 	}
-	if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b {
-		benchmarkEnv = "BENCHMARK_FILE=/kaniko/benchmarks/" + dockerfile
-		benchmarkFile := path.Join(benchmarkDir, dockerfile)
-		fileName := fmt.Sprintf("run_%s_%s", time.Now().Format("2006-01-02-15:04"), dockerfile)
-		dst := path.Join("benchmarks", fileName)
-		defer UploadFileToBucket(gcsBucket, benchmarkFile, dst)
-	}
-
-	// build kaniko image
-	additionalFlags := append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
-	kanikoImage := GetKanikoImage(imageRepo, dockerfile)
-	fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags)
-
-	dockerRunFlags := []string{"run", "--net=host",
-		"-e", benchmarkEnv,
-		"-v", contextDir + ":/workspace",
-		"-v", benchmarkDir + ":/kaniko/benchmarks",
-	}
-
-	if env, ok := envsMap[dockerfile]; ok {
-		for _, envVariable := range env {
-			dockerRunFlags = append(dockerRunFlags, "-e", envVariable)
-		}
-	}
-
-	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
-
-	kanikoDockerfilePath := path.Join(buildContextPath, dockerfilesPath, dockerfile)
-	if dockerfilesPath == "" {
-		kanikoDockerfilePath = path.Join(buildContextPath, "Dockerfile")
-	}
-
-	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
-		"-f", kanikoDockerfilePath,
-		"-d", kanikoImage, reproducibleFlag,
-		contextFlag, contextPath)
-	dockerRunFlags = append(dockerRunFlags, additionalFlags...)
-
-	kanikoCmd := exec.Command("docker", dockerRunFlags...)
-
-	timer = timing.Start(dockerfile + "_kaniko")
-	out, err := RunCommandWithoutTest(kanikoCmd)
 	timing.DefaultRun.Stop(timer)
 
-	if err != nil {
-		return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out))
-	}
-
-	if outputCheck := outputChecks[dockerfile]; outputCheck != nil {
-		if err := outputCheck(dockerfile, out); err != nil {
-			return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out))
-		}
-	}
-
 	d.filesBuilt[dockerfile] = struct{}{}
 
 	return nil
@@ -381,9 +331,7 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac
 			"--cache-dir", cacheDir)
 		kanikoCmd := exec.Command("docker", dockerRunFlags...)
 
-		timer := timing.Start(dockerfile + "_kaniko_cached_" + strconv.Itoa(version))
 		_, err := RunCommandWithoutTest(kanikoCmd)
-		timing.DefaultRun.Stop(timer)
 		if err != nil {
 			return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %s", kanikoImage, kanikoCmd.Args, err)
 		}
@@ -435,3 +383,71 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi
 
 	return nil
 }
+
+func buildKanikoImage(
+	dockerfilesPath string,
+	dockerfile string,
+	buildArgs []string,
+	kanikoArgs []string,
+	kanikoImage string,
+	contextDir string,
+	gcsBucket string,
+	serviceAccount string,
+	shdUpload bool,
+) (string, error) {
+	benchmarkEnv := "BENCHMARK_FILE=false"
+	benchmarkDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return "", err
+	}
+	if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b {
+		benchmarkEnv = "BENCHMARK_FILE=/kaniko/benchmarks/" + dockerfile
+		if shdUpload {
+			benchmarkFile := path.Join(benchmarkDir, dockerfile)
+			fileName := fmt.Sprintf("run_%s_%s", time.Now().Format("2006-01-02-15:04"), dockerfile)
+			dst := path.Join("benchmarks", fileName)
+			defer UploadFileToBucket(gcsBucket, benchmarkFile, dst)
+		}
+	}
+
+	// build kaniko image
+	additionalFlags := append(buildArgs, kanikoArgs...)
+	fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags)
+
+	dockerRunFlags := []string{"run", "--net=host",
+		"-e", benchmarkEnv,
+		"-v", contextDir + ":/workspace",
+		"-v", benchmarkDir + ":/kaniko/benchmarks",
+	}
+
+	if env, ok := envsMap[dockerfile]; ok {
+		for _, envVariable := range env {
+			dockerRunFlags = append(dockerRunFlags, "-e", envVariable)
+		}
+	}
+
+	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
+
+	kanikoDockerfilePath := path.Join(buildContextPath, dockerfilesPath, dockerfile)
+	if dockerfilesPath == "" {
+		kanikoDockerfilePath = path.Join(buildContextPath, "Dockerfile")
+	}
+
+	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
+		"-f", kanikoDockerfilePath,
+		"-d", kanikoImage)
+	dockerRunFlags = append(dockerRunFlags, additionalFlags...)
+
+	kanikoCmd := exec.Command("docker", dockerRunFlags...)
+
+	out, err := RunCommandWithoutTest(kanikoCmd)
+	if err != nil {
+		return "", fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out))
+	}
+	if outputCheck := outputChecks[dockerfile]; outputCheck != nil {
+		if err := outputCheck(dockerfile, out); err != nil {
+			return "", fmt.Errorf("Output check failed for image %s with kaniko command : %s %s", kanikoImage, err, string(out))
+		}
+	}
+	return benchmarkDir, nil
+}

scripts/integration-test.sh

@@ -21,19 +21,6 @@ IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}"
 docker version
 
 # Sets up a kokoro (Google internal integration testing tool) environment
-if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then
-    echo "Installing dependencies..."
-    source "$KOKORO_GFILE_DIR/common.sh"
-    mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/
-    cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/
-    pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko
-    echo "Installing container-diff..."
-    mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff
-    chmod +x $KOKORO_GFILE_DIR/container-diff
-    export PATH=$PATH:$KOKORO_GFILE_DIR
-    cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json
-fi
-
 echo "Running integration tests..."
 make out/executor
 make out/warmer