diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 18b69b3e3..330d9ba56 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -166,6 +166,21 @@ Additionally, the integration tests can output benchmarking information to a `be BENCHMARK=true go test -v --bucket $GCS_BUCKET --repo $IMAGE_REPO ``` +#### Benchmarking your GCB runs +If you are GCB builds are slow, you can check which phases in kaniko are bottlenecks or taking more time. +To do this, add "BENCHMARK_ENV" to your cloudbuild.yaml like this. +```shell script +steps: +- name: 'gcr.io/kaniko-project/executor:latest' + args: + - --build-arg=NUM=${_COUNT} + - --no-push + - --snapshotMode=redo + env: + - 'BENCHMARK_FILE=gs://$PROJECT_ID/gcb/benchmark_file' +``` +You can download the file `gs://$PROJECT_ID/gcb/benchmark_file` using `gsutil cp` command. + ## Creating a PR When you have changes you would like to propose to kaniko, you will need to: diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 77afc92dd..8b60da706 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -115,16 +115,27 @@ var RootCmd = &cobra.Command{ benchmarkFile := os.Getenv("BENCHMARK_FILE") // false is a keyword for integration tests to turn off benchmarking if benchmarkFile != "" && benchmarkFile != "false" { - f, err := os.Create(benchmarkFile) - if err != nil { - logrus.Warnf("Unable to create benchmarking file %s: %s", benchmarkFile, err) - } - defer f.Close() s, err := timing.JSON() if err != nil { logrus.Warnf("Unable to write benchmark file: %s", err) + return + } + if strings.HasPrefix(benchmarkFile, "gs://") { + logrus.Info("uploading to gcs") + if err := buildcontext.UploadToBucket(strings.NewReader(s), benchmarkFile); err != nil { + logrus.Infof("Unable to upload %s due to %v", benchmarkFile, err) + } + logrus.Infof("benchmark file written at %s", benchmarkFile) + } else { + f, err := os.Create(benchmarkFile) + if err != nil { + logrus.Warnf("Unable to create benchmarking file %s: %s", benchmarkFile, err) + return + } + defer f.Close() + f.WriteString(s) + logrus.Infof("benchmark file written at %s", benchmarkFile) } - f.WriteString(s) } }, } diff --git a/integration/benchmark_fs/Dockerfile b/integration/benchmark_fs/Dockerfile new file mode 100644 index 000000000..9a70889f1 --- /dev/null +++ b/integration/benchmark_fs/Dockerfile @@ -0,0 +1,21 @@ +# Copyright 2020 Google, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM bash:4.4 + +ARG NUM +COPY context.txt . +COPY make.sh . +SHELL ["/usr/local/bin/bash", "-c"] +RUN ./make.sh $NUM +RUN ls -al /workdir | wc diff --git a/integration/benchmark_fs/Dockerfile_fs_benchmark b/integration/benchmark_fs/Dockerfile_fs_benchmark deleted file mode 100644 index 4065467c4..000000000 --- a/integration/benchmark_fs/Dockerfile_fs_benchmark +++ /dev/null @@ -1,8 +0,0 @@ -FROM bash:4.4 - -ARG NUM -COPY context.txt . -COPY make.sh . -SHELL ["/usr/local/bin/bash", "-c"] -RUN ./make.sh $NUM -RUN ls -al /workdir | wc diff --git a/integration/benchmark_fs/cloudbuild.yaml b/integration/benchmark_fs/cloudbuild.yaml new file mode 100644 index 000000000..829fb946b --- /dev/null +++ b/integration/benchmark_fs/cloudbuild.yaml @@ -0,0 +1,12 @@ +steps: +- name: 'gcr.io/kaniko-project/executor:latest' + args: + - --build-arg=NUM=${_COUNT} + - --no-push + - --snapshotMode=redo + env: + - 'BENCHMARK_FILE=gs://$PROJECT_ID/gcb/benchmark_file_${_COUNT}' + timeout: 2400s +timeout: 2400s +substitutions: + _COUNT: "10000" # default value \ No newline at end of file diff --git a/integration/benchmark_fs/make.sh b/integration/benchmark_fs/make.sh index 828f420ba..e336af448 100755 --- a/integration/benchmark_fs/make.sh +++ b/integration/benchmark_fs/make.sh @@ -17,7 +17,8 @@ mkdir /workdir i=1 -while [ $i -le $1 ] +targetCnt=$(( $1 + 0 )) +while [ $i -le $targetCnt ] do cat context.txt > /workdir/somefile$i i=$(( $i + 1 )) diff --git a/integration/benchmark_test.go b/integration/benchmark_test.go index 036e96beb..f7364114d 100644 --- a/integration/benchmark_test.go +++ b/integration/benchmark_test.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" "path/filepath" "strconv" "sync" @@ -32,6 +33,7 @@ type result struct { totalBuildTime float64 resolvingFiles float64 walkingFiles float64 + hashingFiles float64 } func TestSnapshotBenchmark(t *testing.T) { @@ -44,7 +46,7 @@ func TestSnapshotBenchmark(t *testing.T) { } contextDir := filepath.Join(cwd, "benchmark_fs") - nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000, 800000} + nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000} var timeMap sync.Map var wg sync.WaitGroup @@ -53,7 +55,7 @@ func TestSnapshotBenchmark(t *testing.T) { wg.Add(1) var err error go func(num int, err *error) { - dockerfile := "Dockerfile_fs_benchmark" + dockerfile := "Dockerfile" kanikoImage := fmt.Sprintf("%s_%d", GetKanikoImage(config.imageRepo, dockerfile), num) buildArgs := []string{"--build-arg", fmt.Sprintf("NUM=%d", num)} var benchmarkDir string @@ -106,5 +108,68 @@ func newResult(t *testing.T, f string) result { if c, ok := current["Total Build Time"]; ok { r.totalBuildTime = c.Seconds() } + if c, ok := current["Hashing files"]; ok { + r.hashingFiles = c.Seconds() + } + fmt.Println(r) return r } + +func TestSnapshotBenchmarkGcloud(t *testing.T) { + if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err != nil || !b { + t.SkipNow() + } + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + contextDir := filepath.Join(cwd, "benchmark_fs") + + nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000} + + var wg sync.WaitGroup + fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files") + for _, num := range nums { + t.Run(fmt.Sprintf("test_benchmark_%d", num), func(t *testing.T) { + wg.Add(1) + go func(num int) { + dir, err := runInGcloud(contextDir, num) + if err != nil { + t.Errorf("error when running in gcloud %v", err) + return + } + r := newResult(t, filepath.Join(dir, "results")) + fmt.Println(fmt.Sprintf("%d,%f,%f,%f, %f", num, r.totalBuildTime, r.walkingFiles, r.resolvingFiles, r.hashingFiles)) + wg.Done() + defer os.Remove(dir) + defer os.Chdir(cwd) + }(num) + }) + } + wg.Wait() +} + +func runInGcloud(dir string, num int) (string, error) { + os.Chdir(dir) + cmd := exec.Command("gcloud", "builds", + "submit", "--config=cloudbuild.yaml", + fmt.Sprintf("--substitutions=_COUNT=%d", num)) + _, err := RunCommandWithoutTest(cmd) + if err != nil { + return "", err + } + + // grab gcs and to temp dir and return + tmpDir, err := ioutil.TempDir("", fmt.Sprintf("%d", num)) + if err != nil { + return "", err + } + src := fmt.Sprintf("%s/gcb/benchmark_file_%d", config.gcsBucket, num) + dest := filepath.Join(tmpDir, "results") + copyCommand := exec.Command("gsutil", "cp", src, dest) + _, err = RunCommandWithoutTest(copyCommand) + if err != nil { + return "", fmt.Errorf("failed to download file to GCS bucket %s: %s", src, err) + } + return tmpDir, nil +} diff --git a/pkg/buildcontext/gcs.go b/pkg/buildcontext/gcs.go index 920028b62..f4e98920d 100644 --- a/pkg/buildcontext/gcs.go +++ b/pkg/buildcontext/gcs.go @@ -17,8 +17,10 @@ limitations under the License. package buildcontext import ( + "io" "os" "path/filepath" + "strings" "cloud.google.com/go/storage" "github.com/GoogleContainerTools/kaniko/pkg/constants" @@ -37,6 +39,25 @@ func (g *GCS) UnpackTarFromBuildContext() (string, error) { return constants.BuildContextDir, unpackTarFromGCSBucket(bucket, item, constants.BuildContextDir) } +func UploadToBucket(r io.Reader, dest string) error { + ctx := context.Background() + context := strings.SplitAfter(dest, "://")[1] + bucketName, item := util.GetBucketAndItem(context) + client, err := storage.NewClient(ctx) + if err != nil { + return err + } + bucket := client.Bucket(bucketName) + w := bucket.Object(item).NewWriter(ctx) + if _, err := io.Copy(w, r); err != nil { + return err + } + if err := w.Close(); err != nil { + return err + } + return nil +} + // unpackTarFromGCSBucket unpacks the context.tar.gz file in the given bucket to the given directory func unpackTarFromGCSBucket(bucketName, item, directory string) error { // Get the tar from the bucket diff --git a/pkg/filesystem/resolve.go b/pkg/filesystem/resolve.go index 64121146a..30a79dc05 100644 --- a/pkg/filesystem/resolve.go +++ b/pkg/filesystem/resolve.go @@ -35,7 +35,6 @@ import ( // output set. // * Add all ancestors of each path to the output set. func ResolvePaths(paths []string, wl []util.IgnoreListEntry) (pathsToAdd []string, err error) { - logrus.Infof("Resolving %d paths", len(paths)) logrus.Tracef("Resolving paths %s", paths) fileSet := make(map[string]bool)