diff --git a/Makefile b/Makefile index 0966fe693..3296e6bd1 100644 --- a/Makefile +++ b/Makefile @@ -88,9 +88,9 @@ integration-test-misc: .PHONY: images images: - docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:latest -f deploy/Dockerfile . - docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug . - docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/warmer:latest -f deploy/Dockerfile_warmer . + docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:perf-latest -f deploy/Dockerfile . + #docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug . + #docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/warmer:latest -f deploy/Dockerfile_warmer . .PHONY: push push: diff --git a/integration/benchmark_fs/cloudbuild.yaml b/integration/benchmark_fs/cloudbuild.yaml index 829fb946b..e8bc93f3e 100644 --- a/integration/benchmark_fs/cloudbuild.yaml +++ b/integration/benchmark_fs/cloudbuild.yaml @@ -1,9 +1,9 @@ steps: -- name: 'gcr.io/kaniko-project/executor:latest' +- name: 'gcr.io/kaniko-project/executor:perf-latest' args: - --build-arg=NUM=${_COUNT} - --no-push - - --snapshotMode=redo + - --use-new-run=true env: - 'BENCHMARK_FILE=gs://$PROJECT_ID/gcb/benchmark_file_${_COUNT}' timeout: 2400s diff --git a/integration/benchmark_test.go b/integration/benchmark_test.go index f7364114d..af3f79eab 100644 --- a/integration/benchmark_test.go +++ b/integration/benchmark_test.go @@ -125,7 +125,7 @@ func TestSnapshotBenchmarkGcloud(t *testing.T) { } contextDir := filepath.Join(cwd, "benchmark_fs") - nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000} + nums := []int{10000} //, 50000, 100000, 200000, 300000, 500000, 700000} var wg sync.WaitGroup fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files") diff --git a/integration/images.go b/integration/images.go index c221d6504..be2ae4546 100644 --- a/integration/images.go +++ b/integration/images.go @@ -76,6 +76,7 @@ var additionalDockerFlagsMap = map[string][]string{ var additionalKanikoFlagsMap = map[string][]string{ "Dockerfile_test_add": {"--single-snapshot"}, "Dockerfile_test_run_new": {"--use-new-run=true"}, + "Dockerfile_test_run": {"-v=debug"}, "Dockerfile_test_scratch": {"--single-snapshot"}, "Dockerfile_test_maintainer": {"--single-snapshot"}, "Dockerfile_test_target": {"--target=second"}, @@ -272,11 +273,13 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, kanikoImage := GetKanikoImage(imageRepo, dockerfile) timer = timing.Start(dockerfile + "_kaniko") - if _, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage, - contextDir, gcsBucket, serviceAccount, true); err != nil { + out, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage, + contextDir, gcsBucket, serviceAccount, true) + if err != nil { return err } timing.DefaultRun.Stop(timer) + fmt.Println(out) d.filesBuilt[dockerfile] = struct{}{} @@ -451,5 +454,6 @@ func buildKanikoImage( return "", fmt.Errorf("Output check failed for image %s with kaniko command : %s %s", kanikoImage, err, string(out)) } } + fmt.Println("kaniko command\n", string(out)) return benchmarkDir, nil } diff --git a/integration/integration_test.go b/integration/integration_test.go index 117ba76d7..39e4590f1 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -142,26 +142,26 @@ func buildRequiredImages() error { name: "Building kaniko image", command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."}, }, - { - name: "Building cache warmer image", - command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."}, - }, - { - name: "Building onbuild base image", - command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."}, - }, - { - name: "Pushing onbuild base image", - command: []string{"docker", "push", config.onbuildBaseImage}, - }, - { - name: "Building hardlink base image", - command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."}, - }, - { - name: "Pushing hardlink base image", - command: []string{"docker", "push", config.hardlinkBaseImage}, - }, + //{ + // name: "Building cache warmer image", + // command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."}, + //}, + //{ + // name: "Building onbuild base image", + // command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."}, + //}, + //{ + // name: "Pushing onbuild base image", + // command: []string{"docker", "push", config.onbuildBaseImage}, + //}, + //{ + // name: "Building hardlink base image", + // command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."}, + //}, + //{ + // name: "Pushing hardlink base image", + // command: []string{"docker", "push", config.hardlinkBaseImage}, + //}, } for _, setupCmd := range setupCommands { @@ -294,6 +294,7 @@ func TestGitBuildcontextSubPath(t *testing.T) { kanikoCmd := exec.Command("docker", dockerRunFlags...) out, err = RunCommandWithoutTest(kanikoCmd) + fmt.Println(out) if err != nil { t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out)) } diff --git a/pkg/commands/run_marker.go b/pkg/commands/run_marker.go index 780ba0814..c468e209b 100644 --- a/pkg/commands/run_marker.go +++ b/pkg/commands/run_marker.go @@ -17,16 +17,15 @@ limitations under the License. package commands import ( + "fmt" "io/ioutil" "os" - "os/exec" - "path/filepath" - "strings" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/pkg/util" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/sirupsen/logrus" ) type RunMarkerCommand struct { @@ -37,32 +36,32 @@ type RunMarkerCommand struct { func (r *RunMarkerCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { // run command `touch filemarker` + logrus.Debugf("using new RunMarker command") markerFile, err := ioutil.TempFile("", "marker") + if err != nil { + return fmt.Errorf("could not place a marker file") + } defer func() { os.Remove(markerFile.Name()) }() - + markerInfo, err := os.Stat(markerFile.Name()) + if err != nil { + return fmt.Errorf("could not place a marker file") + } if err := runCommandInExec(config, buildArgs, r.cmd); err != nil { return err } - // run command find to find all new files generated - find := exec.Command("find", "/", "-newer", markerFile.Name()) - out, err := find.Output() - if err != nil { - r.Files = []string{} - return nil - } - - r.Files = []string{} - s := strings.Split(string(out), "\n") - for _, path := range s { - path = filepath.Clean(path) - if util.IsDestDir(path) || util.CheckIgnoreList(path) { - continue + // run command find to find all new files generate + isNewer := func(p string) (bool, error) { + fi, err := os.Stat(p) + if err != nil { + return false, err } - r.Files = append(r.Files, path) + return fi.ModTime().After(markerInfo.ModTime()), nil } + r.Files = util.WalkFS("/", isNewer) + logrus.Debugf("files changed %s", r.Files) return nil } @@ -72,11 +71,11 @@ func (r *RunMarkerCommand) String() string { } func (r *RunMarkerCommand) FilesToSnapshot() []string { - return nil + return r.Files } func (r *RunMarkerCommand) ProvidesFilesToSnapshot() bool { - return false + return true } // CacheCommand returns true since this command should be cached @@ -101,6 +100,6 @@ func (r *RunMarkerCommand) ShouldCacheOutput() bool { return true } -func (b *BaseCommand) ShouldDetectDelete() bool { +func (r *RunMarkerCommand) ShouldDetectDeletedFiles() bool { return true } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 0ecd60962..d9d00cbb0 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -319,7 +319,7 @@ func (s *stageBuilder) build() error { } initSnapshotTaken := false - if s.opts.SingleSnapshot { + if s.opts.SingleSnapshot || s.opts.RunV2 { if err := s.initSnapshotWithTimings(); err != nil { return err } diff --git a/pkg/snapshot/layered_map.go b/pkg/snapshot/layered_map.go index 0097b57a5..78377f959 100644 --- a/pkg/snapshot/layered_map.go +++ b/pkg/snapshot/layered_map.go @@ -31,7 +31,7 @@ import ( type LayeredMap struct { layers []map[string]string - whiteouts []map[string]string + whiteouts []map[string]struct{} hasher func(string) (string, error) // cacheHasher doesn't include mtime in it's hash so that filesystem cache keys are stable cacheHasher func(string) (string, error) @@ -47,7 +47,7 @@ func NewLayeredMap(h func(string) (string, error), c func(string) (string, error } func (l *LayeredMap) Snapshot() { - l.whiteouts = append(l.whiteouts, map[string]string{}) + l.whiteouts = append(l.whiteouts, map[string]struct{}{}) l.layers = append(l.layers, map[string]string{}) } @@ -82,21 +82,21 @@ func (l *LayeredMap) Get(s string) (string, bool) { return "", false } -func (l *LayeredMap) GetWhiteout(s string) (string, bool) { +func (l *LayeredMap) GetWhiteout(s string) bool { for i := len(l.whiteouts) - 1; i >= 0; i-- { - if v, ok := l.whiteouts[i][s]; ok { - return v, ok + if _, ok := l.whiteouts[i][s]; ok { + return ok } } - return "", false + return false } func (l *LayeredMap) MaybeAddWhiteout(s string) bool { - whiteout, ok := l.GetWhiteout(s) - if ok && whiteout == s { + ok := l.GetWhiteout(s) + if ok { return false } - l.whiteouts[len(l.whiteouts)-1][s] = s + l.whiteouts[len(l.whiteouts)-1][s] = struct{}{} return true } diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 06f10f7d5..c9046514e 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -24,11 +24,9 @@ import ( "sort" "syscall" + "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/filesystem" "github.com/GoogleContainerTools/kaniko/pkg/timing" - "github.com/karrick/godirwalk" - - "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/sirupsen/logrus" @@ -36,6 +34,7 @@ import ( // For testing var snapshotPathPrefix = config.KanikoDir +var allPass = func(s string) (bool, error) { return true, nil } // Snapshotter holds the root directory from which to take snapshots, and a list of snapshots taken type Snapshotter struct { @@ -81,7 +80,7 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string, } logrus.Info("Taking snapshot of files...") - logrus.Debugf("Taking snapshot of files %v", files) + logrus.Debugf("Taking snapshot of files %v", filesToAdd) sort.Strings(filesToAdd) @@ -96,23 +95,23 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string, filesToWhiteout := []string{} if shdCheckDelete { existingPaths := s.l.getFlattenedPathsForWhiteOut() - foundFiles := walkFS(s.directory) + foundFiles := util.WalkFS(s.directory, allPass) for _, file := range foundFiles { delete(existingPaths, file) } // The paths left here are the ones that have been deleted in this layer. - filesToWhiteOut := []string{} for path := range existingPaths { // Only add the whiteout if the directory for the file still exists. dir := filepath.Dir(path) if _, ok := existingPaths[dir]; !ok { if s.l.MaybeAddWhiteout(path) { logrus.Debugf("Adding whiteout for %s", path) - filesToWhiteOut = append(filesToWhiteOut, path) + filesToWhiteout = append(filesToWhiteout, path) } } } } + logrus.Infof("whiteouts %s", filesToWhiteout) t := util.NewTar(f) defer t.Close() if err := writeToTar(t, filesToAdd, filesToWhiteout); err != nil { @@ -154,7 +153,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { s.l.Snapshot() - foundPaths := walkFS(s.directory) + foundPaths := util.WalkFS(s.directory, allPass) timer := timing.Start("Resolving Paths") // First handle whiteouts // Get a list of all the files that existed before this layer @@ -265,29 +264,3 @@ func filesWithLinks(path string) ([]string, error) { } return []string{path, link}, nil } - -func walkFS(dir string) []string { - foundPaths := make([]string, 0) - timer := timing.Start("Walking filesystem") - godirwalk.Walk(dir, &godirwalk.Options{ - Callback: func(path string, ent *godirwalk.Dirent) error { - if util.IsInIgnoreList(path) { - if util.IsDestDir(path) { - logrus.Tracef("Skipping paths under %s, as it is a ignored directory", path) - - return filepath.SkipDir - } - - return nil - } - - foundPaths = append(foundPaths, path) - - return nil - }, - Unsorted: true, - }, - ) - timing.DefaultRun.Stop(timer) - return foundPaths -} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index d957e64be..75f1d3870 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -31,14 +31,16 @@ import ( "syscall" "time" - otiai10Cpy "github.com/otiai10/copy" - - "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/docker/docker/builder/dockerignore" "github.com/docker/docker/pkg/fileutils" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/karrick/godirwalk" + otiai10Cpy "github.com/otiai10/copy" "github.com/pkg/errors" "github.com/sirupsen/logrus" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/pkg/timing" ) const DoNotChangeUID = -1 @@ -875,3 +877,30 @@ func UpdateInitialIgnoreList(ignoreVarRun bool) { PrefixMatchOnly: false, }) } + +func WalkFS(dir string, f func(string) (bool, error)) []string { + foundPaths := make([]string, 0) + timer := timing.Start("Walking filesystem") + godirwalk.Walk(dir, &godirwalk.Options{ + Callback: func(path string, ent *godirwalk.Dirent) error { + if IsInIgnoreList(path) { + if IsDestDir(path) { + logrus.Tracef("Skipping paths under %s, as it is a ignored directory", path) + return filepath.SkipDir + } + + return nil + } + if t, err := f(path); err != nil { + return err + } else if t { + foundPaths = append(foundPaths, path) + } + return nil + }, + Unsorted: true, + }, + ) + timing.DefaultRun.Stop(timer) + return foundPaths +}