better improvements

Makefile

@@ -88,9 +88,9 @@ integration-test-misc:
 
 .PHONY: images
 images:
-	docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:latest -f deploy/Dockerfile .
+	docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:perf-latest -f deploy/Dockerfile .
-	docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug .
+	#docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug .
-	docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/warmer:latest -f deploy/Dockerfile_warmer .
+	#docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/warmer:latest -f deploy/Dockerfile_warmer .
 
 .PHONY: push
 push:

integration/benchmark_fs/cloudbuild.yaml

@@ -1,9 +1,9 @@
 steps:
-- name: 'gcr.io/kaniko-project/executor:latest'
+- name: 'gcr.io/kaniko-project/executor:perf-latest'
   args:
   - --build-arg=NUM=${_COUNT}
   - --no-push
-  - --snapshotMode=redo
+  - --use-new-run=true
   env:
   - 'BENCHMARK_FILE=gs://$PROJECT_ID/gcb/benchmark_file_${_COUNT}'
   timeout: 2400s

integration/benchmark_test.go

@@ -125,7 +125,7 @@ func TestSnapshotBenchmarkGcloud(t *testing.T) {
 	}
 	contextDir := filepath.Join(cwd, "benchmark_fs")
 
-	nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000}
+	nums := []int{10000} //, 50000, 100000, 200000, 300000, 500000, 700000}
 
 	var wg sync.WaitGroup
 	fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")

integration/images.go

@@ -76,6 +76,7 @@ var additionalDockerFlagsMap = map[string][]string{
 var additionalKanikoFlagsMap = map[string][]string{
 	"Dockerfile_test_add":        {"--single-snapshot"},
 	"Dockerfile_test_run_new":    {"--use-new-run=true"},
+	"Dockerfile_test_run":        {"-v=debug"},
 	"Dockerfile_test_scratch":    {"--single-snapshot"},
 	"Dockerfile_test_maintainer": {"--single-snapshot"},
 	"Dockerfile_test_target":     {"--target=second"},
@@ -272,11 +273,13 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
 
 	kanikoImage := GetKanikoImage(imageRepo, dockerfile)
 	timer = timing.Start(dockerfile + "_kaniko")
-	if _, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
+	out, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
-		contextDir, gcsBucket, serviceAccount, true); err != nil {
+		contextDir, gcsBucket, serviceAccount, true)
+	if err != nil {
 		return err
 	}
 	timing.DefaultRun.Stop(timer)
+	fmt.Println(out)
 
 	d.filesBuilt[dockerfile] = struct{}{}
 
@@ -451,5 +454,6 @@ func buildKanikoImage(
 			return "", fmt.Errorf("Output check failed for image %s with kaniko command : %s %s", kanikoImage, err, string(out))
 		}
 	}
+	fmt.Println("kaniko command\n", string(out))
 	return benchmarkDir, nil
 }

integration/integration_test.go

@@ -142,26 +142,26 @@ func buildRequiredImages() error {
 			name:    "Building kaniko image",
 			command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."},
 		},
-		{
+		//{
-			name:    "Building cache warmer image",
+		//	name:    "Building cache warmer image",
-			command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."},
+		//	command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."},
-		},
+		//},
-		{
+		//{
-			name:    "Building onbuild base image",
+		//	name:    "Building onbuild base image",
-			command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."},
+		//	command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."},
-		},
+		//},
-		{
+		//{
-			name:    "Pushing onbuild base image",
+		//	name:    "Pushing onbuild base image",
-			command: []string{"docker", "push", config.onbuildBaseImage},
+		//	command: []string{"docker", "push", config.onbuildBaseImage},
-		},
+		//},
-		{
+		//{
-			name:    "Building hardlink base image",
+		//	name:    "Building hardlink base image",
-			command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."},
+		//	command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."},
-		},
+		//},
-		{
+		//{
-			name:    "Pushing hardlink base image",
+		//	name:    "Pushing hardlink base image",
-			command: []string{"docker", "push", config.hardlinkBaseImage},
+		//	command: []string{"docker", "push", config.hardlinkBaseImage},
-		},
+		//},
 	}
 
 	for _, setupCmd := range setupCommands {
@@ -294,6 +294,7 @@ func TestGitBuildcontextSubPath(t *testing.T) {
 	kanikoCmd := exec.Command("docker", dockerRunFlags...)
 
 	out, err = RunCommandWithoutTest(kanikoCmd)
+	fmt.Println(out)
 	if err != nil {
 		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
 	}

pkg/commands/run_marker.go

@@ -17,16 +17,15 @@ limitations under the License.
 package commands
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
 )
 
 type RunMarkerCommand struct {
@@ -37,32 +36,32 @@ type RunMarkerCommand struct {
 
 func (r *RunMarkerCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
 	// run command `touch filemarker`
+	logrus.Debugf("using new RunMarker command")
 	markerFile, err := ioutil.TempFile("", "marker")
+	if err != nil {
+		return fmt.Errorf("could not place a marker file")
+	}
 	defer func() {
 		os.Remove(markerFile.Name())
 	}()
-
+	markerInfo, err := os.Stat(markerFile.Name())
+	if err != nil {
+		return fmt.Errorf("could not place a marker file")
+	}
 	if err := runCommandInExec(config, buildArgs, r.cmd); err != nil {
 		return err
 	}
 
-	// run command find to find all new files generated
+	// run command find to find all new files generate
-	find := exec.Command("find", "/", "-newer", markerFile.Name())
+	isNewer := func(p string) (bool, error) {
-	out, err := find.Output()
+		fi, err := os.Stat(p)
-	if err != nil {
+		if err != nil {
-		r.Files = []string{}
+			return false, err
-		return nil
-	}
-
-	r.Files = []string{}
-	s := strings.Split(string(out), "\n")
-	for _, path := range s {
-		path = filepath.Clean(path)
-		if util.IsDestDir(path) || util.CheckIgnoreList(path) {
-			continue
 		}
-		r.Files = append(r.Files, path)
+		return fi.ModTime().After(markerInfo.ModTime()), nil
 	}
+	r.Files = util.WalkFS("/", isNewer)
+	logrus.Debugf("files changed %s", r.Files)
 	return nil
 }
 
@@ -72,11 +71,11 @@ func (r *RunMarkerCommand) String() string {
 }
 
 func (r *RunMarkerCommand) FilesToSnapshot() []string {
-	return nil
+	return r.Files
 }
 
 func (r *RunMarkerCommand) ProvidesFilesToSnapshot() bool {
-	return false
+	return true
 }
 
 // CacheCommand returns true since this command should be cached
@@ -101,6 +100,6 @@ func (r *RunMarkerCommand) ShouldCacheOutput() bool {
 	return true
 }
 
-func (b *BaseCommand) ShouldDetectDelete() bool {
+func (r *RunMarkerCommand) ShouldDetectDeletedFiles() bool {
 	return true
 }

pkg/executor/build.go

@@ -319,7 +319,7 @@ func (s *stageBuilder) build() error {
 	}
 
 	initSnapshotTaken := false
-	if s.opts.SingleSnapshot {
+	if s.opts.SingleSnapshot || s.opts.RunV2 {
 		if err := s.initSnapshotWithTimings(); err != nil {
 			return err
 		}

pkg/snapshot/layered_map.go

@@ -31,7 +31,7 @@ import (
 
 type LayeredMap struct {
 	layers    []map[string]string
-	whiteouts []map[string]string
+	whiteouts []map[string]struct{}
 	hasher    func(string) (string, error)
 	// cacheHasher doesn't include mtime in it's hash so that filesystem cache keys are stable
 	cacheHasher func(string) (string, error)
@@ -47,7 +47,7 @@ func NewLayeredMap(h func(string) (string, error), c func(string) (string, error
 }
 
 func (l *LayeredMap) Snapshot() {
-	l.whiteouts = append(l.whiteouts, map[string]string{})
+	l.whiteouts = append(l.whiteouts, map[string]struct{}{})
 	l.layers = append(l.layers, map[string]string{})
 }
 
@@ -82,21 +82,21 @@ func (l *LayeredMap) Get(s string) (string, bool) {
 	return "", false
 }
 
-func (l *LayeredMap) GetWhiteout(s string) (string, bool) {
+func (l *LayeredMap) GetWhiteout(s string) bool {
 	for i := len(l.whiteouts) - 1; i >= 0; i-- {
-		if v, ok := l.whiteouts[i][s]; ok {
+		if _, ok := l.whiteouts[i][s]; ok {
-			return v, ok
+			return ok
 		}
 	}
-	return "", false
+	return false
 }
 
 func (l *LayeredMap) MaybeAddWhiteout(s string) bool {
-	whiteout, ok := l.GetWhiteout(s)
+	ok := l.GetWhiteout(s)
-	if ok && whiteout == s {
+	if ok {
 		return false
 	}
-	l.whiteouts[len(l.whiteouts)-1][s] = s
+	l.whiteouts[len(l.whiteouts)-1][s] = struct{}{}
 	return true
 }
 

pkg/snapshot/snapshot.go

@@ -24,11 +24,9 @@ import (
 	"sort"
 	"syscall"
 
+	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/filesystem"
 	"github.com/GoogleContainerTools/kaniko/pkg/timing"
-	"github.com/karrick/godirwalk"
-
-	"github.com/GoogleContainerTools/kaniko/pkg/config"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/sirupsen/logrus"
@@ -36,6 +34,7 @@ import (
 
 // For testing
 var snapshotPathPrefix = config.KanikoDir
+var allPass = func(s string) (bool, error) { return true, nil }
 
 // Snapshotter holds the root directory from which to take snapshots, and a list of snapshots taken
 type Snapshotter struct {
@@ -81,7 +80,7 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string,
 	}
 
 	logrus.Info("Taking snapshot of files...")
-	logrus.Debugf("Taking snapshot of files %v", files)
+	logrus.Debugf("Taking snapshot of files %v", filesToAdd)
 
 	sort.Strings(filesToAdd)
 
@@ -96,23 +95,23 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string,
 	filesToWhiteout := []string{}
 	if shdCheckDelete {
 		existingPaths := s.l.getFlattenedPathsForWhiteOut()
-		foundFiles := walkFS(s.directory)
+		foundFiles := util.WalkFS(s.directory, allPass)
 		for _, file := range foundFiles {
 			delete(existingPaths, file)
 		}
 		// The paths left here are the ones that have been deleted in this layer.
-		filesToWhiteOut := []string{}
 		for path := range existingPaths {
 			// Only add the whiteout if the directory for the file still exists.
 			dir := filepath.Dir(path)
 			if _, ok := existingPaths[dir]; !ok {
 				if s.l.MaybeAddWhiteout(path) {
 					logrus.Debugf("Adding whiteout for %s", path)
-					filesToWhiteOut = append(filesToWhiteOut, path)
+					filesToWhiteout = append(filesToWhiteout, path)
 				}
 			}
 		}
 	}
+	logrus.Infof("whiteouts %s", filesToWhiteout)
 	t := util.NewTar(f)
 	defer t.Close()
 	if err := writeToTar(t, filesToAdd, filesToWhiteout); err != nil {
@@ -154,7 +153,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
 
 	s.l.Snapshot()
 
-	foundPaths := walkFS(s.directory)
+	foundPaths := util.WalkFS(s.directory, allPass)
 	timer := timing.Start("Resolving Paths")
 	// First handle whiteouts
 	//   Get a list of all the files that existed before this layer
@@ -265,29 +264,3 @@ func filesWithLinks(path string) ([]string, error) {
 	}
 	return []string{path, link}, nil
 }
-
-func walkFS(dir string) []string {
-	foundPaths := make([]string, 0)
-	timer := timing.Start("Walking filesystem")
-	godirwalk.Walk(dir, &godirwalk.Options{
-		Callback: func(path string, ent *godirwalk.Dirent) error {
-			if util.IsInIgnoreList(path) {
-				if util.IsDestDir(path) {
-					logrus.Tracef("Skipping paths under %s, as it is a ignored directory", path)
-
-					return filepath.SkipDir
-				}
-
-				return nil
-			}
-
-			foundPaths = append(foundPaths, path)
-
-			return nil
-		},
-		Unsorted: true,
-	},
-	)
-	timing.DefaultRun.Stop(timer)
-	return foundPaths
-}

pkg/util/fs_util.go

@@ -31,14 +31,16 @@ import (
 	"syscall"
 	"time"
 
-	otiai10Cpy "github.com/otiai10/copy"
-
-	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/docker/docker/builder/dockerignore"
 	"github.com/docker/docker/pkg/fileutils"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/karrick/godirwalk"
+	otiai10Cpy "github.com/otiai10/copy"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/config"
+	"github.com/GoogleContainerTools/kaniko/pkg/timing"
 )
 
 const DoNotChangeUID = -1
@@ -875,3 +877,30 @@ func UpdateInitialIgnoreList(ignoreVarRun bool) {
 		PrefixMatchOnly: false,
 	})
 }
+
+func WalkFS(dir string, f func(string) (bool, error)) []string {
+	foundPaths := make([]string, 0)
+	timer := timing.Start("Walking filesystem")
+	godirwalk.Walk(dir, &godirwalk.Options{
+		Callback: func(path string, ent *godirwalk.Dirent) error {
+			if IsInIgnoreList(path) {
+				if IsDestDir(path) {
+					logrus.Tracef("Skipping paths under %s, as it is a ignored directory", path)
+					return filepath.SkipDir
+				}
+
+				return nil
+			}
+			if t, err := f(path); err != nil {
+				return err
+			} else if t {
+				foundPaths = append(foundPaths, path)
+			}
+			return nil
+		},
+		Unsorted: true,
+	},
+	)
+	timing.DefaultRun.Stop(timer)
+	return foundPaths
+}