Merge pull request #1301 from tejal29/addRedoSnapshotter

Add redo snapshotter.

integration/dockerfiles/Dockerfile_test_run_redo

@@ -0,0 +1,26 @@
+# Copyright 2020 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM debian:9.11
+RUN echo "hey" > /etc/foo
+RUN echo "baz" > /etc/baz
+RUN cp /etc/baz /etc/bar
+RUN rm /etc/baz
+
+# Test with ARG
+ARG file
+RUN echo "run" > $file
+
+RUN echo "test home" > $HOME/file
+COPY context/foo $HOME/foo

integration/images.go

@@ -48,6 +48,7 @@ const (
 // Arguments to build Dockerfiles with, used for both docker and kaniko builds
 var argsMap = map[string][]string{
 	"Dockerfile_test_run":        {"file=/file"},
+	"Dockerfile_test_run_redo":   {"file=/file"},
 	"Dockerfile_test_workdir":    {"workdir=/arg/workdir"},
 	"Dockerfile_test_add":        {"file=context/foo"},
 	"Dockerfile_test_arg_secret": {"SSH_PRIVATE_KEY", "SSH_PUBLIC_KEY=Pµbl1cK€Y"},
@@ -74,6 +75,7 @@ var additionalDockerFlagsMap = map[string][]string{
 // Arguments to build Dockerfiles with when building with kaniko
 var additionalKanikoFlagsMap = map[string][]string{
 	"Dockerfile_test_add":        {"--single-snapshot"},
+	"Dockerfile_test_run_redo":   {"--snapshotMode=redo"},
 	"Dockerfile_test_scratch":    {"--single-snapshot"},
 	"Dockerfile_test_maintainer": {"--single-snapshot"},
 	"Dockerfile_test_target":     {"--target=second"},

pkg/constants/constants.go

@@ -47,6 +47,7 @@ const (
 	// Various snapshot modes:
 	SnapshotModeTime = "time"
 	SnapshotModeFull = "full"
+	SnapshotModeRedo = "redo"
 
 	// NoBaseImage is the scratch image
 	NoBaseImage = "scratch"

pkg/executor/build.go

@@ -808,14 +808,17 @@ func saveStageAsTarball(path string, image v1.Image) error {
 }
 
 func getHasher(snapshotMode string) (func(string) (string, error), error) {
-	if snapshotMode == constants.SnapshotModeTime {
+	switch snapshotMode {
+	case constants.SnapshotModeTime:
 		logrus.Info("Only file modification time will be considered when snapshotting")
 		return util.MtimeHasher(), nil
-	}
-	if snapshotMode == constants.SnapshotModeFull {
+	case constants.SnapshotModeFull:
 		return util.Hasher(), nil
+	case constants.SnapshotModeRedo:
+		return util.RedoHasher(), nil
+	default:
+		return nil, fmt.Errorf("%s is not a valid snapshot mode", snapshotMode)
 	}
-	return nil, fmt.Errorf("%s is not a valid snapshot mode", snapshotMode)
 }
 
 func resolveOnBuild(stage *config.KanikoStage, config *v1.Config, stageNameToIdx map[string]string) error {

pkg/util/util.go

@@ -115,6 +115,27 @@ func MtimeHasher() func(string) (string, error) {
 	return hasher
 }
 
+// RedoHasher returns a hash function, which looks at mtime, size, filemode, owner uid and gid
+// Note that the mtime can lag, so it's possible that a file will have changed but the mtime may look the same.
+func RedoHasher() func(string) (string, error) {
+	hasher := func(p string) (string, error) {
+		h := md5.New()
+		fi, err := os.Lstat(p)
+		if err != nil {
+			return "", err
+		}
+		h.Write([]byte(fi.Mode().String()))
+		h.Write([]byte(fi.ModTime().String()))
+		h.Write([]byte(strconv.FormatInt(fi.Size(), 16)))
+		h.Write([]byte(strconv.FormatUint(uint64(fi.Sys().(*syscall.Stat_t).Uid), 36)))
+		h.Write([]byte(","))
+		h.Write([]byte(strconv.FormatUint(uint64(fi.Sys().(*syscall.Stat_t).Gid), 36)))
+
+		return hex.EncodeToString(h.Sum(nil)), nil
+	}
+	return hasher
+}
+
 // SHA256 returns the shasum of the contents of r
 func SHA256(r io.Reader) (string, error) {
 	hasher := sha256.New()