From e6eb5d1abff7ccf0692be0f2aefb2ca86b90d9b2 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 27 Mar 2018 17:43:35 -0700 Subject: [PATCH 1/2] Add hash function which only considers mtime when snapshotting --- executor/cmd/root.go | 10 ++++++++-- integration_tests/integration_test_yaml.go | 10 +++++++++- pkg/util/util.go | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/executor/cmd/root.go b/executor/cmd/root.go index 80b2c088f..a9244d701 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -33,6 +33,7 @@ var ( dockerfilePath string destination string srcContext string + mtime bool logLevel string ) @@ -40,6 +41,7 @@ func init() { RootCmd.PersistentFlags().StringVarP(&dockerfilePath, "dockerfile", "f", "/workspace/Dockerfile", "Path to the dockerfile to be built.") RootCmd.PersistentFlags().StringVarP(&srcContext, "context", "c", "", "Path to the dockerfile build context.") RootCmd.PersistentFlags().StringVarP(&destination, "destination", "d", "", "Registry the final image should be pushed to (ex: gcr.io/test/example:latest)") + RootCmd.PersistentFlags().BoolVarP(&mtime, "mtime", "", false, "Only look at mtime of a file when snapshotting") RootCmd.PersistentFlags().StringVarP(&logLevel, "verbosity", "v", constants.DefaultLogLevel, "Log level (debug, info, warn, error, fatal, panic") } @@ -74,8 +76,12 @@ func execute() error { if err := util.ExtractFileSystemFromImage(baseImage); err != nil { return err } - - l := snapshot.NewLayeredMap(util.Hasher()) + hasher := util.Hasher() + if mtime { + logrus.Info("Only file modification times will be considered when snapshotting.") + hasher = util.MtimeHasher() + } + l := snapshot.NewLayeredMap(hasher) snapshotter := snapshot.NewSnapshotter(l, constants.RootDir) // Take initial snapshot diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index 3129456ff..39ade9f14 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -28,6 +28,7 @@ var fileTests = []struct { configPath string context string repo string + mtime bool }{ { description: "test extract filesystem", @@ -35,6 +36,7 @@ var fileTests = []struct { configPath: "/workspace/integration_tests/dockerfiles/config_test_extract_fs.json", context: "integration_tests/dockerfiles/", repo: "extract-filesystem", + mtime: true, }, { description: "test run", @@ -49,6 +51,7 @@ var fileTests = []struct { configPath: "/workspace/integration_tests/dockerfiles/config_test_run_2.json", context: "integration_tests/dockerfiles/", repo: "test-run-2", + mtime: true, }, { description: "test copy", @@ -56,6 +59,7 @@ var fileTests = []struct { configPath: "/workspace/integration_tests/dockerfiles/config_test_copy.json", context: "/workspace/integration_tests/", repo: "test-copy", + mtime: true, }, } @@ -139,9 +143,13 @@ func main() { // Then, buld the image with kbuild kbuildImage := testRepo + kbuildPrefix + test.repo + mtime := "" + if test.mtime { + mtime = "--mtime" + } kbuild := step{ Name: executorImage, - Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.context}, + Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.context, mtime}, } // Pull the kbuild image diff --git a/pkg/util/util.go b/pkg/util/util.go index f27ebba5a..2e4a08851 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -61,3 +61,17 @@ func Hasher() func(string) (string, error) { } return hasher } + +// Hasher returns a hash function, which only looks at mtime to determine if a file has changed +func MtimeHasher() func(string) (string, error) { + hasher := func(p string) (string, error) { + h := md5.New() + fi, err := os.Lstat(p) + if err != nil { + return "", err + } + h.Write([]byte(fi.ModTime().String())) + return hex.EncodeToString(h.Sum(nil)), nil + } + return hasher +} From 591fef5d85f82b4966bc46dce07b42a61c4fc318 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 28 Mar 2018 10:11:21 -0700 Subject: [PATCH 2/2] Make mtime flag enum --- executor/cmd/root.go | 20 ++++++++++++-------- integration_tests/integration_test_yaml.go | 12 ++++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/executor/cmd/root.go b/executor/cmd/root.go index a9244d701..2d9e52955 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -33,7 +33,7 @@ var ( dockerfilePath string destination string srcContext string - mtime bool + mtime string logLevel string ) @@ -41,7 +41,7 @@ func init() { RootCmd.PersistentFlags().StringVarP(&dockerfilePath, "dockerfile", "f", "/workspace/Dockerfile", "Path to the dockerfile to be built.") RootCmd.PersistentFlags().StringVarP(&srcContext, "context", "c", "", "Path to the dockerfile build context.") RootCmd.PersistentFlags().StringVarP(&destination, "destination", "d", "", "Registry the final image should be pushed to (ex: gcr.io/test/example:latest)") - RootCmd.PersistentFlags().BoolVarP(&mtime, "mtime", "", false, "Only look at mtime of a file when snapshotting") + RootCmd.PersistentFlags().StringVarP(&mtime, "mtime", "", "full", "Set this flag to change the file attributes inspected during snapshotting") RootCmd.PersistentFlags().StringVarP(&logLevel, "verbosity", "v", constants.DefaultLogLevel, "Log level (debug, info, warn, error, fatal, panic") } @@ -76,12 +76,8 @@ func execute() error { if err := util.ExtractFileSystemFromImage(baseImage); err != nil { return err } - hasher := util.Hasher() - if mtime { - logrus.Info("Only file modification times will be considered when snapshotting.") - hasher = util.MtimeHasher() - } - l := snapshot.NewLayeredMap(hasher) + + l := snapshot.NewLayeredMap(getHasher()) snapshotter := snapshot.NewSnapshotter(l, constants.RootDir) // Take initial snapshot @@ -131,3 +127,11 @@ func execute() error { // Push the image return image.PushImage(sourceImage, destination) } + +func getHasher() func(string) (string, error) { + if mtime == "time" { + logrus.Info("Only file modification time will be considered when snapshotting") + return util.MtimeHasher() + } + return util.Hasher() +} diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index 39ade9f14..f284f746e 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -28,7 +28,7 @@ var fileTests = []struct { configPath string context string repo string - mtime bool + mtime string }{ { description: "test extract filesystem", @@ -36,7 +36,7 @@ var fileTests = []struct { configPath: "/workspace/integration_tests/dockerfiles/config_test_extract_fs.json", context: "integration_tests/dockerfiles/", repo: "extract-filesystem", - mtime: true, + mtime: "time", }, { description: "test run", @@ -51,7 +51,7 @@ var fileTests = []struct { configPath: "/workspace/integration_tests/dockerfiles/config_test_run_2.json", context: "integration_tests/dockerfiles/", repo: "test-run-2", - mtime: true, + mtime: "time", }, { description: "test copy", @@ -59,7 +59,7 @@ var fileTests = []struct { configPath: "/workspace/integration_tests/dockerfiles/config_test_copy.json", context: "/workspace/integration_tests/", repo: "test-copy", - mtime: true, + mtime: "time", }, } @@ -144,8 +144,8 @@ func main() { // Then, buld the image with kbuild kbuildImage := testRepo + kbuildPrefix + test.repo mtime := "" - if test.mtime { - mtime = "--mtime" + if test.mtime != "" { + mtime = "--mtime=" + test.mtime } kbuild := step{ Name: executorImage,