diff --git a/Makefile b/Makefile index e7836b76a..80e9fe5f2 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ EXECUTOR_PACKAGE = $(REPOPATH)/executor KBUILD_PACKAGE = $(REPOPATH)/kbuild out/executor: $(GO_FILES) - GOOS=$* GOARCH=$(GOARCH) CGO_ENABLED=0 go build -ldflags $(GO_LDFLAGS) -tags $(GO_BUILD_TAGS) -o $@ $(EXECUTOR_PACKAGE) + GOOS=$* GOARCH=$(GOARCH) CGO_ENABLED=1 go build -ldflags $(GO_LDFLAGS) -tags $(GO_BUILD_TAGS) -o $@ $(EXECUTOR_PACKAGE) out/kbuild: $(GO_FILES) diff --git a/executor/cmd/root.go b/executor/cmd/root.go index 5fd0c54dc..0afb6dae3 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -19,6 +19,8 @@ package cmd import ( "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/constants" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/dockerfile" + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/image" + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/snapshot" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -28,15 +30,15 @@ import ( var ( dockerfilePath string - name string + destination string srcContext string logLevel string ) func init() { - RootCmd.PersistentFlags().StringVarP(&dockerfilePath, "dockerfile", "d", "/workspace/Dockerfile", "Path to the dockerfile to be built.") + RootCmd.PersistentFlags().StringVarP(&dockerfilePath, "dockerfile", "f", "/workspace/Dockerfile", "Path to the dockerfile to be built.") RootCmd.PersistentFlags().StringVarP(&srcContext, "context", "c", "", "Path to the dockerfile build context.") - RootCmd.PersistentFlags().StringVarP(&name, "name", "n", "", "Registry the final image should be pushed to (ex: gcr.io/test/example:latest)") + RootCmd.PersistentFlags().StringVarP(&destination, "destination", "d", "", "Registry the final image should be pushed to (ex: gcr.io/test/example:latest)") RootCmd.PersistentFlags().StringVarP(&logLevel, "verbosity", "v", constants.DefaultLogLevel, "Log level (debug, info, warn, error, fatal, panic") } @@ -68,5 +70,25 @@ func execute() error { // Unpack file system to root logrus.Infof("Unpacking filesystem of %s...", baseImage) - return util.ExtractFileSystemFromImage(baseImage) + if err := util.ExtractFileSystemFromImage(baseImage); err != nil { + return err + } + + l := snapshot.NewLayeredMap(util.Hasher()) + snapshotter := snapshot.NewSnapshotter(l, constants.RootDir) + + // Take initial snapshot + if err := snapshotter.Init(); err != nil { + return err + } + + // Initialize source image + if err := image.InitializeSourceImage(baseImage); err != nil { + return err + } + + // Execute commands here + + // Push the image + return image.PushImage(destination) } diff --git a/files/policy.json b/files/policy.json deleted file mode 100644 index ab25c67e7..000000000 --- a/files/policy.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "default": [ - { - "type": "insecureAcceptAnything" - } - ], - "transports": { - "docker-daemon": { - "": [ - { - "type": "insecureAcceptAnything" - } - ] - } - } -} diff --git a/integration_tests/dockerfiles/config_test_extract_fs.json b/integration_tests/dockerfiles/config_test_extract_fs.json index 1d201720c..a5b03a8e9 100644 --- a/integration_tests/dockerfiles/config_test_extract_fs.json +++ b/integration_tests/dockerfiles/config_test_extract_fs.json @@ -4,42 +4,8 @@ "Image2": "gcr.io/kbuild-test/kbuild-extract-filesystem:latest", "DiffType": "File", "Diff": { - "Adds": [ - { - "Name": "/workspace", - "Size": 16289150 - }, - { - "Name": "/workspace/Dockerfile", - "Size": 0 - }, - { - "Name": "/workspace/executor", - "Size": 16289150 - } - ], - "Dels": [ - { - "Name": "/dev", - "Size": 0 - }, - { - "Name": "/etc/hosts", - "Size": 109 - }, - { - "Name": "/etc/resolv.conf", - "Size": 38 - }, - { - "Name": "/proc", - "Size": 0 - }, - { - "Name": "/sys", - "Size": 0 - } - ], + "Adds": null, + "Dels": null, "Mods": null } } diff --git a/integration_tests/executor/Dockerfile b/integration_tests/executor/Dockerfile index 94fc7862b..9a4595d48 100644 --- a/integration_tests/executor/Dockerfile +++ b/integration_tests/executor/Dockerfile @@ -15,5 +15,5 @@ # Builds the static Go image to execute in a Kubernetes job FROM scratch -ADD out/executor /workspace/executor +ADD out/executor /work-dir/executor ADD files/ca-certificates.crt /etc/ssl/certs/ diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index 8b896a44d..c11b901ec 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -48,7 +48,7 @@ type testyaml struct { } var executorImage = "executor-image" -var executorCommand = "/workspace/executor" +var executorCommand = "/work-dir/executor" var dockerImage = "gcr.io/cloud-builders/docker" var ubuntuImage = "ubuntu" var testRepo = "gcr.io/kbuild-test/" @@ -73,46 +73,50 @@ func main() { Name: dockerImage, Args: []string{"build", "-t", executorImage, "-f", "integration_tests/executor/Dockerfile", "."}, } - y := testyaml{ Steps: []step{containerDiffStep, containerDiffPermissions, buildExecutorImage}, } for _, test := range tests { // First, build the image with docker + dockerImageTag := testRepo + dockerPrefix + test.repo dockerBuild := step{ Name: dockerImage, - Args: []string{"build", "-t", testRepo + dockerPrefix + test.repo, "-f", test.dockerfilePath, test.context}, + Args: []string{"build", "-t", dockerImageTag, "-f", test.dockerfilePath, test.context}, } - // Then, buld the image with kbuild and commit it - var commitID = "test" + // Then, buld the image with kbuild + kbuildImage := testRepo + kbuildPrefix + test.repo kbuild := step{ - Name: dockerImage, - Args: []string{"run", "-v", test.dockerfilePath + ":/workspace/Dockerfile", "--name", commitID, executorImage, executorCommand}, + Name: executorImage, + Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath}, } - commit := step{ + // Pull the kbuild image + pullKbuildImage := step{ Name: dockerImage, - Args: []string{"commit", commitID, testRepo + kbuildPrefix + test.repo}, + Args: []string{"pull", kbuildImage}, } - dockerImage := daemonPrefix + testRepo + dockerPrefix + test.repo - kbuildImage := daemonPrefix + testRepo + kbuildPrefix + test.repo + daemonDockerImage := daemonPrefix + dockerImageTag + daemonKbuildImage := daemonPrefix + kbuildImage // Run container diff on the images - args := "container-diff-linux-amd64 diff " + dockerImage + " " + kbuildImage + " --type=file -j > " + containerDiffOutputFile + args := "container-diff-linux-amd64 diff " + daemonDockerImage + " " + daemonKbuildImage + " --type=file -j >" + containerDiffOutputFile containerDiff := step{ Name: ubuntuImage, Args: []string{"sh", "-c", args}, Env: []string{"PATH=/workspace:/bin"}, } - // Compare output files + catContainerDiffOutput := step{ + Name: ubuntuImage, + Args: []string{"cat", containerDiffOutputFile}, + } compareOutputs := step{ Name: ubuntuImage, Args: []string{"cmp", test.configPath, containerDiffOutputFile}, } - y.Steps = append(y.Steps, dockerBuild, kbuild, commit, containerDiff, compareOutputs) + y.Steps = append(y.Steps, dockerBuild, kbuild, pullKbuildImage, containerDiff, catContainerDiffOutput, compareOutputs) } d, _ := yaml.Marshal(&y) diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 28362e01f..8a619ed1e 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -23,8 +23,8 @@ const ( // RootDir is the path to the root directory RootDir = "/" - WhitelistPath = "/proc/self/mountinfo" + // WorkspaceDir is the path to the workspace directory + WorkspaceDir = "/workspace" - // PolicyJSONPath is the path to the policy JSON - PolicyJSONPath = "/workspace/policy.json" + WhitelistPath = "/proc/self/mountinfo" ) diff --git a/pkg/image/image.go b/pkg/image/image.go index c7cda3cdc..b76ad1269 100644 --- a/pkg/image/image.go +++ b/pkg/image/image.go @@ -18,7 +18,6 @@ package image import ( img "github.com/GoogleCloudPlatform/container-diff/pkg/image" - "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/constants" "github.com/containers/image/copy" "github.com/containers/image/docker" "github.com/containers/image/signature" @@ -31,6 +30,7 @@ var sourceImage img.MutableSource // InitializeSourceImage initializes the source image with the base image func InitializeSourceImage(srcImg string) error { + logrus.Infof("Initializing source image %s", srcImg) ref, err := docker.ParseReference("//" + srcImg) if err != nil { return err @@ -67,12 +67,9 @@ func PushImage(destImg string) error { } func getPolicyContext() (*signature.PolicyContext, error) { - policy, err := signature.NewPolicyFromFile(constants.PolicyJSONPath) - if err != nil { - logrus.Debugf("Error retrieving policy: %s", err) - return nil, err - } - policyContext, err := signature.NewPolicyContext(policy) + policyContext, err := signature.NewPolicyContext(&signature.Policy{ + Default: signature.PolicyRequirements{signature.NewPRInsecureAcceptAnything()}, + }) if err != nil { logrus.Debugf("Error retrieving policy context: %s", err) return nil, err diff --git a/pkg/snapshot/layered_map.go b/pkg/snapshot/layered_map.go new file mode 100644 index 000000000..608808ac6 --- /dev/null +++ b/pkg/snapshot/layered_map.go @@ -0,0 +1,56 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package snapshot + +type LayeredMap struct { + layers []map[string]string + hasher func(string) (string, error) +} + +func NewLayeredMap(h func(string) (string, error)) *LayeredMap { + l := LayeredMap{ + hasher: h, + } + l.layers = []map[string]string{} + return &l +} + +func (l *LayeredMap) Snapshot() { + l.layers = append(l.layers, map[string]string{}) +} + +func (l *LayeredMap) Get(s string) (string, bool) { + for i := len(l.layers) - 1; i >= 0; i-- { + if v, ok := l.layers[i][s]; ok { + return v, ok + } + } + return "", false +} + +func (l *LayeredMap) MaybeAdd(s string) (bool, error) { + oldV, ok := l.Get(s) + newV, err := l.hasher(s) + if err != nil { + return false, err + } + if ok && newV == oldV { + return false, nil + } + l.layers[len(l.layers)-1][s] = newV + return true, nil +} diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go new file mode 100644 index 000000000..05cdd8a58 --- /dev/null +++ b/pkg/snapshot/snapshot.go @@ -0,0 +1,122 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package snapshot + +import ( + "archive/tar" + "bytes" + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util" + "github.com/sirupsen/logrus" + + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// Snapshotter holds the root directory from which to take snapshots, and a list of snapshots taken +type Snapshotter struct { + l *LayeredMap + directory string +} + +// NewSnapshotter creates a new snapshotter rooted at d +func NewSnapshotter(l *LayeredMap, d string) *Snapshotter { + return &Snapshotter{l: l, directory: d} +} + +// Init initializes a new snapshotter +func (s *Snapshotter) Init() error { + if _, err := s.snapShotFS(ioutil.Discard); err != nil { + return err + } + return nil +} + +// TakeSnapshot takes a snapshot of the filesystem, avoiding directories in the whitelist, and creates +// a tarball of the changed files. Return contents of the tarball, and whether or not any files were changed +func (s *Snapshotter) TakeSnapshot() ([]byte, bool, error) { + buf := bytes.NewBuffer([]byte{}) + filesAdded, err := s.snapShotFS(buf) + if err != nil { + return nil, filesAdded, err + } + contents, err := ioutil.ReadAll(buf) + if err != nil { + return nil, filesAdded, err + } + return contents, filesAdded, err +} + +// TakeSnapshotOfFiles takes a snapshot of specific files +// Used for ADD/COPY commands, when we know which files have changed +func (s *Snapshotter) TakeSnapshotOfFiles(files []string) ([]byte, error) { + logrus.Infof("Taking snapshot of files %s", files) + s.l.Snapshot() + if len(files) == 0 { + logrus.Info("No files changed in this command, skipping snapshotting.") + return nil, nil + } + buf := bytes.NewBuffer([]byte{}) + w := tar.NewWriter(buf) + defer w.Close() + for _, file := range files { + info, err := os.Stat(file) + if err != nil { + return nil, err + } + if util.PathInWhitelist(file, s.directory) { + logrus.Debugf("Not adding %s to layer, as it is whitelisted", file) + continue + } + // Only add to the tar if we add it to the layeredmap. + maybeAdd, err := s.l.MaybeAdd(file) + if err != nil { + return nil, err + } + if maybeAdd { + util.AddToTar(file, info, w) + } + } + return ioutil.ReadAll(buf) +} + +func (s *Snapshotter) snapShotFS(f io.Writer) (bool, error) { + s.l.Snapshot() + filesAdded := false + w := tar.NewWriter(f) + defer w.Close() + + err := filepath.Walk(s.directory, func(path string, info os.FileInfo, err error) error { + if util.PathInWhitelist(path, s.directory) { + logrus.Debugf("Not adding %s to layer, as it's whitelisted", path) + return nil + } + + // Only add to the tar if we add it to the layeredmap. + maybeAdd, err := s.l.MaybeAdd(path) + if err != nil { + return err + } + if maybeAdd { + filesAdded = true + return util.AddToTar(path, info, w) + } + return nil + }) + return filesAdded, err +} diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go new file mode 100644 index 000000000..22263aed3 --- /dev/null +++ b/pkg/snapshot/snapshot_test.go @@ -0,0 +1,216 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package snapshot + +import ( + "archive/tar" + "bytes" + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util" + "github.com/GoogleCloudPlatform/k8s-container-builder/testutil" + "github.com/pkg/errors" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestSnapshotFileChange(t *testing.T) { + + testDir, snapshotter, err := setUpTestDir() + defer os.RemoveAll(testDir) + if err != nil { + t.Fatal(err) + } + // Make some changes to the filesystem + newFiles := map[string]string{ + "foo": "newbaz1", + "bar/bat": "baz", + "work-dir/bat": "bat", + } + if err := testutil.SetupFiles(testDir, newFiles); err != nil { + t.Fatalf("Error setting up fs: %s", err) + } + // Take another snapshot + contents, filesAdded, err := snapshotter.TakeSnapshot() + if err != nil { + t.Fatalf("Error taking snapshot of fs: %s", err) + } + if !filesAdded { + t.Fatal("No files added to snapshot.") + } + // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles + reader := bytes.NewReader(contents) + tr := tar.NewReader(reader) + fooPath := filepath.Join(testDir, "foo") + batPath := filepath.Join(testDir, "bar/bat") + snapshotFiles := map[string]string{ + fooPath: "newbaz1", + batPath: "baz", + } + numFiles := 0 + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + numFiles++ + if _, isFile := snapshotFiles[hdr.Name]; !isFile { + t.Fatalf("File %s unexpectedly in tar", hdr.Name) + } + contents, _ := ioutil.ReadAll(tr) + if string(contents) != snapshotFiles[hdr.Name] { + t.Fatalf("Contents of %s incorrect, expected: %s, actual: %s", hdr.Name, snapshotFiles[hdr.Name], string(contents)) + } + } + if numFiles != 2 { + t.Fatalf("Incorrect number of files were added, expected: 2, actual: %v", numFiles) + } +} + +func TestSnapshotChangePermissions(t *testing.T) { + testDir, snapshotter, err := setUpTestDir() + defer os.RemoveAll(testDir) + if err != nil { + t.Fatal(err) + } + // Change permissions on a file + batPath := filepath.Join(testDir, "bar/bat") + if err := os.Chmod(batPath, 0600); err != nil { + t.Fatalf("Error changing permissions on %s: %v", batPath, err) + } + // Take another snapshot + contents, filesAdded, err := snapshotter.TakeSnapshot() + if err != nil { + t.Fatalf("Error taking snapshot of fs: %s", err) + } + if !filesAdded { + t.Fatal("No files added to snapshot.") + } + // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles + reader := bytes.NewReader(contents) + tr := tar.NewReader(reader) + snapshotFiles := map[string]string{ + batPath: "baz2", + } + numFiles := 0 + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + numFiles++ + if _, isFile := snapshotFiles[hdr.Name]; !isFile { + t.Fatalf("File %s unexpectedly in tar", hdr.Name) + } + contents, _ := ioutil.ReadAll(tr) + if string(contents) != snapshotFiles[hdr.Name] { + t.Fatalf("Contents of %s incorrect, expected: %s, actual: %s", hdr.Name, snapshotFiles[hdr.Name], string(contents)) + } + } + if numFiles != 1 { + t.Fatalf("Incorrect number of files were added, expected: 1, got: %v", numFiles) + } +} + +func TestSnapshotFiles(t *testing.T) { + testDir, snapshotter, err := setUpTestDir() + defer os.RemoveAll(testDir) + if err != nil { + t.Fatal(err) + } + // Make some changes to the filesystem + newFiles := map[string]string{ + "foo": "newbaz1", + "work-dir/file": "bat", + } + if err := testutil.SetupFiles(testDir, newFiles); err != nil { + t.Fatalf("Error setting up fs: %s", err) + } + filesToSnapshot := []string{ + filepath.Join(testDir, "foo"), + filepath.Join(testDir, "work-dir/file"), + } + contents, err := snapshotter.TakeSnapshotOfFiles(filesToSnapshot) + if err != nil { + t.Fatal(err) + } + expectedContents := map[string]string{ + filepath.Join(testDir, "foo"): "newbaz1", + } + // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles + reader := bytes.NewReader(contents) + tr := tar.NewReader(reader) + numFiles := 0 + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + numFiles = numFiles + 1 + if _, isFile := expectedContents[hdr.Name]; !isFile { + t.Fatalf("File %s unexpectedly in tar", hdr.Name) + } + contents, _ := ioutil.ReadAll(tr) + if string(contents) != expectedContents[hdr.Name] { + t.Fatalf("Contents of %s incorrect, expected: %s, actual: %s", hdr.Name, expectedContents[hdr.Name], string(contents)) + } + } + if numFiles != 1 { + t.Fatalf("%s was not added.", filepath.Join(testDir, "foo")) + } +} + +func TestEmptySnapshot(t *testing.T) { + testDir, snapshotter, err := setUpTestDir() + defer os.RemoveAll(testDir) + if err != nil { + t.Fatal(err) + } + // Take snapshot with no changes + _, filesAdded, err := snapshotter.TakeSnapshot() + if err != nil { + t.Fatalf("Error taking snapshot of fs: %s", err) + } + // Since we took a snapshot with no changes, contents should be nil + if filesAdded { + t.Fatal("Files added even though no changes to file system were made.") + } +} + +func setUpTestDir() (string, *Snapshotter, error) { + testDir, err := ioutil.TempDir("", "") + if err != nil { + return testDir, nil, errors.Wrap(err, "setting up temp dir") + } + files := map[string]string{ + "foo": "baz1", + "bar/bat": "baz2", + "work-dir/file": "file", + } + // Set up initial files + if err := testutil.SetupFiles(testDir, files); err != nil { + return testDir, nil, errors.Wrap(err, "setting up file system") + } + + // Take the initial snapshot + l := NewLayeredMap(util.Hasher()) + snapshotter := NewSnapshotter(l, testDir) + if err := snapshotter.Init(); err != nil { + return testDir, nil, errors.Wrap(err, "initializing snapshotter") + } + return testDir, snapshotter, nil +} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index e27e79ffb..7e767b8d5 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -24,10 +24,11 @@ import ( "github.com/sirupsen/logrus" "io" "os" + "path/filepath" "strings" ) -var whitelist = []string{"/workspace"} +var whitelist = []string{"/work-dir"} // ExtractFileSystemFromImage pulls an image and unpacks it to a file system at root func ExtractFileSystemFromImage(img string) error { @@ -47,6 +48,17 @@ func ExtractFileSystemFromImage(img string) error { return pkgutil.GetFileSystemFromReference(ref, imgSrc, constants.RootDir, whitelist) } +// PathInWhitelist returns true if the path is whitelisted +func PathInWhitelist(path, directory string) bool { + for _, d := range whitelist { + dirPath := filepath.Join(directory, d) + if pkgutil.HasFilepathPrefix(path, dirPath) { + return true + } + } + return false +} + // Get whitelist from roots of mounted files // Each line of /proc/self/mountinfo is in the form: // 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index b04c113d6..a474794c4 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -46,7 +46,7 @@ func Test_fileSystemWhitelist(t *testing.T) { } actualWhitelist, err := fileSystemWhitelist(path) - expectedWhitelist := []string{"/workspace", "/proc", "/dev", "/dev/pts", "/sys"} + expectedWhitelist := []string{"/work-dir", "/proc", "/dev", "/dev/pts", "/sys"} sort.Strings(actualWhitelist) sort.Strings(expectedWhitelist) testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist) diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go new file mode 100644 index 000000000..30e036c38 --- /dev/null +++ b/pkg/util/tar_util.go @@ -0,0 +1,52 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "archive/tar" + "io" + "os" +) + +// AddToTar adds the file i to tar w at path p +func AddToTar(p string, i os.FileInfo, w *tar.Writer) error { + linkDst := "" + if i.Mode()&os.ModeSymlink != 0 { + var err error + linkDst, err = os.Readlink(p) + if err != nil { + return err + } + } + hdr, err := tar.FileInfoHeader(i, linkDst) + if err != nil { + return err + } + hdr.Name = p + w.WriteHeader(hdr) + if !i.Mode().IsRegular() { + return nil + } + r, err := os.Open(p) + if err != nil { + return err + } + if _, err := io.Copy(w, r); err != nil { + return err + } + return nil +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 30dacba57..f27ebba5a 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -17,8 +17,12 @@ limitations under the License. package util import ( + "crypto/md5" + "encoding/hex" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "io" + "os" ) // SetLogLevel sets the logrus logging level @@ -30,3 +34,30 @@ func SetLogLevel(logLevel string) error { logrus.SetLevel(lvl) return nil } + +// Hasher returns a hash function, used in snapshotting to determine if a file has changed +func Hasher() func(string) (string, error) { + hasher := func(p string) (string, error) { + h := md5.New() + fi, err := os.Lstat(p) + if err != nil { + return "", err + } + h.Write([]byte(fi.Mode().String())) + h.Write([]byte(fi.ModTime().String())) + + if fi.Mode().IsRegular() { + f, err := os.Open(p) + if err != nil { + return "", err + } + defer f.Close() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + } + + return hex.EncodeToString(h.Sum(nil)), nil + } + return hasher +} diff --git a/testutil/util.go b/testutil/util.go index 30b495405..45607a772 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -18,10 +18,27 @@ package testutil import ( "fmt" + "io/ioutil" + "os" + "path/filepath" "reflect" "testing" ) +// SetupFiles creates files at path +func SetupFiles(path string, files map[string]string) error { + for p, c := range files { + path := filepath.Join(path, p) + if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil { + return err + } + if err := ioutil.WriteFile(path, []byte(c), 0644); err != nil { + return err + } + } + return nil +} + func CheckErrorAndDeepEqual(t *testing.T, shouldErr bool, err error, expected, actual interface{}) { if err := checkErr(shouldErr, err); err != nil { t.Error(err)