diff --git a/Gopkg.lock b/Gopkg.lock index 1fd8e4e4c..8f3631a98 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -15,7 +15,7 @@ "pkg/image", "pkg/util" ] - revision = "ce3228b4afba2b2fb9b42c416c8f59a14ee7c1dd" + revision = "6a521891eafa833a08adf664edb6e67b18220ea7" source = "github.com/GoogleCloudPlatform/container-diff" [[projects]] @@ -119,7 +119,7 @@ "pkg/system", "pkg/truncindex" ] - revision = "66a38bb219e9776482c18e8c02f438e5112916f1" + revision = "1e5ce40cdb84ab66e26186435b1273e04b879fef" [[projects]] branch = "master" @@ -370,7 +370,7 @@ "nfs", "xfs" ] - revision = "d274e363d5759d1c916232217be421f1cc89c5fe" + revision = "1c7ff3de94ae006f58cba483a4c9c6d7c61e1d98" [[projects]] name = "github.com/sirupsen/logrus" @@ -419,7 +419,7 @@ "openpgp/s2k", "ssh/terminal" ] - revision = "91a49db82a88618983a78a06c1cbd4e00ab749ab" + revision = "85f98707c97e11569271e4d9b3d397e079c4f4d0" [[projects]] branch = "master" @@ -433,7 +433,7 @@ "lex/httplex", "proxy" ] - revision = "22ae77b79946ea320088417e4d50825671d82d57" + revision = "07e8617a6db2368fa55d4616f371ee1b1403c817" [[projects]] branch = "master" @@ -442,7 +442,7 @@ "unix", "windows" ] - revision = "f6cff0780e542efa0c8e864dc8fa522808f6a598" + revision = "dd2ff4accc098aceecb86b36eaa7829b2a17b1c9" [[projects]] name = "golang.org/x/text" @@ -486,6 +486,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d806d861c987e81225e9da4da86d9417d01fd42d84dbe34e21daf076651e17b5" + inputs-digest = "fd21de0404336debb893db778210835a27a3612fe9b9e5e412dcdc80d288a986" solver-name = "gps-cdcl" solver-version = 1 diff --git a/executor/cmd/root.go b/executor/cmd/root.go index 68d39d65f..a34a5f3cd 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -17,6 +17,7 @@ limitations under the License. package cmd import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/commands" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/constants" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/dockerfile" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/image" @@ -88,8 +89,39 @@ func execute() error { return err } - // Execute commands here + // Set environment variables within the image + if err := image.SetEnvVariables(sourceImage); err != nil { + return err + } + imageConfig := sourceImage.Config() + // Currently only supports single stage builds + for _, stage := range stages { + for _, cmd := range stage.Commands { + dockerCommand, err := commands.GetCommand(cmd) + if err != nil { + return err + } + if err := dockerCommand.ExecuteCommand(imageConfig); err != nil { + return err + } + // Now, we get the files to snapshot from this command and take the snapshot + snapshotFiles := dockerCommand.FilesToSnapshot() + contents, err := snapshotter.TakeSnapshot(snapshotFiles) + if err != nil { + return err + } + if contents == nil { + logrus.Info("No files were changed, appending empty layer to config.") + sourceImage.AppendConfigHistory(constants.Author, true) + continue + } + // Append the layer to the image + if err := sourceImage.AppendLayer(contents, constants.Author); err != nil { + return err + } + } + } // Push the image return image.PushImage(sourceImage, destination) } diff --git a/integration_tests/dockerfiles/Dockerfile_test_run b/integration_tests/dockerfiles/Dockerfile_test_run new file mode 100644 index 000000000..cd225fc03 --- /dev/null +++ b/integration_tests/dockerfiles/Dockerfile_test_run @@ -0,0 +1,19 @@ +# Copyright 2018 Google, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/google-appengine/debian9 +RUN echo "hey" > /etc/foo +RUN apt-get update && apt-get install -y \ + bzr \ + cvs \ diff --git a/integration_tests/dockerfiles/Dockerfile_test_run_2 b/integration_tests/dockerfiles/Dockerfile_test_run_2 new file mode 100644 index 000000000..b77ea9ea0 --- /dev/null +++ b/integration_tests/dockerfiles/Dockerfile_test_run_2 @@ -0,0 +1,19 @@ +# Copyright 2018 Google, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test to make sure the executor builds an image correctly +# when no files are changed + +FROM gcr.io/google-appengine/debian9 +RUN echo "hey" diff --git a/integration_tests/dockerfiles/config_test_run.json b/integration_tests/dockerfiles/config_test_run.json new file mode 100644 index 000000000..0544a3d22 --- /dev/null +++ b/integration_tests/dockerfiles/config_test_run.json @@ -0,0 +1,48 @@ +[ + { + "Image1": "gcr.io/kbuild-test/docker-test-run:latest", + "Image2": "gcr.io/kbuild-test/kbuild-test-run:latest", + "DiffType": "File", + "Diff": { + "Adds": null, + "Dels": null, + "Mods": [ + { + "Name": "/var/log/dpkg.log", + "Size1": 57425, + "Size2": 57425 + }, + { + "Name": "/var/log/apt/term.log", + "Size1": 24400, + "Size2": 24400 + }, + { + "Name": "/var/cache/ldconfig/aux-cache", + "Size1": 8057, + "Size2": 8057 + }, + { + "Name": "/var/log/apt/history.log", + "Size1": 5089, + "Size2": 5089 + }, + { + "Name": "/var/log/alternatives.log", + "Size1": 2579, + "Size2": 2579 + }, + { + "Name": "/usr/lib/python2.7/dist-packages/keyrings/__init__.pyc", + "Size1": 140, + "Size2": 140 + }, + { + "Name": "/usr/lib/python2.7/dist-packages/lazr/__init__.pyc", + "Size1": 136, + "Size2": 136 + } + ] + } + } +] \ No newline at end of file diff --git a/integration_tests/dockerfiles/config_test_run_2.json b/integration_tests/dockerfiles/config_test_run_2.json new file mode 100644 index 000000000..0301973df --- /dev/null +++ b/integration_tests/dockerfiles/config_test_run_2.json @@ -0,0 +1,12 @@ +[ + { + "Image1": "gcr.io/kbuild-test/docker-test-run-2:latest", + "Image2": "gcr.io/kbuild-test/kbuild-test-run-2:latest", + "DiffType": "File", + "Diff": { + "Adds": null, + "Dels": null, + "Mods": null + } + } +] \ No newline at end of file diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index 7c59e2e08..d13be52e9 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -35,6 +35,20 @@ var tests = []struct { context: "integration_tests/dockerfiles/", repo: "extract-filesystem", }, + { + description: "test run", + dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run", + configPath: "/workspace/integration_tests/dockerfiles/config_test_run.json", + context: "integration_tests/dockerfiles/", + repo: "test-run", + }, + { + description: "test run no files changed", + dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run_2", + configPath: "/workspace/integration_tests/dockerfiles/config_test_run_2.json", + context: "integration_tests/dockerfiles/", + repo: "test-run-2", + }, } type step struct { diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go new file mode 100644 index 000000000..eb2cab5bc --- /dev/null +++ b/pkg/commands/commands.go @@ -0,0 +1,43 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/pkg/errors" +) + +type DockerCommand interface { + // ExecuteCommand is responsible for: + // 1. Making required changes to the filesystem (ex. copying files for ADD/COPY or setting ENV variables) + // 2. Updating metadata fields in the config + // It should not change the config history. + ExecuteCommand(*manifest.Schema2Config) error + // The config history has a "created by" field, should return information about the command + CreatedBy() string + // A list of files to snapshot, empty for metadata commands or nil if we don't know + FilesToSnapshot() []string +} + +func GetCommand(cmd instructions.Command) (DockerCommand, error) { + switch c := cmd.(type) { + case *instructions.RunCommand: + return &RunCommand{cmd: c}, nil + } + return nil, errors.Errorf("%s is not a supported command", cmd.Name()) +} diff --git a/pkg/commands/run.go b/pkg/commands/run.go new file mode 100644 index 000000000..b08cf8800 --- /dev/null +++ b/pkg/commands/run.go @@ -0,0 +1,66 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/sirupsen/logrus" + "os" + "os/exec" + "strings" +) + +type RunCommand struct { + cmd *instructions.RunCommand +} + +func (r *RunCommand) ExecuteCommand(config *manifest.Schema2Config) error { + var newCommand []string + if r.cmd.PrependShell { + // This is the default shell on Linux + // TODO: Support shell command here + shell := []string{"/bin/sh", "-c"} + newCommand = append(shell, strings.Join(r.cmd.CmdLine, " ")) + } else { + newCommand = r.cmd.CmdLine + } + + logrus.Infof("cmd: %s", newCommand[0]) + logrus.Infof("args: %s", newCommand[1:]) + + cmd := exec.Command(newCommand[0], newCommand[1:]...) + cmd.Stdout = os.Stdout + return cmd.Run() +} + +// FilesToSnapshot returns nil for this command because we don't know which files +// have changed, so we snapshot the entire system. +func (r *RunCommand) FilesToSnapshot() []string { + return nil +} + +// Author returns some information about the command for the image config +func (r *RunCommand) CreatedBy() string { + cmdLine := strings.Join(r.cmd.CmdLine, " ") + if r.cmd.PrependShell { + // TODO: Support shell command here + shell := []string{"/bin/sh", "-c"} + return strings.Join(append(shell, cmdLine), " ") + } + return cmdLine +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 8a619ed1e..5845194bd 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -27,4 +27,6 @@ const ( WorkspaceDir = "/workspace" WhitelistPath = "/proc/self/mountinfo" + + Author = "kbuild" ) diff --git a/pkg/image/image.go b/pkg/image/image.go index a07fe973f..f458ab1f1 100644 --- a/pkg/image/image.go +++ b/pkg/image/image.go @@ -23,6 +23,7 @@ import ( "github.com/containers/image/signature" "github.com/containers/image/transports/alltransports" "github.com/sirupsen/logrus" + "os" ) // sourceImage is the image that will be modified by the executor @@ -55,6 +56,18 @@ func PushImage(ms *img.MutableSource, destImg string) error { return copy.Image(policyContext, destRef, srcRef, nil) } +// SetEnvVariables sets environment variables as specified in the image +func SetEnvVariables(ms *img.MutableSource) error { + envVars := ms.Env() + for key, val := range envVars { + if err := os.Setenv(key, val); err != nil { + return err + } + logrus.Debugf("Setting environment variable %s=%s", key, val) + } + return nil +} + func getPolicyContext() (*signature.PolicyContext, error) { policyContext, err := signature.NewPolicyContext(&signature.Policy{ Default: signature.PolicyRequirements{signature.NewPRInsecureAcceptAnything()}, diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 05cdd8a58..c75bf4296 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -49,23 +49,30 @@ func (s *Snapshotter) Init() error { // TakeSnapshot takes a snapshot of the filesystem, avoiding directories in the whitelist, and creates // a tarball of the changed files. Return contents of the tarball, and whether or not any files were changed -func (s *Snapshotter) TakeSnapshot() ([]byte, bool, error) { +func (s *Snapshotter) TakeSnapshot(files []string) ([]byte, error) { + if files != nil { + return s.TakeSnapshotOfFiles(files) + } + logrus.Info("Taking snapshot of full filesystem...") buf := bytes.NewBuffer([]byte{}) filesAdded, err := s.snapShotFS(buf) if err != nil { - return nil, filesAdded, err + return nil, err } contents, err := ioutil.ReadAll(buf) if err != nil { - return nil, filesAdded, err + return nil, err } - return contents, filesAdded, err + if !filesAdded { + return nil, nil + } + return contents, err } // TakeSnapshotOfFiles takes a snapshot of specific files // Used for ADD/COPY commands, when we know which files have changed func (s *Snapshotter) TakeSnapshotOfFiles(files []string) ([]byte, error) { - logrus.Infof("Taking snapshot of files %s", files) + logrus.Infof("Taking snapshot of files %v...", files) s.l.Snapshot() if len(files) == 0 { logrus.Info("No files changed in this command, skipping snapshotting.") diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index 5e51d88bc..ef77e6f81 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -45,11 +45,11 @@ func TestSnapshotFileChange(t *testing.T) { t.Fatalf("Error setting up fs: %s", err) } // Take another snapshot - contents, filesAdded, err := snapshotter.TakeSnapshot() + contents, err := snapshotter.TakeSnapshot(nil) if err != nil { t.Fatalf("Error taking snapshot of fs: %s", err) } - if !filesAdded { + if contents == nil { t.Fatal("No files added to snapshot.") } // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles @@ -93,11 +93,11 @@ func TestSnapshotChangePermissions(t *testing.T) { t.Fatalf("Error changing permissions on %s: %v", batPath, err) } // Take another snapshot - contents, filesAdded, err := snapshotter.TakeSnapshot() + contents, err := snapshotter.TakeSnapshot(nil) if err != nil { t.Fatalf("Error taking snapshot of fs: %s", err) } - if !filesAdded { + if contents == nil { t.Fatal("No files added to snapshot.") } // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles @@ -144,7 +144,7 @@ func TestSnapshotFiles(t *testing.T) { filepath.Join(testDir, "foo"), filepath.Join(testDir, "kbuild/file"), } - contents, err := snapshotter.TakeSnapshotOfFiles(filesToSnapshot) + contents, err := snapshotter.TakeSnapshot(filesToSnapshot) if err != nil { t.Fatal(err) } @@ -181,12 +181,12 @@ func TestEmptySnapshot(t *testing.T) { t.Fatal(err) } // Take snapshot with no changes - _, filesAdded, err := snapshotter.TakeSnapshot() + contents, err := snapshotter.TakeSnapshot(nil) if err != nil { t.Fatalf("Error taking snapshot of fs: %s", err) } // Since we took a snapshot with no changes, contents should be nil - if filesAdded { + if contents != nil { t.Fatal("Files added even though no changes to file system were made.") } } diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go index 30e036c38..417f16f04 100644 --- a/pkg/util/tar_util.go +++ b/pkg/util/tar_util.go @@ -18,10 +18,14 @@ package util import ( "archive/tar" + "github.com/sirupsen/logrus" "io" "os" + "syscall" ) +var hardlinks = make(map[uint64]string) + // AddToTar adds the file i to tar w at path p func AddToTar(p string, i os.FileInfo, w *tar.Writer) error { linkDst := "" @@ -37,16 +41,48 @@ func AddToTar(p string, i os.FileInfo, w *tar.Writer) error { return err } hdr.Name = p - w.WriteHeader(hdr) - if !i.Mode().IsRegular() { + + hardlink, linkDst := checkHardlink(p, i) + if hardlink { + hdr.Linkname = linkDst + hdr.Typeflag = tar.TypeLink + hdr.Size = 0 + } + if err := w.WriteHeader(hdr); err != nil { + return err + } + if !(i.Mode().IsRegular()) || hardlink { return nil } r, err := os.Open(p) if err != nil { return err } + defer r.Close() if _, err := io.Copy(w, r); err != nil { return err } return nil } + +// Returns true if path is hardlink, and the link destination +func checkHardlink(p string, i os.FileInfo) (bool, string) { + hardlink := false + linkDst := "" + if sys := i.Sys(); sys != nil { + if stat, ok := sys.(*syscall.Stat_t); ok { + nlinks := stat.Nlink + if nlinks > 1 { + inode := stat.Ino + if original, exists := hardlinks[inode]; exists && original != p { + hardlink = true + logrus.Debugf("%s inode exists in hardlinks map, linking to %s", p, original) + linkDst = original + } else { + hardlinks[inode] = p + } + } + } + } + return hardlink, linkDst +} diff --git a/vendor/github.com/GoogleCloudPlatform/container-diff/pkg/image/mutable_source.go b/vendor/github.com/GoogleCloudPlatform/container-diff/pkg/image/mutable_source.go index fcd4521da..2c7e2795a 100644 --- a/vendor/github.com/GoogleCloudPlatform/container-diff/pkg/image/mutable_source.go +++ b/vendor/github.com/GoogleCloudPlatform/container-diff/pkg/image/mutable_source.go @@ -142,7 +142,7 @@ func (m *MutableSource) AppendLayer(content []byte, author string) error { // Also add it to the config. diffID := digest.FromBytes(content) m.cfg.RootFS.DiffIDs = append(m.cfg.RootFS.DiffIDs, diffID) - m.appendConfigHistory(author, false) + m.AppendConfigHistory(author, false) return nil } @@ -184,7 +184,7 @@ func (m *MutableSource) SetEnv(envMap map[string]string, author string) { envArray = append(envArray, entry) } m.cfg.Schema2V1Image.Config.Env = envArray - m.appendConfigHistory(author, true) + m.AppendConfigHistory(author, true) } func (m *MutableSource) Config() *manifest.Schema2Config { @@ -193,10 +193,10 @@ func (m *MutableSource) Config() *manifest.Schema2Config { func (m *MutableSource) SetConfig(config *manifest.Schema2Config, author string, emptyLayer bool) { m.cfg.Schema2V1Image.Config = config - m.appendConfigHistory(author, emptyLayer) + m.AppendConfigHistory(author, emptyLayer) } -func (m *MutableSource) appendConfigHistory(author string, emptyLayer bool) { +func (m *MutableSource) AppendConfigHistory(author string, emptyLayer bool) { history := manifest.Schema2History{ Created: time.Now(), Author: author, diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 02dd7a1b3..a31a08b2a 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -2276,14 +2276,15 @@ func (s *store) Shutdown(force bool) ([]string, error) { return mounted, err } + s.graphLock.Lock() + defer s.graphLock.Unlock() + rlstore.Lock() defer rlstore.Unlock() if modified, err := rlstore.Modified(); modified || err != nil { rlstore.Load() } - s.graphLock.Lock() - defer s.graphLock.Unlock() layers, err := rlstore.Layers() if err != nil { return mounted, err diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go b/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go index 9a8e6e411..c464783d8 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go @@ -23,8 +23,11 @@ package unix //sys Seek(fd int, offset int64, whence int) (off int64, err error) = SYS_LSEEK func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err error) { - ts := Timespec{Sec: timeout.Sec, Nsec: timeout.Usec * 1000} - return Pselect(nfd, r, w, e, &ts, nil) + var ts *Timespec + if timeout != nil { + ts = &Timespec{Sec: timeout.Sec, Nsec: timeout.Usec * 1000} + } + return Pselect(nfd, r, w, e, ts, nil) } //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go b/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go index 46aa4ff9c..15a69cbdd 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go @@ -26,8 +26,11 @@ package unix //sys Seek(fd int, offset int64, whence int) (off int64, err error) = SYS_LSEEK func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err error) { - ts := Timespec{Sec: timeout.Sec, Nsec: timeout.Usec * 1000} - return Pselect(nfd, r, w, e, &ts, nil) + var ts *Timespec + if timeout != nil { + ts = &Timespec{Sec: timeout.Sec, Nsec: timeout.Usec * 1000} + } + return Pselect(nfd, r, w, e, ts, nil) } //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error)