Merge pull request #33 from priyawadhwa/run

Run command

Gopkg.lock

@@ -15,7 +15,7 @@
     "pkg/image",
     "pkg/util"
   ]
-  revision = "ce3228b4afba2b2fb9b42c416c8f59a14ee7c1dd"
+  revision = "6a521891eafa833a08adf664edb6e67b18220ea7"
   source = "github.com/GoogleCloudPlatform/container-diff"
 
 [[projects]]
@@ -119,7 +119,7 @@
     "pkg/system",
     "pkg/truncindex"
   ]
-  revision = "66a38bb219e9776482c18e8c02f438e5112916f1"
+  revision = "1e5ce40cdb84ab66e26186435b1273e04b879fef"
 
 [[projects]]
   branch = "master"
@@ -370,7 +370,7 @@
     "nfs",
     "xfs"
   ]
-  revision = "d274e363d5759d1c916232217be421f1cc89c5fe"
+  revision = "1c7ff3de94ae006f58cba483a4c9c6d7c61e1d98"
 
 [[projects]]
   name = "github.com/sirupsen/logrus"
@@ -419,7 +419,7 @@
     "openpgp/s2k",
     "ssh/terminal"
   ]
-  revision = "91a49db82a88618983a78a06c1cbd4e00ab749ab"
+  revision = "85f98707c97e11569271e4d9b3d397e079c4f4d0"
 
 [[projects]]
   branch = "master"
@@ -433,7 +433,7 @@
     "lex/httplex",
     "proxy"
   ]
-  revision = "22ae77b79946ea320088417e4d50825671d82d57"
+  revision = "07e8617a6db2368fa55d4616f371ee1b1403c817"
 
 [[projects]]
   branch = "master"
@@ -442,7 +442,7 @@
     "unix",
     "windows"
   ]
-  revision = "f6cff0780e542efa0c8e864dc8fa522808f6a598"
+  revision = "dd2ff4accc098aceecb86b36eaa7829b2a17b1c9"
 
 [[projects]]
   name = "golang.org/x/text"
@@ -486,6 +486,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "d806d861c987e81225e9da4da86d9417d01fd42d84dbe34e21daf076651e17b5"
+  inputs-digest = "fd21de0404336debb893db778210835a27a3612fe9b9e5e412dcdc80d288a986"
   solver-name = "gps-cdcl"
   solver-version = 1

executor/cmd/root.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cmd
 
 import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/commands"
 	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/constants"
 	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/dockerfile"
 	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/image"
@@ -88,8 +89,39 @@ func execute() error {
 		return err
 	}
 
-	// Execute commands here
+	// Set environment variables within the image
+	if err := image.SetEnvVariables(sourceImage); err != nil {
+		return err
+	}
 
+	imageConfig := sourceImage.Config()
+	// Currently only supports single stage builds
+	for _, stage := range stages {
+		for _, cmd := range stage.Commands {
+			dockerCommand, err := commands.GetCommand(cmd)
+			if err != nil {
+				return err
+			}
+			if err := dockerCommand.ExecuteCommand(imageConfig); err != nil {
+				return err
+			}
+			// Now, we get the files to snapshot from this command and take the snapshot
+			snapshotFiles := dockerCommand.FilesToSnapshot()
+			contents, err := snapshotter.TakeSnapshot(snapshotFiles)
+			if err != nil {
+				return err
+			}
+			if contents == nil {
+				logrus.Info("No files were changed, appending empty layer to config.")
+				sourceImage.AppendConfigHistory(constants.Author, true)
+				continue
+			}
+			// Append the layer to the image
+			if err := sourceImage.AppendLayer(contents, constants.Author); err != nil {
+				return err
+			}
+		}
+	}
 	// Push the image
 	return image.PushImage(sourceImage, destination)
 }

integration_tests/dockerfiles/Dockerfile_test_run

@@ -0,0 +1,19 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/google-appengine/debian9
+RUN echo "hey" > /etc/foo
+RUN apt-get update && apt-get install -y \
+  bzr \
+  cvs \

integration_tests/dockerfiles/Dockerfile_test_run_2

@@ -0,0 +1,19 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test to make sure the executor builds an image correctly
+# when no files are changed
+
+FROM gcr.io/google-appengine/debian9
+RUN echo "hey" 

integration_tests/dockerfiles/config_test_run.json

@@ -0,0 +1,48 @@
+[
+  {
+    "Image1": "gcr.io/kbuild-test/docker-test-run:latest",
+    "Image2": "gcr.io/kbuild-test/kbuild-test-run:latest",
+    "DiffType": "File",
+    "Diff": {
+      "Adds": null,
+      "Dels": null,
+      "Mods": [
+        {
+          "Name": "/var/log/dpkg.log",
+          "Size1": 57425,
+          "Size2": 57425
+        },
+        {
+          "Name": "/var/log/apt/term.log",
+          "Size1": 24400,
+          "Size2": 24400
+        },
+        {
+          "Name": "/var/cache/ldconfig/aux-cache",
+          "Size1": 8057,
+          "Size2": 8057
+        },
+        {
+          "Name": "/var/log/apt/history.log",
+          "Size1": 5089,
+          "Size2": 5089
+        },
+        {
+          "Name": "/var/log/alternatives.log",
+          "Size1": 2579,
+          "Size2": 2579
+        },
+        {
+          "Name": "/usr/lib/python2.7/dist-packages/keyrings/__init__.pyc",
+          "Size1": 140,
+          "Size2": 140
+        },
+        {
+          "Name": "/usr/lib/python2.7/dist-packages/lazr/__init__.pyc",
+          "Size1": 136,
+          "Size2": 136
+        }
+      ]
+    }
+  }
+]

integration_tests/dockerfiles/config_test_run_2.json

@@ -0,0 +1,12 @@
+[
+  {
+    "Image1": "gcr.io/kbuild-test/docker-test-run-2:latest",
+    "Image2": "gcr.io/kbuild-test/kbuild-test-run-2:latest",
+    "DiffType": "File",
+    "Diff": {
+      "Adds": null,
+      "Dels": null,
+      "Mods": null
+    }
+  }
+]

integration_tests/integration_test_yaml.go

@@ -35,6 +35,20 @@ var tests = []struct {
 		context:        "integration_tests/dockerfiles/",
 		repo:           "extract-filesystem",
 	},
+	{
+		description:    "test run",
+		dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run",
+		configPath:     "/workspace/integration_tests/dockerfiles/config_test_run.json",
+		context:        "integration_tests/dockerfiles/",
+		repo:           "test-run",
+	},
+	{
+		description:    "test run no files changed",
+		dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run_2",
+		configPath:     "/workspace/integration_tests/dockerfiles/config_test_run_2.json",
+		context:        "integration_tests/dockerfiles/",
+		repo:           "test-run-2",
+	},
 }
 
 type step struct {

pkg/commands/commands.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/pkg/errors"
+)
+
+type DockerCommand interface {
+	// ExecuteCommand is responsible for:
+	// 	1. Making required changes to the filesystem (ex. copying files for ADD/COPY or setting ENV variables)
+	//  2. Updating metadata fields in the config
+	// It should not change the config history.
+	ExecuteCommand(*manifest.Schema2Config) error
+	// The config history has a "created by" field, should return information about the command
+	CreatedBy() string
+	// A list of files to snapshot, empty for metadata commands or nil if we don't know
+	FilesToSnapshot() []string
+}
+
+func GetCommand(cmd instructions.Command) (DockerCommand, error) {
+	switch c := cmd.(type) {
+	case *instructions.RunCommand:
+		return &RunCommand{cmd: c}, nil
+	}
+	return nil, errors.Errorf("%s is not a supported command", cmd.Name())
+}

pkg/commands/run.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+type RunCommand struct {
+	cmd *instructions.RunCommand
+}
+
+func (r *RunCommand) ExecuteCommand(config *manifest.Schema2Config) error {
+	var newCommand []string
+	if r.cmd.PrependShell {
+		// This is the default shell on Linux
+		// TODO: Support shell command here
+		shell := []string{"/bin/sh", "-c"}
+		newCommand = append(shell, strings.Join(r.cmd.CmdLine, " "))
+	} else {
+		newCommand = r.cmd.CmdLine
+	}
+
+	logrus.Infof("cmd: %s", newCommand[0])
+	logrus.Infof("args: %s", newCommand[1:])
+
+	cmd := exec.Command(newCommand[0], newCommand[1:]...)
+	cmd.Stdout = os.Stdout
+	return cmd.Run()
+}
+
+// FilesToSnapshot returns nil for this command because we don't know which files
+// have changed, so we snapshot the entire system.
+func (r *RunCommand) FilesToSnapshot() []string {
+	return nil
+}
+
+// Author returns some information about the command for the image config
+func (r *RunCommand) CreatedBy() string {
+	cmdLine := strings.Join(r.cmd.CmdLine, " ")
+	if r.cmd.PrependShell {
+		// TODO: Support shell command here
+		shell := []string{"/bin/sh", "-c"}
+		return strings.Join(append(shell, cmdLine), " ")
+	}
+	return cmdLine
+}

pkg/constants/constants.go

@@ -27,4 +27,6 @@ const (
 	WorkspaceDir = "/workspace"
 
 	WhitelistPath = "/proc/self/mountinfo"
+
+	Author = "kbuild"
 )

pkg/image/image.go

@@ -23,6 +23,7 @@ import (
 	"github.com/containers/image/signature"
 	"github.com/containers/image/transports/alltransports"
 	"github.com/sirupsen/logrus"
+	"os"
 )
 
 // sourceImage is the image that will be modified by the executor
@@ -55,6 +56,18 @@ func PushImage(ms *img.MutableSource, destImg string) error {
 	return copy.Image(policyContext, destRef, srcRef, nil)
 }
 
+// SetEnvVariables sets environment variables as specified in the image
+func SetEnvVariables(ms *img.MutableSource) error {
+	envVars := ms.Env()
+	for key, val := range envVars {
+		if err := os.Setenv(key, val); err != nil {
+			return err
+		}
+		logrus.Debugf("Setting environment variable %s=%s", key, val)
+	}
+	return nil
+}
+
 func getPolicyContext() (*signature.PolicyContext, error) {
 	policyContext, err := signature.NewPolicyContext(&signature.Policy{
 		Default: signature.PolicyRequirements{signature.NewPRInsecureAcceptAnything()},

pkg/snapshot/snapshot.go

@@ -49,23 +49,30 @@ func (s *Snapshotter) Init() error {
 
 // TakeSnapshot takes a snapshot of the filesystem, avoiding directories in the whitelist, and creates
 // a tarball of the changed files. Return contents of the tarball, and whether or not any files were changed
-func (s *Snapshotter) TakeSnapshot() ([]byte, bool, error) {
+func (s *Snapshotter) TakeSnapshot(files []string) ([]byte, error) {
+	if files != nil {
+		return s.TakeSnapshotOfFiles(files)
+	}
+	logrus.Info("Taking snapshot of full filesystem...")
 	buf := bytes.NewBuffer([]byte{})
 	filesAdded, err := s.snapShotFS(buf)
 	if err != nil {
-		return nil, filesAdded, err
+		return nil, err
 	}
 	contents, err := ioutil.ReadAll(buf)
 	if err != nil {
-		return nil, filesAdded, err
+		return nil, err
 	}
-	return contents, filesAdded, err
+	if !filesAdded {
+		return nil, nil
+	}
+	return contents, err
 }
 
 // TakeSnapshotOfFiles takes a snapshot of specific files
 // Used for ADD/COPY commands, when we know which files have changed
 func (s *Snapshotter) TakeSnapshotOfFiles(files []string) ([]byte, error) {
-	logrus.Infof("Taking snapshot of files %s", files)
+	logrus.Infof("Taking snapshot of files %v...", files)
 	s.l.Snapshot()
 	if len(files) == 0 {
 		logrus.Info("No files changed in this command, skipping snapshotting.")

pkg/snapshot/snapshot_test.go

@@ -45,11 +45,11 @@ func TestSnapshotFileChange(t *testing.T) {
 		t.Fatalf("Error setting up fs: %s", err)
 	}
 	// Take another snapshot
-	contents, filesAdded, err := snapshotter.TakeSnapshot()
+	contents, err := snapshotter.TakeSnapshot(nil)
 	if err != nil {
 		t.Fatalf("Error taking snapshot of fs: %s", err)
 	}
-	if !filesAdded {
+	if contents == nil {
 		t.Fatal("No files added to snapshot.")
 	}
 	// Check contents of the snapshot, make sure contents is equivalent to snapshotFiles
@@ -93,11 +93,11 @@ func TestSnapshotChangePermissions(t *testing.T) {
 		t.Fatalf("Error changing permissions on %s: %v", batPath, err)
 	}
 	// Take another snapshot
-	contents, filesAdded, err := snapshotter.TakeSnapshot()
+	contents, err := snapshotter.TakeSnapshot(nil)
 	if err != nil {
 		t.Fatalf("Error taking snapshot of fs: %s", err)
 	}
-	if !filesAdded {
+	if contents == nil {
 		t.Fatal("No files added to snapshot.")
 	}
 	// Check contents of the snapshot, make sure contents is equivalent to snapshotFiles
@@ -144,7 +144,7 @@ func TestSnapshotFiles(t *testing.T) {
 		filepath.Join(testDir, "foo"),
 		filepath.Join(testDir, "kbuild/file"),
 	}
-	contents, err := snapshotter.TakeSnapshotOfFiles(filesToSnapshot)
+	contents, err := snapshotter.TakeSnapshot(filesToSnapshot)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -181,12 +181,12 @@ func TestEmptySnapshot(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Take snapshot with no changes
-	_, filesAdded, err := snapshotter.TakeSnapshot()
+	contents, err := snapshotter.TakeSnapshot(nil)
 	if err != nil {
 		t.Fatalf("Error taking snapshot of fs: %s", err)
 	}
 	// Since we took a snapshot with no changes, contents should be nil
-	if filesAdded {
+	if contents != nil {
 		t.Fatal("Files added even though no changes to file system were made.")
 	}
 }

pkg/util/tar_util.go

@@ -18,10 +18,14 @@ package util
 
 import (
 	"archive/tar"
+	"github.com/sirupsen/logrus"
 	"io"
 	"os"
+	"syscall"
 )
 
+var hardlinks = make(map[uint64]string)
+
 // AddToTar adds the file i to tar w at path p
 func AddToTar(p string, i os.FileInfo, w *tar.Writer) error {
 	linkDst := ""
@@ -37,16 +41,48 @@ func AddToTar(p string, i os.FileInfo, w *tar.Writer) error {
 		return err
 	}
 	hdr.Name = p
-	w.WriteHeader(hdr)
+
-	if !i.Mode().IsRegular() {
+	hardlink, linkDst := checkHardlink(p, i)
+	if hardlink {
+		hdr.Linkname = linkDst
+		hdr.Typeflag = tar.TypeLink
+		hdr.Size = 0
+	}
+	if err := w.WriteHeader(hdr); err != nil {
+		return err
+	}
+	if !(i.Mode().IsRegular()) || hardlink {
 		return nil
 	}
 	r, err := os.Open(p)
 	if err != nil {
 		return err
 	}
+	defer r.Close()
 	if _, err := io.Copy(w, r); err != nil {
 		return err
 	}
 	return nil
 }
+
+// Returns true if path is hardlink, and the link destination
+func checkHardlink(p string, i os.FileInfo) (bool, string) {
+	hardlink := false
+	linkDst := ""
+	if sys := i.Sys(); sys != nil {
+		if stat, ok := sys.(*syscall.Stat_t); ok {
+			nlinks := stat.Nlink
+			if nlinks > 1 {
+				inode := stat.Ino
+				if original, exists := hardlinks[inode]; exists && original != p {
+					hardlink = true
+					logrus.Debugf("%s inode exists in hardlinks map, linking to %s", p, original)
+					linkDst = original
+				} else {
+					hardlinks[inode] = p
+				}
+			}
+		}
+	}
+	return hardlink, linkDst
+}

vendor/github.com/GoogleCloudPlatform/container-diff/pkg/image/mutable_source.go

@@ -142,7 +142,7 @@ func (m *MutableSource) AppendLayer(content []byte, author string) error {
 	// Also add it to the config.
 	diffID := digest.FromBytes(content)
 	m.cfg.RootFS.DiffIDs = append(m.cfg.RootFS.DiffIDs, diffID)
-	m.appendConfigHistory(author, false)
+	m.AppendConfigHistory(author, false)
 	return nil
 }
 
@@ -184,7 +184,7 @@ func (m *MutableSource) SetEnv(envMap map[string]string, author string) {
 		envArray = append(envArray, entry)
 	}
 	m.cfg.Schema2V1Image.Config.Env = envArray
-	m.appendConfigHistory(author, true)
+	m.AppendConfigHistory(author, true)
 }
 
 func (m *MutableSource) Config() *manifest.Schema2Config {
@@ -193,10 +193,10 @@ func (m *MutableSource) Config() *manifest.Schema2Config {
 
 func (m *MutableSource) SetConfig(config *manifest.Schema2Config, author string, emptyLayer bool) {
 	m.cfg.Schema2V1Image.Config = config
-	m.appendConfigHistory(author, emptyLayer)
+	m.AppendConfigHistory(author, emptyLayer)
 }
 
-func (m *MutableSource) appendConfigHistory(author string, emptyLayer bool) {
+func (m *MutableSource) AppendConfigHistory(author string, emptyLayer bool) {
 	history := manifest.Schema2History{
 		Created:    time.Now(),
 		Author:     author,

vendor/github.com/containers/storage/store.go

@@ -2276,14 +2276,15 @@ func (s *store) Shutdown(force bool) ([]string, error) {
 		return mounted, err
 	}
 
+	s.graphLock.Lock()
+	defer s.graphLock.Unlock()
+
 	rlstore.Lock()
 	defer rlstore.Unlock()
 	if modified, err := rlstore.Modified(); modified || err != nil {
 		rlstore.Load()
 	}
 
-	s.graphLock.Lock()
-	defer s.graphLock.Unlock()
 	layers, err := rlstore.Layers()
 	if err != nil {
 		return mounted, err

vendor/golang.org/x/sys/unix/syscall_linux_arm64.go

@@ -23,8 +23,11 @@ package unix
 //sys	Seek(fd int, offset int64, whence int) (off int64, err error) = SYS_LSEEK
 
 func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err error) {
-	ts := Timespec{Sec: timeout.Sec, Nsec: timeout.Usec * 1000}
+	var ts *Timespec
-	return Pselect(nfd, r, w, e, &ts, nil)
+	if timeout != nil {
+		ts = &Timespec{Sec: timeout.Sec, Nsec: timeout.Usec * 1000}
+	}
+	return Pselect(nfd, r, w, e, ts, nil)
 }
 
 //sys	sendfile(outfd int, infd int, offset *int64, count int) (written int, err error)

vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go

@@ -26,8 +26,11 @@ package unix
 //sys	Seek(fd int, offset int64, whence int) (off int64, err error) = SYS_LSEEK
 
 func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err error) {
-	ts := Timespec{Sec: timeout.Sec, Nsec: timeout.Usec * 1000}
+	var ts *Timespec
-	return Pselect(nfd, r, w, e, &ts, nil)
+	if timeout != nil {
+		ts = &Timespec{Sec: timeout.Sec, Nsec: timeout.Usec * 1000}
+	}
+	return Pselect(nfd, r, w, e, ts, nil)
 }
 
 //sys	sendfile(outfd int, infd int, offset *int64, count int) (written int, err error)