Integration tests for run cmd

Makefile

@@ -29,13 +29,13 @@ REPOPATH ?= $(ORG)/$(PROJECT)
 
 GO_FILES := $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 GO_LDFLAGS := '-extldflags "-static"'
-GO_BUILD_TAGS := "containers_image_ostree_stub containers_image_openpgp exclude_graphdriver_devicemapper exclude_graphdriver_btrfs"
+GO_BUILD_TAGS := "containers_image_ostree_stub containers_image_openpgp exclude_graphdriver_devicemapper exclude_graphdriver_btrfs exclude_graphdriver_overlay"
 
 EXECUTOR_PACKAGE = $(REPOPATH)/executor
 KBUILD_PACKAGE = $(REPOPATH)/kbuild
 
 out/executor: $(GO_FILES)
-	GOOS=$* GOARCH=$(GOARCH) CGO_ENABLED=1 go build -ldflags $(GO_LDFLAGS) -tags $(GO_BUILD_TAGS) -o $@ $(EXECUTOR_PACKAGE)
+	GOOS=$* GOARCH=$(GOARCH) CGO_ENABLED=0 go build -ldflags $(GO_LDFLAGS) -tags $(GO_BUILD_TAGS) -o $@ $(EXECUTOR_PACKAGE)
 
 
 out/kbuild: $(GO_FILES)

executor/cmd/root.go

@@ -17,11 +17,13 @@ limitations under the License.
 package cmd
 
 import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/commands"
 	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/constants"
 	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/dockerfile"
 	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/image"
 	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/snapshot"
 	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"io/ioutil"
@@ -88,7 +90,57 @@ func execute() error {
 	}
 
 	// Execute commands here
+	if err := image.SetEnvVariables(); err != nil {
+		return err
+	}
 
+	// Currently only supports single stage builds
+	for _, stage := range stages {
+		for _, cmd := range stage.Commands {
+			dockerCommand := commands.GetCommand(cmd)
+			if dockerCommand == nil {
+				return errors.Errorf("Invalid or unsupported docker command: %v", cmd)
+			}
+			if err := dockerCommand.ExecuteCommand(); err != nil {
+				return err
+			}
+			// Now, we get the files to snapshot from this command
+			// If this is nil, snapshot the entire filesystem
+			// Else take a snapshot of the specific files
+			snapshotFiles := dockerCommand.FilesToSnapshot()
+			if snapshotFiles == nil {
+				logrus.Info("Taking snapshot of full filesystem...")
+				contents, filesAdded, err := snapshotter.TakeSnapshot()
+				if err != nil {
+					return err
+				}
+				if !filesAdded {
+					logrus.Info("No files were changed, appending empty layer to config.")
+					image.AppendConfigHistory(dockerCommand.Author(), true)
+					continue
+				}
+				// Append the layer to the image
+				if err := image.AppendLayer(contents, dockerCommand.Author()); err != nil {
+					return err
+				}
+			} else {
+				logrus.Infof("Taking snapshot of files %v...", snapshotFiles)
+				contents, err := snapshotter.TakeSnapshotOfFiles(snapshotFiles)
+				if err != nil {
+					return err
+				}
+				if contents == nil {
+					logrus.Info("No files were changed, appending empty layer to config.")
+					image.AppendConfigHistory(dockerCommand.Author(), true)
+					continue
+				}
+				// Append the layer to the image
+				if err := image.AppendLayer(contents, dockerCommand.Author()); err != nil {
+					return err
+				}
+			}
+		}
+	}
 	// Push the image
 	return image.PushImage(destination)
 }

integration_tests/dockerfiles/Dockerfile_test_run

@@ -0,0 +1,5 @@
+FROM gcr.io/google-appengine/debian9
+RUN echo "hey" > /etc/foo
+RUN apt-get update && apt-get install -y \
+  bzr \
+  cvs \

integration_tests/dockerfiles/config_test_run.json

@@ -0,0 +1,57 @@
+[
+  {
+    "Image1": "gcr.io/kbuild-test/docker-test-run:latest",
+    "Image2": "gcr.io/kbuild-test/kbuild-test-run:latest",
+    "DiffType": "File",
+    "Diff": {
+      "Adds": [
+        {
+          "Name": "/bin/bzcat",
+          "Size": 35448
+        },
+        {
+          "Name": "/bin/bzip2",
+          "Size": 35448
+        }
+      ],
+      "Dels": null,
+      "Mods": [
+        {
+          "Name": "/var/log/dpkg.log",
+          "Size1": 57425,
+          "Size2": 57425
+        },
+        {
+          "Name": "/var/log/apt/term.log",
+          "Size1": 24400,
+          "Size2": 24400
+        },
+        {
+          "Name": "/var/cache/ldconfig/aux-cache",
+          "Size1": 8057,
+          "Size2": 8057
+        },
+        {
+          "Name": "/var/log/apt/history.log",
+          "Size1": 5089,
+          "Size2": 5089
+        },
+        {
+          "Name": "/var/log/alternatives.log",
+          "Size1": 2579,
+          "Size2": 2579
+        },
+        {
+          "Name": "/usr/lib/python2.7/dist-packages/keyrings/__init__.pyc",
+          "Size1": 140,
+          "Size2": 140
+        },
+        {
+          "Name": "/usr/lib/python2.7/dist-packages/lazr/__init__.pyc",
+          "Size1": 136,
+          "Size2": 136
+        }
+      ]
+    }
+  }
+]

integration_tests/integration_test_yaml.go

@@ -35,6 +35,13 @@ var tests = []struct {
 		context:        "integration_tests/dockerfiles/",
 		repo:           "extract-filesystem",
 	},
+	{
+		description:    "test run",
+		dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run",
+		configPath:     "/workspace/integration_tests/dockerfiles/config_test_run.json",
+		context:        "integration_tests/dockerfiles/",
+		repo:           "test-run",
+	},
 }
 
 type step struct {

pkg/commands/commands.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2018 Google LLC
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+)
+
+type DockerCommand interface {
+	ExecuteCommand() error
+	// The config file has an "author" field, should return information about the command
+	Author() string
+	// A list of files to snapshot, empty for metadata commands or nil if we don't know
+	FilesToSnapshot() []string
+}
+
+func GetCommand(cmd instructions.Command) DockerCommand {
+	switch c := cmd.(type) {
+	case *instructions.RunCommand:
+		return &RunCommand{cmd: c}
+	}
+	logrus.Errorf("%s is not a supported command.", cmd.Name())
+	return nil
+}

pkg/commands/run.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2018 Google LLC
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+type RunCommand struct {
+	cmd *instructions.RunCommand
+}
+
+func (r *RunCommand) ExecuteCommand() error {
+	var newCommand []string
+	if r.cmd.PrependShell {
+		// This is the default shell on Linux
+		shell := []string{"/bin/sh", "-c"}
+		newCommand = append(shell, strings.Join(r.cmd.CmdLine, " "))
+	} else {
+		newCommand = r.cmd.CmdLine
+	}
+
+	logrus.Infof("cmd: %s", newCommand[0])
+	logrus.Infof("args: %s", newCommand[1:])
+
+	cmd := exec.Command(newCommand[0], newCommand[1:]...)
+	cmd.Stdout = os.Stdout
+	return cmd.Run()
+}
+
+// FilesToSnapshot returns nil for this command because we don't know which files
+// have changed, so we snapshot the entire system.
+func (r *RunCommand) FilesToSnapshot() []string {
+	return nil
+}
+
+// Author returns some information about the command for the image config
+func (r *RunCommand) Author() string {
+	return "kbuild /bin/sh -c" + strings.Join(r.cmd.CmdLine, " ")
+}

pkg/image/image.go

@@ -23,6 +23,7 @@ import (
 	"github.com/containers/image/signature"
 	"github.com/containers/image/transports/alltransports"
 	"github.com/sirupsen/logrus"
+	"os"
 )
 
 // sourceImage is the image that will be modified by the executor
@@ -45,6 +46,7 @@ func InitializeSourceImage(srcImg string) error {
 
 // AppendLayer appends a layer onto the base image
 func AppendLayer(contents []byte, author string) error {
+	logrus.Info("Appending layer to base image")
 	return sourceImage.AppendLayer(contents, author)
 }
 
@@ -66,6 +68,23 @@ func PushImage(destImg string) error {
 	return copy.Image(policyContext, destRef, srcRef, nil)
 }
 
+// AppendConfigHistory appends to the source image config history
+func AppendConfigHistory(author string, emptyLayer bool) {
+	sourceImage.AppendConfigHistory(author, emptyLayer)
+}
+
+// SetEnvVariables sets environment variables as specified in the image
+func SetEnvVariables() error {
+	envVars := sourceImage.Env()
+	for key, val := range envVars {
+		if err := os.Setenv(key, val); err != nil {
+			return err
+		}
+		logrus.Debugf("Setting environment variable %s=%s", key, val)
+	}
+	return nil
+}
+
 func getPolicyContext() (*signature.PolicyContext, error) {
 	policyContext, err := signature.NewPolicyContext(&signature.Policy{
 		Default: signature.PolicyRequirements{signature.NewPRInsecureAcceptAnything()},