Fixed merge conflict, added validation

pkg/commands/commands.go

@@ -56,6 +56,10 @@ func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, e
 		return &LabelCommand{cmd: c}, nil
 	case *instructions.UserCommand:
 		return &UserCommand{cmd: c}, nil
+	case *instructions.OnbuildCommand:
+		return &OnBuildCommand{cmd: c}, nil
+	case *instructions.VolumeCommand:
+		return &VolumeCommand{cmd: c}, nil
 	}
 	return nil, errors.Errorf("%s is not a supported command", cmd.Name())
 }

pkg/commands/onbuild.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+)
+
+type OnBuildCommand struct {
+	cmd *instructions.OnbuildCommand
+}
+
+//ExecuteCommand adds the specified expression in Onbuild to the config
+func (o *OnBuildCommand) ExecuteCommand(config *manifest.Schema2Config) error {
+	logrus.Info("cmd: ONBUILD")
+	logrus.Infof("args: %s", o.cmd.Expression)
+	resolvedExpression, err := util.ResolveEnvironmentReplacement(o.cmd.Expression, config.Env, false)
+	if err != nil {
+		return err
+	}
+	if config.OnBuild == nil {
+		config.OnBuild = []string{resolvedExpression}
+	} else {
+		config.OnBuild = append(config.OnBuild, resolvedExpression)
+	}
+	return nil
+}
+
+// FilesToSnapshot returns that no files have changed, this command only touches metadata.
+func (o *OnBuildCommand) FilesToSnapshot() []string {
+	return []string{}
+}
+
+// CreatedBy returns some information about the command for the image config history
+func (o *OnBuildCommand) CreatedBy() string {
+	return o.cmd.Expression
+}

pkg/commands/onbuild_test.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/testutil"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"testing"
+)
+
+var onbuildTests = []struct {
+	expression    string
+	onbuildArray  []string
+	expectedArray []string
+}{
+	{
+		expression:   "RUN echo \\\"hi\\\" > $dir",
+		onbuildArray: nil,
+		expectedArray: []string{
+			"RUN echo \"hi\" > /some/dir",
+		},
+	},
+	{
+		expression: "COPY foo foo",
+		onbuildArray: []string{
+			"RUN echo \"hi\" > /some/dir",
+		},
+		expectedArray: []string{
+			"RUN echo \"hi\" > /some/dir",
+			"COPY foo foo",
+		},
+	},
+}
+
+func TestExecuteOnbuild(t *testing.T) {
+	for _, test := range onbuildTests {
+		cfg := &manifest.Schema2Config{
+			Env: []string{
+				"dir=/some/dir",
+			},
+			OnBuild: test.onbuildArray,
+		}
+
+		onbuildCmd := &OnBuildCommand{
+			&instructions.OnbuildCommand{
+				Expression: test.expression,
+			},
+		}
+
+		err := onbuildCmd.ExecuteCommand(cfg)
+		testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedArray, cfg.OnBuild)
+	}
+
+}

pkg/commands/volume.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/sirupsen/logrus"
+	"os"
+	"strings"
+)
+
+type VolumeCommand struct {
+	cmd           *instructions.VolumeCommand
+	snapshotFiles []string
+}
+
+func (v *VolumeCommand) ExecuteCommand(config *manifest.Schema2Config) error {
+	logrus.Info("cmd: VOLUME")
+	volumes := v.cmd.Volumes
+	resolvedVolumes, err := util.ResolveEnvironmentReplacementList(volumes, config.Env, true)
+	if err != nil {
+		return err
+	}
+	existingVolumes := config.Volumes
+	if existingVolumes == nil {
+		existingVolumes = map[string]struct{}{}
+	}
+	for _, volume := range resolvedVolumes {
+		var x struct{}
+		existingVolumes[volume] = x
+		err := util.AddPathToVolumeWhitelist(volume)
+		if err != nil {
+			return err
+		}
+
+		logrus.Infof("Creating directory %s", volume)
+		if err := os.MkdirAll(volume, 0755); err != nil {
+			return err
+		}
+
+		//Check if directory already exists?
+		v.snapshotFiles = append(v.snapshotFiles, volume)
+	}
+	config.Volumes = existingVolumes
+
+	return nil
+}
+
+func (v *VolumeCommand) FilesToSnapshot() []string {
+	return v.snapshotFiles
+}
+
+func (v *VolumeCommand) CreatedBy() string {
+	return strings.Join(append([]string{v.cmd.Name()}, v.cmd.Volumes...), " ")
+}

pkg/commands/volume_test.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package commands
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/testutil"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"testing"
+)
+
+func TestUpdateVolume(t *testing.T) {
+	cfg := &manifest.Schema2Config{
+		Env: []string{
+			"VOLUME=/etc",
+		},
+		Volumes: map[string]struct{}{},
+	}
+
+	volumes := []string{
+		"/tmp",
+		"/var/lib",
+		"$VOLUME",
+	}
+
+	volumeCmd := &VolumeCommand{
+		cmd: &instructions.VolumeCommand{
+			Volumes: volumes,
+		},
+		snapshotFiles: []string{},
+	}
+
+	expectedVolumes := map[string]struct{}{
+		"/tmp":     {},
+		"/var/lib": {},
+		"/etc":     {},
+	}
+
+	err := volumeCmd.ExecuteCommand(cfg)
+	testutil.CheckErrorAndDeepEqual(t, false, err, expectedVolumes, cfg.Volumes)
+}

pkg/constants/constants.go

@@ -26,17 +26,21 @@ const (
 	// WorkspaceDir is the path to the workspace directory
 	WorkspaceDir = "/workspace"
 
-	//KbuildDir is the path to the kbuild directory
-	KbuildDir = "/kbuild"
+	//KanikoDir is the path to the Kaniko directory
+	KanikoDir = "/kaniko"
 
 	WhitelistPath = "/proc/self/mountinfo"
 
-	Author = "kbuild"
+	Author = "kaniko"
 
 	// ContextTar is the default name of the tar uploaded to GCS buckets
 	ContextTar = "context.tar.gz"
 
 	// BuildContextDir is the directory a build context will be unpacked into,
 	// for example, a tarball from a GCS bucket will be unpacked here
-	BuildContextDir = "/kbuild/buildcontext/"
+	BuildContextDir = "/kaniko/buildcontext/"
+
+	// Various snapshot modes:
+	SnapshotModeTime = "time"
+	SnapshotModeFull = "full"
 )

pkg/dockerfile/dockerfile.go

@@ -18,6 +18,7 @@ package dockerfile
 
 import (
 	"bytes"
+	"strings"
 
 	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/builder/dockerfile/parser"
@@ -35,3 +36,21 @@ func Parse(b []byte) ([]instructions.Stage, error) {
 	}
 	return stages, err
 }
+
+// ParseCommands parses an array of commands into an array of instructions.Command; used for onbuild
+func ParseCommands(cmdArray []string) ([]instructions.Command, error) {
+	var cmds []instructions.Command
+	cmdString := strings.Join(cmdArray, "\n")
+	ast, err := parser.Parse(strings.NewReader(cmdString))
+	if err != nil {
+		return nil, err
+	}
+	for _, child := range ast.AST.Children {
+		cmd, err := instructions.ParseCommand(child)
+		if err != nil {
+			return nil, err
+		}
+		cmds = append(cmds, cmd)
+	}
+	return cmds, nil
+}

pkg/snapshot/snapshot_test.go

@@ -39,7 +39,7 @@ func TestSnapshotFileChange(t *testing.T) {
 	newFiles := map[string]string{
 		"foo":        "newbaz1",
 		"bar/bat":    "baz",
-		"kbuild/bat": "bat",
+		"kaniko/bat": "bat",
 	}
 	if err := testutil.SetupFiles(testDir, newFiles); err != nil {
 		t.Fatalf("Error setting up fs: %s", err)
@@ -135,14 +135,14 @@ func TestSnapshotFiles(t *testing.T) {
 	// Make some changes to the filesystem
 	newFiles := map[string]string{
 		"foo":         "newbaz1",
-		"kbuild/file": "bat",
+		"kaniko/file": "bat",
 	}
 	if err := testutil.SetupFiles(testDir, newFiles); err != nil {
 		t.Fatalf("Error setting up fs: %s", err)
 	}
 	filesToSnapshot := []string{
 		filepath.Join(testDir, "foo"),
-		filepath.Join(testDir, "kbuild/file"),
+		filepath.Join(testDir, "kaniko/file"),
 	}
 	contents, err := snapshotter.TakeSnapshot(filesToSnapshot)
 	if err != nil {
@@ -199,7 +199,7 @@ func setUpTestDir() (string, *Snapshotter, error) {
 	files := map[string]string{
 		"foo":         "baz1",
 		"bar/bat":     "baz2",
-		"kbuild/file": "file",
+		"kaniko/file": "file",
 	}
 	// Set up initial files
 	if err := testutil.SetupFiles(testDir, files); err != nil {

pkg/util/bucket_util.go

@@ -25,7 +25,7 @@ import (
 	"path/filepath"
 )
 
-// UnpackTarFromGCSBucket unpacks the kbuild.tar file in the given bucket to the given directory
+// UnpackTarFromGCSBucket unpacks the context.tar.gz file in the given bucket to the given directory
 func UnpackTarFromGCSBucket(bucketName, directory string) error {
 	// Get the tar from the bucket
 	tarPath, err := getTarFromBucket(bucketName, directory)
@@ -41,7 +41,7 @@ func UnpackTarFromGCSBucket(bucketName, directory string) error {
 	return os.Remove(tarPath)
 }
 
-// getTarFromBucket gets kbuild.tar from the GCS bucket and saves it to the filesystem
+// getTarFromBucket gets context.tar.gz from the GCS bucket and saves it to the filesystem
 // It returns the path to the tar file
 func getTarFromBucket(bucketName, directory string) (string, error) {
 	ctx := context.Background()
@@ -50,7 +50,7 @@ func getTarFromBucket(bucketName, directory string) (string, error) {
 		return "", err
 	}
 	bucket := client.Bucket(bucketName)
-	// Get the tarfile kbuild.tar from the GCS bucket, and save it to a tar object
+	// Get the tarfile context.tar.gz from the GCS bucket, and save it to a tar object
 	reader, err := bucket.Object(constants.ContextTar).NewReader(ctx)
 	if err != nil {
 		return "", err

pkg/util/fs_util.go

@@ -30,7 +30,8 @@ import (
 	"time"
 )
 
-var whitelist = []string{"/kbuild"}
+var whitelist = []string{"/kaniko"}
+var volumeWhitelist = []string{}
 
 // ExtractFileSystemFromImage pulls an image and unpacks it to a file system at root
 func ExtractFileSystemFromImage(img string) error {
@@ -156,6 +157,25 @@ func CreateFile(path string, reader io.Reader, perm os.FileMode) error {
 	return dest.Chmod(perm)
 }
 
+// AddPathToVolumeWhitelist adds the given path to the volume whitelist
+// It will get snapshotted when the VOLUME command is run then ignored
+// for subsequent commands.
+func AddPathToVolumeWhitelist(path string) error {
+	logrus.Infof("adding %s to volume whitelist", path)
+	volumeWhitelist = append(volumeWhitelist, path)
+	return nil
+}
+
+// MoveVolumeWhitelistToWhitelist copies over all directories that were volume mounted
+// in this step to be whitelisted for all subsequent docker commands.
+func MoveVolumeWhitelistToWhitelist() error {
+	if len(volumeWhitelist) > 0 {
+		whitelist = append(whitelist, volumeWhitelist...)
+		volumeWhitelist = []string{}
+	}
+	return nil
+}
+
 // DownloadFileToDest downloads the file at rawurl to the given dest for the ADD command
 // From add command docs:
 // 	1. If <src> is a remote file URL:

pkg/util/fs_util_test.go

@@ -46,7 +46,7 @@ func Test_fileSystemWhitelist(t *testing.T) {
 	}
 
 	actualWhitelist, err := fileSystemWhitelist(path)
-	expectedWhitelist := []string{"/kbuild", "/proc", "/dev", "/dev/pts", "/sys"}
+	expectedWhitelist := []string{"/kaniko", "/proc", "/dev", "/dev/pts", "/sys"}
 	sort.Strings(actualWhitelist)
 	sort.Strings(expectedWhitelist)
 	testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist)
@@ -61,7 +61,7 @@ var tests = []struct {
 		files: map[string]string{
 			"/workspace/foo/a": "baz1",
 			"/workspace/foo/b": "baz2",
-			"/kbuild/file":     "file",
+			"/kaniko/file":     "file",
 		},
 		directory: "/workspace/foo/",
 		expectedFiles: []string{
@@ -84,7 +84,7 @@ var tests = []struct {
 			"/workspace/foo/a": "baz1",
 			"/workspace/foo/b": "baz2",
 			"/workspace/baz":   "hey",
-			"/kbuild/file":     "file",
+			"/kaniko/file":     "file",
 		},
 		directory: "/workspace",
 		expectedFiles: []string{
@@ -99,16 +99,16 @@ var tests = []struct {
 		files: map[string]string{
 			"/workspace/foo/a": "baz1",
 			"/workspace/foo/b": "baz2",
-			"/kbuild/file":     "file",
+			"/kaniko/file":     "file",
 		},
 		directory: "",
 		expectedFiles: []string{
 			"workspace/foo/a",
 			"workspace/foo/b",
-			"kbuild/file",
+			"kaniko/file",
 			"workspace",
 			"workspace/foo",
-			"kbuild",
+			"kaniko",
 			".",
 		},
 	},