Merge branch 'master' of github.com:GoogleCloudPlatform/k8s-container-builder into expose-cmd

Gopkg.lock

@@ -161,6 +161,7 @@
     "builder/dockerfile/command",
     "builder/dockerfile/instructions",
     "builder/dockerfile/parser",
+    "builder/dockerfile/shell",
     "client",
     "pkg/homedir",
     "pkg/idtools",
@@ -486,6 +487,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "fd21de0404336debb893db778210835a27a3612fe9b9e5e412dcdc80d288a986"
+  inputs-digest = "eadec1feacc8473e54622d5f3a25fbc9c7fb1f9bd38776475c3e2d283bd80d2a"
   solver-name = "gps-cdcl"
   solver-version = 1

integration_tests/dockerfiles/Dockerfile_test_env

@@ -0,0 +1,19 @@
+FROM gcr.io/google-appengine/debian9
+ENV hey hey
+ENV PATH /usr/local
+ENV hey hello
+ENV first=foo second=foo2
+ENV third $second:/third
+ENV myName="John Doe" myDog=Rex\ The\ Dog \
+    myCat=fluffy
+ENV PATH something
+ENV test=value\ value2
+ENV test1="a'b'c"
+ENV test2='a"b"c'
+ENV test3=a\ b name2=b\ c
+ENV test4="a\"b"
+ENV test5='a\"b'
+ENV test6="a\'b"
+ENV atomic=one
+ENV atomic=two newatomic=$atomic
+ENV newenv=$doesntexist/newenv

integration_tests/dockerfiles/config_test_run.json

@@ -9,13 +9,13 @@
       "Mods": [
         {
           "Name": "/var/log/dpkg.log",
-          "Size1": 57425,
-          "Size2": 57425
+          "Size1": 57481,
+          "Size2": 57481
         },
         {
           "Name": "/var/log/apt/term.log",
-          "Size1": 24400,
-          "Size2": 24400
+          "Size1": 24421,
+          "Size2": 24421
         },
         {
           "Name": "/var/cache/ldconfig/aux-cache",
@@ -24,8 +24,8 @@
         },
         {
           "Name": "/var/log/apt/history.log",
-          "Size1": 5089,
-          "Size2": 5089
+          "Size1": 5415,
+          "Size2": 5415
         },
         {
           "Name": "/var/log/alternatives.log",

integration_tests/dockerfiles/test_env.yaml

@@ -0,0 +1,41 @@
+schemaVersion: '2.0.0'
+metadataTest:
+  env:
+    - key: hey
+      value: hello
+    - key: PATH
+      value: something
+    - key: first
+      value: foo
+    - key: second
+      value: foo2
+    - key: third
+      value: foo2:/third
+    - key: myName
+      value: John Doe
+    - key: myDog
+      value: Rex The Dog
+    - key: myCat
+      value: fluffy
+    - key: test
+      value: value value2
+    - key: test1
+      value: a'b'c
+    - key: test2
+      value: a"b"c
+    - key: test3
+      value: a b
+    - key: name2
+      value: b c
+    - key: test4
+      value: a"b
+    - key: test5
+      value: a\"b
+    - key: test6
+      value: a\'b
+    - key: atomic
+      value: two
+    - key: newatomic
+      value: one
+    - key: newenv
+      value: /newenv

integration_tests/integration_test_yaml.go

@@ -21,7 +21,7 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-var tests = []struct {
+var fileTests = []struct {
 	description    string
 	dockerfilePath string
 	configPath     string
@@ -51,6 +51,22 @@ var tests = []struct {
 	},
 }
 
+var structureTests = []struct {
+	description           string
+	dockerfilePath        string
+	structureTestYamlPath string
+	dockerBuildContext    string
+	repo                  string
+}{
+	{
+		description:           "test env",
+		dockerfilePath:        "/workspace/integration_tests/dockerfiles/Dockerfile_test_env",
+		repo:                  "test-env",
+		dockerBuildContext:    "/workspace/integration_tests/dockerfiles/",
+		structureTestYamlPath: "/workspace/integration_tests/dockerfiles/test_env.yaml",
+	},
+}
+
 type step struct {
 	Name string
 	Args []string
@@ -82,15 +98,23 @@ func main() {
 		Name: ubuntuImage,
 		Args: []string{"chmod", "+x", "container-diff-linux-amd64"},
 	}
+	structureTestsStep := step{
+		Name: "gcr.io/cloud-builders/gsutil",
+		Args: []string{"cp", "gs://container-structure-test/latest/container-structure-test", "."},
+	}
+	structureTestPermissions := step{
+		Name: ubuntuImage,
+		Args: []string{"chmod", "+x", "container-structure-test"},
+	}
 	// Build executor image
 	buildExecutorImage := step{
 		Name: dockerImage,
 		Args: []string{"build", "-t", executorImage, "-f", "integration_tests/executor/Dockerfile", "."},
 	}
 	y := testyaml{
-		Steps: []step{containerDiffStep, containerDiffPermissions, buildExecutorImage},
+		Steps: []step{containerDiffStep, containerDiffPermissions, structureTestsStep, structureTestPermissions, buildExecutorImage},
 	}
-	for _, test := range tests {
+	for _, test := range fileTests {
 		// First, build the image with docker
 		dockerImageTag := testRepo + dockerPrefix + test.repo
 		dockerBuild := step{
@@ -133,6 +157,43 @@ func main() {
 		y.Steps = append(y.Steps, dockerBuild, kbuild, pullKbuildImage, containerDiff, catContainerDiffOutput, compareOutputs)
 	}
 
+	for _, test := range structureTests {
+
+		// First, build the image with docker
+		dockerImageTag := testRepo + dockerPrefix + test.repo
+		dockerBuild := step{
+			Name: dockerImage,
+			Args: []string{"build", "-t", dockerImageTag, "-f", test.dockerfilePath, test.dockerBuildContext},
+		}
+
+		// Build the image with kbuild
+		kbuildImage := testRepo + kbuildPrefix + test.repo
+		kbuild := step{
+			Name: executorImage,
+			Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath},
+		}
+		// Pull the kbuild image
+		pullKbuildImage := step{
+			Name: dockerImage,
+			Args: []string{"pull", kbuildImage},
+		}
+		// Run structure tests on the kbuild and docker image
+		args := "container-structure-test -image " + kbuildImage + " " + test.structureTestYamlPath
+		structureTest := step{
+			Name: ubuntuImage,
+			Args: []string{"sh", "-c", args},
+			Env:  []string{"PATH=/workspace:/bin"},
+		}
+		args = "container-structure-test -image " + dockerImageTag + " " + test.structureTestYamlPath
+		dockerStructureTest := step{
+			Name: ubuntuImage,
+			Args: []string{"sh", "-c", args},
+			Env:  []string{"PATH=/workspace:/bin"},
+		}
+
+		y.Steps = append(y.Steps, dockerBuild, kbuild, pullKbuildImage, structureTest, dockerStructureTest)
+	}
+
 	d, _ := yaml.Marshal(&y)
 	fmt.Println(string(d))
 }

pkg/commands/commands.go

@@ -40,6 +40,8 @@ func GetCommand(cmd instructions.Command) (DockerCommand, error) {
 		return &RunCommand{cmd: c}, nil
 	case *instructions.ExposeCommand:
 		return &ExposeCommand{cmd: c}, nil
+	case *instructions.EnvCommand:
+		return &EnvCommand{cmd: c}, nil
 	}
 	return nil, errors.Errorf("%s is not a supported command", cmd.Name())
 }

pkg/commands/env.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"bytes"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/builder/dockerfile/shell"
+	"github.com/sirupsen/logrus"
+	"os"
+	"strings"
+)
+
+type EnvCommand struct {
+	cmd *instructions.EnvCommand
+}
+
+func (e *EnvCommand) ExecuteCommand(config *manifest.Schema2Config) error {
+	logrus.Info("cmd: ENV")
+	// The dockerfile/shell package handles processing env values
+	// It handles escape characters and supports expansion from the config.Env array
+	// Shlex handles some of the following use cases (these and more are tested in integration tests)
+	// ""a'b'c"" -> "a'b'c"
+	// "Rex\ The\ Dog \" -> "Rex The Dog"
+	// "a\"b" -> "a"b"
+	envString := envToString(e.cmd)
+	p, err := parser.Parse(bytes.NewReader([]byte(envString)))
+	if err != nil {
+		return err
+	}
+	shlex := shell.NewLex(p.EscapeToken)
+	newEnvs := e.cmd.Env
+	for index, pair := range newEnvs {
+		expandedValue, err := shlex.ProcessWord(pair.Value, config.Env)
+		if err != nil {
+			return err
+		}
+		newEnvs[index] = instructions.KeyValuePair{
+			Key:   pair.Key,
+			Value: expandedValue,
+		}
+		logrus.Infof("Setting environment variable %s=%s", pair.Key, expandedValue)
+		if err := os.Setenv(pair.Key, expandedValue); err != nil {
+			return err
+		}
+	}
+	return updateConfigEnv(newEnvs, config)
+}
+
+func updateConfigEnv(newEnvs []instructions.KeyValuePair, config *manifest.Schema2Config) error {
+	// First, convert config.Env array to []instruction.KeyValuePair
+	var kvps []instructions.KeyValuePair
+	for _, env := range config.Env {
+		entry := strings.Split(env, "=")
+		kvps = append(kvps, instructions.KeyValuePair{
+			Key:   entry[0],
+			Value: entry[1],
+		})
+	}
+	// Iterate through new environment variables, and replace existing keys
+	// We can't use a map because we need to preserve the order of the environment variables
+Loop:
+	for _, newEnv := range newEnvs {
+		for index, kvp := range kvps {
+			// If key exists, replace the KeyValuePair...
+			if kvp.Key == newEnv.Key {
+				logrus.Debugf("Replacing environment variable %v with %v in config", kvp, newEnv)
+				kvps[index] = newEnv
+				continue Loop
+			}
+		}
+		// ... Else, append it as a new env variable
+		kvps = append(kvps, newEnv)
+	}
+	// Convert back to array and set in config
+	envArray := []string{}
+	for _, kvp := range kvps {
+		entry := kvp.Key + "=" + kvp.Value
+		envArray = append(envArray, entry)
+	}
+	config.Env = envArray
+	return nil
+}
+
+func envToString(cmd *instructions.EnvCommand) string {
+	env := []string{"ENV"}
+	for _, kvp := range cmd.Env {
+		env = append(env, kvp.Key+"="+kvp.Value)
+	}
+	return strings.Join(env, " ")
+}
+
+// We know that no files have changed, so return an empty array
+func (e *EnvCommand) FilesToSnapshot() []string {
+	return []string{}
+}
+
+// CreatedBy returns some information about the command for the image config history
+func (e *EnvCommand) CreatedBy() string {
+	envArray := []string{e.cmd.Name()}
+	for _, pair := range e.cmd.Env {
+		envArray = append(envArray, pair.Key+"="+pair.Value)
+	}
+	return strings.Join(envArray, " ")
+}

pkg/commands/env_test.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package commands
+
+import (
+	"github.com/GoogleCloudPlatform/k8s-container-builder/testutil"
+	"github.com/containers/image/manifest"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"testing"
+)
+
+func TestUpdateEnvConfig(t *testing.T) {
+	cfg := &manifest.Schema2Config{
+		Env: []string{
+			"PATH=/path/to/dir",
+			"hey=hey",
+		},
+	}
+
+	newEnvs := []instructions.KeyValuePair{
+		{
+			Key:   "foo",
+			Value: "foo2",
+		},
+		{
+			Key:   "PATH",
+			Value: "/new/path/",
+		},
+		{
+			Key:   "foo",
+			Value: "newfoo",
+		},
+	}
+
+	expectedEnvArray := []string{
+		"PATH=/new/path/",
+		"hey=hey",
+		"foo=newfoo",
+	}
+	updateConfigEnv(newEnvs, cfg)
+	testutil.CheckErrorAndDeepEqual(t, false, nil, expectedEnvArray, cfg.Env)
+}
+
+func TestEnvToString(t *testing.T) {
+	envCmd := &instructions.EnvCommand{
+		Env: []instructions.KeyValuePair{
+			{
+				Key:   "PATH",
+				Value: "/some/path",
+			},
+			{
+				Key:   "HOME",
+				Value: "/root",
+			},
+		},
+	}
+	expectedString := "ENV PATH=/some/path HOME=/root"
+	actualString := envToString(envCmd)
+	testutil.CheckErrorAndDeepEqual(t, false, nil, expectedString, actualString)
+}

vendor/github.com/docker/docker/builder/dockerfile/shell/equal_env_unix.go

@@ -0,0 +1,9 @@
+// +build !windows
+
+package shell // import "github.com/docker/docker/builder/dockerfile/shell"
+
+// EqualEnvKeys compare two strings and returns true if they are equal. On
+// Windows this comparison is case insensitive.
+func EqualEnvKeys(from, to string) bool {
+	return from == to
+}

vendor/github.com/docker/docker/builder/dockerfile/shell/equal_env_windows.go

@@ -0,0 +1,9 @@
+package shell // import "github.com/docker/docker/builder/dockerfile/shell"
+
+import "strings"
+
+// EqualEnvKeys compare two strings and returns true if they are equal. On
+// Windows this comparison is case insensitive.
+func EqualEnvKeys(from, to string) bool {
+	return strings.ToUpper(from) == strings.ToUpper(to)
+}

vendor/github.com/docker/docker/builder/dockerfile/shell/lex.go

@@ -0,0 +1,344 @@
+package shell // import "github.com/docker/docker/builder/dockerfile/shell"
+
+import (
+	"bytes"
+	"strings"
+	"text/scanner"
+	"unicode"
+
+	"github.com/pkg/errors"
+)
+
+// Lex performs shell word splitting and variable expansion.
+//
+// Lex takes a string and an array of env variables and
+// process all quotes (" and ') as well as $xxx and ${xxx} env variable
+// tokens.  Tries to mimic bash shell process.
+// It doesn't support all flavors of ${xx:...} formats but new ones can
+// be added by adding code to the "special ${} format processing" section
+type Lex struct {
+	escapeToken rune
+}
+
+// NewLex creates a new Lex which uses escapeToken to escape quotes.
+func NewLex(escapeToken rune) *Lex {
+	return &Lex{escapeToken: escapeToken}
+}
+
+// ProcessWord will use the 'env' list of environment variables,
+// and replace any env var references in 'word'.
+func (s *Lex) ProcessWord(word string, env []string) (string, error) {
+	word, _, err := s.process(word, env)
+	return word, err
+}
+
+// ProcessWords will use the 'env' list of environment variables,
+// and replace any env var references in 'word' then it will also
+// return a slice of strings which represents the 'word'
+// split up based on spaces - taking into account quotes.  Note that
+// this splitting is done **after** the env var substitutions are done.
+// Note, each one is trimmed to remove leading and trailing spaces (unless
+// they are quoted", but ProcessWord retains spaces between words.
+func (s *Lex) ProcessWords(word string, env []string) ([]string, error) {
+	_, words, err := s.process(word, env)
+	return words, err
+}
+
+func (s *Lex) process(word string, env []string) (string, []string, error) {
+	sw := &shellWord{
+		envs:        env,
+		escapeToken: s.escapeToken,
+	}
+	sw.scanner.Init(strings.NewReader(word))
+	return sw.process(word)
+}
+
+type shellWord struct {
+	scanner     scanner.Scanner
+	envs        []string
+	escapeToken rune
+}
+
+func (sw *shellWord) process(source string) (string, []string, error) {
+	word, words, err := sw.processStopOn(scanner.EOF)
+	if err != nil {
+		err = errors.Wrapf(err, "failed to process %q", source)
+	}
+	return word, words, err
+}
+
+type wordsStruct struct {
+	word   string
+	words  []string
+	inWord bool
+}
+
+func (w *wordsStruct) addChar(ch rune) {
+	if unicode.IsSpace(ch) && w.inWord {
+		if len(w.word) != 0 {
+			w.words = append(w.words, w.word)
+			w.word = ""
+			w.inWord = false
+		}
+	} else if !unicode.IsSpace(ch) {
+		w.addRawChar(ch)
+	}
+}
+
+func (w *wordsStruct) addRawChar(ch rune) {
+	w.word += string(ch)
+	w.inWord = true
+}
+
+func (w *wordsStruct) addString(str string) {
+	var scan scanner.Scanner
+	scan.Init(strings.NewReader(str))
+	for scan.Peek() != scanner.EOF {
+		w.addChar(scan.Next())
+	}
+}
+
+func (w *wordsStruct) addRawString(str string) {
+	w.word += str
+	w.inWord = true
+}
+
+func (w *wordsStruct) getWords() []string {
+	if len(w.word) > 0 {
+		w.words = append(w.words, w.word)
+
+		// Just in case we're called again by mistake
+		w.word = ""
+		w.inWord = false
+	}
+	return w.words
+}
+
+// Process the word, starting at 'pos', and stop when we get to the
+// end of the word or the 'stopChar' character
+func (sw *shellWord) processStopOn(stopChar rune) (string, []string, error) {
+	var result bytes.Buffer
+	var words wordsStruct
+
+	var charFuncMapping = map[rune]func() (string, error){
+		'\'': sw.processSingleQuote,
+		'"':  sw.processDoubleQuote,
+		'$':  sw.processDollar,
+	}
+
+	for sw.scanner.Peek() != scanner.EOF {
+		ch := sw.scanner.Peek()
+
+		if stopChar != scanner.EOF && ch == stopChar {
+			sw.scanner.Next()
+			break
+		}
+		if fn, ok := charFuncMapping[ch]; ok {
+			// Call special processing func for certain chars
+			tmp, err := fn()
+			if err != nil {
+				return "", []string{}, err
+			}
+			result.WriteString(tmp)
+
+			if ch == rune('$') {
+				words.addString(tmp)
+			} else {
+				words.addRawString(tmp)
+			}
+		} else {
+			// Not special, just add it to the result
+			ch = sw.scanner.Next()
+
+			if ch == sw.escapeToken {
+				// '\' (default escape token, but ` allowed) escapes, except end of line
+				ch = sw.scanner.Next()
+
+				if ch == scanner.EOF {
+					break
+				}
+
+				words.addRawChar(ch)
+			} else {
+				words.addChar(ch)
+			}
+
+			result.WriteRune(ch)
+		}
+	}
+
+	return result.String(), words.getWords(), nil
+}
+
+func (sw *shellWord) processSingleQuote() (string, error) {
+	// All chars between single quotes are taken as-is
+	// Note, you can't escape '
+	//
+	// From the "sh" man page:
+	// Single Quotes
+	//   Enclosing characters in single quotes preserves the literal meaning of
+	//   all the characters (except single quotes, making it impossible to put
+	//   single-quotes in a single-quoted string).
+
+	var result bytes.Buffer
+
+	sw.scanner.Next()
+
+	for {
+		ch := sw.scanner.Next()
+		switch ch {
+		case scanner.EOF:
+			return "", errors.New("unexpected end of statement while looking for matching single-quote")
+		case '\'':
+			return result.String(), nil
+		}
+		result.WriteRune(ch)
+	}
+}
+
+func (sw *shellWord) processDoubleQuote() (string, error) {
+	// All chars up to the next " are taken as-is, even ', except any $ chars
+	// But you can escape " with a \ (or ` if escape token set accordingly)
+	//
+	// From the "sh" man page:
+	// Double Quotes
+	//  Enclosing characters within double quotes preserves the literal meaning
+	//  of all characters except dollarsign ($), backquote (`), and backslash
+	//  (\).  The backslash inside double quotes is historically weird, and
+	//  serves to quote only the following characters:
+	//    $ ` " \ <newline>.
+	//  Otherwise it remains literal.
+
+	var result bytes.Buffer
+
+	sw.scanner.Next()
+
+	for {
+		switch sw.scanner.Peek() {
+		case scanner.EOF:
+			return "", errors.New("unexpected end of statement while looking for matching double-quote")
+		case '"':
+			sw.scanner.Next()
+			return result.String(), nil
+		case '$':
+			value, err := sw.processDollar()
+			if err != nil {
+				return "", err
+			}
+			result.WriteString(value)
+		default:
+			ch := sw.scanner.Next()
+			if ch == sw.escapeToken {
+				switch sw.scanner.Peek() {
+				case scanner.EOF:
+					// Ignore \ at end of word
+					continue
+				case '"', '$', sw.escapeToken:
+					// These chars can be escaped, all other \'s are left as-is
+					// Note: for now don't do anything special with ` chars.
+					// Not sure what to do with them anyway since we're not going
+					// to execute the text in there (not now anyway).
+					ch = sw.scanner.Next()
+				}
+			}
+			result.WriteRune(ch)
+		}
+	}
+}
+
+func (sw *shellWord) processDollar() (string, error) {
+	sw.scanner.Next()
+
+	// $xxx case
+	if sw.scanner.Peek() != '{' {
+		name := sw.processName()
+		if name == "" {
+			return "$", nil
+		}
+		return sw.getEnv(name), nil
+	}
+
+	sw.scanner.Next()
+	name := sw.processName()
+	ch := sw.scanner.Peek()
+	if ch == '}' {
+		// Normal ${xx} case
+		sw.scanner.Next()
+		return sw.getEnv(name), nil
+	}
+	if ch == ':' {
+		// Special ${xx:...} format processing
+		// Yes it allows for recursive $'s in the ... spot
+
+		sw.scanner.Next() // skip over :
+		modifier := sw.scanner.Next()
+
+		word, _, err := sw.processStopOn('}')
+		if err != nil {
+			return "", err
+		}
+
+		// Grab the current value of the variable in question so we
+		// can use to to determine what to do based on the modifier
+		newValue := sw.getEnv(name)
+
+		switch modifier {
+		case '+':
+			if newValue != "" {
+				newValue = word
+			}
+			return newValue, nil
+
+		case '-':
+			if newValue == "" {
+				newValue = word
+			}
+			return newValue, nil
+
+		default:
+			return "", errors.Errorf("unsupported modifier (%c) in substitution", modifier)
+		}
+	}
+	return "", errors.Errorf("missing ':' in substitution")
+}
+
+func (sw *shellWord) processName() string {
+	// Read in a name (alphanumeric or _)
+	// If it starts with a numeric then just return $#
+	var name bytes.Buffer
+
+	for sw.scanner.Peek() != scanner.EOF {
+		ch := sw.scanner.Peek()
+		if name.Len() == 0 && unicode.IsDigit(ch) {
+			ch = sw.scanner.Next()
+			return string(ch)
+		}
+		if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' {
+			break
+		}
+		ch = sw.scanner.Next()
+		name.WriteRune(ch)
+	}
+
+	return name.String()
+}
+
+func (sw *shellWord) getEnv(name string) string {
+	for _, env := range sw.envs {
+		i := strings.Index(env, "=")
+		if i < 0 {
+			if EqualEnvKeys(name, env) {
+				// Should probably never get here, but just in case treat
+				// it like "var" and "var=" are the same
+				return ""
+			}
+			continue
+		}
+		compareName := env[:i]
+		if !EqualEnvKeys(name, compareName) {
+			continue
+		}
+		return env[i+1:]
+	}
+	return ""
+}