diff --git a/Gopkg.lock b/Gopkg.lock index 8f3631a98..d6be583a7 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -161,6 +161,7 @@ "builder/dockerfile/command", "builder/dockerfile/instructions", "builder/dockerfile/parser", + "builder/dockerfile/shell", "client", "pkg/homedir", "pkg/idtools", @@ -486,6 +487,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "fd21de0404336debb893db778210835a27a3612fe9b9e5e412dcdc80d288a986" + inputs-digest = "eadec1feacc8473e54622d5f3a25fbc9c7fb1f9bd38776475c3e2d283bd80d2a" solver-name = "gps-cdcl" solver-version = 1 diff --git a/integration_tests/dockerfiles/Dockerfile_test_env b/integration_tests/dockerfiles/Dockerfile_test_env new file mode 100644 index 000000000..ce37081cb --- /dev/null +++ b/integration_tests/dockerfiles/Dockerfile_test_env @@ -0,0 +1,19 @@ +FROM gcr.io/google-appengine/debian9 +ENV hey hey +ENV PATH /usr/local +ENV hey hello +ENV first=foo second=foo2 +ENV third $second:/third +ENV myName="John Doe" myDog=Rex\ The\ Dog \ + myCat=fluffy +ENV PATH something +ENV test=value\ value2 +ENV test1="a'b'c" +ENV test2='a"b"c' +ENV test3=a\ b name2=b\ c +ENV test4="a\"b" +ENV test5='a\"b' +ENV test6="a\'b" +ENV atomic=one +ENV atomic=two newatomic=$atomic +ENV newenv=$doesntexist/newenv diff --git a/integration_tests/dockerfiles/config_test_run.json b/integration_tests/dockerfiles/config_test_run.json index 0544a3d22..19cab3219 100644 --- a/integration_tests/dockerfiles/config_test_run.json +++ b/integration_tests/dockerfiles/config_test_run.json @@ -9,13 +9,13 @@ "Mods": [ { "Name": "/var/log/dpkg.log", - "Size1": 57425, - "Size2": 57425 + "Size1": 57481, + "Size2": 57481 }, { "Name": "/var/log/apt/term.log", - "Size1": 24400, - "Size2": 24400 + "Size1": 24421, + "Size2": 24421 }, { "Name": "/var/cache/ldconfig/aux-cache", @@ -24,8 +24,8 @@ }, { "Name": "/var/log/apt/history.log", - "Size1": 5089, - "Size2": 5089 + "Size1": 5415, + "Size2": 5415 }, { "Name": "/var/log/alternatives.log", diff --git a/integration_tests/dockerfiles/test_env.yaml b/integration_tests/dockerfiles/test_env.yaml new file mode 100644 index 000000000..c23c7a516 --- /dev/null +++ b/integration_tests/dockerfiles/test_env.yaml @@ -0,0 +1,41 @@ +schemaVersion: '2.0.0' +metadataTest: + env: + - key: hey + value: hello + - key: PATH + value: something + - key: first + value: foo + - key: second + value: foo2 + - key: third + value: foo2:/third + - key: myName + value: John Doe + - key: myDog + value: Rex The Dog + - key: myCat + value: fluffy + - key: test + value: value value2 + - key: test1 + value: a'b'c + - key: test2 + value: a"b"c + - key: test3 + value: a b + - key: name2 + value: b c + - key: test4 + value: a"b + - key: test5 + value: a\"b + - key: test6 + value: a\'b + - key: atomic + value: two + - key: newatomic + value: one + - key: newenv + value: /newenv diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index d13be52e9..5471e207e 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -21,7 +21,7 @@ import ( "gopkg.in/yaml.v2" ) -var tests = []struct { +var fileTests = []struct { description string dockerfilePath string configPath string @@ -51,6 +51,22 @@ var tests = []struct { }, } +var structureTests = []struct { + description string + dockerfilePath string + structureTestYamlPath string + dockerBuildContext string + repo string +}{ + { + description: "test env", + dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_env", + repo: "test-env", + dockerBuildContext: "/workspace/integration_tests/dockerfiles/", + structureTestYamlPath: "/workspace/integration_tests/dockerfiles/test_env.yaml", + }, +} + type step struct { Name string Args []string @@ -82,15 +98,23 @@ func main() { Name: ubuntuImage, Args: []string{"chmod", "+x", "container-diff-linux-amd64"}, } + structureTestsStep := step{ + Name: "gcr.io/cloud-builders/gsutil", + Args: []string{"cp", "gs://container-structure-test/latest/container-structure-test", "."}, + } + structureTestPermissions := step{ + Name: ubuntuImage, + Args: []string{"chmod", "+x", "container-structure-test"}, + } // Build executor image buildExecutorImage := step{ Name: dockerImage, Args: []string{"build", "-t", executorImage, "-f", "integration_tests/executor/Dockerfile", "."}, } y := testyaml{ - Steps: []step{containerDiffStep, containerDiffPermissions, buildExecutorImage}, + Steps: []step{containerDiffStep, containerDiffPermissions, structureTestsStep, structureTestPermissions, buildExecutorImage}, } - for _, test := range tests { + for _, test := range fileTests { // First, build the image with docker dockerImageTag := testRepo + dockerPrefix + test.repo dockerBuild := step{ @@ -133,6 +157,43 @@ func main() { y.Steps = append(y.Steps, dockerBuild, kbuild, pullKbuildImage, containerDiff, catContainerDiffOutput, compareOutputs) } + for _, test := range structureTests { + + // First, build the image with docker + dockerImageTag := testRepo + dockerPrefix + test.repo + dockerBuild := step{ + Name: dockerImage, + Args: []string{"build", "-t", dockerImageTag, "-f", test.dockerfilePath, test.dockerBuildContext}, + } + + // Build the image with kbuild + kbuildImage := testRepo + kbuildPrefix + test.repo + kbuild := step{ + Name: executorImage, + Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath}, + } + // Pull the kbuild image + pullKbuildImage := step{ + Name: dockerImage, + Args: []string{"pull", kbuildImage}, + } + // Run structure tests on the kbuild and docker image + args := "container-structure-test -image " + kbuildImage + " " + test.structureTestYamlPath + structureTest := step{ + Name: ubuntuImage, + Args: []string{"sh", "-c", args}, + Env: []string{"PATH=/workspace:/bin"}, + } + args = "container-structure-test -image " + dockerImageTag + " " + test.structureTestYamlPath + dockerStructureTest := step{ + Name: ubuntuImage, + Args: []string{"sh", "-c", args}, + Env: []string{"PATH=/workspace:/bin"}, + } + + y.Steps = append(y.Steps, dockerBuild, kbuild, pullKbuildImage, structureTest, dockerStructureTest) + } + d, _ := yaml.Marshal(&y) fmt.Println(string(d)) } diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index ad4b111ae..742fd5ef3 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -40,6 +40,8 @@ func GetCommand(cmd instructions.Command) (DockerCommand, error) { return &RunCommand{cmd: c}, nil case *instructions.ExposeCommand: return &ExposeCommand{cmd: c}, nil + case *instructions.EnvCommand: + return &EnvCommand{cmd: c}, nil } return nil, errors.Errorf("%s is not a supported command", cmd.Name()) } diff --git a/pkg/commands/env.go b/pkg/commands/env.go new file mode 100644 index 000000000..f7f004940 --- /dev/null +++ b/pkg/commands/env.go @@ -0,0 +1,121 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "bytes" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/docker/docker/builder/dockerfile/parser" + "github.com/docker/docker/builder/dockerfile/shell" + "github.com/sirupsen/logrus" + "os" + "strings" +) + +type EnvCommand struct { + cmd *instructions.EnvCommand +} + +func (e *EnvCommand) ExecuteCommand(config *manifest.Schema2Config) error { + logrus.Info("cmd: ENV") + // The dockerfile/shell package handles processing env values + // It handles escape characters and supports expansion from the config.Env array + // Shlex handles some of the following use cases (these and more are tested in integration tests) + // ""a'b'c"" -> "a'b'c" + // "Rex\ The\ Dog \" -> "Rex The Dog" + // "a\"b" -> "a"b" + envString := envToString(e.cmd) + p, err := parser.Parse(bytes.NewReader([]byte(envString))) + if err != nil { + return err + } + shlex := shell.NewLex(p.EscapeToken) + newEnvs := e.cmd.Env + for index, pair := range newEnvs { + expandedValue, err := shlex.ProcessWord(pair.Value, config.Env) + if err != nil { + return err + } + newEnvs[index] = instructions.KeyValuePair{ + Key: pair.Key, + Value: expandedValue, + } + logrus.Infof("Setting environment variable %s=%s", pair.Key, expandedValue) + if err := os.Setenv(pair.Key, expandedValue); err != nil { + return err + } + } + return updateConfigEnv(newEnvs, config) +} + +func updateConfigEnv(newEnvs []instructions.KeyValuePair, config *manifest.Schema2Config) error { + // First, convert config.Env array to []instruction.KeyValuePair + var kvps []instructions.KeyValuePair + for _, env := range config.Env { + entry := strings.Split(env, "=") + kvps = append(kvps, instructions.KeyValuePair{ + Key: entry[0], + Value: entry[1], + }) + } + // Iterate through new environment variables, and replace existing keys + // We can't use a map because we need to preserve the order of the environment variables +Loop: + for _, newEnv := range newEnvs { + for index, kvp := range kvps { + // If key exists, replace the KeyValuePair... + if kvp.Key == newEnv.Key { + logrus.Debugf("Replacing environment variable %v with %v in config", kvp, newEnv) + kvps[index] = newEnv + continue Loop + } + } + // ... Else, append it as a new env variable + kvps = append(kvps, newEnv) + } + // Convert back to array and set in config + envArray := []string{} + for _, kvp := range kvps { + entry := kvp.Key + "=" + kvp.Value + envArray = append(envArray, entry) + } + config.Env = envArray + return nil +} + +func envToString(cmd *instructions.EnvCommand) string { + env := []string{"ENV"} + for _, kvp := range cmd.Env { + env = append(env, kvp.Key+"="+kvp.Value) + } + return strings.Join(env, " ") +} + +// We know that no files have changed, so return an empty array +func (e *EnvCommand) FilesToSnapshot() []string { + return []string{} +} + +// CreatedBy returns some information about the command for the image config history +func (e *EnvCommand) CreatedBy() string { + envArray := []string{e.cmd.Name()} + for _, pair := range e.cmd.Env { + envArray = append(envArray, pair.Key+"="+pair.Value) + } + return strings.Join(envArray, " ") +} diff --git a/pkg/commands/env_test.go b/pkg/commands/env_test.go new file mode 100644 index 000000000..f0cccdaa4 --- /dev/null +++ b/pkg/commands/env_test.go @@ -0,0 +1,73 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package commands + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/testutil" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "testing" +) + +func TestUpdateEnvConfig(t *testing.T) { + cfg := &manifest.Schema2Config{ + Env: []string{ + "PATH=/path/to/dir", + "hey=hey", + }, + } + + newEnvs := []instructions.KeyValuePair{ + { + Key: "foo", + Value: "foo2", + }, + { + Key: "PATH", + Value: "/new/path/", + }, + { + Key: "foo", + Value: "newfoo", + }, + } + + expectedEnvArray := []string{ + "PATH=/new/path/", + "hey=hey", + "foo=newfoo", + } + updateConfigEnv(newEnvs, cfg) + testutil.CheckErrorAndDeepEqual(t, false, nil, expectedEnvArray, cfg.Env) +} + +func TestEnvToString(t *testing.T) { + envCmd := &instructions.EnvCommand{ + Env: []instructions.KeyValuePair{ + { + Key: "PATH", + Value: "/some/path", + }, + { + Key: "HOME", + Value: "/root", + }, + }, + } + expectedString := "ENV PATH=/some/path HOME=/root" + actualString := envToString(envCmd) + testutil.CheckErrorAndDeepEqual(t, false, nil, expectedString, actualString) +} diff --git a/vendor/github.com/docker/docker/builder/dockerfile/shell/equal_env_unix.go b/vendor/github.com/docker/docker/builder/dockerfile/shell/equal_env_unix.go new file mode 100644 index 000000000..ac97a1e16 --- /dev/null +++ b/vendor/github.com/docker/docker/builder/dockerfile/shell/equal_env_unix.go @@ -0,0 +1,9 @@ +// +build !windows + +package shell // import "github.com/docker/docker/builder/dockerfile/shell" + +// EqualEnvKeys compare two strings and returns true if they are equal. On +// Windows this comparison is case insensitive. +func EqualEnvKeys(from, to string) bool { + return from == to +} diff --git a/vendor/github.com/docker/docker/builder/dockerfile/shell/equal_env_windows.go b/vendor/github.com/docker/docker/builder/dockerfile/shell/equal_env_windows.go new file mode 100644 index 000000000..4baa771bd --- /dev/null +++ b/vendor/github.com/docker/docker/builder/dockerfile/shell/equal_env_windows.go @@ -0,0 +1,9 @@ +package shell // import "github.com/docker/docker/builder/dockerfile/shell" + +import "strings" + +// EqualEnvKeys compare two strings and returns true if they are equal. On +// Windows this comparison is case insensitive. +func EqualEnvKeys(from, to string) bool { + return strings.ToUpper(from) == strings.ToUpper(to) +} diff --git a/vendor/github.com/docker/docker/builder/dockerfile/shell/lex.go b/vendor/github.com/docker/docker/builder/dockerfile/shell/lex.go new file mode 100644 index 000000000..bd3fac525 --- /dev/null +++ b/vendor/github.com/docker/docker/builder/dockerfile/shell/lex.go @@ -0,0 +1,344 @@ +package shell // import "github.com/docker/docker/builder/dockerfile/shell" + +import ( + "bytes" + "strings" + "text/scanner" + "unicode" + + "github.com/pkg/errors" +) + +// Lex performs shell word splitting and variable expansion. +// +// Lex takes a string and an array of env variables and +// process all quotes (" and ') as well as $xxx and ${xxx} env variable +// tokens. Tries to mimic bash shell process. +// It doesn't support all flavors of ${xx:...} formats but new ones can +// be added by adding code to the "special ${} format processing" section +type Lex struct { + escapeToken rune +} + +// NewLex creates a new Lex which uses escapeToken to escape quotes. +func NewLex(escapeToken rune) *Lex { + return &Lex{escapeToken: escapeToken} +} + +// ProcessWord will use the 'env' list of environment variables, +// and replace any env var references in 'word'. +func (s *Lex) ProcessWord(word string, env []string) (string, error) { + word, _, err := s.process(word, env) + return word, err +} + +// ProcessWords will use the 'env' list of environment variables, +// and replace any env var references in 'word' then it will also +// return a slice of strings which represents the 'word' +// split up based on spaces - taking into account quotes. Note that +// this splitting is done **after** the env var substitutions are done. +// Note, each one is trimmed to remove leading and trailing spaces (unless +// they are quoted", but ProcessWord retains spaces between words. +func (s *Lex) ProcessWords(word string, env []string) ([]string, error) { + _, words, err := s.process(word, env) + return words, err +} + +func (s *Lex) process(word string, env []string) (string, []string, error) { + sw := &shellWord{ + envs: env, + escapeToken: s.escapeToken, + } + sw.scanner.Init(strings.NewReader(word)) + return sw.process(word) +} + +type shellWord struct { + scanner scanner.Scanner + envs []string + escapeToken rune +} + +func (sw *shellWord) process(source string) (string, []string, error) { + word, words, err := sw.processStopOn(scanner.EOF) + if err != nil { + err = errors.Wrapf(err, "failed to process %q", source) + } + return word, words, err +} + +type wordsStruct struct { + word string + words []string + inWord bool +} + +func (w *wordsStruct) addChar(ch rune) { + if unicode.IsSpace(ch) && w.inWord { + if len(w.word) != 0 { + w.words = append(w.words, w.word) + w.word = "" + w.inWord = false + } + } else if !unicode.IsSpace(ch) { + w.addRawChar(ch) + } +} + +func (w *wordsStruct) addRawChar(ch rune) { + w.word += string(ch) + w.inWord = true +} + +func (w *wordsStruct) addString(str string) { + var scan scanner.Scanner + scan.Init(strings.NewReader(str)) + for scan.Peek() != scanner.EOF { + w.addChar(scan.Next()) + } +} + +func (w *wordsStruct) addRawString(str string) { + w.word += str + w.inWord = true +} + +func (w *wordsStruct) getWords() []string { + if len(w.word) > 0 { + w.words = append(w.words, w.word) + + // Just in case we're called again by mistake + w.word = "" + w.inWord = false + } + return w.words +} + +// Process the word, starting at 'pos', and stop when we get to the +// end of the word or the 'stopChar' character +func (sw *shellWord) processStopOn(stopChar rune) (string, []string, error) { + var result bytes.Buffer + var words wordsStruct + + var charFuncMapping = map[rune]func() (string, error){ + '\'': sw.processSingleQuote, + '"': sw.processDoubleQuote, + '$': sw.processDollar, + } + + for sw.scanner.Peek() != scanner.EOF { + ch := sw.scanner.Peek() + + if stopChar != scanner.EOF && ch == stopChar { + sw.scanner.Next() + break + } + if fn, ok := charFuncMapping[ch]; ok { + // Call special processing func for certain chars + tmp, err := fn() + if err != nil { + return "", []string{}, err + } + result.WriteString(tmp) + + if ch == rune('$') { + words.addString(tmp) + } else { + words.addRawString(tmp) + } + } else { + // Not special, just add it to the result + ch = sw.scanner.Next() + + if ch == sw.escapeToken { + // '\' (default escape token, but ` allowed) escapes, except end of line + ch = sw.scanner.Next() + + if ch == scanner.EOF { + break + } + + words.addRawChar(ch) + } else { + words.addChar(ch) + } + + result.WriteRune(ch) + } + } + + return result.String(), words.getWords(), nil +} + +func (sw *shellWord) processSingleQuote() (string, error) { + // All chars between single quotes are taken as-is + // Note, you can't escape ' + // + // From the "sh" man page: + // Single Quotes + // Enclosing characters in single quotes preserves the literal meaning of + // all the characters (except single quotes, making it impossible to put + // single-quotes in a single-quoted string). + + var result bytes.Buffer + + sw.scanner.Next() + + for { + ch := sw.scanner.Next() + switch ch { + case scanner.EOF: + return "", errors.New("unexpected end of statement while looking for matching single-quote") + case '\'': + return result.String(), nil + } + result.WriteRune(ch) + } +} + +func (sw *shellWord) processDoubleQuote() (string, error) { + // All chars up to the next " are taken as-is, even ', except any $ chars + // But you can escape " with a \ (or ` if escape token set accordingly) + // + // From the "sh" man page: + // Double Quotes + // Enclosing characters within double quotes preserves the literal meaning + // of all characters except dollarsign ($), backquote (`), and backslash + // (\). The backslash inside double quotes is historically weird, and + // serves to quote only the following characters: + // $ ` " \ . + // Otherwise it remains literal. + + var result bytes.Buffer + + sw.scanner.Next() + + for { + switch sw.scanner.Peek() { + case scanner.EOF: + return "", errors.New("unexpected end of statement while looking for matching double-quote") + case '"': + sw.scanner.Next() + return result.String(), nil + case '$': + value, err := sw.processDollar() + if err != nil { + return "", err + } + result.WriteString(value) + default: + ch := sw.scanner.Next() + if ch == sw.escapeToken { + switch sw.scanner.Peek() { + case scanner.EOF: + // Ignore \ at end of word + continue + case '"', '$', sw.escapeToken: + // These chars can be escaped, all other \'s are left as-is + // Note: for now don't do anything special with ` chars. + // Not sure what to do with them anyway since we're not going + // to execute the text in there (not now anyway). + ch = sw.scanner.Next() + } + } + result.WriteRune(ch) + } + } +} + +func (sw *shellWord) processDollar() (string, error) { + sw.scanner.Next() + + // $xxx case + if sw.scanner.Peek() != '{' { + name := sw.processName() + if name == "" { + return "$", nil + } + return sw.getEnv(name), nil + } + + sw.scanner.Next() + name := sw.processName() + ch := sw.scanner.Peek() + if ch == '}' { + // Normal ${xx} case + sw.scanner.Next() + return sw.getEnv(name), nil + } + if ch == ':' { + // Special ${xx:...} format processing + // Yes it allows for recursive $'s in the ... spot + + sw.scanner.Next() // skip over : + modifier := sw.scanner.Next() + + word, _, err := sw.processStopOn('}') + if err != nil { + return "", err + } + + // Grab the current value of the variable in question so we + // can use to to determine what to do based on the modifier + newValue := sw.getEnv(name) + + switch modifier { + case '+': + if newValue != "" { + newValue = word + } + return newValue, nil + + case '-': + if newValue == "" { + newValue = word + } + return newValue, nil + + default: + return "", errors.Errorf("unsupported modifier (%c) in substitution", modifier) + } + } + return "", errors.Errorf("missing ':' in substitution") +} + +func (sw *shellWord) processName() string { + // Read in a name (alphanumeric or _) + // If it starts with a numeric then just return $# + var name bytes.Buffer + + for sw.scanner.Peek() != scanner.EOF { + ch := sw.scanner.Peek() + if name.Len() == 0 && unicode.IsDigit(ch) { + ch = sw.scanner.Next() + return string(ch) + } + if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' { + break + } + ch = sw.scanner.Next() + name.WriteRune(ch) + } + + return name.String() +} + +func (sw *shellWord) getEnv(name string) string { + for _, env := range sw.envs { + i := strings.Index(env, "=") + if i < 0 { + if EqualEnvKeys(name, env) { + // Should probably never get here, but just in case treat + // it like "var" and "var=" are the same + return "" + } + continue + } + compareName := env[:i] + if !EqualEnvKeys(name, compareName) { + continue + } + return env[i+1:] + } + return "" +}