From ce2b515d4949082b1db94e10cdc0235c6e3d985b Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 6 Apr 2018 12:02:57 -0700 Subject: [PATCH] adding VOLUME command (#62) * adding VOLUME command * proper test project * general fixes * fixing project name * fixing volume unit test * fixing integration test * adding tests * adding util test * fixing test * actually create the volume mounted directory * fix test --- executor/cmd/root.go | 1 + .../dockerfiles/Dockerfile_test_volume | 7 ++ .../dockerfiles/config_test_volume.json | 12 ++++ integration_tests/integration_test_yaml.go | 8 +++ pkg/commands/commands.go | 2 + pkg/commands/volume.go | 71 +++++++++++++++++++ pkg/commands/volume_test.go | 54 ++++++++++++++ pkg/util/fs_util.go | 20 ++++++ 8 files changed, 175 insertions(+) create mode 100644 integration_tests/dockerfiles/Dockerfile_test_volume create mode 100644 integration_tests/dockerfiles/config_test_volume.json create mode 100644 pkg/commands/volume.go create mode 100644 pkg/commands/volume_test.go diff --git a/executor/cmd/root.go b/executor/cmd/root.go index 42ca57e1c..cc6e37815 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -146,6 +146,7 @@ func execute() error { if err != nil { return err } + util.MoveVolumeWhitelistToWhitelist() if contents == nil { logrus.Info("No files were changed, appending empty layer to config.") sourceImage.AppendConfigHistory(constants.Author, true) diff --git a/integration_tests/dockerfiles/Dockerfile_test_volume b/integration_tests/dockerfiles/Dockerfile_test_volume new file mode 100644 index 000000000..f5a255cfe --- /dev/null +++ b/integration_tests/dockerfiles/Dockerfile_test_volume @@ -0,0 +1,7 @@ +FROM gcr.io/google-appengine/debian9 +RUN mkdir /foo +RUN echo "hello" > /foo/hey +VOLUME /foo/bar /tmp +ENV VOL /baz/bat +VOLUME ["${VOL}"] +RUN echo "hello again" > /tmp/hey diff --git a/integration_tests/dockerfiles/config_test_volume.json b/integration_tests/dockerfiles/config_test_volume.json new file mode 100644 index 000000000..72e958d85 --- /dev/null +++ b/integration_tests/dockerfiles/config_test_volume.json @@ -0,0 +1,12 @@ +[ + { + "Image1": "gcr.io/kbuild-test/docker-test-volume:latest", + "Image2": "gcr.io/kbuild-test/kbuild-test-volume:latest", + "DiffType": "File", + "Diff": { + "Adds": null, + "Dels": null, + "Mods": null + } + } +] diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index 3de05f43d..3616d219a 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -94,6 +94,14 @@ var fileTests = []struct { kbuildContext: buildcontextPath, repo: "test-workdir", }, + { + description: "test volume", + dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_volume", + configPath: "/workspace/integration_tests/dockerfiles/config_test_volume.json", + dockerContext: buildcontextPath, + kbuildContext: buildcontextPath, + repo: "test-volume", + }, { description: "test add", dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_add", diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index fd040decc..1c56faaa5 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -56,6 +56,8 @@ func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, e return &LabelCommand{cmd: c}, nil case *instructions.UserCommand: return &UserCommand{cmd: c}, nil + case *instructions.VolumeCommand: + return &VolumeCommand{cmd: c}, nil } return nil, errors.Errorf("%s is not a supported command", cmd.Name()) } diff --git a/pkg/commands/volume.go b/pkg/commands/volume.go new file mode 100644 index 000000000..d5fd8ad05 --- /dev/null +++ b/pkg/commands/volume.go @@ -0,0 +1,71 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/sirupsen/logrus" + "os" + "strings" +) + +type VolumeCommand struct { + cmd *instructions.VolumeCommand + snapshotFiles []string +} + +func (v *VolumeCommand) ExecuteCommand(config *manifest.Schema2Config) error { + logrus.Info("cmd: VOLUME") + volumes := v.cmd.Volumes + resolvedVolumes, err := util.ResolveEnvironmentReplacementList(volumes, config.Env, true) + if err != nil { + return err + } + existingVolumes := config.Volumes + if existingVolumes == nil { + existingVolumes = map[string]struct{}{} + } + for _, volume := range resolvedVolumes { + var x struct{} + existingVolumes[volume] = x + err := util.AddPathToVolumeWhitelist(volume) + if err != nil { + return err + } + + logrus.Infof("Creating directory %s", volume) + if err := os.MkdirAll(volume, 0755); err != nil { + return err + } + + //Check if directory already exists? + v.snapshotFiles = append(v.snapshotFiles, volume) + } + config.Volumes = existingVolumes + + return nil +} + +func (v *VolumeCommand) FilesToSnapshot() []string { + return v.snapshotFiles +} + +func (v *VolumeCommand) CreatedBy() string { + return strings.Join(append([]string{v.cmd.Name()}, v.cmd.Volumes...), " ") +} diff --git a/pkg/commands/volume_test.go b/pkg/commands/volume_test.go new file mode 100644 index 000000000..69bd5b9c4 --- /dev/null +++ b/pkg/commands/volume_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package commands + +import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/testutil" + "github.com/containers/image/manifest" + "github.com/docker/docker/builder/dockerfile/instructions" + "testing" +) + +func TestUpdateVolume(t *testing.T) { + cfg := &manifest.Schema2Config{ + Env: []string{ + "VOLUME=/etc", + }, + Volumes: map[string]struct{}{}, + } + + volumes := []string{ + "/tmp", + "/var/lib", + "$VOLUME", + } + + volumeCmd := &VolumeCommand{ + cmd: &instructions.VolumeCommand{ + Volumes: volumes, + }, + snapshotFiles: []string{}, + } + + expectedVolumes := map[string]struct{}{ + "/tmp": {}, + "/var/lib": {}, + "/etc": {}, + } + + err := volumeCmd.ExecuteCommand(cfg) + testutil.CheckErrorAndDeepEqual(t, false, err, expectedVolumes, cfg.Volumes) +} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index b3b020c15..a97c75964 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -31,6 +31,7 @@ import ( ) var whitelist = []string{"/kbuild"} +var volumeWhitelist = []string{} // ExtractFileSystemFromImage pulls an image and unpacks it to a file system at root func ExtractFileSystemFromImage(img string) error { @@ -156,6 +157,25 @@ func CreateFile(path string, reader io.Reader, perm os.FileMode) error { return dest.Chmod(perm) } +// AddPathToVolumeWhitelist adds the given path to the volume whitelist +// It will get snapshotted when the VOLUME command is run then ignored +// for subsequent commands. +func AddPathToVolumeWhitelist(path string) error { + logrus.Infof("adding %s to volume whitelist", path) + volumeWhitelist = append(volumeWhitelist, path) + return nil +} + +// MoveVolumeWhitelistToWhitelist copies over all directories that were volume mounted +// in this step to be whitelisted for all subsequent docker commands. +func MoveVolumeWhitelistToWhitelist() error { + if len(volumeWhitelist) > 0 { + whitelist = append(whitelist, volumeWhitelist...) + volumeWhitelist = []string{} + } + return nil +} + // DownloadFileToDest downloads the file at rawurl to the given dest for the ADD command // From add command docs: // 1. If is a remote file URL: