diff --git a/Makefile b/Makefile index 80e9fe5f2..9d1064eb8 100644 --- a/Makefile +++ b/Makefile @@ -29,13 +29,13 @@ REPOPATH ?= $(ORG)/$(PROJECT) GO_FILES := $(shell find . -type f -name '*.go' -not -path "./vendor/*") GO_LDFLAGS := '-extldflags "-static"' -GO_BUILD_TAGS := "containers_image_ostree_stub containers_image_openpgp exclude_graphdriver_devicemapper exclude_graphdriver_btrfs" +GO_BUILD_TAGS := "containers_image_ostree_stub containers_image_openpgp exclude_graphdriver_devicemapper exclude_graphdriver_btrfs exclude_graphdriver_overlay" EXECUTOR_PACKAGE = $(REPOPATH)/executor KBUILD_PACKAGE = $(REPOPATH)/kbuild out/executor: $(GO_FILES) - GOOS=$* GOARCH=$(GOARCH) CGO_ENABLED=1 go build -ldflags $(GO_LDFLAGS) -tags $(GO_BUILD_TAGS) -o $@ $(EXECUTOR_PACKAGE) + GOOS=$* GOARCH=$(GOARCH) CGO_ENABLED=0 go build -ldflags $(GO_LDFLAGS) -tags $(GO_BUILD_TAGS) -o $@ $(EXECUTOR_PACKAGE) out/kbuild: $(GO_FILES) diff --git a/executor/cmd/root.go b/executor/cmd/root.go index 0afb6dae3..f0c5be45b 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -17,11 +17,13 @@ limitations under the License. package cmd import ( + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/commands" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/constants" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/dockerfile" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/image" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/snapshot" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/util" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "io/ioutil" @@ -88,7 +90,57 @@ func execute() error { } // Execute commands here + if err := image.SetEnvVariables(); err != nil { + return err + } + // Currently only supports single stage builds + for _, stage := range stages { + for _, cmd := range stage.Commands { + dockerCommand := commands.GetCommand(cmd) + if dockerCommand == nil { + return errors.Errorf("Invalid or unsupported docker command: %v", cmd) + } + if err := dockerCommand.ExecuteCommand(); err != nil { + return err + } + // Now, we get the files to snapshot from this command + // If this is nil, snapshot the entire filesystem + // Else take a snapshot of the specific files + snapshotFiles := dockerCommand.FilesToSnapshot() + if snapshotFiles == nil { + logrus.Info("Taking snapshot of full filesystem...") + contents, filesAdded, err := snapshotter.TakeSnapshot() + if err != nil { + return err + } + if !filesAdded { + logrus.Info("No files were changed, appending empty layer to config.") + image.AppendConfigHistory(dockerCommand.Author(), true) + continue + } + // Append the layer to the image + if err := image.AppendLayer(contents, dockerCommand.Author()); err != nil { + return err + } + } else { + logrus.Infof("Taking snapshot of files %v...", snapshotFiles) + contents, err := snapshotter.TakeSnapshotOfFiles(snapshotFiles) + if err != nil { + return err + } + if contents == nil { + logrus.Info("No files were changed, appending empty layer to config.") + image.AppendConfigHistory(dockerCommand.Author(), true) + continue + } + // Append the layer to the image + if err := image.AppendLayer(contents, dockerCommand.Author()); err != nil { + return err + } + } + } + } // Push the image return image.PushImage(destination) } diff --git a/integration_tests/dockerfiles/Dockerfile_test_run b/integration_tests/dockerfiles/Dockerfile_test_run new file mode 100644 index 000000000..f928750c3 --- /dev/null +++ b/integration_tests/dockerfiles/Dockerfile_test_run @@ -0,0 +1,5 @@ +FROM gcr.io/google-appengine/debian9 +RUN echo "hey" > /etc/foo +RUN apt-get update && apt-get install -y \ + bzr \ + cvs \ diff --git a/integration_tests/dockerfiles/config_test_run.json b/integration_tests/dockerfiles/config_test_run.json new file mode 100644 index 000000000..53f0a3eb2 --- /dev/null +++ b/integration_tests/dockerfiles/config_test_run.json @@ -0,0 +1,57 @@ +[ + { + "Image1": "gcr.io/kbuild-test/docker-test-run:latest", + "Image2": "gcr.io/kbuild-test/kbuild-test-run:latest", + "DiffType": "File", + "Diff": { + "Adds": [ + { + "Name": "/bin/bzcat", + "Size": 35448 + }, + { + "Name": "/bin/bzip2", + "Size": 35448 + } + ], + "Dels": null, + "Mods": [ + { + "Name": "/var/log/dpkg.log", + "Size1": 57425, + "Size2": 57425 + }, + { + "Name": "/var/log/apt/term.log", + "Size1": 24400, + "Size2": 24400 + }, + { + "Name": "/var/cache/ldconfig/aux-cache", + "Size1": 8057, + "Size2": 8057 + }, + { + "Name": "/var/log/apt/history.log", + "Size1": 5089, + "Size2": 5089 + }, + { + "Name": "/var/log/alternatives.log", + "Size1": 2579, + "Size2": 2579 + }, + { + "Name": "/usr/lib/python2.7/dist-packages/keyrings/__init__.pyc", + "Size1": 140, + "Size2": 140 + }, + { + "Name": "/usr/lib/python2.7/dist-packages/lazr/__init__.pyc", + "Size1": 136, + "Size2": 136 + } + ] + } + } +] \ No newline at end of file diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index c11b901ec..f422c9989 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -35,6 +35,13 @@ var tests = []struct { context: "integration_tests/dockerfiles/", repo: "extract-filesystem", }, + { + description: "test run", + dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run", + configPath: "/workspace/integration_tests/dockerfiles/config_test_run.json", + context: "integration_tests/dockerfiles/", + repo: "test-run", + }, } type step struct { diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go new file mode 100644 index 000000000..e8cfa8ebb --- /dev/null +++ b/pkg/commands/commands.go @@ -0,0 +1,36 @@ +/* +Copyright 2018 Google LLC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/sirupsen/logrus" +) + +type DockerCommand interface { + ExecuteCommand() error + // The config file has an "author" field, should return information about the command + Author() string + // A list of files to snapshot, empty for metadata commands or nil if we don't know + FilesToSnapshot() []string +} + +func GetCommand(cmd instructions.Command) DockerCommand { + switch c := cmd.(type) { + case *instructions.RunCommand: + return &RunCommand{cmd: c} + } + logrus.Errorf("%s is not a supported command.", cmd.Name()) + return nil +} diff --git a/pkg/commands/run.go b/pkg/commands/run.go new file mode 100644 index 000000000..9b85387b4 --- /dev/null +++ b/pkg/commands/run.go @@ -0,0 +1,55 @@ +/* +Copyright 2018 Google LLC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/docker/docker/builder/dockerfile/instructions" + "github.com/sirupsen/logrus" + "os" + "os/exec" + "strings" +) + +type RunCommand struct { + cmd *instructions.RunCommand +} + +func (r *RunCommand) ExecuteCommand() error { + var newCommand []string + if r.cmd.PrependShell { + // This is the default shell on Linux + shell := []string{"/bin/sh", "-c"} + newCommand = append(shell, strings.Join(r.cmd.CmdLine, " ")) + } else { + newCommand = r.cmd.CmdLine + } + + logrus.Infof("cmd: %s", newCommand[0]) + logrus.Infof("args: %s", newCommand[1:]) + + cmd := exec.Command(newCommand[0], newCommand[1:]...) + cmd.Stdout = os.Stdout + return cmd.Run() +} + +// FilesToSnapshot returns nil for this command because we don't know which files +// have changed, so we snapshot the entire system. +func (r *RunCommand) FilesToSnapshot() []string { + return nil +} + +// Author returns some information about the command for the image config +func (r *RunCommand) Author() string { + return "kbuild /bin/sh -c" + strings.Join(r.cmd.CmdLine, " ") +} diff --git a/pkg/image/image.go b/pkg/image/image.go index b76ad1269..a55c122f8 100644 --- a/pkg/image/image.go +++ b/pkg/image/image.go @@ -23,6 +23,7 @@ import ( "github.com/containers/image/signature" "github.com/containers/image/transports/alltransports" "github.com/sirupsen/logrus" + "os" ) // sourceImage is the image that will be modified by the executor @@ -45,6 +46,7 @@ func InitializeSourceImage(srcImg string) error { // AppendLayer appends a layer onto the base image func AppendLayer(contents []byte, author string) error { + logrus.Info("Appending layer to base image") return sourceImage.AppendLayer(contents, author) } @@ -66,6 +68,23 @@ func PushImage(destImg string) error { return copy.Image(policyContext, destRef, srcRef, nil) } +// AppendConfigHistory appends to the source image config history +func AppendConfigHistory(author string, emptyLayer bool) { + sourceImage.AppendConfigHistory(author, emptyLayer) +} + +// SetEnvVariables sets environment variables as specified in the image +func SetEnvVariables() error { + envVars := sourceImage.Env() + for key, val := range envVars { + if err := os.Setenv(key, val); err != nil { + return err + } + logrus.Debugf("Setting environment variable %s=%s", key, val) + } + return nil +} + func getPolicyContext() (*signature.PolicyContext, error) { policyContext, err := signature.NewPolicyContext(&signature.Policy{ Default: signature.PolicyRequirements{signature.NewPRInsecureAcceptAnything()},