From 85bbb6edffdf155f2da67db42c35944fce65deab Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 26 Mar 2018 13:59:56 -0700 Subject: [PATCH] Unpack tar from GCS bucket --- executor/cmd/root.go | 39 ++++++++++ .../dockerfiles/Dockerfile_test_bucket | 2 + .../config_test_bucket_buildcontext.json | 12 +++ integration_tests/integration_test_yaml.go | 70 +++++++++++++----- pkg/constants/constants.go | 10 +++ pkg/util/bucket_util.go | 73 +++++++++++++++++++ 6 files changed, 186 insertions(+), 20 deletions(-) create mode 100644 integration_tests/dockerfiles/Dockerfile_test_bucket create mode 100644 integration_tests/dockerfiles/config_test_bucket_buildcontext.json create mode 100644 pkg/util/bucket_util.go diff --git a/executor/cmd/root.go b/executor/cmd/root.go index 80b2c088f..116380f6b 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -49,6 +49,10 @@ var RootCmd = &cobra.Command{ return util.SetLogLevel(logLevel) }, Run: func(cmd *cobra.Command, args []string) { + if err := resolveSourceContext(); err != nil { + logrus.Error(err) + os.Exit(1) + } if err := execute(); err != nil { logrus.Error(err) os.Exit(1) @@ -56,6 +60,23 @@ var RootCmd = &cobra.Command{ }, } +// resolveSourceContext unpacks the source context if it is a tar in a GCS bucket +// it resets srcContext to be the path to the unpacked build context within the image +func resolveSourceContext() error { + if util.FilepathExists(srcContext) { + return nil + } + // Else, assume the source context is the name of a bucket + logrus.Infof("Using GCS bucket %s as source context", srcContext) + buildContextPath := constants.BuildContextDir + if err := util.UnpackTarFromGCSBucket(srcContext, buildContextPath); err != nil { + return err + } + logrus.Debugf("Unpacked tar from %s to path %s", srcContext, buildContextPath) + srcContext = buildContextPath + return nil +} + func execute() error { // Parse dockerfile and unpack base image to root d, err := ioutil.ReadFile(dockerfilePath) @@ -123,5 +144,23 @@ func execute() error { } } // Push the image + if err := setDefaultEnv(); err != nil { + return err + } return image.PushImage(sourceImage, destination) } + +// setDefaultEnv sets default values for HOME and PATH so that +// config.json and docker-credential-gcr can be accessed +func setDefaultEnv() error { + defaultEnvs := map[string]string{ + "HOME": "/root", + "PATH": "/usr/local/bin/", + } + for key, val := range defaultEnvs { + if err := os.Setenv(key, val); err != nil { + return err + } + } + return nil +} diff --git a/integration_tests/dockerfiles/Dockerfile_test_bucket b/integration_tests/dockerfiles/Dockerfile_test_bucket new file mode 100644 index 000000000..356eb5e06 --- /dev/null +++ b/integration_tests/dockerfiles/Dockerfile_test_bucket @@ -0,0 +1,2 @@ +FROM gcr.io/distroless/base:latest +COPY baz /tmp/baz \ No newline at end of file diff --git a/integration_tests/dockerfiles/config_test_bucket_buildcontext.json b/integration_tests/dockerfiles/config_test_bucket_buildcontext.json new file mode 100644 index 000000000..690bbea5a --- /dev/null +++ b/integration_tests/dockerfiles/config_test_bucket_buildcontext.json @@ -0,0 +1,12 @@ +[ + { + "Image1": "gcr.io/kbuild-test/docker-test-bucket-buildcontext:latest", + "Image2": "gcr.io/kbuild-test/kbuild-test-bucket-buildcontext:latest", + "DiffType": "File", + "Diff": { + "Adds": null, + "Dels": null, + "Mods": null + } + } +] \ No newline at end of file diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index e81d4269d..26a2c9b07 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -22,41 +22,69 @@ import ( "gopkg.in/yaml.v2" ) +const ( + executorImage = "executor-image" + executorCommand = "/kbuild/executor" + dockerImage = "gcr.io/cloud-builders/docker" + ubuntuImage = "ubuntu" + testRepo = "gcr.io/kbuild-test/" + dockerPrefix = "docker-" + kbuildPrefix = "kbuild-" + daemonPrefix = "daemon://" + containerDiffOutputFile = "container-diff.json" + kbuildTestBucket = "kbuild-test-bucket" + buildcontextPath = "/workspace/integration_tests" + dockerfilesPath = "/workspace/integration_tests/dockerfiles" +) + var fileTests = []struct { description string dockerfilePath string configPath string - context string + dockerContext string + kbuildContext string repo string }{ { description: "test extract filesystem", dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_extract_fs", configPath: "/workspace/integration_tests/dockerfiles/config_test_extract_fs.json", - context: "integration_tests/dockerfiles/", + dockerContext: dockerfilesPath, + kbuildContext: dockerfilesPath, repo: "extract-filesystem", }, { description: "test run", dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run", configPath: "/workspace/integration_tests/dockerfiles/config_test_run.json", - context: "integration_tests/dockerfiles/", + dockerContext: dockerfilesPath, + kbuildContext: dockerfilesPath, repo: "test-run", }, { description: "test run no files changed", dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run_2", configPath: "/workspace/integration_tests/dockerfiles/config_test_run_2.json", - context: "integration_tests/dockerfiles/", + dockerContext: dockerfilesPath, + kbuildContext: dockerfilesPath, repo: "test-run-2", }, { description: "test copy", dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_copy", configPath: "/workspace/integration_tests/dockerfiles/config_test_copy.json", - context: "/workspace/integration_tests/", + dockerContext: buildcontextPath, + kbuildContext: buildcontextPath, repo: "test-copy", }, + { + description: "test bucket build context", + dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_copy", + configPath: "/workspace/integration_tests/dockerfiles/config_test_bucket_buildcontext.json", + dockerContext: buildcontextPath, + kbuildContext: kbuildTestBucket, + repo: "test-bucket-buildcontext", + }, } var structureTests = []struct { @@ -64,13 +92,15 @@ var structureTests = []struct { dockerfilePath string structureTestYamlPath string dockerBuildContext string + kbuildContext string repo string }{ { description: "test env", dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_env", repo: "test-env", - dockerBuildContext: "/workspace/integration_tests/dockerfiles/", + dockerBuildContext: dockerfilesPath, + kbuildContext: dockerfilesPath, structureTestYamlPath: "/workspace/integration_tests/dockerfiles/test_env.yaml", }, } @@ -85,16 +115,6 @@ type testyaml struct { Steps []step } -var executorImage = "executor-image" -var executorCommand = "/kbuild/executor" -var dockerImage = "gcr.io/cloud-builders/docker" -var ubuntuImage = "ubuntu" -var testRepo = "gcr.io/kbuild-test/" -var dockerPrefix = "docker-" -var kbuildPrefix = "kbuild-" -var daemonPrefix = "daemon://" -var containerDiffOutputFile = "container-diff.json" - func main() { // First, copy container-diff in @@ -114,27 +134,37 @@ func main() { Name: ubuntuImage, Args: []string{"chmod", "+x", "container-structure-test"}, } + + GCSBucketTarBuildContext := step{ + Name: ubuntuImage, + Args: []string{"tar", "-C", "/workspace/integration_tests/", "-cf", "/workspace/kbuild.tar", "."}, + } + uploadTarBuildContext := step{ + Name: "gcr.io/cloud-builders/gsutil", + Args: []string{"cp", "/workspace/kbuild.tar", "gs://kbuild-test-bucket/"}, + } + // Build executor image buildExecutorImage := step{ Name: dockerImage, Args: []string{"build", "-t", executorImage, "-f", "deploy/Dockerfile", "."}, } y := testyaml{ - Steps: []step{containerDiffStep, containerDiffPermissions, structureTestsStep, structureTestPermissions, buildExecutorImage}, + Steps: []step{containerDiffStep, containerDiffPermissions, structureTestsStep, structureTestPermissions, GCSBucketTarBuildContext, uploadTarBuildContext, buildExecutorImage}, } for _, test := range fileTests { // First, build the image with docker dockerImageTag := testRepo + dockerPrefix + test.repo dockerBuild := step{ Name: dockerImage, - Args: []string{"build", "-t", dockerImageTag, "-f", test.dockerfilePath, test.context}, + Args: []string{"build", "-t", dockerImageTag, "-f", test.dockerfilePath, test.dockerContext}, } // Then, buld the image with kbuild kbuildImage := testRepo + kbuildPrefix + test.repo kbuild := step{ Name: executorImage, - Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.context}, + Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.kbuildContext}, } // Pull the kbuild image @@ -178,7 +208,7 @@ func main() { kbuildImage := testRepo + kbuildPrefix + test.repo kbuild := step{ Name: executorImage, - Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath}, + Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.kbuildContext}, } // Pull the kbuild image pullKbuildImage := step{ diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 5845194bd..06d76dd15 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -26,7 +26,17 @@ const ( // WorkspaceDir is the path to the workspace directory WorkspaceDir = "/workspace" + //KbuildDir is the path to the kbuild directory + KbuildDir = "/kbuild" + WhitelistPath = "/proc/self/mountinfo" Author = "kbuild" + + // KbuildTar is the default name of the tar uploaded to GCS buckets + KbuildTar = "kbuild.tar" + + // BuildContextDir is the directory a build context will be unpacked into, + // for example, a tarball from a GCS bucket will be unpacked here + BuildContextDir = "/kbuild/buildcontext/" ) diff --git a/pkg/util/bucket_util.go b/pkg/util/bucket_util.go new file mode 100644 index 000000000..8df1d3839 --- /dev/null +++ b/pkg/util/bucket_util.go @@ -0,0 +1,73 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "cloud.google.com/go/storage" + pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util" + "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/constants" + "github.com/sirupsen/logrus" + "golang.org/x/net/context" + "os" + "path/filepath" +) + +// UnpackTarFromGCSBucket unpacks the kbuild.tar file in the given bucket to the given directory +func UnpackTarFromGCSBucket(bucketName, directory string) error { + // Get the tar from the bucket + tarPath, err := getTarFromBucket(bucketName, directory) + if err != nil { + return err + } + logrus.Debug("Unpacking source context tar...") + // Now, unpack the tar to a build context, and return the path to the build context + file, err := os.Open(tarPath) + if err != nil { + return err + } + if err := pkgutil.UnTar(file, directory, nil); err != nil { + return err + } + // Remove the tar so it doesn't interfere with subsequent commands + logrus.Debugf("Deleting %s", tarPath) + return os.Remove(tarPath) +} + +// getTarFromBucket gets kbuild.tar from the GCS bucket and saves it to the filesystem +// It returns the path to the tar file +func getTarFromBucket(bucketName, directory string) (string, error) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return "", err + } + bucket := client.Bucket(bucketName) + + // Get the tarfile kbuild.tar from the GCS bucket, and save it to a tar object + + reader, err := bucket.Object(constants.KbuildTar).NewReader(ctx) + if err != nil { + return "", err + } + defer reader.Close() + tarPath := filepath.Join(directory, constants.KbuildTar) + if err := CreateFile(tarPath, reader, 0600); err != nil { + return "", err + } + logrus.Debugf("Copied tarball %s from GCS bucket %s to %s", constants.KbuildTar, bucketName, tarPath) + return tarPath, nil +}