From 74c4a6629de97626bbcc8a68e7126c4f3d066384 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 3 Apr 2018 14:38:17 -0700 Subject: [PATCH] Unpack context.tar.gz from bucket --- Gopkg.lock | 2 +- executor/cmd/root.go | 8 +++++++- integration_tests/integration_test_yaml.go | 7 +++---- pkg/constants/constants.go | 4 ++-- pkg/util/bucket_util.go | 14 ++++---------- pkg/util/tar_util.go | 17 +++++++++++++++++ 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index e331fd5f0..9c77abebd 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -620,6 +620,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "8538050f7de37808cc15a8a347740aa73ccb80f0302b9dda86c929a6d8b0db35" + inputs-digest = "51fdf15812acb4ba731695ce038965ca38f136449addbebf03a6f13cd91d4077" solver-name = "gps-cdcl" solver-version = 1 diff --git a/executor/cmd/root.go b/executor/cmd/root.go index 6df579254..42ca57e1c 100644 --- a/executor/cmd/root.go +++ b/executor/cmd/root.go @@ -28,6 +28,7 @@ import ( "github.com/spf13/cobra" "io/ioutil" "os" + "path/filepath" ) var ( @@ -39,7 +40,7 @@ var ( ) func init() { - RootCmd.PersistentFlags().StringVarP(&dockerfilePath, "dockerfile", "f", "/workspace/Dockerfile", "Path to the dockerfile to be built.") + RootCmd.PersistentFlags().StringVarP(&dockerfilePath, "dockerfile", "f", "Dockerfile", "Path to the dockerfile to be built.") RootCmd.PersistentFlags().StringVarP(&srcContext, "context", "c", "", "Path to the dockerfile build context.") RootCmd.PersistentFlags().StringVarP(&bucket, "bucket", "b", "", "Name of the GCS bucket from which to access build context as tarball.") RootCmd.PersistentFlags().StringVarP(&destination, "destination", "d", "", "Registry the final image should be pushed to (ex: gcr.io/test/example:latest)") @@ -82,6 +83,11 @@ func resolveSourceContext() error { } logrus.Debugf("Unpacked tar from %s to path %s", bucket, buildContextPath) srcContext = buildContextPath + // If path to dockerfile doesn't exist, assume it is in the unpacked tar + if !util.FilepathExists(dockerfilePath) { + logrus.Debugf("Expecting dockerfile to be located at %s within the tar build context", dockerfilePath) + dockerfilePath = filepath.Join(srcContext, dockerfilePath) + } return nil } diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index 9ec5ab58e..0e3505677 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -24,7 +24,6 @@ import ( const ( executorImage = "executor-image" - executorCommand = "/kbuild/executor" dockerImage = "gcr.io/cloud-builders/docker" ubuntuImage = "ubuntu" testRepo = "gcr.io/kbuild-test/" @@ -163,11 +162,11 @@ func main() { GCSBucketTarBuildContext := step{ Name: ubuntuImage, - Args: []string{"tar", "-C", "/workspace/integration_tests/", "-cf", "/workspace/kbuild.tar", "."}, + Args: []string{"tar", "-C", "/workspace/integration_tests/", "-zcvf", "/workspace/context.tar.gz", "."}, } uploadTarBuildContext := step{ Name: "gcr.io/cloud-builders/gsutil", - Args: []string{"cp", "/workspace/kbuild.tar", "gs://kbuild-test-bucket/"}, + Args: []string{"cp", "/workspace/context.tar.gz", "gs://kbuild-test-bucket/"}, } // Build executor image @@ -238,7 +237,7 @@ func main() { kbuildImage := testRepo + kbuildPrefix + test.repo kbuild := step{ Name: executorImage, - Args: []string{"--destination", kbuildImage, "--dockerfile", test.dockerfilePath}, + Args: []string{"--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.kbuildContext}, } // Pull the kbuild image pullKbuildImage := step{ diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 06d76dd15..f2f1709dd 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -33,8 +33,8 @@ const ( Author = "kbuild" - // KbuildTar is the default name of the tar uploaded to GCS buckets - KbuildTar = "kbuild.tar" + // ContextTar is the default name of the tar uploaded to GCS buckets + ContextTar = "context.tar.gz" // BuildContextDir is the directory a build context will be unpacked into, // for example, a tarball from a GCS bucket will be unpacked here diff --git a/pkg/util/bucket_util.go b/pkg/util/bucket_util.go index 46ed9a9ef..c96bb715c 100644 --- a/pkg/util/bucket_util.go +++ b/pkg/util/bucket_util.go @@ -18,7 +18,6 @@ package util import ( "cloud.google.com/go/storage" - pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util" "github.com/GoogleCloudPlatform/k8s-container-builder/pkg/constants" "github.com/sirupsen/logrus" "golang.org/x/net/context" @@ -34,12 +33,7 @@ func UnpackTarFromGCSBucket(bucketName, directory string) error { return err } logrus.Debug("Unpacking source context tar...") - // Now, unpack the tar to a build context, and return the path to the build context - file, err := os.Open(tarPath) - if err != nil { - return err - } - if err := pkgutil.UnTar(file, directory, nil); err != nil { + if err := UnpackCompressedTar(tarPath, directory); err != nil { return err } // Remove the tar so it doesn't interfere with subsequent commands @@ -57,15 +51,15 @@ func getTarFromBucket(bucketName, directory string) (string, error) { } bucket := client.Bucket(bucketName) // Get the tarfile kbuild.tar from the GCS bucket, and save it to a tar object - reader, err := bucket.Object(constants.KbuildTar).NewReader(ctx) + reader, err := bucket.Object(constants.ContextTar).NewReader(ctx) if err != nil { return "", err } defer reader.Close() - tarPath := filepath.Join(directory, constants.KbuildTar) + tarPath := filepath.Join(directory, constants.ContextTar) if err := CreateFile(tarPath, reader, 0600); err != nil { return "", err } - logrus.Debugf("Copied tarball %s from GCS bucket %s to %s", constants.KbuildTar, bucketName, tarPath) + logrus.Debugf("Copied tarball %s from GCS bucket %s to %s", constants.ContextTar, bucketName, tarPath) return tarPath, nil } diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go index 417f16f04..f44b8208c 100644 --- a/pkg/util/tar_util.go +++ b/pkg/util/tar_util.go @@ -18,6 +18,8 @@ package util import ( "archive/tar" + "compress/gzip" + pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util" "github.com/sirupsen/logrus" "io" "os" @@ -86,3 +88,18 @@ func checkHardlink(p string, i os.FileInfo) (bool, string) { } return hardlink, linkDst } + +// UnpackCompressedTar unpacks the compressed tar at path to dir +func UnpackCompressedTar(path, dir string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + gzr, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gzr.Close() + return pkgutil.UnTar(gzr, dir, nil) +}