Feature/contextsources (#195)

* added switch to extract different sources as build context

* first rough implementation of aws s3

* added buildcontext package and interface

* added GetBuildContext func to buildcontext.go
added fallback to gcs
renamed GC struct to GCS

* improved the default behavior of build context retrieval

* renamed gc:// to gs:// in order to follow common standards

* renamed struct File to Dir and some cleanup work

* moved context.tar suffix to the buildcontext processors where it is needed

* added buildcontext retrieval as struct variable

added fallback if prefix in bucket specifier is present

* cleanup if structures

* added prefix to s3

* WIP

* Fixed build context bugs

* refactored build context

pkg/buildcontext/buildcontext.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"errors"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"strings"
+)
+
+// BuildContext unifies calls to download and unpack the build context.
+type BuildContext interface {
+	// Unpacks a build context and returns the directory where it resides
+	UnpackTarFromBuildContext() (string, error)
+}
+
+// GetBuildContext parses srcContext for the prefix and returns related buildcontext
+// parser
+func GetBuildContext(srcContext string) (BuildContext, error) {
+	split := strings.SplitAfter(srcContext, "://")
+	prefix := split[0]
+	context := split[1]
+	switch prefix {
+	case constants.GCSBuildContextPrefix:
+		return &GCS{context: context}, nil
+	case constants.S3BuildContextPrefix:
+		return &S3{context: context}, nil
+	case constants.LocalDirBuildContextPrefix:
+		return &Dir{context: context}, nil
+	}
+	return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://")
+}

pkg/buildcontext/dir.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+// Dir unifies calls to download and unpack the build context.
+type Dir struct {
+	context string
+}
+
+// UnpackTarFromBuildContext just provides a directory with already extracted content
+func (f *Dir) UnpackTarFromBuildContext() (string, error) {
+	return f.context, nil
+}

pkg/buildcontext/gcs.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"os"
+	"path/filepath"
+
+	"cloud.google.com/go/storage"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+)
+
+// GCS struct for Google Cloud Storage processing
+type GCS struct {
+	context string
+}
+
+func (g *GCS) UnpackTarFromBuildContext() (string, error) {
+	bucket, item := util.GetBucketAndItem(g.context)
+	return constants.BuildContextDir, unpackTarFromGCSBucket(bucket, item, constants.BuildContextDir)
+}
+
+// unpackTarFromGCSBucket unpacks the context.tar.gz file in the given bucket to the given directory
+func unpackTarFromGCSBucket(bucketName, item, directory string) error {
+	// Get the tar from the bucket
+	tarPath, err := getTarFromBucket(bucketName, item, directory)
+	if err != nil {
+		return err
+	}
+	logrus.Debug("Unpacking source context tar...")
+	if err := util.UnpackCompressedTar(tarPath, directory); err != nil {
+		return err
+	}
+	// Remove the tar so it doesn't interfere with subsequent commands
+	logrus.Debugf("Deleting %s", tarPath)
+	return os.Remove(tarPath)
+}
+
+// getTarFromBucket gets context.tar.gz from the GCS bucket and saves it to the filesystem
+// It returns the path to the tar file
+func getTarFromBucket(bucketName, item, directory string) (string, error) {
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		return "", err
+	}
+	bucket := client.Bucket(bucketName)
+	// Get the tarfile context.tar.gz from the GCS bucket, and save it to a tar object
+	reader, err := bucket.Object(item).NewReader(ctx)
+	if err != nil {
+		return "", err
+	}
+	defer reader.Close()
+	tarPath := filepath.Join(directory, constants.ContextTar)
+	if err := util.CreateFile(tarPath, reader, 0600, 0, 0); err != nil {
+		return "", err
+	}
+	logrus.Debugf("Copied tarball %s from GCS bucket %s to %s", constants.ContextTar, bucketName, tarPath)
+	return tarPath, nil
+}

pkg/buildcontext/s3.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"os"
+	"path/filepath"
+)
+
+// S3 unifies calls to download and unpack the build context.
+type S3 struct {
+	context string
+}
+
+// UnpackTarFromBuildContext download and untar a file from s3
+func (s *S3) UnpackTarFromBuildContext() (string, error) {
+	bucket, item := util.GetBucketAndItem(s.context)
+	downloader := s3manager.NewDownloader(session.New())
+	directory := constants.BuildContextDir
+	tarPath := filepath.Join(directory, constants.ContextTar)
+	if err := os.MkdirAll(directory, 0750); err != nil {
+		return directory, err
+	}
+	file, err := os.Create(tarPath)
+	if err != nil {
+		return directory, err
+	}
+	_, err = downloader.Download(file,
+		&s3.GetObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(item),
+		})
+	if err != nil {
+		return directory, err
+	}
+
+	return directory, util.UnpackCompressedTar(tarPath, directory)
+}

pkg/constants/constants.go

@@ -46,6 +46,10 @@ const (
 
 	// NoBaseImage is the scratch image
 	NoBaseImage = "scratch"
+
+	GCSBuildContextPrefix      = "gs://"
+	S3BuildContextPrefix       = "s3://"
+	LocalDirBuildContextPrefix = "dir://"
 )
 
 // KanikoBuildFiles is the list of files required to build kaniko

pkg/util/bucket_util.go

@@ -17,50 +17,14 @@ limitations under the License.
 package util
 
 import (
-	"os"
-	"path/filepath"
-
-	"cloud.google.com/go/storage"
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
-	"github.com/sirupsen/logrus"
-	"golang.org/x/net/context"
+	"strings"
 )
 
-// UnpackTarFromGCSBucket unpacks the context.tar.gz file in the given bucket to the given directory
-func UnpackTarFromGCSBucket(bucketName, directory string) error {
-	// Get the tar from the bucket
-	tarPath, err := getTarFromBucket(bucketName, directory)
-	if err != nil {
-		return err
+func GetBucketAndItem(context string) (string, string) {
+	split := strings.SplitN(context, "/", 2)
+	if len(split) == 2 {
+		return split[0], split[1]
 	}
-	logrus.Debug("Unpacking source context tar...")
-	if err := UnpackCompressedTar(tarPath, directory); err != nil {
-		return err
-	}
-	// Remove the tar so it doesn't interfere with subsequent commands
-	logrus.Debugf("Deleting %s", tarPath)
-	return os.Remove(tarPath)
-}
-
-// getTarFromBucket gets context.tar.gz from the GCS bucket and saves it to the filesystem
-// It returns the path to the tar file
-func getTarFromBucket(bucketName, directory string) (string, error) {
-	ctx := context.Background()
-	client, err := storage.NewClient(ctx)
-	if err != nil {
-		return "", err
-	}
-	bucket := client.Bucket(bucketName)
-	// Get the tarfile context.tar.gz from the GCS bucket, and save it to a tar object
-	reader, err := bucket.Object(constants.ContextTar).NewReader(ctx)
-	if err != nil {
-		return "", err
-	}
-	defer reader.Close()
-	tarPath := filepath.Join(directory, constants.ContextTar)
-	if err := CreateFile(tarPath, reader, 0600, 0, 0); err != nil {
-		return "", err
-	}
-	logrus.Debugf("Copied tarball %s from GCS bucket %s to %s", constants.ContextTar, bucketName, tarPath)
-	return tarPath, nil
+	return split[0], constants.ContextTar
 }