Merge remote-tracking branch 'upstream/master'

pkg/buildcontext/azureblob.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
+)
+
+// AzureBlob struct for Azure Blob Storage processing
+type AzureBlob struct {
+	context string
+}
+
+// Download context file from given azure blob storage url and unpack it to BuildContextDir
+func (b *AzureBlob) UnpackTarFromBuildContext() (string, error) {
+
+	// Get Azure_STORAGE_ACCESS_KEY from environment variables
+	accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if len(accountKey) == 0 {
+		return "", errors.New("AZURE_STORAGE_ACCESS_KEY environment variable is not set")
+	}
+
+	// Get storage accoutname for Azure Blob Storage
+	u, _ := url.Parse(b.context)
+	parts := azblob.NewBlobURLParts(*u)
+	accountName := strings.Split(parts.Host, ".")[0]
+
+	// Generate credentail with accountname and accountkey
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return parts.Host, err
+	}
+
+	// Create directory and target file for downloading the context file
+	directory := constants.BuildContextDir
+	tarPath := filepath.Join(directory, constants.ContextTar)
+	file, err := util.CreateTargetTarfile(tarPath)
+	if err != nil {
+		return tarPath, err
+	}
+
+	// Downloading contextfile from Azure Blob Storage
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+	blobURL := azblob.NewBlobURL(*u, p)
+	ctx := context.Background()
+
+	if err := azblob.DownloadBlobToFile(ctx, blobURL, 0, 0, file, azblob.DownloadFromBlobOptions{}); err != nil {
+		return parts.Host, err
+	}
+
+	if err := util.UnpackCompressedTar(tarPath, directory); err != nil {
+		return tarPath, err
+	}
+	// Remove the tar so it doesn't interfere with subsequent commands
+	return directory, os.Remove(tarPath)
+}

pkg/buildcontext/buildcontext.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
 )
 
 // BuildContext unifies calls to download and unpack the build context.
@@ -35,6 +36,7 @@ func GetBuildContext(srcContext string) (BuildContext, error) {
 	split := strings.SplitAfter(srcContext, "://")
 	prefix := split[0]
 	context := split[1]
+
 	switch prefix {
 	case constants.GCSBuildContextPrefix:
 		return &GCS{context: context}, nil
@@ -44,6 +46,11 @@ func GetBuildContext(srcContext string) (BuildContext, error) {
 		return &Dir{context: context}, nil
 	case constants.GitBuildContextPrefix:
 		return &Git{context: context}, nil
+	case constants.HTTPSBuildContextPrefix:
+		if util.ValidAzureBlobStorageHost(srcContext) {
+			return &AzureBlob{context: srcContext}, nil
+		}
+		return nil, errors.New("url provided for https context is not in a supported format, please use the https url for Azure Blob Storage")
 	}
-	return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://, git://")
+	return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://, git://, https://")
 }

pkg/constants/constants.go

@@ -58,6 +58,7 @@ const (
 	S3BuildContextPrefix       = "s3://"
 	LocalDirBuildContextPrefix = "dir://"
 	GitBuildContextPrefix      = "git://"
+	HTTPSBuildContextPrefix    = "https://"
 
 	HOME = "HOME"
 	// DefaultHOMEValue is the default value Docker sets for $HOME
@@ -78,3 +79,10 @@ const (
 
 // ScratchEnvVars are the default environment variables needed for a scratch image.
 var ScratchEnvVars = []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}
+
+// AzureBlobStorageHostRegEx is ReqEX for Valid azure blob storage host suffix in url for AzureCloud, AzureChinaCloud, AzureGermanCloud and AzureUSGovernment
+var AzureBlobStorageHostRegEx = []string{"https://(.+?).blob.core.windows.net/(.+)",
+	"https://(.+?).blob.core.chinacloudapi.cn/(.+)",
+	"https://(.+?).blob.core.cloudapi.de/(.+)",
+	"https://(.+?).blob.core.usgovcloudapi.net/(.+)",
+}

pkg/util/azureblob_util.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"regexp"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+)
+
+// Validate if the host url provided is with correct suffix for AzureCloud, AzureChinaCloud, AzureGermanCloud and AzureUSGovernment
+// RegEX for supported suffix defined in constants.AzureBlobStorageHostRegEx
+func ValidAzureBlobStorageHost(context string) bool {
+	for _, re := range constants.AzureBlobStorageHostRegEx {
+		validBlobURL := regexp.MustCompile(re)
+		if validBlobURL.MatchString(context) {
+			return true
+		}
+	}
+
+	return false
+}

pkg/util/azureblob_util_test.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	"github.com/GoogleContainerTools/kaniko/testutil"
+)
+
+func Test_ValidAzureBlobStorageHost(t *testing.T) {
+	tests := []struct {
+		name           string
+		context        string
+		expectedResult bool
+	}{
+		{
+			name:           "AzureCloud",
+			context:        "https://myaccount.blob.core.windows.net/fairingcontext/context.tar.gz",
+			expectedResult: true,
+		},
+		{
+			name:           "AzureChinaCloud",
+			context:        "https://myaccount.blob.core.chinacloudapi.cn/fairingcontext/context.tar.gz",
+			expectedResult: true,
+		},
+		{
+			name:           "AzureGermanCloud",
+			context:        "https://myaccount.blob.core.cloudapi.de/fairingcontext/context.tar.gz",
+			expectedResult: true,
+		},
+		{
+			name:           "AzureUSGovernment",
+			context:        "https://myaccount.blob.core.usgovcloudapi.net/fairingcontext/context.tar.gz",
+			expectedResult: true,
+		},
+		{
+			name:           "Invalid Azure Blob Storage Hostname",
+			context:        "https://myaccount.anything.core.windows.net/fairingcontext/context.tar.gz",
+			expectedResult: false,
+		},
+		{
+			name:           "URL Missing Accountname",
+			context:        "https://blob.core.windows.net/fairingcontext/context.tar.gz",
+			expectedResult: false,
+		},
+		{
+			name:           "URL Missing Containername",
+			context:        "https://myaccount.blob.core.windows.net/",
+			expectedResult: false,
+		},
+		{
+			name:           "URL with folder structure",
+			context:        "https://myaccount.blob.core.windows.net/fairingcontext/path/to/context.tar.gz",
+			expectedResult: true,
+		},
+		{
+			name:           "URL with $root container",
+			context:        "https://myaccount.blob.core.windows.net/$root/context.tar.gz",
+			expectedResult: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			result := ValidAzureBlobStorageHost(test.context)
+			testutil.CheckDeepEqual(t, test.expectedResult, result)
+
+		})
+	}
+}

pkg/util/fs_util.go

@@ -84,9 +84,9 @@ func GetFSFromImage(root string, img v1.Image) ([]string, error) {
 
 	for i, l := range layers {
 		if mediaType, err := l.MediaType(); err == nil {
-			logrus.Debugf("Extracting layer %d of media type %s", mediaType)
+			logrus.Tracef("Extracting layer %d of media type %s", mediaType)
 		} else {
-			logrus.Debugf("Extracting layer %d", i)
+			logrus.Tracef("Extracting layer %d", i)
 		}
 
 		r, err := l.Uncompressed()
@@ -206,10 +206,10 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error {
 	}
 	switch hdr.Typeflag {
 	case tar.TypeReg:
-		logrus.Debugf("creating file %s", path)
+		logrus.Tracef("creating file %s", path)
 		// It's possible a file is in the tar before its directory.
 		if _, err := os.Stat(dir); os.IsNotExist(err) {
-			logrus.Debugf("base %s for file %s does not exist. Creating.", base, path)
+			logrus.Tracef("base %s for file %s does not exist. Creating.", base, path)
 			if err := os.MkdirAll(dir, 0755); err != nil {
 				return err
 			}
@@ -233,19 +233,19 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error {
 		}
 		currFile.Close()
 	case tar.TypeDir:
-		logrus.Debugf("creating dir %s", path)
+		logrus.Tracef("creating dir %s", path)
 		if err := mkdirAllWithPermissions(path, mode, uid, gid); err != nil {
 			return err
 		}
 
 	case tar.TypeLink:
-		logrus.Debugf("link from %s to %s", hdr.Linkname, path)
+		logrus.Tracef("link from %s to %s", hdr.Linkname, path)
 		abs, err := filepath.Abs(hdr.Linkname)
 		if err != nil {
 			return err
 		}
 		if CheckWhitelist(abs) {
-			logrus.Debugf("skipping symlink from %s to %s because %s is whitelisted", hdr.Linkname, path, hdr.Linkname)
+			logrus.Tracef("skipping symlink from %s to %s because %s is whitelisted", hdr.Linkname, path, hdr.Linkname)
 			return nil
 		}
 		// The base directory for a link may not exist before it is created.
@@ -265,7 +265,7 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error {
 		}
 
 	case tar.TypeSymlink:
-		logrus.Debugf("symlink from %s to %s", hdr.Linkname, path)
+		logrus.Tracef("symlink from %s to %s", hdr.Linkname, path)
 		// The base directory for a symlink may not exist before it is created.
 		if err := os.MkdirAll(dir, 0755); err != nil {
 			return err
@@ -327,27 +327,27 @@ func DetectFilesystemWhitelist(path string) error {
 	reader := bufio.NewReader(f)
 	for {
 		line, err := reader.ReadString('\n')
-		logrus.Debugf("Read the following line from %s: %s", path, line)
+		logrus.Tracef("Read the following line from %s: %s", path, line)
 		if err != nil && err != io.EOF {
 			return err
 		}
 		lineArr := strings.Split(line, " ")
 		if len(lineArr) < 5 {
 			if err == io.EOF {
-				logrus.Debugf("Reached end of file %s", path)
+				logrus.Tracef("Reached end of file %s", path)
 				break
 			}
 			continue
 		}
 		if lineArr[4] != constants.RootDir {
-			logrus.Debugf("Appending %s from line: %s", lineArr[4], line)
+			logrus.Tracef("Appending %s from line: %s", lineArr[4], line)
 			whitelist = append(whitelist, WhitelistEntry{
 				Path:            lineArr[4],
 				PrefixMatchOnly: false,
 			})
 		}
 		if err == io.EOF {
-			logrus.Debugf("Reached end of file %s", path)
+			logrus.Tracef("Reached end of file %s", path)
 			break
 		}
 	}
@@ -422,7 +422,7 @@ func CreateFile(path string, reader io.Reader, perm os.FileMode, uid uint32, gid
 	// Create directory path if it doesn't exist
 	baseDir := filepath.Dir(path)
 	if _, err := os.Lstat(baseDir); os.IsNotExist(err) {
-		logrus.Debugf("baseDir %s for file %s does not exist. Creating.", baseDir, path)
+		logrus.Tracef("baseDir %s for file %s does not exist. Creating.", baseDir, path)
 		if err := os.MkdirAll(baseDir, 0755); err != nil {
 			return err
 		}
@@ -640,3 +640,17 @@ func setFilePermissions(path string, mode os.FileMode, uid, gid int) error {
 	// Must chmod after chown because chown resets the file mode.
 	return os.Chmod(path, mode)
 }
+
+// CreateTargetTarfile creates target tar file for downloading the context file.
+// Make directory if directory does not exist
+func CreateTargetTarfile(tarpath string) (*os.File, error) {
+	baseDir := filepath.Dir(tarpath)
+	if _, err := os.Lstat(baseDir); os.IsNotExist(err) {
+		logrus.Debugf("baseDir %s for file %s does not exist. Creating.", baseDir, tarpath)
+		if err := os.MkdirAll(baseDir, 0755); err != nil {
+			return nil, err
+		}
+	}
+	return os.Create(tarpath)
+
+}