public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

Unpack tar from GCS bucket

This commit is contained in:
Priya Wadhwa 2018-03-26 13:59:56 -07:00
parent d9c31044b9
commit 85bbb6edff
No known key found for this signature in database
GPG Key ID: 0D0DAFD8F7AA73AE
6 changed files with 186 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
executor/cmd/root.go
View File

@ -49,6 +49,10 @@ var RootCmd = &cobra.Command{
return util.SetLogLevel(logLevel)
},
Run: func(cmd *cobra.Command, args []string) {
if err := resolveSourceContext(); err != nil {
logrus.Error(err)
os.Exit(1)
}
if err := execute(); err != nil {
logrus.Error(err)
os.Exit(1)
@ -56,6 +60,23 @@ var RootCmd = &cobra.Command{
},
}
// resolveSourceContext unpacks the source context if it is a tar in a GCS bucket
// it resets srcContext to be the path to the unpacked build context within the image
func resolveSourceContext() error {
if util.FilepathExists(srcContext) {
return nil
}
// Else, assume the source context is the name of a bucket
logrus.Infof("Using GCS bucket %s as source context", srcContext)
buildContextPath := constants.BuildContextDir
if err := util.UnpackTarFromGCSBucket(srcContext, buildContextPath); err != nil {
return err
}
logrus.Debugf("Unpacked tar from %s to path %s", srcContext, buildContextPath)
srcContext = buildContextPath
return nil
}
func execute() error {
// Parse dockerfile and unpack base image to root
d, err := ioutil.ReadFile(dockerfilePath)
@ -123,5 +144,23 @@ func execute() error {
}
}
// Push the image
if err := setDefaultEnv(); err != nil {
return err
}
return image.PushImage(sourceImage, destination)
}
// setDefaultEnv sets default values for HOME and PATH so that
// config.json and docker-credential-gcr can be accessed
func setDefaultEnv() error {
defaultEnvs := map[string]string{
"HOME": "/root",
"PATH": "/usr/local/bin/",
}
for key, val := range defaultEnvs {
if err := os.Setenv(key, val); err != nil {
return err
}
}
return nil
}

2
integration_tests/dockerfiles/Dockerfile_test_bucket Normal file
View File

@ -0,0 +1,2 @@
FROM gcr.io/distroless/base:latest
COPY baz /tmp/baz

12
integration_tests/dockerfiles/config_test_bucket_buildcontext.json Normal file
View File

@ -0,0 +1,12 @@
[
{
"Image1": "gcr.io/kbuild-test/docker-test-bucket-buildcontext:latest",
"Image2": "gcr.io/kbuild-test/kbuild-test-bucket-buildcontext:latest",
"DiffType": "File",
"Diff": {
"Adds": null,
"Dels": null,
"Mods": null
}
}
]

70
integration_tests/integration_test_yaml.go
View File

@ -22,41 +22,69 @@ import (
"gopkg.in/yaml.v2"
)
const (
executorImage = "executor-image"
executorCommand = "/kbuild/executor"
dockerImage = "gcr.io/cloud-builders/docker"
ubuntuImage = "ubuntu"
testRepo = "gcr.io/kbuild-test/"
dockerPrefix = "docker-"
kbuildPrefix = "kbuild-"
daemonPrefix = "daemon://"
containerDiffOutputFile = "container-diff.json"
kbuildTestBucket = "kbuild-test-bucket"
buildcontextPath = "/workspace/integration_tests"
dockerfilesPath = "/workspace/integration_tests/dockerfiles"
)
var fileTests = []struct {
description string
dockerfilePath string
configPath string
context string
dockerContext string
kbuildContext string
repo string
}{
{
description: "test extract filesystem",
dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_extract_fs",
configPath: "/workspace/integration_tests/dockerfiles/config_test_extract_fs.json",
context: "integration_tests/dockerfiles/",
dockerContext: dockerfilesPath,
kbuildContext: dockerfilesPath,
repo: "extract-filesystem",
},
{
description: "test run",
dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run",
configPath: "/workspace/integration_tests/dockerfiles/config_test_run.json",
context: "integration_tests/dockerfiles/",
dockerContext: dockerfilesPath,
kbuildContext: dockerfilesPath,
repo: "test-run",
},
{
description: "test run no files changed",
dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_run_2",
configPath: "/workspace/integration_tests/dockerfiles/config_test_run_2.json",
context: "integration_tests/dockerfiles/",
dockerContext: dockerfilesPath,
kbuildContext: dockerfilesPath,
repo: "test-run-2",
},
{
description: "test copy",
dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_copy",
configPath: "/workspace/integration_tests/dockerfiles/config_test_copy.json",
context: "/workspace/integration_tests/",
dockerContext: buildcontextPath,
kbuildContext: buildcontextPath,
repo: "test-copy",
},
{
description: "test bucket build context",
dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_copy",
configPath: "/workspace/integration_tests/dockerfiles/config_test_bucket_buildcontext.json",
dockerContext: buildcontextPath,
kbuildContext: kbuildTestBucket,
repo: "test-bucket-buildcontext",
},
}
var structureTests = []struct {
@ -64,13 +92,15 @@ var structureTests = []struct {
dockerfilePath string
structureTestYamlPath string
dockerBuildContext string
kbuildContext string
repo string
}{
{
description: "test env",
dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_env",
repo: "test-env",
dockerBuildContext: "/workspace/integration_tests/dockerfiles/",
dockerBuildContext: dockerfilesPath,
kbuildContext: dockerfilesPath,
structureTestYamlPath: "/workspace/integration_tests/dockerfiles/test_env.yaml",
},
}
@ -85,16 +115,6 @@ type testyaml struct {
Steps []step
}
var executorImage = "executor-image"
var executorCommand = "/kbuild/executor"
var dockerImage = "gcr.io/cloud-builders/docker"
var ubuntuImage = "ubuntu"
var testRepo = "gcr.io/kbuild-test/"
var dockerPrefix = "docker-"
var kbuildPrefix = "kbuild-"
var daemonPrefix = "daemon://"
var containerDiffOutputFile = "container-diff.json"
func main() {
// First, copy container-diff in
@ -114,27 +134,37 @@ func main() {
Name: ubuntuImage,
Args: []string{"chmod", "+x", "container-structure-test"},
}
GCSBucketTarBuildContext := step{
Name: ubuntuImage,
Args: []string{"tar", "-C", "/workspace/integration_tests/", "-cf", "/workspace/kbuild.tar", "."},
}
uploadTarBuildContext := step{
Name: "gcr.io/cloud-builders/gsutil",
Args: []string{"cp", "/workspace/kbuild.tar", "gs://kbuild-test-bucket/"},
}
// Build executor image
buildExecutorImage := step{
Name: dockerImage,
Args: []string{"build", "-t", executorImage, "-f", "deploy/Dockerfile", "."},
}
y := testyaml{
Steps: []step{containerDiffStep, containerDiffPermissions, structureTestsStep, structureTestPermissions, buildExecutorImage},
Steps: []step{containerDiffStep, containerDiffPermissions, structureTestsStep, structureTestPermissions, GCSBucketTarBuildContext, uploadTarBuildContext, buildExecutorImage},
}
for _, test := range fileTests {
// First, build the image with docker
dockerImageTag := testRepo + dockerPrefix + test.repo
dockerBuild := step{
Name: dockerImage,
Args: []string{"build", "-t", dockerImageTag, "-f", test.dockerfilePath, test.context},
Args: []string{"build", "-t", dockerImageTag, "-f", test.dockerfilePath, test.dockerContext},
}
// Then, buld the image with kbuild
kbuildImage := testRepo + kbuildPrefix + test.repo
kbuild := step{
Name: executorImage,
Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.context},
Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.kbuildContext},
}
// Pull the kbuild image
@ -178,7 +208,7 @@ func main() {
kbuildImage := testRepo + kbuildPrefix + test.repo
kbuild := step{
Name: executorImage,
Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath},
Args: []string{executorCommand, "--destination", kbuildImage, "--dockerfile", test.dockerfilePath, "--context", test.kbuildContext},
}
// Pull the kbuild image
pullKbuildImage := step{

10
pkg/constants/constants.go
View File

@ -26,7 +26,17 @@ const (
// WorkspaceDir is the path to the workspace directory
WorkspaceDir = "/workspace"
//KbuildDir is the path to the kbuild directory
KbuildDir = "/kbuild"
WhitelistPath = "/proc/self/mountinfo"
Author = "kbuild"
// KbuildTar is the default name of the tar uploaded to GCS buckets
KbuildTar = "kbuild.tar"
// BuildContextDir is the directory a build context will be unpacked into,
// for example, a tarball from a GCS bucket will be unpacked here
BuildContextDir = "/kbuild/buildcontext/"
)

73
pkg/util/bucket_util.go Normal file
View File

@ -0,0 +1,73 @@
/*
Copyright 2018 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"cloud.google.com/go/storage"
pkgutil "github.com/GoogleCloudPlatform/container-diff/pkg/util"
"github.com/GoogleCloudPlatform/k8s-container-builder/pkg/constants"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
"os"
"path/filepath"
)
// UnpackTarFromGCSBucket unpacks the kbuild.tar file in the given bucket to the given directory
func UnpackTarFromGCSBucket(bucketName, directory string) error {
// Get the tar from the bucket
tarPath, err := getTarFromBucket(bucketName, directory)
if err != nil {
return err
}
logrus.Debug("Unpacking source context tar...")
// Now, unpack the tar to a build context, and return the path to the build context
file, err := os.Open(tarPath)
if err != nil {
return err
}
if err := pkgutil.UnTar(file, directory, nil); err != nil {
return err
}
// Remove the tar so it doesn't interfere with subsequent commands
logrus.Debugf("Deleting %s", tarPath)
return os.Remove(tarPath)
}
// getTarFromBucket gets kbuild.tar from the GCS bucket and saves it to the filesystem
// It returns the path to the tar file
func getTarFromBucket(bucketName, directory string) (string, error) {
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
return "", err
}
bucket := client.Bucket(bucketName)
// Get the tarfile kbuild.tar from the GCS bucket, and save it to a tar object
reader, err := bucket.Object(constants.KbuildTar).NewReader(ctx)
if err != nil {
return "", err
}
defer reader.Close()
tarPath := filepath.Join(directory, constants.KbuildTar)
if err := CreateFile(tarPath, reader, 0600); err != nil {
return "", err
}
logrus.Debugf("Copied tarball %s from GCS bucket %s to %s", constants.KbuildTar, bucketName, tarPath)
return tarPath, nil
}