diff --git a/.gcloudignore b/.gcloudignore deleted file mode 100644 index 48b8bf907..000000000 --- a/.gcloudignore +++ /dev/null @@ -1 +0,0 @@ -vendor/ diff --git a/.travis.yml b/.travis.yml index df05f880a..364f2b7c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,10 @@ language: go os: linux dist: bionic env: - - IMAGE_REPO=localhost:5000 + global: + - IMAGE_REPO=localhost:5000 REGISTRY=localhost:5000 go: - - "1.13.3" + - "1.14" go_import_path: github.com/GoogleContainerTools/kaniko jobs: include: @@ -21,11 +22,16 @@ jobs: - make travis-setup script: - make integration-test-layers + - name: build-image-and-k8s-integration-test + before_install: + - make travis-setup + - make minikube-setup + script: + - make images + - make push + - make integration-test-k8s - name: integration-test-misc before_install: - make travis-setup script: - make integration-test-misc - - stage: build-images - script: - - make images diff --git a/Makefile b/Makefile index 7d94dee5c..c7b50a3af 100644 --- a/Makefile +++ b/Makefile @@ -55,28 +55,36 @@ out/warmer: $(GO_FILES) .PHONY: travis-setup travis-setup: - @ ./travis-setup.sh + @ ./scripts/travis-setup.sh + +.PHONY: minikube-setup +minikube-setup: + @ ./scripts/minikube-setup.sh .PHONY: test test: out/executor - @ ./test.sh + @ ./scripts/test.sh .PHONY: integration-test integration-test: - @ ./integration-test.sh + @ ./scripts/integration-test.sh .PHONY: integration-test-run integration-test-run: - @ ./integration-test.sh -run "TestRun" + @ ./scripts/integration-test.sh -run "TestRun" .PHONY: integration-test-layers integration-test-layers: - @ ./integration-test.sh -run "TestLayers" + @ ./scripts/integration-test.sh -run "TestLayers" + +.PHONY: integration-test-k8s +integration-test-k8s: + @ ./scripts/integration-test.sh -run "TestK8s" .PHONY: integration-test-misc integration-test-misc: - $(eval RUN_ARG=$(shell ./misc-integration-test.sh)) - @ ./integration-test.sh -run "$(RUN_ARG)" + $(eval RUN_ARG=$(shell ./scripts/misc-integration-test.sh)) + @ ./scripts/integration-test.sh -run "$(RUN_ARG)" .PHONY: images images: @@ -86,6 +94,6 @@ images: .PHONY: push push: - docker push $(REGISTRY)/executor:latest - docker push $(REGISTRY)/executor:debug - docker push $(REGISTRY)/warmer:latest + docker push $(REGISTRY)/executor:latest + docker push $(REGISTRY)/executor:debug + docker push $(REGISTRY)/warmer:latest diff --git a/README.md b/README.md index b79611724..2e0de1929 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ When running kaniko, use the `--context` flag with the appropriate prefix to spe | Source | Prefix | Example | |---------|---------|---------| | Local Directory | dir://[path to a directory in the kaniko container] | `dir:///workspace` | +| Local Tar Gz | tar://[path to a .tar.gz in the kaniko container] | `tar://path/to/context.tar.gz` | | GCS Bucket | gs://[bucket name]/[path to .tar.gz] | `gs://kaniko-bucket/path/to/context.tar.gz` | | S3 Bucket | s3://[bucket name]/[path to .tar.gz] | `s3://kaniko-bucket/path/to/context.tar.gz` | | Azure Blob Storage| https://[account].[azureblobhostsuffix]/[container]/[path to .tar.gz] | `https://myaccount.blob.core.windows.net/container/path/to/context.tar.gz` | @@ -260,21 +261,24 @@ kaniko will build and push the final image in this build step. Requirements: - [Docker](https://docs.docker.com/install/) -- [gcloud](https://cloud.google.com/sdk/install) We can run the kaniko executor image locally in a Docker daemon to build and push an image from a Dockerfile. -1. Load the executor image into the Docker daemon by running: +For example, when using gcloud and GCR you could run Kaniko as follows: +```shell +docker run \ + -v "$HOME"/.config/gcloud:/root/.config/gcloud \ + -v /path/to/context:/workspace \ + gcr.io/kaniko-project/executor:latest \ + --dockerfile /workspace/Dockerfile + --destination "gcr.io/$PROJECT_ID/$IMAGE_NAME:$TAG" + --context dir:///workspace/" +``` - ```shell - make images - ``` - -2. Run kaniko in Docker using [`run_in_docker.sh`](./run_in_docker.sh): - - ```shell - ./run_in_docker.sh - ``` +There is also a utility script [`run_in_docker.sh`](./run_in_docker.sh) that can be used as follows: +```shell +./run_in_docker.sh +``` _NOTE: `run_in_docker.sh` expects a path to a Dockerfile relative to the absolute path of the build context._ @@ -284,7 +288,7 @@ context in the local directory `/home/user/kaniko-project`, and a Google Contain as a remote image destination: ```shell -./run_in_docker.sh /workspace/Dockerfile /home/user/kaniko-project gcr.io/// +./run_in_docker.sh /workspace/Dockerfile /home/user/kaniko-project gcr.io/$PROJECT_ID/$TAG ``` ### Caching diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 87c661e29..ff198060d 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -233,7 +233,7 @@ func copyDockerfile() error { return nil } -// resolveSourceContext unpacks the source context if it is a tar in a bucket +// resolveSourceContext unpacks the source context if it is a tar in a bucket or in kaniko container // it resets srcContext to be the path to the unpacked build context within the image func resolveSourceContext() error { if opts.SrcContext == "" && opts.Bucket == "" { diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 2e1b1fb9b..b96714548 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -14,7 +14,7 @@ # Builds the static Go image to execute in a Kubernetes job -FROM golang:1.13 +FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper @@ -35,7 +35,7 @@ FROM scratch COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/executor /kaniko/executor COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login -COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr-linux +COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /root/.docker/config.json /kaniko/.docker/config.json ENV HOME /root diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index e9ccf15e3..57562a90f 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -15,7 +15,7 @@ # Builds the static Go image to execute in a Kubernetes job # Stage 0: Build the executor binary and get credential helpers -FROM golang:1.13 +FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper @@ -25,6 +25,10 @@ RUN docker-credential-gcr configure-docker # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 +# ACR docker credential helper +ADD https://aadacr.blob.core.windows.net/acr-docker-credential-helper/docker-credential-acr-linux-amd64.tar.gz /usr/local/bin +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-acr-linux-amd64.tar.gz + COPY . . RUN make GOARCH=${GOARCH} && make out/warmer @@ -39,6 +43,7 @@ FROM scratch COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/* /kaniko/ COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login +COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY --from=1 /distroless/bazel-bin/experimental/busybox/busybox/ /busybox/ # Declare /busybox as a volume to get it automatically whitelisted VOLUME /busybox diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer index d33d2d74b..f1e6cb074 100644 --- a/deploy/Dockerfile_warmer +++ b/deploy/Dockerfile_warmer @@ -14,7 +14,7 @@ # Builds the static Go image to execute in a Kubernetes job -FROM golang:1.13 +FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper @@ -24,6 +24,9 @@ RUN docker-credential-gcr configure-docker # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 +# ACR docker credential helper +ADD https://aadacr.blob.core.windows.net/acr-docker-credential-helper/docker-credential-acr-linux-amd64.tar.gz /usr/local/bin +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-acr-linux-amd64.tar.gz COPY . . RUN make GOARCH=${GOARCH} out/warmer @@ -32,6 +35,7 @@ FROM scratch COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/warmer /kaniko/warmer COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login +COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /root/.docker/config.json /kaniko/.docker/config.json ENV HOME /root diff --git a/deploy/cloudbuild.yaml b/deploy/cloudbuild.yaml index 44b30aa49..47a9a0ab2 100644 --- a/deploy/cloudbuild.yaml +++ b/deploy/cloudbuild.yaml @@ -2,18 +2,21 @@ steps: # First, build kaniko - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile", - "-t", "gcr.io/kaniko-project/executor:${COMMIT_SHA}", "."] + "-t", "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:${COMMIT_SHA}", "."] # Then, we want to build kaniko:debug - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile_debug", - "-t", "gcr.io/kaniko-project/executor:debug-${COMMIT_SHA}", "."] + "-t", "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:debug-${COMMIT_SHA}", "."] - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile_debug", - "-t", "gcr.io/kaniko-project/executor:debug", "."] + "-t", "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:debug", "."] # Then, we want to build the cache warmer - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile_warmer", - "-t", "gcr.io/kaniko-project/warmer:${COMMIT_SHA}", "."] -images: ["gcr.io/kaniko-project/executor:${COMMIT_SHA}", - "gcr.io/kaniko-project/executor:debug-${COMMIT_SHA}", - "gcr.io/kaniko-project/warmer:${COMMIT_SHA}"] + "-t", "gcr.io/$PROJECT_ID/${_WARMER_IMAGE_NAME}:${COMMIT_SHA}", "."] +images: ["gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:${COMMIT_SHA}", + "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:debug-${COMMIT_SHA}", + "gcr.io/$PROJECT_ID/${_WARMER_IMAGE_NAME}:${COMMIT_SHA}"] +substitutions: + _EXECUTOR_IMAGE_NAME: executor + _WARMER_IMAGE_NAME: warmer diff --git a/go.mod b/go.mod index 1e16f41d6..f870edf65 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/GoogleContainerTools/kaniko -go 1.13 +go 1.14 replace ( github.com/containerd/containerd v1.4.0-0.20191014053712-acdcf13d5eaf => github.com/containerd/containerd v0.0.0-20191014053712-acdcf13d5eaf diff --git a/hack/linter.sh b/hack/linter.sh index 6b61e1aa6..41d539b19 100755 --- a/hack/linter.sh +++ b/hack/linter.sh @@ -20,7 +20,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" if ! [ -x "$(command -v golangci-lint)" ]; then echo "Installing GolangCI-Lint" - ${DIR}/install_golint.sh -b $GOPATH/bin v1.21.0 + ${DIR}/install_golint.sh -b $GOPATH/bin v1.23.7 fi golangci-lint run diff --git a/integration-test.sh b/integration-test.sh deleted file mode 100755 index 3af24c922..000000000 --- a/integration-test.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -GCS_BUCKET="${GCS_BUCKET:-gs://kaniko-test-bucket}" -IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}" - -docker version - -# Sets up a kokoro (Google internal integration testing tool) environment -if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then - echo "Installing dependencies..." - source "$KOKORO_GFILE_DIR/common.sh" - mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/ - cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/ - pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko - echo "Installing container-diff..." - mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff - chmod +x $KOKORO_GFILE_DIR/container-diff - export PATH=$PATH:$KOKORO_GFILE_DIR - cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json -fi - -echo "Running integration tests..." -make out/executor -make out/warmer -go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@" diff --git a/integration-test.sh b/integration-test.sh new file mode 120000 index 000000000..8a77788df --- /dev/null +++ b/integration-test.sh @@ -0,0 +1 @@ +scripts/integration-test.sh \ No newline at end of file diff --git a/integration/images.go b/integration/images.go index 593433abc..e6af511fc 100644 --- a/integration/images.go +++ b/integration/images.go @@ -185,23 +185,8 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string { return flags } -// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. -// The resulting image will be tagged with imageRepo. If the dockerfile will be built with -// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. -func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { - _, ex, _, _ := runtime.Caller(0) - cwd := filepath.Dir(ex) - - return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd) -} - -func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { - if _, present := d.filesBuilt[dockerfile]; present { - return nil - } - gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo - - fmt.Printf("Building images for Dockerfile %s\n", dockerfile) +func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir string) error { + fmt.Printf("Building image for Dockerfile %s\n", dockerfile) var buildArgs []string buildArgFlag := "--build-arg" @@ -230,13 +215,39 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerCmd.Env = append(dockerCmd.Env, env...) } - timer := timing.Start(dockerfile + "_docker") out, err := RunCommandWithoutTest(dockerCmd) - timing.DefaultRun.Stop(timer) if err != nil { return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) } fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out) + return nil +} + +// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. +// The resulting image will be tagged with imageRepo. If the dockerfile will be built with +// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. +func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + + return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd) +} + +func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { + if _, present := d.filesBuilt[dockerfile]; present { + return nil + } + gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo + + var buildArgs []string + buildArgFlag := "--build-arg" + for _, arg := range argsMap[dockerfile] { + buildArgs = append(buildArgs, buildArgFlag, arg) + } + + timer := timing.Start(dockerfile + "_docker") + d.BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir) + timing.DefaultRun.Stop(timer) contextFlag := "-c" contextPath := buildContextPath @@ -269,7 +280,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, } // build kaniko image - additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) + additionalFlags := append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) kanikoImage := GetKanikoImage(imageRepo, dockerfile) fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) @@ -301,16 +312,16 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, kanikoCmd := exec.Command("docker", dockerRunFlags...) timer = timing.Start(dockerfile + "_kaniko") - out, err = RunCommandWithoutTest(kanikoCmd) + out, err := RunCommandWithoutTest(kanikoCmd) timing.DefaultRun.Stop(timer) if err != nil { - return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out)) + return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) } if outputCheck := outputChecks[dockerfile]; outputCheck != nil { if err := outputCheck(dockerfile, out); err != nil { - return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out)) + return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) } } diff --git a/integration/integration_test.go b/integration/integration_test.go index 357b46cb0..6781f7fe7 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -33,11 +33,10 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/daemon" - "github.com/pkg/errors" - "github.com/GoogleContainerTools/kaniko/pkg/timing" "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/GoogleContainerTools/kaniko/testutil" + "github.com/pkg/errors" ) var config *integrationTestConfig diff --git a/integration/k8s-job.yaml b/integration/k8s-job.yaml new file mode 100644 index 000000000..ca84047f1 --- /dev/null +++ b/integration/k8s-job.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: kaniko-test-{{.Name}} +spec: + template: + spec: + hostNetwork: true + containers: + - name: kaniko + image: localhost:5000/executor:debug + args: [ "--context=dir:///workspace", + "--destination={{.KanikoImage}}"] + volumeMounts: + - name: context + mountPath: /workspace + restartPolicy: Never + volumes: + - name: context + hostPath: + path: {{.Context}} + backoffLimit: 1 diff --git a/integration/k8s_test.go b/integration/k8s_test.go new file mode 100644 index 000000000..d7e6235cf --- /dev/null +++ b/integration/k8s_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "testing" + "text/template" +) + +type K8sConfig struct { + KanikoImage string + Context string + Name string +} + +func TestK8s(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + dir := filepath.Join(cwd, "dockerfiles-with-context") + + testDirs, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + builder := NewDockerFileBuilder() + + for _, tdInfo := range testDirs { + name := tdInfo.Name() + testDir := filepath.Join(dir, name) + + t.Run("test_k8s_with_context_"+name, func(t *testing.T) { + t.Parallel() + + if err := builder.BuildDockerImage( + config.imageRepo, "", name, testDir, + ); err != nil { + t.Fatal(err) + } + + dockerImage := GetDockerImage(config.imageRepo, name) + kanikoImage := GetKanikoImage(config.imageRepo, name) + + tmpfile, err := ioutil.TempFile("", "k8s-job-*.yaml") + if err != nil { + log.Fatal(err) + } + defer os.Remove(tmpfile.Name()) // clean up + tmpl := template.Must(template.ParseFiles("k8s-job.yaml")) + job := K8sConfig{KanikoImage: kanikoImage, Context: testDir, Name: name} + if err := tmpl.Execute(tmpfile, job); err != nil { + t.Fatal(err) + } + + fmt.Printf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n", + testDir, kanikoImage) + content, err := ioutil.ReadFile(tmpfile.Name()) + if err != nil { + log.Fatal(err) + } + fmt.Printf("K8s template %s:\n%s\n", tmpfile.Name(), content) + + kubeCmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name()) + RunCommand(kubeCmd, t) + + fmt.Printf("Waiting for K8s kaniko build job to finish: %s\n", + "job/kaniko-test-"+job.Name) + + kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=60s", + "job/kaniko-test-"+job.Name) + RunCommand(kubeWaitCmd, t) + + diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache") + + expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) + checkContainerDiffOutput(t, diff, expected) + }) + } + + if err := logBenchmarks("benchmark"); err != nil { + t.Logf("Failed to create benchmark file: %v", err) + } +} diff --git a/pkg/buildcontext/azureblob.go b/pkg/buildcontext/azureblob.go index 6af468015..5f6817a90 100644 --- a/pkg/buildcontext/azureblob.go +++ b/pkg/buildcontext/azureblob.go @@ -43,12 +43,12 @@ func (b *AzureBlob) UnpackTarFromBuildContext() (string, error) { return "", errors.New("AZURE_STORAGE_ACCESS_KEY environment variable is not set") } - // Get storage accoutname for Azure Blob Storage + // Get storage accountName for Azure Blob Storage u, _ := url.Parse(b.context) parts := azblob.NewBlobURLParts(*u) accountName := strings.Split(parts.Host, ".")[0] - // Generate credentail with accountname and accountkey + // Generate credential with accountName and accountKey credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) if err != nil { return parts.Host, err @@ -62,7 +62,7 @@ func (b *AzureBlob) UnpackTarFromBuildContext() (string, error) { return tarPath, err } - // Downloading contextfile from Azure Blob Storage + // Downloading context file from Azure Blob Storage p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) blobURL := azblob.NewBlobURL(*u, p) ctx := context.Background() diff --git a/pkg/buildcontext/buildcontext.go b/pkg/buildcontext/buildcontext.go index 3142239a8..ef39c21e2 100644 --- a/pkg/buildcontext/buildcontext.go +++ b/pkg/buildcontext/buildcontext.go @@ -24,6 +24,10 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/util" ) +const ( + TarBuildContextPrefix = "tar://" +) + // BuildContext unifies calls to download and unpack the build context. type BuildContext interface { // Unpacks a build context and returns the directory where it resides @@ -51,6 +55,8 @@ func GetBuildContext(srcContext string) (BuildContext, error) { return &AzureBlob{context: srcContext}, nil } return nil, errors.New("url provided for https context is not in a supported format, please use the https url for Azure Blob Storage") + case TarBuildContextPrefix: + return &Tar{context: context}, nil } - return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://, git://, https://") + return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, tar://, s3://, git://, https://") } diff --git a/pkg/buildcontext/tar.go b/pkg/buildcontext/tar.go new file mode 100644 index 000000000..d5d1c250a --- /dev/null +++ b/pkg/buildcontext/tar.go @@ -0,0 +1,40 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package buildcontext + +import ( + "os" + + "github.com/GoogleContainerTools/kaniko/pkg/constants" + "github.com/GoogleContainerTools/kaniko/pkg/util" + "github.com/pkg/errors" +) + +// Tar unifies calls to download and unpack the build context. +type Tar struct { + context string +} + +// UnpackTarFromBuildContext unpack the compressed tar file +func (t *Tar) UnpackTarFromBuildContext() (string, error) { + directory := constants.BuildContextDir + if err := os.MkdirAll(directory, 0750); err != nil { + return "", errors.Wrap(err, "unpacking tar from build context") + } + + return directory, util.UnpackCompressedTar(t.context, directory) +} diff --git a/pkg/buildcontext/tar_test.go b/pkg/buildcontext/tar_test.go new file mode 100644 index 000000000..5904b4e60 --- /dev/null +++ b/pkg/buildcontext/tar_test.go @@ -0,0 +1,158 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package buildcontext + +import ( + "bytes" + "compress/gzip" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sync" + "testing" + + "github.com/GoogleContainerTools/kaniko/pkg/util" + "github.com/GoogleContainerTools/kaniko/testutil" +) + +func TestBuildWithLocalTar(t *testing.T) { + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + + testDir := "test_dir" + testDirLongPath := filepath.Join(cwd, testDir) + dirUnpack := filepath.Join(testDirLongPath, "dir_where_to_unpack") + + if err := os.MkdirAll(dirUnpack, 0750); err != nil { + t.Errorf("Failed to create dir_where_to_extract: %v", err) + } + + validDockerfile := "Dockerfile_valid" + invalidDockerfile := "Dockerfile_invalid" + nonExistingDockerfile := "Dockerfile_non_existing" + + files := map[string]string{ + validDockerfile: "FROM debian:9.11\nRUN echo \"valid\"", + invalidDockerfile: "FROM debian:9.11\nRUN echo \"invalid\"", + } + + if err := testutil.SetupFiles(testDir, files); err != nil { + t.Errorf("Failed to setup files %v on %s: %v", files, testDir, err) + } + + if err := os.Chdir(testDir); err != nil { + t.Fatalf("Failed to Chdir on %s: %v", testDir, err) + } + + validTarPath := fmt.Sprintf("%s.tar.gz", validDockerfile) + invalidTarPath := fmt.Sprintf("%s.tar.gz", invalidDockerfile) + nonExistingTarPath := fmt.Sprintf("%s.tar.gz", nonExistingDockerfile) + + var wg sync.WaitGroup + wg.Add(1) + // Create Tar Gz File with dockerfile inside + go func(wg *sync.WaitGroup) { + defer wg.Done() + validTarFile, err := os.Create(validTarPath) + if err != nil { + t.Errorf("Failed to create %s: %v", validTarPath, err) + } + defer validTarFile.Close() + + invalidTarFile, err := os.Create(invalidTarPath) + if err != nil { + t.Errorf("Failed to create %s: %v", invalidTarPath, err) + } + defer invalidTarFile.Close() + + gw := gzip.NewWriter(validTarFile) + defer gw.Close() + + tw := util.NewTar(gw) + defer tw.Close() + + if err := tw.AddFileToTar(validDockerfile); err != nil { + t.Errorf("Failed to add %s to %s: %v", validDockerfile, validTarPath, err) + } + }(&wg) + + // Waiting for the Tar Gz file creation to be done before moving on + wg.Wait() + + tests := []struct { + dockerfile string + srcContext string + unpackShouldErr bool + srcShaShouldErr bool + destShaShouldErr bool + }{ + { + dockerfile: validDockerfile, + srcContext: filepath.Join(testDir, validTarPath), + unpackShouldErr: false, + srcShaShouldErr: false, + destShaShouldErr: false, + }, + { + dockerfile: invalidDockerfile, + srcContext: filepath.Join(testDir, invalidTarPath), + unpackShouldErr: true, + srcShaShouldErr: false, + destShaShouldErr: true, + }, + { + dockerfile: nonExistingDockerfile, + srcContext: filepath.Join(testDir, nonExistingTarPath), + unpackShouldErr: true, + srcShaShouldErr: true, + destShaShouldErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.dockerfile, func(t *testing.T) { + err := util.UnpackCompressedTar(filepath.Join(cwd, tt.srcContext), dirUnpack) + testutil.CheckError(t, tt.unpackShouldErr, err) + srcSHA, err := getSHAFromFilePath(tt.dockerfile) + testutil.CheckError(t, tt.srcShaShouldErr, err) + destSHA, err := getSHAFromFilePath(filepath.Join(dirUnpack, tt.dockerfile)) + testutil.CheckError(t, tt.destShaShouldErr, err) + if err == nil { + testutil.CheckDeepEqual(t, srcSHA, destSHA) + } + }) + } + + if err := os.RemoveAll(testDirLongPath); err != nil { + t.Errorf("Failed to remove %s: %v", testDirLongPath, err) + } +} + +func getSHAFromFilePath(f string) (string, error) { + data, err := ioutil.ReadFile(f) + if err != nil { + return "", err + } + sha, err := util.SHA256(bytes.NewReader(data)) + if err != nil { + return "", err + } + + return sha, nil +} diff --git a/pkg/executor/build.go b/pkg/executor/build.go index ecfcd51d8..e8d09d6f0 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -73,7 +73,7 @@ type stageBuilder struct { stageIdxToDigest map[string]string snapshotter snapShotter layerCache cache.LayerCache - pushCache cachePusher + pushLayerToCache cachePusher } // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage @@ -116,7 +116,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross layerCache: &cache.RegistryCache{ Opts: opts, }, - pushCache: pushLayerToCache, + pushLayerToCache: pushLayerToCache, } for _, cmd := range s.stage.Commands { @@ -166,9 +166,15 @@ func initializeConfig(img partial.WithConfigFile, opts *config.KanikoOptions) (* return imageConfig, nil } -func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string, compositeKey CompositeCache) (CompositeCache, error) { +func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string, compositeKey CompositeCache, args *dockerfile.BuildArgs, env []string) (CompositeCache, error) { + // First replace all the environment variables or args in the command + replacementEnvs := args.ReplacementEnvs(env) + resolvedCmd, err := util.ResolveEnvironmentReplacement(command.String(), replacementEnvs, false) + if err != nil { + return compositeKey, err + } // Add the next command to the cache key. - compositeKey.AddKey(command.String()) + compositeKey.AddKey(resolvedCmd) switch v := command.(type) { case *commands.CopyCommand: compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey) @@ -221,7 +227,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro return errors.Wrap(err, "failed to get files used from context") } - compositeKey, err = s.populateCompositeKey(command, files, compositeKey) + compositeKey, err = s.populateCompositeKey(command, files, compositeKey, s.args, cfg.Env) if err != nil { return err } @@ -271,8 +277,6 @@ func (s *stageBuilder) build() error { compositeKey = NewCompositeCache(s.baseImageDigest) } - compositeKey.AddKey(s.opts.BuildArgs...) - // Apply optimizations to the instructions. if err := s.optimize(*compositeKey, s.cf.Config); err != nil { return errors.Wrap(err, "failed to optimize instructions") @@ -329,7 +333,7 @@ func (s *stageBuilder) build() error { return errors.Wrap(err, "failed to get files used from context") } - *compositeKey, err = s.populateCompositeKey(command, files, *compositeKey) + *compositeKey, err = s.populateCompositeKey(command, files, *compositeKey, s.args, s.cf.Config.Env) if err != nil { return err } @@ -378,7 +382,7 @@ func (s *stageBuilder) build() error { // Push layer to cache (in parallel) now along with new config file if s.opts.Cache && command.ShouldCacheOutput() { cacheGroup.Go(func() error { - return s.pushCache(s.opts, ck, tarPath, command.String()) + return s.pushLayerToCache(s.opts, ck, tarPath, command.String()) }) } if err := s.saveSnapshotToImage(command.String(), tarPath); err != nil { diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index d468aee19..831067173 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -497,7 +497,8 @@ func Test_stageBuilder_optimize(t *testing.T) { cf := &v1.ConfigFile{} snap := fakeSnapShotter{} lc := &fakeLayerCache{retrieve: tc.retrieve} - sb := &stageBuilder{opts: tc.opts, cf: cf, snapshotter: snap, layerCache: lc} + sb := &stageBuilder{opts: tc.opts, cf: cf, snapshotter: snap, layerCache: lc, + args: dockerfile.NewBuildArgs([]string{})} ck := CompositeCache{} file, err := ioutil.TempFile("", "foo") if err != nil { @@ -517,10 +518,135 @@ func Test_stageBuilder_optimize(t *testing.T) { } } +type stageContext struct { + command fmt.Stringer + args *dockerfile.BuildArgs + env []string +} + +func newStageContext(command string, args map[string]string, env []string) stageContext { + dockerArgs := dockerfile.NewBuildArgs([]string{}) + for k, v := range args { + dockerArgs.AddArg(k, &v) + } + return stageContext{MockDockerCommand{command: command}, dockerArgs, env} +} + +func Test_stageBuilder_populateCompositeKey(t *testing.T) { + testCases := []struct { + description string + cmd1 stageContext + cmd2 stageContext + shdEqual bool + }{ + { + description: "cache key for same command, different buildargs, args not used in command", + cmd1: newStageContext( + "RUN echo const > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=foo1"}, + ), + cmd2: newStageContext( + "RUN echo const > test", + map[string]string{"ARG": "bar"}, + []string{"ENV=bar1"}, + ), + shdEqual: true, + }, + { + description: "cache key for same command with same build args", + cmd1: newStageContext( + "RUN echo $ARG > test", + map[string]string{"ARG": "foo"}, + []string{}, + ), + cmd2: newStageContext( + "RUN echo $ARG > test", + map[string]string{"ARG": "foo"}, + []string{}, + ), + shdEqual: true, + }, + { + description: "cache key for same command with same env", + cmd1: newStageContext( + "RUN echo $ENV > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=same"}, + ), + cmd2: newStageContext( + "RUN echo $ENV > test", + map[string]string{"ARG": "bar"}, + []string{"ENV=same"}, + ), + shdEqual: true, + }, + { + description: "cache key for same command with a build arg values", + cmd1: newStageContext( + "RUN echo $ARG > test", + map[string]string{"ARG": "foo"}, + []string{}, + ), + cmd2: newStageContext( + "RUN echo $ARG > test", + map[string]string{"ARG": "bar"}, + []string{}, + ), + }, + { + description: "cache key for same command with different env values", + cmd1: newStageContext( + "RUN echo $ENV > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=1"}, + ), + cmd2: newStageContext( + "RUN echo $ENV > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=2"}, + ), + }, + { + description: "cache key for different command same context", + cmd1: newStageContext( + "RUN echo other > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=1"}, + ), + cmd2: newStageContext( + "RUN echo another > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=1"}, + ), + }, + } + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + sb := &stageBuilder{opts: &config.KanikoOptions{SrcContext: "workspace"}} + ck := CompositeCache{} + + ck1, err := sb.populateCompositeKey(tc.cmd1.command, []string{}, ck, tc.cmd1.args, tc.cmd1.env) + if err != nil { + t.Errorf("Expected error to be nil but was %v", err) + } + ck2, err := sb.populateCompositeKey(tc.cmd2.command, []string{}, ck, tc.cmd2.args, tc.cmd2.env) + if err != nil { + t.Errorf("Expected error to be nil but was %v", err) + } + key1, key2 := hashCompositeKeys(t, ck1, ck2) + if b := key1 == key2; b != tc.shdEqual { + t.Errorf("expected keys to be equal as %t but found %t", tc.shdEqual, !tc.shdEqual) + } + }) + } +} + func Test_stageBuilder_build(t *testing.T) { type testcase struct { description string opts *config.KanikoOptions + args map[string]string layerCache *fakeLayerCache expectedCacheKeys []string pushedCacheKeys []string @@ -544,6 +670,7 @@ func Test_stageBuilder_build(t *testing.T) { t.Errorf("couldn't create hash %v", err) } command := MockDockerCommand{ + command: "meow", contextFiles: []string{filePath}, cacheCommand: MockCachedDockerCommand{ contextFiles: []string{filePath}, @@ -576,6 +703,7 @@ func Test_stageBuilder_build(t *testing.T) { t.Errorf("couldn't create hash %v", err) } command := MockDockerCommand{ + command: "meow", contextFiles: []string{filePath}, cacheCommand: MockCachedDockerCommand{ contextFiles: []string{filePath}, @@ -838,6 +966,117 @@ COPY %s bar.txt commands: getCommands(dir, cmds), } }(), + func() testcase { + dir, _ := tempDirAndFile(t) + ch := NewCompositeCache("") + ch.AddKey("RUN foobar") + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + + command := MockDockerCommand{ + command: "RUN foobar", + contextFiles: []string{}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{}, + }, + } + + return testcase{ + description: "cached run command with no build arg value used uses cached layer and does not push anything", + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: dir}}, + opts: &config.KanikoOptions{Cache: true}, + args: map[string]string{ + "test": "value", + }, + expectedCacheKeys: []string{hash}, + commands: []commands.DockerCommand{command}, + // layer key needs to be read. + layerCache: &fakeLayerCache{ + img: &fakeImage{ImageLayers: []v1.Layer{fakeLayer{}}}, + keySequence: []string{hash}, + }, + rootDir: dir, + } + }(), + func() testcase { + dir, _ := tempDirAndFile(t) + + ch := NewCompositeCache("") + ch.AddKey("RUN value") + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + + command := MockDockerCommand{ + command: "RUN $arg", + contextFiles: []string{}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{}, + }, + } + + return testcase{ + description: "cached run command with same build arg does not push layer", + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: dir}}, + opts: &config.KanikoOptions{Cache: true}, + args: map[string]string{ + "arg": "value", + }, + // layer key that exists + layerCache: &fakeLayerCache{ + img: &fakeImage{ImageLayers: []v1.Layer{fakeLayer{}}}, + keySequence: []string{hash}, + }, + expectedCacheKeys: []string{hash}, + commands: []commands.DockerCommand{command}, + rootDir: dir, + } + }(), + func() testcase { + dir, _ := tempDirAndFile(t) + + ch1 := NewCompositeCache("") + ch1.AddKey("RUN value") + hash1, err := ch1.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + + ch2 := NewCompositeCache("") + ch2.AddKey("RUN anotherValue") + hash2, err := ch2.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + command := MockDockerCommand{ + command: "RUN $arg", + contextFiles: []string{}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{}, + }, + } + + return testcase{ + description: "cached run command with another build arg pushes layer", + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: dir}}, + opts: &config.KanikoOptions{Cache: true}, + args: map[string]string{ + "arg": "anotherValue", + }, + // layer for arg=value already exists + layerCache: &fakeLayerCache{ + img: &fakeImage{ImageLayers: []v1.Layer{fakeLayer{}}}, + keySequence: []string{hash1}, + }, + expectedCacheKeys: []string{hash2}, + pushedCacheKeys: []string{hash2}, + commands: []commands.DockerCommand{command}, + rootDir: dir, + } + }(), } for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { @@ -875,18 +1114,21 @@ COPY %s bar.txt } keys := []string{} sb := &stageBuilder{ - args: &dockerfile.BuildArgs{}, //required or code will panic + args: dockerfile.NewBuildArgs([]string{}), //required or code will panic image: tc.image, opts: tc.opts, cf: cf, snapshotter: snap, layerCache: lc, - pushCache: func(_ *config.KanikoOptions, cacheKey, _, _ string) error { + pushLayerToCache: func(_ *config.KanikoOptions, cacheKey, _, _ string) error { keys = append(keys, cacheKey) return nil }, } sb.cmds = tc.commands + for key, value := range tc.args { + sb.args.AddArg(key, &value) + } tmp := commands.RootDir if tc.rootDir != "" { commands.RootDir = tc.rootDir @@ -993,3 +1235,15 @@ func generateTar(t *testing.T, dir string, fileNames ...string) []byte { } return buf.Bytes() } + +func hashCompositeKeys(t *testing.T, ck1 CompositeCache, ck2 CompositeCache) (string, string) { + key1, err := ck1.Hash() + if err != nil { + t.Errorf("could not hash composite key due to %s", err) + } + key2, err := ck2.Hash() + if err != nil { + t.Errorf("could not hash composite key due to %s", err) + } + return key1, key2 +} diff --git a/pkg/executor/fakes.go b/pkg/executor/fakes.go index e9cfdc694..e89d5ac47 100644 --- a/pkg/executor/fakes.go +++ b/pkg/executor/fakes.go @@ -43,13 +43,14 @@ func (f fakeSnapShotter) TakeSnapshot(_ []string) (string, error) { } type MockDockerCommand struct { + command string contextFiles []string cacheCommand commands.DockerCommand } func (m MockDockerCommand) ExecuteCommand(c *v1.Config, args *dockerfile.BuildArgs) error { return nil } func (m MockDockerCommand) String() string { - return "meow" + return m.command } func (m MockDockerCommand) FilesToSnapshot() []string { return []string{"meow-snapshot-no-cache"} diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100755 index 000000000..3af24c922 --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +GCS_BUCKET="${GCS_BUCKET:-gs://kaniko-test-bucket}" +IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}" + +docker version + +# Sets up a kokoro (Google internal integration testing tool) environment +if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then + echo "Installing dependencies..." + source "$KOKORO_GFILE_DIR/common.sh" + mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/ + cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/ + pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko + echo "Installing container-diff..." + mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff + chmod +x $KOKORO_GFILE_DIR/container-diff + export PATH=$PATH:$KOKORO_GFILE_DIR + cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json +fi + +echo "Running integration tests..." +make out/executor +make out/warmer +go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@" diff --git a/scripts/minikube-setup.sh b/scripts/minikube-setup.sh new file mode 100755 index 000000000..c1b03d806 --- /dev/null +++ b/scripts/minikube-setup.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl +chmod +x kubectl +sudo mv kubectl /usr/local/bin/ +kubectl version --client + +curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +chmod +x minikube +sudo mv minikube /usr/local/bin/ + +sudo apt-get update +sudo apt-get install -y liblz4-tool + +sudo minikube start --vm-driver=none +sudo minikube status +sudo chown -R $USER $HOME/.kube $HOME/.minikube +kubectl cluster-info diff --git a/misc-integration-test.sh b/scripts/misc-integration-test.sh similarity index 89% rename from misc-integration-test.sh rename to scripts/misc-integration-test.sh index c227fcb14..b989152b3 100755 --- a/misc-integration-test.sh +++ b/scripts/misc-integration-test.sh @@ -17,9 +17,9 @@ # TestRun and TestLayers set -e -TESTS=$(./integration-test.sh -list=Test -mod=vendor) +TESTS=$(./scripts/integration-test.sh -list=Test -mod=vendor) -TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers') +TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers' | grep -v 'TestK8s') RUN_ARG='' count=0 diff --git a/test.sh b/scripts/test.sh similarity index 100% rename from test.sh rename to scripts/test.sh diff --git a/travis-setup.sh b/scripts/travis-setup.sh similarity index 100% rename from travis-setup.sh rename to scripts/travis-setup.sh diff --git a/vendor/modules.txt b/vendor/modules.txt index 20c9622ea..34ed98d65 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,5 @@ # cloud.google.com/go v0.26.0 +## explicit cloud.google.com/go/compute/metadata cloud.google.com/go/iam cloud.google.com/go/internal @@ -7,11 +8,13 @@ cloud.google.com/go/internal/trace cloud.google.com/go/internal/version cloud.google.com/go/storage # github.com/Azure/azure-pipeline-go v0.2.2 +## explicit github.com/Azure/azure-pipeline-go/pipeline # github.com/Azure/azure-sdk-for-go v19.1.1+incompatible github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2017-10-01/containerregistry github.com/Azure/azure-sdk-for-go/version # github.com/Azure/azure-storage-blob-go v0.8.0 +## explicit github.com/Azure/azure-storage-blob-go/azblob # github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 github.com/Azure/go-ansiterm @@ -28,7 +31,12 @@ github.com/Azure/go-autorest/version # github.com/Microsoft/go-winio v0.4.14 github.com/Microsoft/go-winio github.com/Microsoft/go-winio/pkg/guid +# github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 +## explicit +# github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 +## explicit # github.com/aws/aws-sdk-go v1.25.19 +## explicit github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/awserr github.com/aws/aws-sdk-go/aws/awsutil @@ -71,6 +79,7 @@ github.com/aws/aws-sdk-go/service/s3/s3manager github.com/aws/aws-sdk-go/service/sts github.com/aws/aws-sdk-go/service/sts/stsiface # github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 +## explicit github.com/beorn7/perks/quantile # github.com/containerd/containerd v1.4.0-0.20191014053712-acdcf13d5eaf => github.com/containerd/containerd v0.0.0-20191014053712-acdcf13d5eaf github.com/containerd/containerd/cio @@ -102,6 +111,7 @@ github.com/docker/distribution/digestset github.com/docker/distribution/reference github.com/docker/distribution/registry/api/errcode # github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c => github.com/docker/docker v0.0.0-20190319215453-e7b5f7dbe98c +## explicit github.com/docker/docker/api github.com/docker/docker/api/types github.com/docker/docker/api/types/backend @@ -194,10 +204,12 @@ github.com/docker/go-connections/tlsconfig # github.com/docker/go-events v0.0.0-20170721190031-9461782956ad github.com/docker/go-events # github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916 +## explicit github.com/docker/go-metrics # github.com/docker/go-units v0.4.0 github.com/docker/go-units # github.com/docker/swarmkit v1.12.1-0.20180726190244-7567d47988d8 +## explicit github.com/docker/swarmkit/agent/exec github.com/docker/swarmkit/api github.com/docker/swarmkit/api/deepcopy @@ -207,18 +219,24 @@ github.com/docker/swarmkit/manager/raftselector github.com/docker/swarmkit/protobuf/plugin github.com/docker/swarmkit/protobuf/ptypes # github.com/emirpasic/gods v1.9.0 +## explicit github.com/emirpasic/gods/containers github.com/emirpasic/gods/lists github.com/emirpasic/gods/lists/arraylist github.com/emirpasic/gods/trees github.com/emirpasic/gods/trees/binaryheap github.com/emirpasic/gods/utils +# github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 +## explicit # github.com/fsnotify/fsnotify v1.4.7 github.com/fsnotify/fsnotify # github.com/genuinetools/amicontained v0.4.3 +## explicit github.com/genuinetools/amicontained/container # github.com/ghodss/yaml v1.0.0 github.com/ghodss/yaml +# github.com/gliderlabs/ssh v0.2.2 +## explicit # github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d github.com/gogo/protobuf/gogoproto github.com/gogo/protobuf/proto @@ -228,6 +246,7 @@ github.com/gogo/protobuf/types # github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/glog # github.com/golang/mock v1.3.1 +## explicit github.com/golang/mock/gomock # github.com/golang/protobuf v1.3.2 github.com/golang/protobuf/proto @@ -239,12 +258,14 @@ github.com/golang/protobuf/ptypes/timestamp # github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a github.com/google/btree # github.com/google/go-cmp v0.3.0 +## explicit github.com/google/go-cmp/cmp github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value # github.com/google/go-containerregistry v0.0.0-20191218175032-34fb8ff33bed +## explicit github.com/google/go-containerregistry/pkg/authn github.com/google/go-containerregistry/pkg/authn/k8schain github.com/google/go-containerregistry/pkg/internal/retry @@ -266,12 +287,19 @@ github.com/google/go-containerregistry/pkg/v1/types github.com/google/go-containerregistry/pkg/v1/v1util github.com/google/go-containerregistry/pkg/v1/validate # github.com/google/go-github v17.0.0+incompatible +## explicit github.com/google/go-github/github # github.com/google/go-querystring v1.0.0 +## explicit github.com/google/go-querystring/query # github.com/google/gofuzz v1.0.0 github.com/google/gofuzz +# github.com/google/martian v2.1.0+incompatible +## explicit +# github.com/google/uuid v1.0.0 +## explicit # github.com/googleapis/gax-go v2.0.0+incompatible +## explicit github.com/googleapis/gax-go # github.com/googleapis/gnostic v0.2.2 github.com/googleapis/gnostic/OpenAPIv2 @@ -285,34 +313,45 @@ github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc # github.com/hashicorp/go-immutable-radix v1.0.0 github.com/hashicorp/go-immutable-radix # github.com/hashicorp/go-memdb v0.0.0-20180223233045-1289e7fffe71 +## explicit github.com/hashicorp/go-memdb +# github.com/hashicorp/go-uuid v1.0.1 +## explicit # github.com/hashicorp/golang-lru v0.5.0 github.com/hashicorp/golang-lru/simplelru # github.com/inconshreveable/mousetrap v1.0.0 github.com/inconshreveable/mousetrap # github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 +## explicit github.com/jbenet/go-context/io # github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/jmespath/go-jmespath # github.com/json-iterator/go v1.1.8 github.com/json-iterator/go # github.com/karrick/godirwalk v1.7.7 +## explicit github.com/karrick/godirwalk # github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e +## explicit github.com/kevinburke/ssh_config # github.com/konsorten/go-windows-terminal-sequences v1.0.2 github.com/konsorten/go-windows-terminal-sequences # github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb +## explicit github.com/mattn/go-ieproxy # github.com/mattn/go-shellwords v1.0.3 +## explicit github.com/mattn/go-shellwords # github.com/matttproud/golang_protobuf_extensions v1.0.1 +## explicit github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/minio/highwayhash v1.0.0 +## explicit github.com/minio/highwayhash # github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir # github.com/moby/buildkit v0.0.0-20191111154543-00bfbab0390c +## explicit github.com/moby/buildkit/frontend/dockerfile/command github.com/moby/buildkit/frontend/dockerfile/instructions github.com/moby/buildkit/frontend/dockerfile/parser @@ -338,28 +377,37 @@ github.com/opencontainers/runc/libcontainer/devices github.com/opencontainers/runc/libcontainer/system github.com/opencontainers/runc/libcontainer/user # github.com/opencontainers/runtime-spec v1.0.1 +## explicit github.com/opencontainers/runtime-spec/specs-go # github.com/opencontainers/selinux v1.0.0-rc1 +## explicit github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label # github.com/opentracing/opentracing-go v1.0.2 +## explicit github.com/opentracing/opentracing-go github.com/opentracing/opentracing-go/ext github.com/opentracing/opentracing-go/log # github.com/otiai10/copy v1.0.2 +## explicit github.com/otiai10/copy # github.com/pelletier/go-buffruneio v0.2.0 +## explicit github.com/pelletier/go-buffruneio # github.com/peterbourgon/diskv v2.0.1+incompatible github.com/peterbourgon/diskv # github.com/pkg/errors v0.9.1 +## explicit github.com/pkg/errors # github.com/prometheus/client_golang v0.9.0-pre1.0.20180210140205-a40133b69fbd +## explicit github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/promhttp # github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 +## explicit github.com/prometheus/client_model/go # github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1 +## explicit github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/model @@ -367,17 +415,23 @@ github.com/prometheus/common/model github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs # github.com/sergi/go-diff v1.0.0 +## explicit github.com/sergi/go-diff/diffmatchpatch # github.com/sirupsen/logrus v1.4.2 +## explicit github.com/sirupsen/logrus # github.com/spf13/afero v1.2.1 +## explicit github.com/spf13/afero github.com/spf13/afero/mem # github.com/spf13/cobra v0.0.5 +## explicit github.com/spf13/cobra # github.com/spf13/pflag v1.0.5 +## explicit github.com/spf13/pflag # github.com/src-d/gcfg v1.3.0 +## explicit github.com/src-d/gcfg github.com/src-d/gcfg/scanner github.com/src-d/gcfg/token @@ -385,17 +439,21 @@ github.com/src-d/gcfg/types # github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 github.com/syndtr/gocapability/capability # github.com/tonistiigi/fsutil v0.0.0-20191018213012-0f039a052ca1 +## explicit github.com/tonistiigi/fsutil github.com/tonistiigi/fsutil/types # github.com/vbatts/tar-split v0.10.2 +## explicit github.com/vbatts/tar-split/archive/tar github.com/vbatts/tar-split/tar/asm github.com/vbatts/tar-split/tar/storage # github.com/xanzy/ssh-agent v0.2.0 +## explicit github.com/xanzy/ssh-agent # go.etcd.io/bbolt v1.3.3 go.etcd.io/bbolt # go.opencensus.io v0.14.0 +## explicit go.opencensus.io go.opencensus.io/exporter/stackdriver/propagation go.opencensus.io/internal @@ -430,6 +488,7 @@ golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/knownhosts golang.org/x/crypto/ssh/terminal # golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 +## explicit golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/http/httpguts @@ -442,12 +501,14 @@ golang.org/x/net/internal/timeseries golang.org/x/net/proxy golang.org/x/net/trace # golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be +## explicit golang.org/x/oauth2 golang.org/x/oauth2/google golang.org/x/oauth2/internal golang.org/x/oauth2/jws golang.org/x/oauth2/jwt # golang.org/x/sync v0.0.0-20190423024810-112230192c58 +## explicit golang.org/x/sync/errgroup golang.org/x/sync/singleflight golang.org/x/sync/syncmap @@ -464,6 +525,7 @@ golang.org/x/text/unicode/norm # golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 golang.org/x/time/rate # google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79 +## explicit google.golang.org/api/gensupport google.golang.org/api/googleapi google.golang.org/api/googleapi/internal/uritemplates @@ -528,12 +590,16 @@ google.golang.org/grpc/tap # gopkg.in/inf.v0 v0.9.1 gopkg.in/inf.v0 # gopkg.in/src-d/go-billy.v4 v4.2.0 +## explicit gopkg.in/src-d/go-billy.v4 gopkg.in/src-d/go-billy.v4/helper/chroot gopkg.in/src-d/go-billy.v4/helper/polyfill gopkg.in/src-d/go-billy.v4/osfs gopkg.in/src-d/go-billy.v4/util +# gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 +## explicit # gopkg.in/src-d/go-git.v4 v4.6.0 +## explicit gopkg.in/src-d/go-git.v4 gopkg.in/src-d/go-git.v4/config gopkg.in/src-d/go-git.v4/internal/revision @@ -575,6 +641,7 @@ gopkg.in/src-d/go-git.v4/utils/merkletrie/index gopkg.in/src-d/go-git.v4/utils/merkletrie/internal/frame gopkg.in/src-d/go-git.v4/utils/merkletrie/noder # gopkg.in/warnings.v0 v0.1.2 +## explicit gopkg.in/warnings.v0 # gopkg.in/yaml.v2 v2.2.4 gopkg.in/yaml.v2 @@ -699,3 +766,4 @@ k8s.io/kubernetes/pkg/credentialprovider/aws k8s.io/kubernetes/pkg/credentialprovider/azure k8s.io/kubernetes/pkg/credentialprovider/gcp k8s.io/kubernetes/pkg/credentialprovider/secrets +# github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d => github.com/tonistiigi/fsutil v0.0.0-20191018213012-0f039a052ca1