diff --git a/.travis.yml b/.travis.yml index df05f880a..fc5c56032 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,8 @@ language: go os: linux dist: bionic env: - - IMAGE_REPO=localhost:5000 + global: + - IMAGE_REPO=localhost:5000 REGISTRY=localhost:5000 go: - "1.13.3" go_import_path: github.com/GoogleContainerTools/kaniko @@ -21,11 +22,16 @@ jobs: - make travis-setup script: - make integration-test-layers + - name: build-image-and-k8s-integration-test + before_install: + - make travis-setup + - make minikube-setup + script: + - make images + - make push + - make integration-test-k8s - name: integration-test-misc before_install: - make travis-setup script: - make integration-test-misc - - stage: build-images - script: - - make images diff --git a/Makefile b/Makefile index 2eebc829d..c7b50a3af 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,10 @@ out/warmer: $(GO_FILES) travis-setup: @ ./scripts/travis-setup.sh +.PHONY: minikube-setup +minikube-setup: + @ ./scripts/minikube-setup.sh + .PHONY: test test: out/executor @ ./scripts/test.sh @@ -73,6 +77,10 @@ integration-test-run: integration-test-layers: @ ./scripts/integration-test.sh -run "TestLayers" +.PHONY: integration-test-k8s +integration-test-k8s: + @ ./scripts/integration-test.sh -run "TestK8s" + .PHONY: integration-test-misc integration-test-misc: $(eval RUN_ARG=$(shell ./scripts/misc-integration-test.sh)) @@ -86,6 +94,6 @@ images: .PHONY: push push: - docker push $(REGISTRY)/executor:latest - docker push $(REGISTRY)/executor:debug - docker push $(REGISTRY)/warmer:latest + docker push $(REGISTRY)/executor:latest + docker push $(REGISTRY)/executor:debug + docker push $(REGISTRY)/warmer:latest diff --git a/integration/images.go b/integration/images.go index 593433abc..e6af511fc 100644 --- a/integration/images.go +++ b/integration/images.go @@ -185,23 +185,8 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string { return flags } -// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. -// The resulting image will be tagged with imageRepo. If the dockerfile will be built with -// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. -func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { - _, ex, _, _ := runtime.Caller(0) - cwd := filepath.Dir(ex) - - return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd) -} - -func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { - if _, present := d.filesBuilt[dockerfile]; present { - return nil - } - gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo - - fmt.Printf("Building images for Dockerfile %s\n", dockerfile) +func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir string) error { + fmt.Printf("Building image for Dockerfile %s\n", dockerfile) var buildArgs []string buildArgFlag := "--build-arg" @@ -230,13 +215,39 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerCmd.Env = append(dockerCmd.Env, env...) } - timer := timing.Start(dockerfile + "_docker") out, err := RunCommandWithoutTest(dockerCmd) - timing.DefaultRun.Stop(timer) if err != nil { return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) } fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out) + return nil +} + +// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. +// The resulting image will be tagged with imageRepo. If the dockerfile will be built with +// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. +func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + + return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd) +} + +func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { + if _, present := d.filesBuilt[dockerfile]; present { + return nil + } + gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo + + var buildArgs []string + buildArgFlag := "--build-arg" + for _, arg := range argsMap[dockerfile] { + buildArgs = append(buildArgs, buildArgFlag, arg) + } + + timer := timing.Start(dockerfile + "_docker") + d.BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir) + timing.DefaultRun.Stop(timer) contextFlag := "-c" contextPath := buildContextPath @@ -269,7 +280,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, } // build kaniko image - additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) + additionalFlags := append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) kanikoImage := GetKanikoImage(imageRepo, dockerfile) fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) @@ -301,16 +312,16 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, kanikoCmd := exec.Command("docker", dockerRunFlags...) timer = timing.Start(dockerfile + "_kaniko") - out, err = RunCommandWithoutTest(kanikoCmd) + out, err := RunCommandWithoutTest(kanikoCmd) timing.DefaultRun.Stop(timer) if err != nil { - return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out)) + return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) } if outputCheck := outputChecks[dockerfile]; outputCheck != nil { if err := outputCheck(dockerfile, out); err != nil { - return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out)) + return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) } } diff --git a/integration/k8s-job.yaml b/integration/k8s-job.yaml new file mode 100644 index 000000000..ca84047f1 --- /dev/null +++ b/integration/k8s-job.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: kaniko-test-{{.Name}} +spec: + template: + spec: + hostNetwork: true + containers: + - name: kaniko + image: localhost:5000/executor:debug + args: [ "--context=dir:///workspace", + "--destination={{.KanikoImage}}"] + volumeMounts: + - name: context + mountPath: /workspace + restartPolicy: Never + volumes: + - name: context + hostPath: + path: {{.Context}} + backoffLimit: 1 diff --git a/integration/k8s_test.go b/integration/k8s_test.go new file mode 100644 index 000000000..d7e6235cf --- /dev/null +++ b/integration/k8s_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "testing" + "text/template" +) + +type K8sConfig struct { + KanikoImage string + Context string + Name string +} + +func TestK8s(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + dir := filepath.Join(cwd, "dockerfiles-with-context") + + testDirs, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + builder := NewDockerFileBuilder() + + for _, tdInfo := range testDirs { + name := tdInfo.Name() + testDir := filepath.Join(dir, name) + + t.Run("test_k8s_with_context_"+name, func(t *testing.T) { + t.Parallel() + + if err := builder.BuildDockerImage( + config.imageRepo, "", name, testDir, + ); err != nil { + t.Fatal(err) + } + + dockerImage := GetDockerImage(config.imageRepo, name) + kanikoImage := GetKanikoImage(config.imageRepo, name) + + tmpfile, err := ioutil.TempFile("", "k8s-job-*.yaml") + if err != nil { + log.Fatal(err) + } + defer os.Remove(tmpfile.Name()) // clean up + tmpl := template.Must(template.ParseFiles("k8s-job.yaml")) + job := K8sConfig{KanikoImage: kanikoImage, Context: testDir, Name: name} + if err := tmpl.Execute(tmpfile, job); err != nil { + t.Fatal(err) + } + + fmt.Printf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n", + testDir, kanikoImage) + content, err := ioutil.ReadFile(tmpfile.Name()) + if err != nil { + log.Fatal(err) + } + fmt.Printf("K8s template %s:\n%s\n", tmpfile.Name(), content) + + kubeCmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name()) + RunCommand(kubeCmd, t) + + fmt.Printf("Waiting for K8s kaniko build job to finish: %s\n", + "job/kaniko-test-"+job.Name) + + kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=60s", + "job/kaniko-test-"+job.Name) + RunCommand(kubeWaitCmd, t) + + diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache") + + expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) + checkContainerDiffOutput(t, diff, expected) + }) + } + + if err := logBenchmarks("benchmark"); err != nil { + t.Logf("Failed to create benchmark file: %v", err) + } +} diff --git a/scripts/minikube-setup.sh b/scripts/minikube-setup.sh new file mode 100755 index 000000000..c1b03d806 --- /dev/null +++ b/scripts/minikube-setup.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl +chmod +x kubectl +sudo mv kubectl /usr/local/bin/ +kubectl version --client + +curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +chmod +x minikube +sudo mv minikube /usr/local/bin/ + +sudo apt-get update +sudo apt-get install -y liblz4-tool + +sudo minikube start --vm-driver=none +sudo minikube status +sudo chown -R $USER $HOME/.kube $HOME/.minikube +kubectl cluster-info diff --git a/scripts/misc-integration-test.sh b/scripts/misc-integration-test.sh index e02c700c0..b989152b3 100755 --- a/scripts/misc-integration-test.sh +++ b/scripts/misc-integration-test.sh @@ -19,7 +19,7 @@ set -e TESTS=$(./scripts/integration-test.sh -list=Test -mod=vendor) -TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers') +TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers' | grep -v 'TestK8s') RUN_ARG='' count=0