From cdc8c60c529874ed5ab4903b1c0eb4f5f0f6edb3 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Fri, 6 Mar 2020 17:21:43 -0800 Subject: [PATCH 1/2] Move testing scripts to scripts directory Since we recently started adding more scripts it makes sense to move to a separate directory. This aligns with a common practice across golang projects: https://github.com/golang-standards/project-layout --- Makefile | 14 +++---- integration-test.sh | 41 +------------------ scripts/integration-test.sh | 40 ++++++++++++++++++ .../misc-integration-test.sh | 2 +- test.sh => scripts/test.sh | 0 travis-setup.sh => scripts/travis-setup.sh | 0 6 files changed, 49 insertions(+), 48 deletions(-) mode change 100755 => 120000 integration-test.sh create mode 100755 scripts/integration-test.sh rename misc-integration-test.sh => scripts/misc-integration-test.sh (93%) rename test.sh => scripts/test.sh (100%) rename travis-setup.sh => scripts/travis-setup.sh (100%) diff --git a/Makefile b/Makefile index 7d94dee5c..2eebc829d 100644 --- a/Makefile +++ b/Makefile @@ -55,28 +55,28 @@ out/warmer: $(GO_FILES) .PHONY: travis-setup travis-setup: - @ ./travis-setup.sh + @ ./scripts/travis-setup.sh .PHONY: test test: out/executor - @ ./test.sh + @ ./scripts/test.sh .PHONY: integration-test integration-test: - @ ./integration-test.sh + @ ./scripts/integration-test.sh .PHONY: integration-test-run integration-test-run: - @ ./integration-test.sh -run "TestRun" + @ ./scripts/integration-test.sh -run "TestRun" .PHONY: integration-test-layers integration-test-layers: - @ ./integration-test.sh -run "TestLayers" + @ ./scripts/integration-test.sh -run "TestLayers" .PHONY: integration-test-misc integration-test-misc: - $(eval RUN_ARG=$(shell ./misc-integration-test.sh)) - @ ./integration-test.sh -run "$(RUN_ARG)" + $(eval RUN_ARG=$(shell ./scripts/misc-integration-test.sh)) + @ ./scripts/integration-test.sh -run "$(RUN_ARG)" .PHONY: images images: diff --git a/integration-test.sh b/integration-test.sh deleted file mode 100755 index 3af24c922..000000000 --- a/integration-test.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -GCS_BUCKET="${GCS_BUCKET:-gs://kaniko-test-bucket}" -IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}" - -docker version - -# Sets up a kokoro (Google internal integration testing tool) environment -if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then - echo "Installing dependencies..." - source "$KOKORO_GFILE_DIR/common.sh" - mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/ - cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/ - pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko - echo "Installing container-diff..." - mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff - chmod +x $KOKORO_GFILE_DIR/container-diff - export PATH=$PATH:$KOKORO_GFILE_DIR - cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json -fi - -echo "Running integration tests..." -make out/executor -make out/warmer -go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@" diff --git a/integration-test.sh b/integration-test.sh new file mode 120000 index 000000000..8a77788df --- /dev/null +++ b/integration-test.sh @@ -0,0 +1 @@ +scripts/integration-test.sh \ No newline at end of file diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100755 index 000000000..3af24c922 --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +GCS_BUCKET="${GCS_BUCKET:-gs://kaniko-test-bucket}" +IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}" + +docker version + +# Sets up a kokoro (Google internal integration testing tool) environment +if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then + echo "Installing dependencies..." + source "$KOKORO_GFILE_DIR/common.sh" + mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/ + cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/ + pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko + echo "Installing container-diff..." + mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff + chmod +x $KOKORO_GFILE_DIR/container-diff + export PATH=$PATH:$KOKORO_GFILE_DIR + cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json +fi + +echo "Running integration tests..." +make out/executor +make out/warmer +go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@" diff --git a/misc-integration-test.sh b/scripts/misc-integration-test.sh similarity index 93% rename from misc-integration-test.sh rename to scripts/misc-integration-test.sh index c227fcb14..e02c700c0 100755 --- a/misc-integration-test.sh +++ b/scripts/misc-integration-test.sh @@ -17,7 +17,7 @@ # TestRun and TestLayers set -e -TESTS=$(./integration-test.sh -list=Test -mod=vendor) +TESTS=$(./scripts/integration-test.sh -list=Test -mod=vendor) TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers') diff --git a/test.sh b/scripts/test.sh similarity index 100% rename from test.sh rename to scripts/test.sh diff --git a/travis-setup.sh b/scripts/travis-setup.sh similarity index 100% rename from travis-setup.sh rename to scripts/travis-setup.sh From 882793b75583e9471a664414573352584f128190 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 7 Mar 2020 14:09:29 -0800 Subject: [PATCH 2/2] Add K8s integration tests - Use minikube for deploying a lightweight K8s on Travis CI - Build and push Kaniko image to local docker registry - Build dockerfiles with context on both docker and K8s and check the diff between the 2. --- .travis.yml | 14 ++-- Makefile | 14 +++- integration/images.go | 57 ++++++++++------- integration/k8s-job.yaml | 22 +++++++ integration/k8s_test.go | 106 +++++++++++++++++++++++++++++++ scripts/minikube-setup.sh | 33 ++++++++++ scripts/misc-integration-test.sh | 2 +- 7 files changed, 217 insertions(+), 31 deletions(-) create mode 100644 integration/k8s-job.yaml create mode 100644 integration/k8s_test.go create mode 100755 scripts/minikube-setup.sh diff --git a/.travis.yml b/.travis.yml index df05f880a..fc5c56032 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,8 @@ language: go os: linux dist: bionic env: - - IMAGE_REPO=localhost:5000 + global: + - IMAGE_REPO=localhost:5000 REGISTRY=localhost:5000 go: - "1.13.3" go_import_path: github.com/GoogleContainerTools/kaniko @@ -21,11 +22,16 @@ jobs: - make travis-setup script: - make integration-test-layers + - name: build-image-and-k8s-integration-test + before_install: + - make travis-setup + - make minikube-setup + script: + - make images + - make push + - make integration-test-k8s - name: integration-test-misc before_install: - make travis-setup script: - make integration-test-misc - - stage: build-images - script: - - make images diff --git a/Makefile b/Makefile index 2eebc829d..c7b50a3af 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,10 @@ out/warmer: $(GO_FILES) travis-setup: @ ./scripts/travis-setup.sh +.PHONY: minikube-setup +minikube-setup: + @ ./scripts/minikube-setup.sh + .PHONY: test test: out/executor @ ./scripts/test.sh @@ -73,6 +77,10 @@ integration-test-run: integration-test-layers: @ ./scripts/integration-test.sh -run "TestLayers" +.PHONY: integration-test-k8s +integration-test-k8s: + @ ./scripts/integration-test.sh -run "TestK8s" + .PHONY: integration-test-misc integration-test-misc: $(eval RUN_ARG=$(shell ./scripts/misc-integration-test.sh)) @@ -86,6 +94,6 @@ images: .PHONY: push push: - docker push $(REGISTRY)/executor:latest - docker push $(REGISTRY)/executor:debug - docker push $(REGISTRY)/warmer:latest + docker push $(REGISTRY)/executor:latest + docker push $(REGISTRY)/executor:debug + docker push $(REGISTRY)/warmer:latest diff --git a/integration/images.go b/integration/images.go index 593433abc..e6af511fc 100644 --- a/integration/images.go +++ b/integration/images.go @@ -185,23 +185,8 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string { return flags } -// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. -// The resulting image will be tagged with imageRepo. If the dockerfile will be built with -// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. -func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { - _, ex, _, _ := runtime.Caller(0) - cwd := filepath.Dir(ex) - - return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd) -} - -func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { - if _, present := d.filesBuilt[dockerfile]; present { - return nil - } - gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo - - fmt.Printf("Building images for Dockerfile %s\n", dockerfile) +func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir string) error { + fmt.Printf("Building image for Dockerfile %s\n", dockerfile) var buildArgs []string buildArgFlag := "--build-arg" @@ -230,13 +215,39 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerCmd.Env = append(dockerCmd.Env, env...) } - timer := timing.Start(dockerfile + "_docker") out, err := RunCommandWithoutTest(dockerCmd) - timing.DefaultRun.Stop(timer) if err != nil { return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) } fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out) + return nil +} + +// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. +// The resulting image will be tagged with imageRepo. If the dockerfile will be built with +// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. +func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + + return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd) +} + +func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { + if _, present := d.filesBuilt[dockerfile]; present { + return nil + } + gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo + + var buildArgs []string + buildArgFlag := "--build-arg" + for _, arg := range argsMap[dockerfile] { + buildArgs = append(buildArgs, buildArgFlag, arg) + } + + timer := timing.Start(dockerfile + "_docker") + d.BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir) + timing.DefaultRun.Stop(timer) contextFlag := "-c" contextPath := buildContextPath @@ -269,7 +280,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, } // build kaniko image - additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) + additionalFlags := append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) kanikoImage := GetKanikoImage(imageRepo, dockerfile) fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) @@ -301,16 +312,16 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, kanikoCmd := exec.Command("docker", dockerRunFlags...) timer = timing.Start(dockerfile + "_kaniko") - out, err = RunCommandWithoutTest(kanikoCmd) + out, err := RunCommandWithoutTest(kanikoCmd) timing.DefaultRun.Stop(timer) if err != nil { - return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out)) + return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) } if outputCheck := outputChecks[dockerfile]; outputCheck != nil { if err := outputCheck(dockerfile, out); err != nil { - return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out)) + return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) } } diff --git a/integration/k8s-job.yaml b/integration/k8s-job.yaml new file mode 100644 index 000000000..ca84047f1 --- /dev/null +++ b/integration/k8s-job.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: kaniko-test-{{.Name}} +spec: + template: + spec: + hostNetwork: true + containers: + - name: kaniko + image: localhost:5000/executor:debug + args: [ "--context=dir:///workspace", + "--destination={{.KanikoImage}}"] + volumeMounts: + - name: context + mountPath: /workspace + restartPolicy: Never + volumes: + - name: context + hostPath: + path: {{.Context}} + backoffLimit: 1 diff --git a/integration/k8s_test.go b/integration/k8s_test.go new file mode 100644 index 000000000..d7e6235cf --- /dev/null +++ b/integration/k8s_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "testing" + "text/template" +) + +type K8sConfig struct { + KanikoImage string + Context string + Name string +} + +func TestK8s(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + dir := filepath.Join(cwd, "dockerfiles-with-context") + + testDirs, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + builder := NewDockerFileBuilder() + + for _, tdInfo := range testDirs { + name := tdInfo.Name() + testDir := filepath.Join(dir, name) + + t.Run("test_k8s_with_context_"+name, func(t *testing.T) { + t.Parallel() + + if err := builder.BuildDockerImage( + config.imageRepo, "", name, testDir, + ); err != nil { + t.Fatal(err) + } + + dockerImage := GetDockerImage(config.imageRepo, name) + kanikoImage := GetKanikoImage(config.imageRepo, name) + + tmpfile, err := ioutil.TempFile("", "k8s-job-*.yaml") + if err != nil { + log.Fatal(err) + } + defer os.Remove(tmpfile.Name()) // clean up + tmpl := template.Must(template.ParseFiles("k8s-job.yaml")) + job := K8sConfig{KanikoImage: kanikoImage, Context: testDir, Name: name} + if err := tmpl.Execute(tmpfile, job); err != nil { + t.Fatal(err) + } + + fmt.Printf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n", + testDir, kanikoImage) + content, err := ioutil.ReadFile(tmpfile.Name()) + if err != nil { + log.Fatal(err) + } + fmt.Printf("K8s template %s:\n%s\n", tmpfile.Name(), content) + + kubeCmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name()) + RunCommand(kubeCmd, t) + + fmt.Printf("Waiting for K8s kaniko build job to finish: %s\n", + "job/kaniko-test-"+job.Name) + + kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=60s", + "job/kaniko-test-"+job.Name) + RunCommand(kubeWaitCmd, t) + + diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache") + + expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) + checkContainerDiffOutput(t, diff, expected) + }) + } + + if err := logBenchmarks("benchmark"); err != nil { + t.Logf("Failed to create benchmark file: %v", err) + } +} diff --git a/scripts/minikube-setup.sh b/scripts/minikube-setup.sh new file mode 100755 index 000000000..c1b03d806 --- /dev/null +++ b/scripts/minikube-setup.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl +chmod +x kubectl +sudo mv kubectl /usr/local/bin/ +kubectl version --client + +curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +chmod +x minikube +sudo mv minikube /usr/local/bin/ + +sudo apt-get update +sudo apt-get install -y liblz4-tool + +sudo minikube start --vm-driver=none +sudo minikube status +sudo chown -R $USER $HOME/.kube $HOME/.minikube +kubectl cluster-info diff --git a/scripts/misc-integration-test.sh b/scripts/misc-integration-test.sh index e02c700c0..b989152b3 100755 --- a/scripts/misc-integration-test.sh +++ b/scripts/misc-integration-test.sh @@ -19,7 +19,7 @@ set -e TESTS=$(./scripts/integration-test.sh -list=Test -mod=vendor) -TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers') +TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers' | grep -v 'TestK8s') RUN_ARG='' count=0