From 2ba7d25090b999303e8026750af0a889e63ea61a Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 28 Feb 2020 01:51:31 -0800 Subject: [PATCH 01/14] remove build args from composite key and replace all build args before adding command to the cache --- pkg/executor/build.go | 22 +++--- pkg/executor/build_test.go | 138 ++++++++++++++++++++++++++++++++++++- pkg/executor/fakes.go | 3 +- 3 files changed, 152 insertions(+), 11 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index dd7021a77..c0add7e86 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -72,7 +72,7 @@ type stageBuilder struct { stageIdxToDigest map[string]string snapshotter snapShotter layerCache cache.LayerCache - pushCache cachePusher + pushLayerToCache cachePusher } // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage @@ -115,7 +115,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross layerCache: &cache.RegistryCache{ Opts: opts, }, - pushCache: pushLayerToCache, + pushLayerToCache: pushLayerToCache, } for _, cmd := range s.stage.Commands { @@ -146,9 +146,15 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { return imageConfig, nil } -func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string, compositeKey CompositeCache) (CompositeCache, error) { +func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string, compositeKey CompositeCache, args *dockerfile.BuildArgs, env []string) (CompositeCache, error) { + // First replace all the environment variables or args in the command + replacementEnvs := args.ReplacementEnvs(env) + resolvedCmd, err := util.ResolveEnvironmentReplacement(command.String(), replacementEnvs, false) + if err != nil { + return compositeKey, err + } // Add the next command to the cache key. - compositeKey.AddKey(command.String()) + compositeKey.AddKey(resolvedCmd) switch v := command.(type) { case *commands.CopyCommand: compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey) @@ -201,7 +207,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro return errors.Wrap(err, "failed to get files used from context") } - compositeKey, err = s.populateCompositeKey(command, files, compositeKey) + compositeKey, err = s.populateCompositeKey(command, files, compositeKey, s.args, cfg.Env) if err != nil { return err } @@ -251,8 +257,6 @@ func (s *stageBuilder) build() error { compositeKey = NewCompositeCache(s.baseImageDigest) } - compositeKey.AddKey(s.opts.BuildArgs...) - // Apply optimizations to the instructions. if err := s.optimize(*compositeKey, s.cf.Config); err != nil { return errors.Wrap(err, "failed to optimize instructions") @@ -309,7 +313,7 @@ func (s *stageBuilder) build() error { return errors.Wrap(err, "failed to get files used from context") } - *compositeKey, err = s.populateCompositeKey(command, files, *compositeKey) + *compositeKey, err = s.populateCompositeKey(command, files, *compositeKey, s.args, s.cf.Config.Env) if err != nil { return err } @@ -358,7 +362,7 @@ func (s *stageBuilder) build() error { // Push layer to cache (in parallel) now along with new config file if s.opts.Cache && command.ShouldCacheOutput() { cacheGroup.Go(func() error { - return s.pushCache(s.opts, ck, tarPath, command.String()) + return s.pushLayerToCache(s.opts, ck, tarPath, command.String()) }) } if err := s.saveSnapshotToImage(command.String(), tarPath); err != nil { diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index 64046bc29..ef09de5f7 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -517,6 +517,130 @@ func Test_stageBuilder_optimize(t *testing.T) { } } +type stageContext struct { + command fmt.Stringer + args *dockerfile.BuildArgs + env []string +} + +func newStageContext(command string, args map[string]string, env []string) stageContext { + dockerArgs := dockerfile.NewBuildArgs([]string{}) + for k, v := range args { + dockerArgs.AddArg(k, &v) + } + return stageContext{MockDockerCommand{command: command}, dockerArgs, env} +} + +func Test_stageBuilder_populateCompositeKey(t *testing.T) { + testCases := []struct { + description string + cmd1 stageContext + cmd2 stageContext + shdEqual bool + }{ + { + description: "cache key for same command, different buildargs, args not used in command", + cmd1: newStageContext( + "RUN echo const > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=foo1"}, + ), + cmd2: newStageContext( + "RUN echo const > test", + map[string]string{"ARG": "bar"}, + []string{"ENV=bar1"}, + ), + shdEqual: true, + }, + { + description: "cache key for same command with same build args", + cmd1: newStageContext( + "RUN echo $ARG > test", + map[string]string{"ARG": "foo"}, + []string{}, + ), + cmd2: newStageContext( + "RUN echo $ARG > test", + map[string]string{"ARG": "foo"}, + []string{}, + ), + shdEqual: true, + }, + { + description: "cache key for same command with same env", + cmd1: newStageContext( + "RUN echo $ENV > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=same"}, + ), + cmd2: newStageContext( + "RUN echo $ENV > test", + map[string]string{"ARG": "bar"}, + []string{"ENV=same"}, + ), + shdEqual: true, + }, + { + description: "cache key for same command with a build arg values", + cmd1: newStageContext( + "RUN echo $ARG > test", + map[string]string{"ARG": "foo"}, + []string{}, + ), + cmd2: newStageContext( + "RUN echo $ARG > test", + map[string]string{"ARG": "bar"}, + []string{}, + ), + }, + { + description: "cache key for same command with different env values", + cmd1: newStageContext( + "RUN echo $ENV > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=1"}, + ), + cmd2: newStageContext( + "RUN echo $ENV > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=2"}, + ), + }, + { + description: "cache key for different command same context", + cmd1: newStageContext( + "RUN echo other > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=1"}, + ), + cmd2: newStageContext( + "RUN echo another > test", + map[string]string{"ARG": "foo"}, + []string{"ENV=1"}, + ), + }, + } + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + sb := &stageBuilder{opts: &config.KanikoOptions{SrcContext: "workspace"}} + ck := CompositeCache{} + + ck1, err := sb.populateCompositeKey(tc.cmd1.command, []string{}, ck, tc.cmd1.args, tc.cmd1.env) + if err != nil { + t.Errorf("Expected error to be nil but was %v", err) + } + ck2, err := sb.populateCompositeKey(tc.cmd2.command, []string{}, ck, tc.cmd2.args, tc.cmd2.env) + if err != nil { + t.Errorf("Expected error to be nil but was %v", err) + } + key1, key2 := hashCompositeKeys(t, ck1, ck2) + if b := key1 == key2; b != tc.shdEqual { + t.Errorf("expected keys to be equal as %t but found %t", tc.shdEqual, !tc.shdEqual) + } + }) + } +} + func Test_stageBuilder_build(t *testing.T) { type testcase struct { description string @@ -881,7 +1005,7 @@ COPY %s bar.txt cf: cf, snapshotter: snap, layerCache: lc, - pushCache: func(_ *config.KanikoOptions, cacheKey, _, _ string) error { + pushLayerToCache: func(_ *config.KanikoOptions, cacheKey, _, _ string) error { keys = append(keys, cacheKey) return nil }, @@ -993,3 +1117,15 @@ func generateTar(t *testing.T, dir string, fileNames ...string) []byte { } return buf.Bytes() } + +func hashCompositeKeys(t *testing.T, ck1 CompositeCache, ck2 CompositeCache) (string, string) { + key1, err := ck1.Hash() + if err != nil { + t.Errorf("could not hash composite key due to %s", err) + } + key2, err := ck2.Hash() + if err != nil { + t.Errorf("could not hash composite key due to %s", err) + } + return key1, key2 +} diff --git a/pkg/executor/fakes.go b/pkg/executor/fakes.go index e9cfdc694..e89d5ac47 100644 --- a/pkg/executor/fakes.go +++ b/pkg/executor/fakes.go @@ -43,13 +43,14 @@ func (f fakeSnapShotter) TakeSnapshot(_ []string) (string, error) { } type MockDockerCommand struct { + command string contextFiles []string cacheCommand commands.DockerCommand } func (m MockDockerCommand) ExecuteCommand(c *v1.Config, args *dockerfile.BuildArgs) error { return nil } func (m MockDockerCommand) String() string { - return "meow" + return m.command } func (m MockDockerCommand) FilesToSnapshot() []string { return []string{"meow-snapshot-no-cache"} From 58c0d47fa60f09e5708220e53a6e7d2ce5e7dcf3 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 28 Feb 2020 02:44:43 -0800 Subject: [PATCH 02/14] fix npe --- pkg/executor/build_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index ef09de5f7..e968e3eae 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -497,7 +497,8 @@ func Test_stageBuilder_optimize(t *testing.T) { cf := &v1.ConfigFile{} snap := fakeSnapShotter{} lc := &fakeLayerCache{retrieve: tc.retrieve} - sb := &stageBuilder{opts: tc.opts, cf: cf, snapshotter: snap, layerCache: lc} + sb := &stageBuilder{opts: tc.opts, cf: cf, snapshotter: snap, layerCache: lc, + args: dockerfile.NewBuildArgs([]string{})} ck := CompositeCache{} file, err := ioutil.TempFile("", "foo") if err != nil { From ae5a7bf1586d10e95e4f7a5df56148bb8e079908 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Fri, 28 Feb 2020 02:53:42 -0800 Subject: [PATCH 03/14] fix tests --- pkg/executor/build_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index e968e3eae..f18626581 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -669,6 +669,7 @@ func Test_stageBuilder_build(t *testing.T) { t.Errorf("couldn't create hash %v", err) } command := MockDockerCommand{ + command: "meow", contextFiles: []string{filePath}, cacheCommand: MockCachedDockerCommand{ contextFiles: []string{filePath}, @@ -701,6 +702,7 @@ func Test_stageBuilder_build(t *testing.T) { t.Errorf("couldn't create hash %v", err) } command := MockDockerCommand{ + command: "meow", contextFiles: []string{filePath}, cacheCommand: MockCachedDockerCommand{ contextFiles: []string{filePath}, From b9f4c4bab5052f24a151145e58c6563bf2667d40 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Wed, 4 Mar 2020 16:57:26 -0800 Subject: [PATCH 04/14] add more tests for stageBuilder_build --- pkg/executor/build_test.go | 71 +++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index f18626581..f76aabbf7 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -646,6 +646,7 @@ func Test_stageBuilder_build(t *testing.T) { type testcase struct { description string opts *config.KanikoOptions + args map[string]string layerCache *fakeLayerCache expectedCacheKeys []string pushedCacheKeys []string @@ -965,6 +966,71 @@ COPY %s bar.txt commands: getCommands(dir, cmds), } }(), + func() testcase { + dir, _ := tempDirAndFile(t) + ch := NewCompositeCache("") + ch.AddKey("RUN foobar") + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + + command := MockDockerCommand{ + command: "RUN foobar", + contextFiles: []string{}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{}, + }, + } + + return testcase{ + description: "cached run command with no build arg value used uses cached layer and does not push anything", + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: dir}}, + opts: &config.KanikoOptions{Cache: true}, + args: map[string]string{ + "test": "value", + }, + expectedCacheKeys: []string{hash}, + commands: []commands.DockerCommand{command}, + // layer key needs to be read. + layerCache: &fakeLayerCache{ + img: &fakeImage{ImageLayers: []v1.Layer{fakeLayer{}}}, + keySequence: []string{hash}, + }, + rootDir: dir, + } + }(), + func() testcase { + dir, _ := tempDirAndFile(t) + + ch := NewCompositeCache("") + ch.AddKey("RUN anotherValue") + hash, err := ch.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + + command := MockDockerCommand{ + command: "RUN $arg", + contextFiles: []string{}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{}, + }, + } + + return testcase{ + description: "cached run command with another build arg pushes layer", + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: dir}}, + opts: &config.KanikoOptions{Cache: true}, + args: map[string]string{ + "arg": "anotherValue", + }, + expectedCacheKeys: []string{hash}, + pushedCacheKeys: []string{hash}, + commands: []commands.DockerCommand{command}, + rootDir: dir, + } + }(), } for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { @@ -1002,7 +1068,7 @@ COPY %s bar.txt } keys := []string{} sb := &stageBuilder{ - args: &dockerfile.BuildArgs{}, //required or code will panic + args: dockerfile.NewBuildArgs([]string{}), //required or code will panic image: tc.image, opts: tc.opts, cf: cf, @@ -1014,6 +1080,9 @@ COPY %s bar.txt }, } sb.cmds = tc.commands + for key, value := range tc.args { + sb.args.AddArg(key, &value) + } tmp := commands.RootDir if tc.rootDir != "" { commands.RootDir = tc.rootDir From 579ec52a304a8338d4e5121da148c118ccb96e34 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Wed, 4 Mar 2020 17:04:42 -0800 Subject: [PATCH 05/14] add more tests for stageBuilder_build --- pkg/executor/build_test.go | 52 +++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index f76aabbf7..f5a4caa7a 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -1004,7 +1004,7 @@ COPY %s bar.txt dir, _ := tempDirAndFile(t) ch := NewCompositeCache("") - ch.AddKey("RUN anotherValue") + ch.AddKey("RUN value") hash, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) @@ -1018,6 +1018,47 @@ COPY %s bar.txt }, } + return testcase{ + description: "cached run command with same build arg does not push layer", + config: &v1.ConfigFile{Config: v1.Config{WorkingDir: dir}}, + opts: &config.KanikoOptions{Cache: true}, + args: map[string]string{ + "arg": "value", + }, + // layer key that exists + layerCache: &fakeLayerCache{ + img: &fakeImage{ImageLayers: []v1.Layer{fakeLayer{}}}, + keySequence: []string{hash}, + }, + expectedCacheKeys: []string{hash}, + commands: []commands.DockerCommand{command}, + rootDir: dir, + } + }(), + func() testcase { + dir, _ := tempDirAndFile(t) + + ch1 := NewCompositeCache("") + ch1.AddKey("RUN value") + hash1, err := ch1.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + + ch2 := NewCompositeCache("") + ch2.AddKey("RUN anotherValue") + hash2, err := ch2.Hash() + if err != nil { + t.Errorf("couldn't create hash %v", err) + } + command := MockDockerCommand{ + command: "RUN $arg", + contextFiles: []string{}, + cacheCommand: MockCachedDockerCommand{ + contextFiles: []string{}, + }, + } + return testcase{ description: "cached run command with another build arg pushes layer", config: &v1.ConfigFile{Config: v1.Config{WorkingDir: dir}}, @@ -1025,8 +1066,13 @@ COPY %s bar.txt args: map[string]string{ "arg": "anotherValue", }, - expectedCacheKeys: []string{hash}, - pushedCacheKeys: []string{hash}, + // layer for arg=value already exists + layerCache: &fakeLayerCache{ + img: &fakeImage{ImageLayers: []v1.Layer{fakeLayer{}}}, + keySequence: []string{hash1}, + }, + expectedCacheKeys: []string{hash2}, + pushedCacheKeys: []string{hash2}, commands: []commands.DockerCommand{command}, rootDir: dir, } From cdc8c60c529874ed5ab4903b1c0eb4f5f0f6edb3 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Fri, 6 Mar 2020 17:21:43 -0800 Subject: [PATCH 06/14] Move testing scripts to scripts directory Since we recently started adding more scripts it makes sense to move to a separate directory. This aligns with a common practice across golang projects: https://github.com/golang-standards/project-layout --- Makefile | 14 +++---- integration-test.sh | 41 +------------------ scripts/integration-test.sh | 40 ++++++++++++++++++ .../misc-integration-test.sh | 2 +- test.sh => scripts/test.sh | 0 travis-setup.sh => scripts/travis-setup.sh | 0 6 files changed, 49 insertions(+), 48 deletions(-) mode change 100755 => 120000 integration-test.sh create mode 100755 scripts/integration-test.sh rename misc-integration-test.sh => scripts/misc-integration-test.sh (93%) rename test.sh => scripts/test.sh (100%) rename travis-setup.sh => scripts/travis-setup.sh (100%) diff --git a/Makefile b/Makefile index 7d94dee5c..2eebc829d 100644 --- a/Makefile +++ b/Makefile @@ -55,28 +55,28 @@ out/warmer: $(GO_FILES) .PHONY: travis-setup travis-setup: - @ ./travis-setup.sh + @ ./scripts/travis-setup.sh .PHONY: test test: out/executor - @ ./test.sh + @ ./scripts/test.sh .PHONY: integration-test integration-test: - @ ./integration-test.sh + @ ./scripts/integration-test.sh .PHONY: integration-test-run integration-test-run: - @ ./integration-test.sh -run "TestRun" + @ ./scripts/integration-test.sh -run "TestRun" .PHONY: integration-test-layers integration-test-layers: - @ ./integration-test.sh -run "TestLayers" + @ ./scripts/integration-test.sh -run "TestLayers" .PHONY: integration-test-misc integration-test-misc: - $(eval RUN_ARG=$(shell ./misc-integration-test.sh)) - @ ./integration-test.sh -run "$(RUN_ARG)" + $(eval RUN_ARG=$(shell ./scripts/misc-integration-test.sh)) + @ ./scripts/integration-test.sh -run "$(RUN_ARG)" .PHONY: images images: diff --git a/integration-test.sh b/integration-test.sh deleted file mode 100755 index 3af24c922..000000000 --- a/integration-test.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -GCS_BUCKET="${GCS_BUCKET:-gs://kaniko-test-bucket}" -IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}" - -docker version - -# Sets up a kokoro (Google internal integration testing tool) environment -if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then - echo "Installing dependencies..." - source "$KOKORO_GFILE_DIR/common.sh" - mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/ - cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/ - pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko - echo "Installing container-diff..." - mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff - chmod +x $KOKORO_GFILE_DIR/container-diff - export PATH=$PATH:$KOKORO_GFILE_DIR - cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json -fi - -echo "Running integration tests..." -make out/executor -make out/warmer -go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@" diff --git a/integration-test.sh b/integration-test.sh new file mode 120000 index 000000000..8a77788df --- /dev/null +++ b/integration-test.sh @@ -0,0 +1 @@ +scripts/integration-test.sh \ No newline at end of file diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100755 index 000000000..3af24c922 --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +GCS_BUCKET="${GCS_BUCKET:-gs://kaniko-test-bucket}" +IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}" + +docker version + +# Sets up a kokoro (Google internal integration testing tool) environment +if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then + echo "Installing dependencies..." + source "$KOKORO_GFILE_DIR/common.sh" + mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/ + cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/ + pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko + echo "Installing container-diff..." + mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff + chmod +x $KOKORO_GFILE_DIR/container-diff + export PATH=$PATH:$KOKORO_GFILE_DIR + cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json +fi + +echo "Running integration tests..." +make out/executor +make out/warmer +go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@" diff --git a/misc-integration-test.sh b/scripts/misc-integration-test.sh similarity index 93% rename from misc-integration-test.sh rename to scripts/misc-integration-test.sh index c227fcb14..e02c700c0 100755 --- a/misc-integration-test.sh +++ b/scripts/misc-integration-test.sh @@ -17,7 +17,7 @@ # TestRun and TestLayers set -e -TESTS=$(./integration-test.sh -list=Test -mod=vendor) +TESTS=$(./scripts/integration-test.sh -list=Test -mod=vendor) TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers') diff --git a/test.sh b/scripts/test.sh similarity index 100% rename from test.sh rename to scripts/test.sh diff --git a/travis-setup.sh b/scripts/travis-setup.sh similarity index 100% rename from travis-setup.sh rename to scripts/travis-setup.sh From 7d345f13d22e067e1f1605644f74040506ff9b31 Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Sun, 26 Jan 2020 13:38:45 +0100 Subject: [PATCH 07/14] Use the correct name for acr helper Add it to debug and warmer images too --- deploy/Dockerfile | 2 +- deploy/Dockerfile_debug | 5 +++++ deploy/Dockerfile_warmer | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 1ed34a4d8..2c35ecae6 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -35,7 +35,7 @@ FROM scratch COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/executor /kaniko/executor COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login -COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr-linux +COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /root/.docker/config.json /kaniko/.docker/config.json ENV HOME /root diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index a57151bab..ee1c84188 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -25,6 +25,10 @@ RUN docker-credential-gcr configure-docker # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 +# ACR docker credential helper +ADD https://aadacr.blob.core.windows.net/acr-docker-credential-helper/docker-credential-acr-linux-amd64.tar.gz /usr/local/bin +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-acr-linux-amd64.tar.gz + COPY . . RUN make GOARCH=${GOARCH} && make out/warmer @@ -39,6 +43,7 @@ FROM scratch COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/* /kaniko/ COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login +COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY --from=1 /distroless/bazel-bin/experimental/busybox/busybox/ /busybox/ # Declare /busybox as a volume to get it automatically whitelisted VOLUME /busybox diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer index b81df4413..181975369 100644 --- a/deploy/Dockerfile_warmer +++ b/deploy/Dockerfile_warmer @@ -24,6 +24,9 @@ RUN docker-credential-gcr configure-docker # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 +# ACR docker credential helper +ADD https://aadacr.blob.core.windows.net/acr-docker-credential-helper/docker-credential-acr-linux-amd64.tar.gz /usr/local/bin +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-acr-linux-amd64.tar.gz COPY . . RUN make GOARCH=${GOARCH} out/warmer @@ -32,6 +35,7 @@ FROM scratch COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/warmer /kaniko/warmer COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login +COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /root/.docker/config.json /kaniko/.docker/config.json ENV HOME /root From 882793b75583e9471a664414573352584f128190 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 7 Mar 2020 14:09:29 -0800 Subject: [PATCH 08/14] Add K8s integration tests - Use minikube for deploying a lightweight K8s on Travis CI - Build and push Kaniko image to local docker registry - Build dockerfiles with context on both docker and K8s and check the diff between the 2. --- .travis.yml | 14 ++-- Makefile | 14 +++- integration/images.go | 57 ++++++++++------- integration/k8s-job.yaml | 22 +++++++ integration/k8s_test.go | 106 +++++++++++++++++++++++++++++++ scripts/minikube-setup.sh | 33 ++++++++++ scripts/misc-integration-test.sh | 2 +- 7 files changed, 217 insertions(+), 31 deletions(-) create mode 100644 integration/k8s-job.yaml create mode 100644 integration/k8s_test.go create mode 100755 scripts/minikube-setup.sh diff --git a/.travis.yml b/.travis.yml index df05f880a..fc5c56032 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,8 @@ language: go os: linux dist: bionic env: - - IMAGE_REPO=localhost:5000 + global: + - IMAGE_REPO=localhost:5000 REGISTRY=localhost:5000 go: - "1.13.3" go_import_path: github.com/GoogleContainerTools/kaniko @@ -21,11 +22,16 @@ jobs: - make travis-setup script: - make integration-test-layers + - name: build-image-and-k8s-integration-test + before_install: + - make travis-setup + - make minikube-setup + script: + - make images + - make push + - make integration-test-k8s - name: integration-test-misc before_install: - make travis-setup script: - make integration-test-misc - - stage: build-images - script: - - make images diff --git a/Makefile b/Makefile index 2eebc829d..c7b50a3af 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,10 @@ out/warmer: $(GO_FILES) travis-setup: @ ./scripts/travis-setup.sh +.PHONY: minikube-setup +minikube-setup: + @ ./scripts/minikube-setup.sh + .PHONY: test test: out/executor @ ./scripts/test.sh @@ -73,6 +77,10 @@ integration-test-run: integration-test-layers: @ ./scripts/integration-test.sh -run "TestLayers" +.PHONY: integration-test-k8s +integration-test-k8s: + @ ./scripts/integration-test.sh -run "TestK8s" + .PHONY: integration-test-misc integration-test-misc: $(eval RUN_ARG=$(shell ./scripts/misc-integration-test.sh)) @@ -86,6 +94,6 @@ images: .PHONY: push push: - docker push $(REGISTRY)/executor:latest - docker push $(REGISTRY)/executor:debug - docker push $(REGISTRY)/warmer:latest + docker push $(REGISTRY)/executor:latest + docker push $(REGISTRY)/executor:debug + docker push $(REGISTRY)/warmer:latest diff --git a/integration/images.go b/integration/images.go index 593433abc..e6af511fc 100644 --- a/integration/images.go +++ b/integration/images.go @@ -185,23 +185,8 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string { return flags } -// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. -// The resulting image will be tagged with imageRepo. If the dockerfile will be built with -// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. -func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { - _, ex, _, _ := runtime.Caller(0) - cwd := filepath.Dir(ex) - - return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd) -} - -func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { - if _, present := d.filesBuilt[dockerfile]; present { - return nil - } - gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo - - fmt.Printf("Building images for Dockerfile %s\n", dockerfile) +func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir string) error { + fmt.Printf("Building image for Dockerfile %s\n", dockerfile) var buildArgs []string buildArgFlag := "--build-arg" @@ -230,13 +215,39 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerCmd.Env = append(dockerCmd.Env, env...) } - timer := timing.Start(dockerfile + "_docker") out, err := RunCommandWithoutTest(dockerCmd) - timing.DefaultRun.Stop(timer) if err != nil { return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) } fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out) + return nil +} + +// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. +// The resulting image will be tagged with imageRepo. If the dockerfile will be built with +// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. +func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + + return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd) +} + +func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { + if _, present := d.filesBuilt[dockerfile]; present { + return nil + } + gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo + + var buildArgs []string + buildArgFlag := "--build-arg" + for _, arg := range argsMap[dockerfile] { + buildArgs = append(buildArgs, buildArgFlag, arg) + } + + timer := timing.Start(dockerfile + "_docker") + d.BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir) + timing.DefaultRun.Stop(timer) contextFlag := "-c" contextPath := buildContextPath @@ -269,7 +280,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, } // build kaniko image - additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) + additionalFlags := append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) kanikoImage := GetKanikoImage(imageRepo, dockerfile) fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) @@ -301,16 +312,16 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, kanikoCmd := exec.Command("docker", dockerRunFlags...) timer = timing.Start(dockerfile + "_kaniko") - out, err = RunCommandWithoutTest(kanikoCmd) + out, err := RunCommandWithoutTest(kanikoCmd) timing.DefaultRun.Stop(timer) if err != nil { - return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out)) + return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) } if outputCheck := outputChecks[dockerfile]; outputCheck != nil { if err := outputCheck(dockerfile, out); err != nil { - return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out)) + return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) } } diff --git a/integration/k8s-job.yaml b/integration/k8s-job.yaml new file mode 100644 index 000000000..ca84047f1 --- /dev/null +++ b/integration/k8s-job.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: kaniko-test-{{.Name}} +spec: + template: + spec: + hostNetwork: true + containers: + - name: kaniko + image: localhost:5000/executor:debug + args: [ "--context=dir:///workspace", + "--destination={{.KanikoImage}}"] + volumeMounts: + - name: context + mountPath: /workspace + restartPolicy: Never + volumes: + - name: context + hostPath: + path: {{.Context}} + backoffLimit: 1 diff --git a/integration/k8s_test.go b/integration/k8s_test.go new file mode 100644 index 000000000..d7e6235cf --- /dev/null +++ b/integration/k8s_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "testing" + "text/template" +) + +type K8sConfig struct { + KanikoImage string + Context string + Name string +} + +func TestK8s(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + dir := filepath.Join(cwd, "dockerfiles-with-context") + + testDirs, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + builder := NewDockerFileBuilder() + + for _, tdInfo := range testDirs { + name := tdInfo.Name() + testDir := filepath.Join(dir, name) + + t.Run("test_k8s_with_context_"+name, func(t *testing.T) { + t.Parallel() + + if err := builder.BuildDockerImage( + config.imageRepo, "", name, testDir, + ); err != nil { + t.Fatal(err) + } + + dockerImage := GetDockerImage(config.imageRepo, name) + kanikoImage := GetKanikoImage(config.imageRepo, name) + + tmpfile, err := ioutil.TempFile("", "k8s-job-*.yaml") + if err != nil { + log.Fatal(err) + } + defer os.Remove(tmpfile.Name()) // clean up + tmpl := template.Must(template.ParseFiles("k8s-job.yaml")) + job := K8sConfig{KanikoImage: kanikoImage, Context: testDir, Name: name} + if err := tmpl.Execute(tmpfile, job); err != nil { + t.Fatal(err) + } + + fmt.Printf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n", + testDir, kanikoImage) + content, err := ioutil.ReadFile(tmpfile.Name()) + if err != nil { + log.Fatal(err) + } + fmt.Printf("K8s template %s:\n%s\n", tmpfile.Name(), content) + + kubeCmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name()) + RunCommand(kubeCmd, t) + + fmt.Printf("Waiting for K8s kaniko build job to finish: %s\n", + "job/kaniko-test-"+job.Name) + + kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=60s", + "job/kaniko-test-"+job.Name) + RunCommand(kubeWaitCmd, t) + + diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache") + + expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) + checkContainerDiffOutput(t, diff, expected) + }) + } + + if err := logBenchmarks("benchmark"); err != nil { + t.Logf("Failed to create benchmark file: %v", err) + } +} diff --git a/scripts/minikube-setup.sh b/scripts/minikube-setup.sh new file mode 100755 index 000000000..c1b03d806 --- /dev/null +++ b/scripts/minikube-setup.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl +chmod +x kubectl +sudo mv kubectl /usr/local/bin/ +kubectl version --client + +curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +chmod +x minikube +sudo mv minikube /usr/local/bin/ + +sudo apt-get update +sudo apt-get install -y liblz4-tool + +sudo minikube start --vm-driver=none +sudo minikube status +sudo chown -R $USER $HOME/.kube $HOME/.minikube +kubectl cluster-info diff --git a/scripts/misc-integration-test.sh b/scripts/misc-integration-test.sh index e02c700c0..b989152b3 100755 --- a/scripts/misc-integration-test.sh +++ b/scripts/misc-integration-test.sh @@ -19,7 +19,7 @@ set -e TESTS=$(./scripts/integration-test.sh -list=Test -mod=vendor) -TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers') +TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers' | grep -v 'TestK8s') RUN_ARG='' count=0 From 8eef58939b270c22cdf70c50d6d33d79b16bdd2e Mon Sep 17 00:00:00 2001 From: Yuheng Zhang Date: Thu, 12 Mar 2020 14:31:15 +0800 Subject: [PATCH 09/14] fix: credential typo --- pkg/buildcontext/azureblob.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/buildcontext/azureblob.go b/pkg/buildcontext/azureblob.go index 6af468015..5f6817a90 100644 --- a/pkg/buildcontext/azureblob.go +++ b/pkg/buildcontext/azureblob.go @@ -43,12 +43,12 @@ func (b *AzureBlob) UnpackTarFromBuildContext() (string, error) { return "", errors.New("AZURE_STORAGE_ACCESS_KEY environment variable is not set") } - // Get storage accoutname for Azure Blob Storage + // Get storage accountName for Azure Blob Storage u, _ := url.Parse(b.context) parts := azblob.NewBlobURLParts(*u) accountName := strings.Split(parts.Host, ".")[0] - // Generate credentail with accountname and accountkey + // Generate credential with accountName and accountKey credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) if err != nil { return parts.Host, err @@ -62,7 +62,7 @@ func (b *AzureBlob) UnpackTarFromBuildContext() (string, error) { return tarPath, err } - // Downloading contextfile from Azure Blob Storage + // Downloading context file from Azure Blob Storage p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) blobURL := azblob.NewBlobURL(*u, p) ctx := context.Background() From fda7ed4f747a29512f0d5e0b4db7d55360cb1119 Mon Sep 17 00:00:00 2001 From: Jordan GOASDOUE Date: Sat, 7 Mar 2020 16:48:07 +0100 Subject: [PATCH 10/14] feat: add support of local '.tar.gz' file inside the kaniko container --- README.md | 1 + cmd/executor/cmd/root.go | 2 +- integration/integration_test.go | 3 +- pkg/buildcontext/buildcontext.go | 8 +- pkg/buildcontext/tar.go | 40 ++++++++ pkg/buildcontext/tar_test.go | 158 +++++++++++++++++++++++++++++++ 6 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 pkg/buildcontext/tar.go create mode 100644 pkg/buildcontext/tar_test.go diff --git a/README.md b/README.md index f285a6080..0a6d3c8a5 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ When running kaniko, use the `--context` flag with the appropriate prefix to spe | Source | Prefix | Example | |---------|---------|---------| | Local Directory | dir://[path to a directory in the kaniko container] | `dir:///workspace` | +| Local Tar Gz | tar://[path to a .tar.gz in the kaniko container] | `tar://path/to/context.tar.gz` | | GCS Bucket | gs://[bucket name]/[path to .tar.gz] | `gs://kaniko-bucket/path/to/context.tar.gz` | | S3 Bucket | s3://[bucket name]/[path to .tar.gz] | `s3://kaniko-bucket/path/to/context.tar.gz` | | Azure Blob Storage| https://[account].[azureblobhostsuffix]/[container]/[path to .tar.gz] | `https://myaccount.blob.core.windows.net/container/path/to/context.tar.gz` | diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index efda975b5..9ba1178dd 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -230,7 +230,7 @@ func copyDockerfile() error { return nil } -// resolveSourceContext unpacks the source context if it is a tar in a bucket +// resolveSourceContext unpacks the source context if it is a tar in a bucket or in kaniko container // it resets srcContext to be the path to the unpacked build context within the image func resolveSourceContext() error { if opts.SrcContext == "" && opts.Bucket == "" { diff --git a/integration/integration_test.go b/integration/integration_test.go index f70522b35..5fc4541ee 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -33,11 +33,10 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/daemon" - "github.com/pkg/errors" - "github.com/GoogleContainerTools/kaniko/pkg/timing" "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/GoogleContainerTools/kaniko/testutil" + "github.com/pkg/errors" ) var config *integrationTestConfig diff --git a/pkg/buildcontext/buildcontext.go b/pkg/buildcontext/buildcontext.go index 3142239a8..ef39c21e2 100644 --- a/pkg/buildcontext/buildcontext.go +++ b/pkg/buildcontext/buildcontext.go @@ -24,6 +24,10 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/util" ) +const ( + TarBuildContextPrefix = "tar://" +) + // BuildContext unifies calls to download and unpack the build context. type BuildContext interface { // Unpacks a build context and returns the directory where it resides @@ -51,6 +55,8 @@ func GetBuildContext(srcContext string) (BuildContext, error) { return &AzureBlob{context: srcContext}, nil } return nil, errors.New("url provided for https context is not in a supported format, please use the https url for Azure Blob Storage") + case TarBuildContextPrefix: + return &Tar{context: context}, nil } - return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, s3://, git://, https://") + return nil, errors.New("unknown build context prefix provided, please use one of the following: gs://, dir://, tar://, s3://, git://, https://") } diff --git a/pkg/buildcontext/tar.go b/pkg/buildcontext/tar.go new file mode 100644 index 000000000..d5d1c250a --- /dev/null +++ b/pkg/buildcontext/tar.go @@ -0,0 +1,40 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package buildcontext + +import ( + "os" + + "github.com/GoogleContainerTools/kaniko/pkg/constants" + "github.com/GoogleContainerTools/kaniko/pkg/util" + "github.com/pkg/errors" +) + +// Tar unifies calls to download and unpack the build context. +type Tar struct { + context string +} + +// UnpackTarFromBuildContext unpack the compressed tar file +func (t *Tar) UnpackTarFromBuildContext() (string, error) { + directory := constants.BuildContextDir + if err := os.MkdirAll(directory, 0750); err != nil { + return "", errors.Wrap(err, "unpacking tar from build context") + } + + return directory, util.UnpackCompressedTar(t.context, directory) +} diff --git a/pkg/buildcontext/tar_test.go b/pkg/buildcontext/tar_test.go new file mode 100644 index 000000000..5904b4e60 --- /dev/null +++ b/pkg/buildcontext/tar_test.go @@ -0,0 +1,158 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package buildcontext + +import ( + "bytes" + "compress/gzip" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sync" + "testing" + + "github.com/GoogleContainerTools/kaniko/pkg/util" + "github.com/GoogleContainerTools/kaniko/testutil" +) + +func TestBuildWithLocalTar(t *testing.T) { + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + + testDir := "test_dir" + testDirLongPath := filepath.Join(cwd, testDir) + dirUnpack := filepath.Join(testDirLongPath, "dir_where_to_unpack") + + if err := os.MkdirAll(dirUnpack, 0750); err != nil { + t.Errorf("Failed to create dir_where_to_extract: %v", err) + } + + validDockerfile := "Dockerfile_valid" + invalidDockerfile := "Dockerfile_invalid" + nonExistingDockerfile := "Dockerfile_non_existing" + + files := map[string]string{ + validDockerfile: "FROM debian:9.11\nRUN echo \"valid\"", + invalidDockerfile: "FROM debian:9.11\nRUN echo \"invalid\"", + } + + if err := testutil.SetupFiles(testDir, files); err != nil { + t.Errorf("Failed to setup files %v on %s: %v", files, testDir, err) + } + + if err := os.Chdir(testDir); err != nil { + t.Fatalf("Failed to Chdir on %s: %v", testDir, err) + } + + validTarPath := fmt.Sprintf("%s.tar.gz", validDockerfile) + invalidTarPath := fmt.Sprintf("%s.tar.gz", invalidDockerfile) + nonExistingTarPath := fmt.Sprintf("%s.tar.gz", nonExistingDockerfile) + + var wg sync.WaitGroup + wg.Add(1) + // Create Tar Gz File with dockerfile inside + go func(wg *sync.WaitGroup) { + defer wg.Done() + validTarFile, err := os.Create(validTarPath) + if err != nil { + t.Errorf("Failed to create %s: %v", validTarPath, err) + } + defer validTarFile.Close() + + invalidTarFile, err := os.Create(invalidTarPath) + if err != nil { + t.Errorf("Failed to create %s: %v", invalidTarPath, err) + } + defer invalidTarFile.Close() + + gw := gzip.NewWriter(validTarFile) + defer gw.Close() + + tw := util.NewTar(gw) + defer tw.Close() + + if err := tw.AddFileToTar(validDockerfile); err != nil { + t.Errorf("Failed to add %s to %s: %v", validDockerfile, validTarPath, err) + } + }(&wg) + + // Waiting for the Tar Gz file creation to be done before moving on + wg.Wait() + + tests := []struct { + dockerfile string + srcContext string + unpackShouldErr bool + srcShaShouldErr bool + destShaShouldErr bool + }{ + { + dockerfile: validDockerfile, + srcContext: filepath.Join(testDir, validTarPath), + unpackShouldErr: false, + srcShaShouldErr: false, + destShaShouldErr: false, + }, + { + dockerfile: invalidDockerfile, + srcContext: filepath.Join(testDir, invalidTarPath), + unpackShouldErr: true, + srcShaShouldErr: false, + destShaShouldErr: true, + }, + { + dockerfile: nonExistingDockerfile, + srcContext: filepath.Join(testDir, nonExistingTarPath), + unpackShouldErr: true, + srcShaShouldErr: true, + destShaShouldErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.dockerfile, func(t *testing.T) { + err := util.UnpackCompressedTar(filepath.Join(cwd, tt.srcContext), dirUnpack) + testutil.CheckError(t, tt.unpackShouldErr, err) + srcSHA, err := getSHAFromFilePath(tt.dockerfile) + testutil.CheckError(t, tt.srcShaShouldErr, err) + destSHA, err := getSHAFromFilePath(filepath.Join(dirUnpack, tt.dockerfile)) + testutil.CheckError(t, tt.destShaShouldErr, err) + if err == nil { + testutil.CheckDeepEqual(t, srcSHA, destSHA) + } + }) + } + + if err := os.RemoveAll(testDirLongPath); err != nil { + t.Errorf("Failed to remove %s: %v", testDirLongPath, err) + } +} + +func getSHAFromFilePath(f string) (string, error) { + data, err := ioutil.ReadFile(f) + if err != nil { + return "", err + } + sha, err := util.SHA256(bytes.NewReader(data)) + if err != nil { + return "", err + } + + return sha, nil +} From fa3318a273407b7a79afc3f2a676cadfaab54b09 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 14 Mar 2020 18:05:00 -0700 Subject: [PATCH 11/14] Make cloudbuild.yaml re-usable for anyone Cloud build is also really useful for devs that have limited resources to build Kaniko locally. This commit makes adds parameters to cloudbuild to make it easy to adjust to your own project. --- deploy/cloudbuild.yaml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/deploy/cloudbuild.yaml b/deploy/cloudbuild.yaml index 44b30aa49..47a9a0ab2 100644 --- a/deploy/cloudbuild.yaml +++ b/deploy/cloudbuild.yaml @@ -2,18 +2,21 @@ steps: # First, build kaniko - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile", - "-t", "gcr.io/kaniko-project/executor:${COMMIT_SHA}", "."] + "-t", "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:${COMMIT_SHA}", "."] # Then, we want to build kaniko:debug - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile_debug", - "-t", "gcr.io/kaniko-project/executor:debug-${COMMIT_SHA}", "."] + "-t", "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:debug-${COMMIT_SHA}", "."] - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile_debug", - "-t", "gcr.io/kaniko-project/executor:debug", "."] + "-t", "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:debug", "."] # Then, we want to build the cache warmer - name: "gcr.io/cloud-builders/docker" args: ["build", "-f", "deploy/Dockerfile_warmer", - "-t", "gcr.io/kaniko-project/warmer:${COMMIT_SHA}", "."] -images: ["gcr.io/kaniko-project/executor:${COMMIT_SHA}", - "gcr.io/kaniko-project/executor:debug-${COMMIT_SHA}", - "gcr.io/kaniko-project/warmer:${COMMIT_SHA}"] + "-t", "gcr.io/$PROJECT_ID/${_WARMER_IMAGE_NAME}:${COMMIT_SHA}", "."] +images: ["gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:${COMMIT_SHA}", + "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:debug-${COMMIT_SHA}", + "gcr.io/$PROJECT_ID/${_WARMER_IMAGE_NAME}:${COMMIT_SHA}"] +substitutions: + _EXECUTOR_IMAGE_NAME: executor + _WARMER_IMAGE_NAME: warmer From 8b8be5c4187f675b602a109a2b23141c3290c0ef Mon Sep 17 00:00:00 2001 From: Ben Einaudi Date: Tue, 3 Mar 2020 10:18:12 +0100 Subject: [PATCH 12/14] Migrate to golang 1.14 * Upgrade go.mod file * Upgrade travis file * Run rm -rf vendor && go mod tidy && go mod vendor to upgrade vendor/modules.txt according to golang 1.14 * Upgrade golangci-lint version --- .travis.yml | 2 +- deploy/Dockerfile | 2 +- deploy/Dockerfile_debug | 2 +- deploy/Dockerfile_warmer | 2 +- go.mod | 2 +- hack/linter.sh | 2 +- vendor/modules.txt | 68 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 74 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index df05f880a..4b3d90651 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ dist: bionic env: - IMAGE_REPO=localhost:5000 go: - - "1.13.3" + - "1.14" go_import_path: github.com/GoogleContainerTools/kaniko jobs: include: diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 2e1b1fb9b..c83d0ab08 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -14,7 +14,7 @@ # Builds the static Go image to execute in a Kubernetes job -FROM golang:1.13 +FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index e9ccf15e3..e0e85a538 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -15,7 +15,7 @@ # Builds the static Go image to execute in a Kubernetes job # Stage 0: Build the executor binary and get credential helpers -FROM golang:1.13 +FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer index d33d2d74b..6628ed8bc 100644 --- a/deploy/Dockerfile_warmer +++ b/deploy/Dockerfile_warmer @@ -14,7 +14,7 @@ # Builds the static Go image to execute in a Kubernetes job -FROM golang:1.13 +FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper diff --git a/go.mod b/go.mod index c336c7152..954031907 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/GoogleContainerTools/kaniko -go 1.13 +go 1.14 replace ( github.com/containerd/containerd v1.4.0-0.20191014053712-acdcf13d5eaf => github.com/containerd/containerd v0.0.0-20191014053712-acdcf13d5eaf diff --git a/hack/linter.sh b/hack/linter.sh index 6b61e1aa6..41d539b19 100755 --- a/hack/linter.sh +++ b/hack/linter.sh @@ -20,7 +20,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" if ! [ -x "$(command -v golangci-lint)" ]; then echo "Installing GolangCI-Lint" - ${DIR}/install_golint.sh -b $GOPATH/bin v1.21.0 + ${DIR}/install_golint.sh -b $GOPATH/bin v1.23.7 fi golangci-lint run diff --git a/vendor/modules.txt b/vendor/modules.txt index d8704fce8..28ccf9be6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,5 @@ # cloud.google.com/go v0.26.0 +## explicit cloud.google.com/go/compute/metadata cloud.google.com/go/iam cloud.google.com/go/internal @@ -7,11 +8,13 @@ cloud.google.com/go/internal/trace cloud.google.com/go/internal/version cloud.google.com/go/storage # github.com/Azure/azure-pipeline-go v0.2.2 +## explicit github.com/Azure/azure-pipeline-go/pipeline # github.com/Azure/azure-sdk-for-go v19.1.1+incompatible github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2017-10-01/containerregistry github.com/Azure/azure-sdk-for-go/version # github.com/Azure/azure-storage-blob-go v0.8.0 +## explicit github.com/Azure/azure-storage-blob-go/azblob # github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 github.com/Azure/go-ansiterm @@ -28,7 +31,12 @@ github.com/Azure/go-autorest/version # github.com/Microsoft/go-winio v0.4.14 github.com/Microsoft/go-winio github.com/Microsoft/go-winio/pkg/guid +# github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 +## explicit +# github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 +## explicit # github.com/aws/aws-sdk-go v1.25.19 +## explicit github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/awserr github.com/aws/aws-sdk-go/aws/awsutil @@ -71,6 +79,7 @@ github.com/aws/aws-sdk-go/service/s3/s3manager github.com/aws/aws-sdk-go/service/sts github.com/aws/aws-sdk-go/service/sts/stsiface # github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 +## explicit github.com/beorn7/perks/quantile # github.com/containerd/containerd v1.4.0-0.20191014053712-acdcf13d5eaf => github.com/containerd/containerd v0.0.0-20191014053712-acdcf13d5eaf github.com/containerd/containerd/cio @@ -102,6 +111,7 @@ github.com/docker/distribution/digestset github.com/docker/distribution/reference github.com/docker/distribution/registry/api/errcode # github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c => github.com/docker/docker v0.0.0-20190319215453-e7b5f7dbe98c +## explicit github.com/docker/docker/api github.com/docker/docker/api/types github.com/docker/docker/api/types/backend @@ -194,10 +204,12 @@ github.com/docker/go-connections/tlsconfig # github.com/docker/go-events v0.0.0-20170721190031-9461782956ad github.com/docker/go-events # github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916 +## explicit github.com/docker/go-metrics # github.com/docker/go-units v0.4.0 github.com/docker/go-units # github.com/docker/swarmkit v1.12.1-0.20180726190244-7567d47988d8 +## explicit github.com/docker/swarmkit/agent/exec github.com/docker/swarmkit/api github.com/docker/swarmkit/api/deepcopy @@ -207,18 +219,24 @@ github.com/docker/swarmkit/manager/raftselector github.com/docker/swarmkit/protobuf/plugin github.com/docker/swarmkit/protobuf/ptypes # github.com/emirpasic/gods v1.9.0 +## explicit github.com/emirpasic/gods/containers github.com/emirpasic/gods/lists github.com/emirpasic/gods/lists/arraylist github.com/emirpasic/gods/trees github.com/emirpasic/gods/trees/binaryheap github.com/emirpasic/gods/utils +# github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 +## explicit # github.com/fsnotify/fsnotify v1.4.7 github.com/fsnotify/fsnotify # github.com/genuinetools/amicontained v0.4.3 +## explicit github.com/genuinetools/amicontained/container # github.com/ghodss/yaml v1.0.0 github.com/ghodss/yaml +# github.com/gliderlabs/ssh v0.2.2 +## explicit # github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d github.com/gogo/protobuf/gogoproto github.com/gogo/protobuf/proto @@ -228,6 +246,7 @@ github.com/gogo/protobuf/types # github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/glog # github.com/golang/mock v1.3.1 +## explicit github.com/golang/mock/gomock # github.com/golang/protobuf v1.3.2 github.com/golang/protobuf/proto @@ -239,12 +258,14 @@ github.com/golang/protobuf/ptypes/timestamp # github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a github.com/google/btree # github.com/google/go-cmp v0.3.0 +## explicit github.com/google/go-cmp/cmp github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value # github.com/google/go-containerregistry v0.0.0-20191218175032-34fb8ff33bed +## explicit github.com/google/go-containerregistry/pkg/authn github.com/google/go-containerregistry/pkg/authn/k8schain github.com/google/go-containerregistry/pkg/internal/retry @@ -266,12 +287,19 @@ github.com/google/go-containerregistry/pkg/v1/types github.com/google/go-containerregistry/pkg/v1/v1util github.com/google/go-containerregistry/pkg/v1/validate # github.com/google/go-github v17.0.0+incompatible +## explicit github.com/google/go-github/github # github.com/google/go-querystring v1.0.0 +## explicit github.com/google/go-querystring/query # github.com/google/gofuzz v1.0.0 github.com/google/gofuzz +# github.com/google/martian v2.1.0+incompatible +## explicit +# github.com/google/uuid v1.0.0 +## explicit # github.com/googleapis/gax-go v2.0.0+incompatible +## explicit github.com/googleapis/gax-go # github.com/googleapis/gnostic v0.2.2 github.com/googleapis/gnostic/OpenAPIv2 @@ -285,34 +313,45 @@ github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc # github.com/hashicorp/go-immutable-radix v1.0.0 github.com/hashicorp/go-immutable-radix # github.com/hashicorp/go-memdb v0.0.0-20180223233045-1289e7fffe71 +## explicit github.com/hashicorp/go-memdb +# github.com/hashicorp/go-uuid v1.0.1 +## explicit # github.com/hashicorp/golang-lru v0.5.0 github.com/hashicorp/golang-lru/simplelru # github.com/inconshreveable/mousetrap v1.0.0 github.com/inconshreveable/mousetrap # github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 +## explicit github.com/jbenet/go-context/io # github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/jmespath/go-jmespath # github.com/json-iterator/go v1.1.8 github.com/json-iterator/go # github.com/karrick/godirwalk v1.7.7 +## explicit github.com/karrick/godirwalk # github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e +## explicit github.com/kevinburke/ssh_config # github.com/konsorten/go-windows-terminal-sequences v1.0.2 github.com/konsorten/go-windows-terminal-sequences # github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb +## explicit github.com/mattn/go-ieproxy # github.com/mattn/go-shellwords v1.0.3 +## explicit github.com/mattn/go-shellwords # github.com/matttproud/golang_protobuf_extensions v1.0.1 +## explicit github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/minio/highwayhash v1.0.0 +## explicit github.com/minio/highwayhash # github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir # github.com/moby/buildkit v0.0.0-20191111154543-00bfbab0390c +## explicit github.com/moby/buildkit/frontend/dockerfile/command github.com/moby/buildkit/frontend/dockerfile/instructions github.com/moby/buildkit/frontend/dockerfile/parser @@ -338,28 +377,37 @@ github.com/opencontainers/runc/libcontainer/devices github.com/opencontainers/runc/libcontainer/system github.com/opencontainers/runc/libcontainer/user # github.com/opencontainers/runtime-spec v1.0.1 +## explicit github.com/opencontainers/runtime-spec/specs-go # github.com/opencontainers/selinux v1.0.0-rc1 +## explicit github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label # github.com/opentracing/opentracing-go v1.0.2 +## explicit github.com/opentracing/opentracing-go github.com/opentracing/opentracing-go/ext github.com/opentracing/opentracing-go/log # github.com/otiai10/copy v1.0.2 +## explicit github.com/otiai10/copy # github.com/pelletier/go-buffruneio v0.2.0 +## explicit github.com/pelletier/go-buffruneio # github.com/peterbourgon/diskv v2.0.1+incompatible github.com/peterbourgon/diskv # github.com/pkg/errors v0.8.1 +## explicit github.com/pkg/errors # github.com/prometheus/client_golang v0.9.0-pre1.0.20180210140205-a40133b69fbd +## explicit github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/promhttp # github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 +## explicit github.com/prometheus/client_model/go # github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1 +## explicit github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/model @@ -367,17 +415,23 @@ github.com/prometheus/common/model github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs # github.com/sergi/go-diff v1.0.0 +## explicit github.com/sergi/go-diff/diffmatchpatch # github.com/sirupsen/logrus v1.4.2 +## explicit github.com/sirupsen/logrus # github.com/spf13/afero v1.2.1 +## explicit github.com/spf13/afero github.com/spf13/afero/mem # github.com/spf13/cobra v0.0.5 +## explicit github.com/spf13/cobra # github.com/spf13/pflag v1.0.5 +## explicit github.com/spf13/pflag # github.com/src-d/gcfg v1.3.0 +## explicit github.com/src-d/gcfg github.com/src-d/gcfg/scanner github.com/src-d/gcfg/token @@ -385,17 +439,21 @@ github.com/src-d/gcfg/types # github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 github.com/syndtr/gocapability/capability # github.com/tonistiigi/fsutil v0.0.0-20191018213012-0f039a052ca1 +## explicit github.com/tonistiigi/fsutil github.com/tonistiigi/fsutil/types # github.com/vbatts/tar-split v0.10.2 +## explicit github.com/vbatts/tar-split/archive/tar github.com/vbatts/tar-split/tar/asm github.com/vbatts/tar-split/tar/storage # github.com/xanzy/ssh-agent v0.2.0 +## explicit github.com/xanzy/ssh-agent # go.etcd.io/bbolt v1.3.3 go.etcd.io/bbolt # go.opencensus.io v0.14.0 +## explicit go.opencensus.io go.opencensus.io/exporter/stackdriver/propagation go.opencensus.io/internal @@ -430,6 +488,7 @@ golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/knownhosts golang.org/x/crypto/ssh/terminal # golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 +## explicit golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/http/httpguts @@ -442,12 +501,14 @@ golang.org/x/net/internal/timeseries golang.org/x/net/proxy golang.org/x/net/trace # golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be +## explicit golang.org/x/oauth2 golang.org/x/oauth2/google golang.org/x/oauth2/internal golang.org/x/oauth2/jws golang.org/x/oauth2/jwt # golang.org/x/sync v0.0.0-20190423024810-112230192c58 +## explicit golang.org/x/sync/errgroup golang.org/x/sync/singleflight golang.org/x/sync/syncmap @@ -464,6 +525,7 @@ golang.org/x/text/unicode/norm # golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 golang.org/x/time/rate # google.golang.org/api v0.0.0-20180730000901-31ca0e01cd79 +## explicit google.golang.org/api/gensupport google.golang.org/api/googleapi google.golang.org/api/googleapi/internal/uritemplates @@ -528,12 +590,16 @@ google.golang.org/grpc/tap # gopkg.in/inf.v0 v0.9.1 gopkg.in/inf.v0 # gopkg.in/src-d/go-billy.v4 v4.2.0 +## explicit gopkg.in/src-d/go-billy.v4 gopkg.in/src-d/go-billy.v4/helper/chroot gopkg.in/src-d/go-billy.v4/helper/polyfill gopkg.in/src-d/go-billy.v4/osfs gopkg.in/src-d/go-billy.v4/util +# gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 +## explicit # gopkg.in/src-d/go-git.v4 v4.6.0 +## explicit gopkg.in/src-d/go-git.v4 gopkg.in/src-d/go-git.v4/config gopkg.in/src-d/go-git.v4/internal/revision @@ -575,6 +641,7 @@ gopkg.in/src-d/go-git.v4/utils/merkletrie/index gopkg.in/src-d/go-git.v4/utils/merkletrie/internal/frame gopkg.in/src-d/go-git.v4/utils/merkletrie/noder # gopkg.in/warnings.v0 v0.1.2 +## explicit gopkg.in/warnings.v0 # gopkg.in/yaml.v2 v2.2.4 gopkg.in/yaml.v2 @@ -699,3 +766,4 @@ k8s.io/kubernetes/pkg/credentialprovider/aws k8s.io/kubernetes/pkg/credentialprovider/azure k8s.io/kubernetes/pkg/credentialprovider/gcp k8s.io/kubernetes/pkg/credentialprovider/secrets +# github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d => github.com/tonistiigi/fsutil v0.0.0-20191018213012-0f039a052ca1 From 1dbefa20aee7e2e154509a9c9760c3bb997fc841 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Mon, 16 Mar 2020 09:37:41 -0700 Subject: [PATCH 13/14] Remove .gcloudignore to get cloud build to work Cloud build requires vendors directory to be uploaded when using a build submitted from local directory. However .gcloudignore prevents the vendor directory to be uploaded and causes issues. --- .gcloudignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gcloudignore diff --git a/.gcloudignore b/.gcloudignore deleted file mode 100644 index 48b8bf907..000000000 --- a/.gcloudignore +++ /dev/null @@ -1 +0,0 @@ -vendor/ From bb76d8c3cc51af475f266d3d45daeb24423288fd Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Mon, 16 Mar 2020 18:49:27 -0700 Subject: [PATCH 14/14] Update README on running in Docker Make the running in Docker section more clear by showing the actual docker run command instead of only the utility. --- README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b79611724..689abb40e 100644 --- a/README.md +++ b/README.md @@ -260,21 +260,24 @@ kaniko will build and push the final image in this build step. Requirements: - [Docker](https://docs.docker.com/install/) -- [gcloud](https://cloud.google.com/sdk/install) We can run the kaniko executor image locally in a Docker daemon to build and push an image from a Dockerfile. -1. Load the executor image into the Docker daemon by running: +For example, when using gcloud and GCR you could run Kaniko as follows: +```shell +docker run \ + -v "$HOME"/.config/gcloud:/root/.config/gcloud \ + -v /path/to/context:/workspace \ + gcr.io/kaniko-project/executor:latest \ + --dockerfile /workspace/Dockerfile + --destination "gcr.io/$PROJECT_ID/$IMAGE_NAME:$TAG" + --context dir:///workspace/" +``` - ```shell - make images - ``` - -2. Run kaniko in Docker using [`run_in_docker.sh`](./run_in_docker.sh): - - ```shell - ./run_in_docker.sh - ``` +There is also a utility script [`run_in_docker.sh`](./run_in_docker.sh) that can be used as follows: +```shell +./run_in_docker.sh +``` _NOTE: `run_in_docker.sh` expects a path to a Dockerfile relative to the absolute path of the build context._ @@ -284,7 +287,7 @@ context in the local directory `/home/user/kaniko-project`, and a Google Contain as a remote image destination: ```shell -./run_in_docker.sh /workspace/Dockerfile /home/user/kaniko-project gcr.io/// +./run_in_docker.sh /workspace/Dockerfile /home/user/kaniko-project gcr.io/$PROJECT_ID/$TAG ``` ### Caching