From 5e275d982472e8e5bef15a1f4b9ab4881a491849 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Tue, 5 May 2020 21:27:28 -0700 Subject: [PATCH 01/14] Apply dockefile exclude only for first stage --- integration/dockerfiles/Dockerfile_test_dockerignore | 8 +++++++- pkg/executor/build.go | 2 ++ pkg/util/fs_util.go | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/integration/dockerfiles/Dockerfile_test_dockerignore b/integration/dockerfiles/Dockerfile_test_dockerignore index d646872ef..70667623d 100644 --- a/integration/dockerfiles/Dockerfile_test_dockerignore +++ b/integration/dockerfiles/Dockerfile_test_dockerignore @@ -1,5 +1,11 @@ # This dockerfile makes sure the .dockerignore is working # If so then ignore/foo should copy to /foo # If not, then this image won't build because it will attempt to copy three files to /foo, which is a file not a directory -FROM scratch +FROM scratch as base COPY ignore/* /foo + +From base as first +COPY --from=base /foo ignore/bar + +FROM first +COPY --from=first ignore/* /fooAnother/ \ No newline at end of file diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 87a44df43..ea8bf8107 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -596,6 +596,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { } logrus.Infof("Built cross stage deps: %v", crossStageDependencies) + util.IsFirstStage = true for index, stage := range kanikoStages { sb, err := newStageBuilder(opts, stage, crossStageDependencies, digestToCacheKey, stageIdxToDigest, stageNameToIdx) if err != nil { @@ -604,6 +605,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { if err := sb.build(); err != nil { return nil, errors.Wrap(err, "error building stage") } + util.IsFirstStage = false reviewConfig(stage, &sb.cf.Config) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 230d87669..714408656 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -73,6 +73,7 @@ var whitelist = initialWhitelist var volumes = []string{} var excluded []string +var IsFirstStage = true type ExtractFunction func(string, *tar.Header, io.Reader) error @@ -675,6 +676,10 @@ func GetExcludedFiles(dockerfilepath string, buildcontext string) error { // ExcludeFile returns true if the .dockerignore specified this file should be ignored func ExcludeFile(path, buildcontext string) bool { + // Do not apply dockerfile excludes for first stage + if !IsFirstStage { + return false + } if HasFilepathPrefix(path, buildcontext, false) { var err error path, err = filepath.Rel(buildcontext, path) From 113c2397ea367d8f09330a5ed103b4d996c6a9d8 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Tue, 5 May 2020 22:27:28 -0700 Subject: [PATCH 02/14] Update comment. --- pkg/util/fs_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 714408656..2b44f97ce 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -676,7 +676,7 @@ func GetExcludedFiles(dockerfilepath string, buildcontext string) error { // ExcludeFile returns true if the .dockerignore specified this file should be ignored func ExcludeFile(path, buildcontext string) bool { - // Do not apply dockerfile excludes for first stage + // Apply dockerfile excludes for first stage only if !IsFirstStage { return false } From c271f989f576b6ac1042f49a87b0b0e0d98d4c6d Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Thu, 7 May 2020 18:58:27 -0700 Subject: [PATCH 03/14] fix switching to non existent workdir --- pkg/commands/run.go | 10 +++++++++- pkg/commands/run_test.go | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/commands/run.go b/pkg/commands/run.go index f09c9c79f..e0dd1beef 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -65,7 +65,8 @@ func (r *RunCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui logrus.Infof("args: %s", newCommand[1:]) cmd := exec.Command(newCommand[0], newCommand[1:]...) - cmd.Dir = config.WorkingDir + + cmd.Dir = setWorkDirIfExists(config.WorkingDir) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr replacementEnvs := buildArgs.ReplacementEnvs(config.Env) @@ -236,3 +237,10 @@ func (cr *CachingRunCommand) String() string { func (cr *CachingRunCommand) MetadataOnly() bool { return false } + +func setWorkDirIfExists(workdir string) string { + if _, err := os.Lstat(workdir); err == nil { + return workdir + } + return "" +} diff --git a/pkg/commands/run_test.go b/pkg/commands/run_test.go index 5a7139a6b..81d213b04 100644 --- a/pkg/commands/run_test.go +++ b/pkg/commands/run_test.go @@ -316,3 +316,12 @@ func Test_CachingRunCommand_ExecuteCommand(t *testing.T) { }) } } + +func TestSetWorkDirIfExists(t *testing.T) { + testDir, err := ioutil.TempDir("", "workdir") + if err != nil { + t.Error(err) + } + testutil.CheckDeepEqual(t, testDir, setWorkDirIfExists(testDir)) + testutil.CheckDeepEqual(t, "", setWorkDirIfExists("doesnot-exists")) +} From ec770e207de830facc17f2b2f6ef7f74717c0a84 Mon Sep 17 00:00:00 2001 From: Chris Sng Date: Fri, 8 May 2020 12:44:44 +0900 Subject: [PATCH 04/14] Update docker-credentials-gcr to support auth with GCP Artifact Registry --- deploy/Dockerfile | 4 ++-- deploy/Dockerfile_debug | 6 +++--- deploy/Dockerfile_warmer | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 24b201b1b..9b4ec3c4b 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -18,8 +18,8 @@ FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper -ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.5.0/docker-credential-gcr_linux_amd64-1.5.0.tar.gz /usr/local/bin/ -RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-1.5.0.tar.gz +ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.0.1/docker-credential-gcr_linux_amd64-2.0.1.tar.gz /usr/local/bin/ +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-2.0.1.tar.gz # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index 725b0467a..6774e7cc7 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -19,11 +19,11 @@ FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper -ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.5.0/docker-credential-gcr_linux_amd64-1.5.0.tar.gz /usr/local/bin/ -RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-1.5.0.tar.gz +ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.0.1/docker-credential-gcr_linux_amd64-2.0.1.tar.gz /usr/local/bin/ +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-2.0.1.tar.gz # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login -RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 +RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 # ACR docker credential helper ADD https://aadacr.blob.core.windows.net/acr-docker-credential-helper/docker-credential-acr-linux-amd64.tar.gz /usr/local/bin RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-acr-linux-amd64.tar.gz diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer index dd83c378c..48499ef92 100644 --- a/deploy/Dockerfile_warmer +++ b/deploy/Dockerfile_warmer @@ -18,8 +18,8 @@ FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper -ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.5.0/docker-credential-gcr_linux_amd64-1.5.0.tar.gz /usr/local/bin/ -RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-1.5.0.tar.gz +ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.0.1/docker-credential-gcr_linux_amd64-2.0.1.tar.gz /usr/local/bin/ +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-2.0.1.tar.gz # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 From 6978fab45ce1161a94c7b2f68d93523d43fdd6fe Mon Sep 17 00:00:00 2001 From: Mitchell Friedman Date: Fri, 8 May 2020 08:45:54 +0100 Subject: [PATCH 05/14] Add retries to image push. This uses the default provided retry transport by go-containerregistry as this originally had no retries built in. This is useful to avoid intermittent failures of image registries when returning a retryable status code. --- pkg/executor/push.go | 3 ++- pkg/executor/push_test.go | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/executor/push.go b/pkg/executor/push.go index ff81409e3..7bc69f8ca 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -41,6 +41,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -311,7 +312,7 @@ func makeTransport(opts *config.KanikoOptions, registryName string, loader syste } } } - return tr + return transport.NewRetry(tr) } // pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go index 7d919db7b..2147b73f8 100644 --- a/pkg/executor/push_test.go +++ b/pkg/executor/push_test.go @@ -283,6 +283,14 @@ func (m *mockedCertPool) append(path string) error { return nil } +type retryTransport struct { + inner http.RoundTripper +} + +func (t *retryTransport) RoundTrip(in *http.Request) (out *http.Response, err error) { + return nil, nil +} + func Test_makeTransport(t *testing.T) { registryName := "my.registry.name" @@ -347,7 +355,7 @@ func Test_makeTransport(t *testing.T) { return &certPool } transport := makeTransport(tt.opts, registryName, mockedSystemCertLoader) - tt.check(transport.(*http.Transport).TLSClientConfig, &certPool) + tt.check(transport.(*retryTransport).inner.(*http.Transport).TLSClientConfig, &certPool) }) } } From 6e615a8ff932cee03d767ebeae523aaa72e34fa5 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Mon, 11 May 2020 10:56:53 -0500 Subject: [PATCH 06/14] Add ability to set git auth token using environment variables Currently the only way to set the git auth is by including it in the context git URL. This can be problematic for certain environments such as ECS where the command args for are stored in logs. Introduces updates to allow setting the authentication info using the environment variables `GIT_USERNAME` and `GIT_PASSWORD`. --- README.md | 3 ++ pkg/buildcontext/git.go | 18 ++++++++ pkg/buildcontext/git_test.go | 87 ++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/README.md b/README.md index 1575397a9..e20114ce0 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,9 @@ If you are using Azure Blob Storage for context file, you will need to pass [Azu ### Using Private Git Repository You can use `Personal Access Tokens` for Build Contexts from Private Repositories from [GitHub](https://blog.github.com/2012-09-21-easier-builds-and-deployments-using-git-over-https-and-oauth/). +You can either pass this in as part of the git URL (e.g., `git://TOKEN@github.com/acme/myproject.git#refs/heads/mybranch`) +or using the environment variable `GIT_USERNAME`. + ### Using Standard Input If running kaniko and using Standard Input build context, you will need to add the docker or kubernetes `-i, --interactive` flag. Once running, kaniko will then get the data from `STDIN` and create the build context as a compressed tar. diff --git a/pkg/buildcontext/git.go b/pkg/buildcontext/git.go index 7c2dddc6d..18d6e2ded 100644 --- a/pkg/buildcontext/git.go +++ b/pkg/buildcontext/git.go @@ -23,12 +23,17 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/constants" git "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) const ( gitPullMethodEnvKey = "GIT_PULL_METHOD" gitPullMethodHTTPS = "https" gitPullMethodHTTP = "http" + + gitAuthUsernameEnvKey = "GIT_USERNAME" + gitAuthPasswordEnvKey = "GIT_PASSWORD" ) var ( @@ -46,6 +51,7 @@ func (g *Git) UnpackTarFromBuildContext() (string, error) { parts := strings.Split(g.context, "#") options := git.CloneOptions{ URL: getGitPullMethod() + "://" + parts[0], + Auth: getGitAuth(), Progress: os.Stdout, } if len(parts) > 1 { @@ -55,6 +61,18 @@ func (g *Git) UnpackTarFromBuildContext() (string, error) { return directory, err } +func getGitAuth() transport.AuthMethod { + username := os.Getenv(gitAuthUsernameEnvKey) + password := os.Getenv(gitAuthPasswordEnvKey) + if username != "" || password != "" { + return &http.BasicAuth{ + Username: username, + Password: password, + } + } + return nil +} + func getGitPullMethod() string { gitPullMethod := os.Getenv(gitPullMethodEnvKey) if ok := supportedGitPullMethods[gitPullMethod]; !ok { diff --git a/pkg/buildcontext/git_test.go b/pkg/buildcontext/git_test.go index ccc53fcb9..aaa487c92 100644 --- a/pkg/buildcontext/git_test.go +++ b/pkg/buildcontext/git_test.go @@ -21,6 +21,8 @@ import ( "testing" "github.com/GoogleContainerTools/kaniko/testutil" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) func TestGetGitPullMethod(t *testing.T) { @@ -80,3 +82,88 @@ func TestGetGitPullMethod(t *testing.T) { }) } } + +func TestGetGitAuth(t *testing.T) { + tests := []struct { + testName string + setEnv func() (expectedValue transport.AuthMethod) + }{ + { + testName: "noEnv", + setEnv: func() (expectedValue transport.AuthMethod) { + expectedValue = nil + return + }, + }, + { + testName: "emptyUsernameEnv", + setEnv: func() (expectedValue transport.AuthMethod) { + _ = os.Setenv(gitAuthUsernameEnvKey, "") + expectedValue = nil + return + }, + }, + { + testName: "emptyPasswordEnv", + setEnv: func() (expectedValue transport.AuthMethod) { + _ = os.Setenv(gitAuthPasswordEnvKey, "") + expectedValue = nil + return + }, + }, + { + testName: "emptyEnv", + setEnv: func() (expectedValue transport.AuthMethod) { + _ = os.Setenv(gitAuthUsernameEnvKey, "") + _ = os.Setenv(gitAuthPasswordEnvKey, "") + expectedValue = nil + return + }, + }, + { + testName: "withUsername", + setEnv: func() (expectedValue transport.AuthMethod) { + username := "foo" + _ = os.Setenv(gitAuthUsernameEnvKey, username) + expectedValue = &http.BasicAuth{Username: username} + return + }, + }, + { + testName: "withPassword", + setEnv: func() (expectedValue transport.AuthMethod) { + pass := "super-secret-password-1234" + _ = os.Setenv(gitAuthPasswordEnvKey, pass) + expectedValue = &http.BasicAuth{Password: pass} + return + }, + }, + { + testName: "withUsernamePassword", + setEnv: func() (expectedValue transport.AuthMethod) { + username := "foo" + pass := "super-secret-password-1234" + _ = os.Setenv(gitAuthUsernameEnvKey, username) + _ = os.Setenv(gitAuthPasswordEnvKey, pass) + expectedValue = &http.BasicAuth{Username: username, Password: pass} + return + }, + }, + } + + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + // Make sure to unset environment vars to get a clean test each time + defer clearTestAuthEnv() + + expectedValue := tt.setEnv() + testutil.CheckDeepEqual(t, expectedValue, getGitAuth()) + }) + } + +} + +func clearTestAuthEnv() { + _ = os.Unsetenv(gitAuthUsernameEnvKey) + _ = os.Unsetenv(gitAuthPasswordEnvKey) +} From 864e6c2ae7c7ac4af7f15bfeb9308cf1a5a09b83 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Tue, 19 May 2020 14:26:15 -0700 Subject: [PATCH 07/14] Make support clause more bold. Add that kaniko is not officially support more evident. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1575397a9..3b9147c94 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # kaniko - Build Images In Kubernetes +`NOTE: This is not an officially supported Google product` + [![Build Status](https://travis-ci.org/GoogleContainerTools/kaniko.svg?branch=master)](https://travis-ci.org/GoogleContainerTools/kaniko) [![Go Report Card](https://goreportcard.com/badge/github.com/GoogleContainerTools/kaniko)](https://goreportcard.com/report/github.com/GoogleContainerTools/kaniko) ![kaniko logo](logo/Kaniko-Logo.png) @@ -15,7 +17,6 @@ We'd love to hear from you! Join us on [#kaniko Kubernetes Slack](https://kuber :mega: **Please fill out our [quick 5-question survey](https://forms.gle/HhZGEM33x4FUz9Qa6)** so that we can learn how satisfied you are with Kaniko, and what improvements we should make. Thank you! :dancers: -Kaniko is not an officially supported Google project. _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPMENT.md) and [CONTRIBUTING.md](CONTRIBUTING.md)._ From 37e454dab442c8bfc73525840389ff92240d4e36 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Tue, 19 May 2020 14:27:13 -0700 Subject: [PATCH 08/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b9147c94..e2939909a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # kaniko - Build Images In Kubernetes -`NOTE: This is not an officially supported Google product` +`NOTE: Kaniko is not an officially supported Google product` [![Build Status](https://travis-ci.org/GoogleContainerTools/kaniko.svg?branch=master)](https://travis-ci.org/GoogleContainerTools/kaniko) [![Go Report Card](https://goreportcard.com/badge/github.com/GoogleContainerTools/kaniko)](https://goreportcard.com/report/github.com/GoogleContainerTools/kaniko) From da5c420ee9f57eed5f71431cb990a59d304674fd Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Tue, 19 May 2020 15:05:46 -0700 Subject: [PATCH 09/14] use a mock --- pkg/executor/push.go | 7 ++++++- pkg/executor/push_test.go | 13 +++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 7bc69f8ca..f894bcda8 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -48,6 +48,11 @@ import ( "github.com/spf13/afero" ) +// for testing +var ( + newRetry = transport.NewRetry +) + type withUserAgent struct { t http.RoundTripper } @@ -312,7 +317,7 @@ func makeTransport(opts *config.KanikoOptions, registryName string, loader syste } } } - return transport.NewRetry(tr) + return newRetry(tr) } // pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go index 2147b73f8..f4581955c 100644 --- a/pkg/executor/push_test.go +++ b/pkg/executor/push_test.go @@ -34,7 +34,9 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/validate" + "github.com/spf13/afero" ) @@ -347,6 +349,13 @@ func Test_makeTransport(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + originalRetry := newRetry + newRetry = func(inner http.RoundTripper, _ ...transport.Option) http.RoundTripper { + return &retryTransport{ + inner: inner, + } + } + defer func() { newRetry = originalRetry }() var certificatesPath []string certPool := mockedCertPool{ certificatesPath: certificatesPath, @@ -354,8 +363,8 @@ func Test_makeTransport(t *testing.T) { var mockedSystemCertLoader systemCertLoader = func() CertPool { return &certPool } - transport := makeTransport(tt.opts, registryName, mockedSystemCertLoader) - tt.check(transport.(*retryTransport).inner.(*http.Transport).TLSClientConfig, &certPool) + actual := makeTransport(tt.opts, registryName, mockedSystemCertLoader) + tt.check(actual.(*retryTransport).inner.(*http.Transport).TLSClientConfig, &certPool) }) } } From 2f6090dcd79c8c30b124347d920152fbf9ed3de0 Mon Sep 17 00:00:00 2001 From: Ben Einaudi Date: Mon, 4 May 2020 15:03:17 +0200 Subject: [PATCH 10/14] Remove use of DefaultTransport Using DefaultTransport and manipulating its tls configuration may lead to unexpected behaviour --- pkg/cache/cache.go | 10 +-- pkg/executor/build.go | 7 +- pkg/executor/push.go | 70 ++-------------- pkg/executor/push_test.go | 100 ---------------------- pkg/image/image.go | 43 ---------- pkg/{util => image}/image_util.go | 34 ++++---- pkg/{util => image}/image_util_test.go | 7 +- pkg/util/transport_util.go | 82 ++++++++++++++++++ pkg/util/transport_util_test.go | 110 +++++++++++++++++++++++++ pkg/util/util.go | 11 --- 10 files changed, 227 insertions(+), 247 deletions(-) delete mode 100644 pkg/image/image.go rename pkg/{util => image}/image_util.go (92%) rename pkg/{util => image}/image_util_test.go (99%) create mode 100644 pkg/util/transport_util.go create mode 100644 pkg/util/transport_util_test.go diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 893b1142d..70fca8b1d 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -17,9 +17,7 @@ limitations under the License. package cache import ( - "crypto/tls" "fmt" - "net/http" "os" "path" "path/filepath" @@ -27,6 +25,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/creds" + "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -67,12 +66,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) { cacheRef.Repository.Registry = newReg } - tr := http.DefaultTransport.(*http.Transport) - if rc.Opts.SkipTLSVerifyRegistries.Contains(registryName) { - tr.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } + tr := util.MakeTransport(rc.Opts, registryName) img, err := remote.Image(cacheRef, remote.WithTransport(tr), remote.WithAuthFromKeychain(creds.GetKeychain())) if err != nil { diff --git a/pkg/executor/build.go b/pkg/executor/build.go index cd953ecc7..761c09e72 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -44,6 +44,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + image_util "github.com/GoogleContainerTools/kaniko/pkg/image" "github.com/GoogleContainerTools/kaniko/pkg/snapshot" "github.com/GoogleContainerTools/kaniko/pkg/timing" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -84,7 +85,7 @@ type stageBuilder struct { // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string, dcm map[string]string, sid map[string]string, stageNameToIdx map[string]string) (*stageBuilder, error) { - sourceImage, err := util.RetrieveSourceImage(stage, opts) + sourceImage, err := image_util.RetrieveSourceImage(stage, opts) if err != nil { return nil, err } @@ -517,7 +518,7 @@ func CalculateDependencies(stages []config.KanikoStage, opts *config.KanikoOptio } else if s.Name == constants.NoBaseImage { image = empty.Image } else { - image, err = util.RetrieveSourceImage(s, opts) + image, err = image_util.RetrieveSourceImage(s, opts) if err != nil { return nil, err } @@ -751,7 +752,7 @@ func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) e // This must be an image name, fetch it. logrus.Debugf("Found extra base image stage %s", c.From) - sourceImage, err := util.RetrieveRemoteImage(c.From, opts) + sourceImage, err := image_util.RetrieveRemoteImage(c.From, opts) if err != nil { return err } diff --git a/pkg/executor/push.go b/pkg/executor/push.go index f894bcda8..623c92525 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -17,8 +17,6 @@ limitations under the License. package executor import ( - "crypto/tls" - "crypto/x509" "encoding/json" "fmt" "io/ioutil" @@ -34,6 +32,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/creds" "github.com/GoogleContainerTools/kaniko/pkg/timing" + "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/GoogleContainerTools/kaniko/pkg/version" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -48,15 +47,15 @@ import ( "github.com/spf13/afero" ) +type withUserAgent struct { + t http.RoundTripper +} + // for testing var ( newRetry = transport.NewRetry ) -type withUserAgent struct { - t http.RoundTripper -} - const ( UpstreamClientUaKey = "UPSTREAM_CLIENT_TYPE" ) @@ -82,41 +81,6 @@ func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) { return w.t.RoundTrip(r) } -type CertPool interface { - value() *x509.CertPool - append(path string) error -} - -type X509CertPool struct { - inner x509.CertPool -} - -func (p *X509CertPool) value() *x509.CertPool { - return &p.inner -} - -func (p *X509CertPool) append(path string) error { - pem, err := ioutil.ReadFile(path) - if err != nil { - return err - } - p.inner.AppendCertsFromPEM(pem) - return nil -} - -type systemCertLoader func() CertPool - -var defaultX509Handler systemCertLoader = func() CertPool { - systemCertPool, err := x509.SystemCertPool() - if err != nil { - logrus.Warn("Failed to load system cert pool. Loading empty one instead.") - systemCertPool = x509.NewCertPool() - } - return &X509CertPool{ - inner: *systemCertPool, - } -} - // for testing var ( fs = afero.NewOsFs() @@ -161,7 +125,7 @@ func CheckPushPermissions(opts *config.KanikoOptions) error { } destRef.Repository.Registry = newReg } - tr := makeTransport(opts, registryName, defaultX509Handler) + tr := newRetry(util.MakeTransport(opts, registryName)) if err := checkRemotePushPermission(destRef, creds.GetKeychain(), tr); err != nil { return errors.Wrapf(err, "checking push permission for %q", destRef) } @@ -258,7 +222,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { return errors.Wrap(err, "resolving pushAuth") } - tr := makeTransport(opts, registryName, defaultX509Handler) + tr := newRetry(util.MakeTransport(opts, registryName)) rt := &withUserAgent{t: tr} if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil { @@ -300,26 +264,6 @@ func writeImageOutputs(image v1.Image, destRefs []name.Tag) error { return nil } -func makeTransport(opts *config.KanikoOptions, registryName string, loader systemCertLoader) http.RoundTripper { - // Create a transport to set our user-agent. - var tr http.RoundTripper = http.DefaultTransport.(*http.Transport).Clone() - if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) { - tr.(*http.Transport).TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } else if certificatePath := opts.RegistriesCertificates[registryName]; certificatePath != "" { - systemCertPool := loader() - if err := systemCertPool.append(certificatePath); err != nil { - logrus.WithError(err).Warnf("Failed to load certificate %s for %s\n", certificatePath, registryName) - } else { - tr.(*http.Transport).TLSClientConfig = &tls.Config{ - RootCAs: systemCertPool.value(), - } - } - } - return newRetry(tr) -} - // pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache // if opts.Cache doesn't exist, infer the cache from the given destination func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error { diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go index f4581955c..2fd36f23c 100644 --- a/pkg/executor/push_test.go +++ b/pkg/executor/push_test.go @@ -18,8 +18,6 @@ package executor import ( "bytes" - "crypto/tls" - "crypto/x509" "fmt" "io/ioutil" "net/http" @@ -34,7 +32,6 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/random" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/validate" "github.com/spf13/afero" @@ -272,103 +269,6 @@ func TestImageNameDigestFile(t *testing.T) { } -type mockedCertPool struct { - certificatesPath []string -} - -func (m *mockedCertPool) value() *x509.CertPool { - return &x509.CertPool{} -} - -func (m *mockedCertPool) append(path string) error { - m.certificatesPath = append(m.certificatesPath, path) - return nil -} - -type retryTransport struct { - inner http.RoundTripper -} - -func (t *retryTransport) RoundTrip(in *http.Request) (out *http.Response, err error) { - return nil, nil -} - -func Test_makeTransport(t *testing.T) { - registryName := "my.registry.name" - - tests := []struct { - name string - opts *config.KanikoOptions - check func(*tls.Config, *mockedCertPool) - }{ - { - name: "SkipTLSVerify set", - opts: &config.KanikoOptions{SkipTLSVerify: true}, - check: func(config *tls.Config, pool *mockedCertPool) { - if !config.InsecureSkipVerify { - t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify not set while SkipTLSVerify set") - } - }, - }, - { - name: "SkipTLSVerifyRegistries set with expected registry", - opts: &config.KanikoOptions{SkipTLSVerifyRegistries: []string{registryName}}, - check: func(config *tls.Config, pool *mockedCertPool) { - if !config.InsecureSkipVerify { - t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify not set while SkipTLSVerifyRegistries set with registry name") - } - }, - }, - { - name: "SkipTLSVerifyRegistries set with other registry", - opts: &config.KanikoOptions{SkipTLSVerifyRegistries: []string{fmt.Sprintf("other.%s", registryName)}}, - check: func(config *tls.Config, pool *mockedCertPool) { - if config.InsecureSkipVerify { - t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify set while SkipTLSVerifyRegistries not set with registry name") - } - }, - }, - { - name: "RegistriesCertificates set for registry", - opts: &config.KanikoOptions{RegistriesCertificates: map[string]string{registryName: "/path/to/the/certificate.cert"}}, - check: func(config *tls.Config, pool *mockedCertPool) { - if len(pool.certificatesPath) != 1 || pool.certificatesPath[0] != "/path/to/the/certificate.cert" { - t.Errorf("makeTransport().RegistriesCertificates certificate not appended to system certificates") - } - }, - }, - { - name: "RegistriesCertificates set for another registry", - opts: &config.KanikoOptions{RegistriesCertificates: map[string]string{fmt.Sprintf("other.%s=", registryName): "/path/to/the/certificate.cert"}}, - check: func(config *tls.Config, pool *mockedCertPool) { - if len(pool.certificatesPath) != 0 { - t.Errorf("makeTransport().RegistriesCertificates certificate appended to system certificates while added for other registry") - } - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - originalRetry := newRetry - newRetry = func(inner http.RoundTripper, _ ...transport.Option) http.RoundTripper { - return &retryTransport{ - inner: inner, - } - } - defer func() { newRetry = originalRetry }() - var certificatesPath []string - certPool := mockedCertPool{ - certificatesPath: certificatesPath, - } - var mockedSystemCertLoader systemCertLoader = func() CertPool { - return &certPool - } - actual := makeTransport(tt.opts, registryName, mockedSystemCertLoader) - tt.check(actual.(*retryTransport).inner.(*http.Transport).TLSClientConfig, &certPool) - }) - } -} - var calledExecCommand = false var calledCheckPushPermission = false diff --git a/pkg/image/image.go b/pkg/image/image.go deleted file mode 100644 index dbb5a9e9d..000000000 --- a/pkg/image/image.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package image - -import ( - "os" - "strings" - - v1 "github.com/google/go-containerregistry/pkg/v1" - - "github.com/sirupsen/logrus" -) - -// SetEnvVariables sets environment variables as specified in the image -func SetEnvVariables(img v1.Image) error { - cfg, err := img.ConfigFile() - if err != nil { - return err - } - envVars := cfg.Config.Env - for _, envVar := range envVars { - split := strings.SplitN(envVar, "=", 2) - if err := os.Setenv(split[0], split[1]); err != nil { - return err - } - logrus.Infof("Setting environment variable %s", envVar) - } - return nil -} diff --git a/pkg/util/image_util.go b/pkg/image/image_util.go similarity index 92% rename from pkg/util/image_util.go rename to pkg/image/image_util.go index 9f1e68e51..fb7a7ce89 100644 --- a/pkg/util/image_util.go +++ b/pkg/image/image_util.go @@ -14,29 +14,28 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package image import ( - "crypto/tls" "fmt" - "net/http" "path/filepath" + "runtime" "strconv" - "github.com/GoogleContainerTools/kaniko/pkg/timing" - + "github.com/GoogleContainerTools/kaniko/pkg/cache" + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/creds" + "github.com/GoogleContainerTools/kaniko/pkg/timing" + "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" - "github.com/sirupsen/logrus" - "github.com/GoogleContainerTools/kaniko/pkg/cache" - "github.com/GoogleContainerTools/kaniko/pkg/config" - "github.com/GoogleContainerTools/kaniko/pkg/constants" + "github.com/sirupsen/logrus" ) var ( @@ -55,7 +54,7 @@ func RetrieveSourceImage(stage config.KanikoStage, opts *config.KanikoOptions) ( buildArgs = append(buildArgs, fmt.Sprintf("%s=%s", arg.Key, arg.ValueString())) } buildArgs = append(buildArgs, opts.BuildArgs...) - currentBaseName, err := ResolveEnvironmentReplacement(stage.BaseName, buildArgs, false) + currentBaseName, err := util.ResolveEnvironmentReplacement(stage.BaseName, buildArgs, false) if err != nil { return nil, err } @@ -146,12 +145,7 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { } func remoteOptions(registryName string, opts *config.KanikoOptions) []remote.Option { - tr := http.DefaultTransport.(*http.Transport) - if opts.SkipTLSVerifyPull || opts.SkipTLSVerifyRegistries.Contains(registryName) { - tr.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } + tr := util.MakeTransport(opts, registryName) // on which v1.Platform is this currently running? platform := currentPlatform() @@ -182,3 +176,11 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { } return cache.LocalSource(&opts.CacheOptions, cacheKey) } + +// CurrentPlatform returns the v1.Platform on which the code runs +func currentPlatform() v1.Platform { + return v1.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + } +} diff --git a/pkg/util/image_util_test.go b/pkg/image/image_util_test.go similarity index 99% rename from pkg/util/image_util_test.go rename to pkg/image/image_util_test.go index 2ce681500..3b9e039cf 100644 --- a/pkg/util/image_util_test.go +++ b/pkg/image/image_util_test.go @@ -14,18 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package image import ( "bytes" "testing" - "github.com/GoogleContainerTools/kaniko/pkg/config" - "github.com/GoogleContainerTools/kaniko/testutil" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/parser" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/testutil" ) var ( diff --git a/pkg/util/transport_util.go b/pkg/util/transport_util.go new file mode 100644 index 000000000..a8e3f1e3f --- /dev/null +++ b/pkg/util/transport_util.go @@ -0,0 +1,82 @@ +/* +Copyright 2020 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "crypto/tls" + "crypto/x509" + + "io/ioutil" + "net/http" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/sirupsen/logrus" +) + +type CertPool interface { + value() *x509.CertPool + append(path string) error +} + +type X509CertPool struct { + inner x509.CertPool +} + +func (p *X509CertPool) value() *x509.CertPool { + return &p.inner +} + +func (p *X509CertPool) append(path string) error { + pem, err := ioutil.ReadFile(path) + if err != nil { + return err + } + p.inner.AppendCertsFromPEM(pem) + return nil +} + +var systemCertLoader CertPool + +func init() { + systemCertPool, err := x509.SystemCertPool() + if err != nil { + logrus.Warn("Failed to load system cert pool. Loading empty one instead.") + systemCertPool = x509.NewCertPool() + } + systemCertLoader = &X509CertPool{ + inner: *systemCertPool, + } +} + +func MakeTransport(opts *config.KanikoOptions, registryName string) http.RoundTripper { + // Create a transport to set our user-agent. + var tr http.RoundTripper = http.DefaultTransport.(*http.Transport).Clone() + if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) { + tr.(*http.Transport).TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } else if certificatePath := opts.RegistriesCertificates[registryName]; certificatePath != "" { + if err := systemCertLoader.append(certificatePath); err != nil { + logrus.WithError(err).Warnf("Failed to load certificate %s for %s\n", certificatePath, registryName) + } else { + tr.(*http.Transport).TLSClientConfig = &tls.Config{ + RootCAs: systemCertLoader.value(), + } + } + } + return tr +} diff --git a/pkg/util/transport_util_test.go b/pkg/util/transport_util_test.go new file mode 100644 index 000000000..b10be6995 --- /dev/null +++ b/pkg/util/transport_util_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2020 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "testing" + + "github.com/GoogleContainerTools/kaniko/pkg/config" +) + +type mockedCertPool struct { + certificatesPath []string +} + +func (m *mockedCertPool) value() *x509.CertPool { + return &x509.CertPool{} +} + +func (m *mockedCertPool) append(path string) error { + m.certificatesPath = append(m.certificatesPath, path) + return nil +} + +func Test_makeTransport(t *testing.T) { + registryName := "my.registry.name" + + tests := []struct { + name string + opts *config.KanikoOptions + check func(*tls.Config, *mockedCertPool) + }{ + { + name: "SkipTLSVerify set", + opts: &config.KanikoOptions{SkipTLSVerify: true}, + check: func(config *tls.Config, pool *mockedCertPool) { + if !config.InsecureSkipVerify { + t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify not set while SkipTLSVerify set") + } + }, + }, + { + name: "SkipTLSVerifyRegistries set with expected registry", + opts: &config.KanikoOptions{SkipTLSVerifyRegistries: []string{registryName}}, + check: func(config *tls.Config, pool *mockedCertPool) { + if !config.InsecureSkipVerify { + t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify not set while SkipTLSVerifyRegistries set with registry name") + } + }, + }, + { + name: "SkipTLSVerifyRegistries set with other registry", + opts: &config.KanikoOptions{SkipTLSVerifyRegistries: []string{fmt.Sprintf("other.%s", registryName)}}, + check: func(config *tls.Config, pool *mockedCertPool) { + if config.InsecureSkipVerify { + t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify set while SkipTLSVerifyRegistries not set with registry name") + } + }, + }, + { + name: "RegistriesCertificates set for registry", + opts: &config.KanikoOptions{RegistriesCertificates: map[string]string{registryName: "/path/to/the/certificate.cert"}}, + check: func(config *tls.Config, pool *mockedCertPool) { + if len(pool.certificatesPath) != 1 || pool.certificatesPath[0] != "/path/to/the/certificate.cert" { + t.Errorf("makeTransport().RegistriesCertificates certificate not appended to system certificates") + } + }, + }, + { + name: "RegistriesCertificates set for another registry", + opts: &config.KanikoOptions{RegistriesCertificates: map[string]string{fmt.Sprintf("other.%s=", registryName): "/path/to/the/certificate.cert"}}, + check: func(config *tls.Config, pool *mockedCertPool) { + if len(pool.certificatesPath) != 0 { + t.Errorf("makeTransport().RegistriesCertificates certificate appended to system certificates while added for other registry") + } + }, + }, + } + savedSystemCertLoader := systemCertLoader + defer func() { systemCertLoader = savedSystemCertLoader }() + for _, tt := range tests { + var certificatesPath []string + certPool := &mockedCertPool{ + certificatesPath: certificatesPath, + } + systemCertLoader = certPool + t.Run(tt.name, func(t *testing.T) { + tr := MakeTransport(tt.opts, registryName) + tt.check(tr.(*http.Transport).TLSClientConfig, certPool) + }) + + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 27340e413..a1dd77f4f 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -23,14 +23,11 @@ import ( "io" "io/ioutil" "os" - "runtime" "strconv" "sync" "syscall" "github.com/minio/highwayhash" - - v1 "github.com/google/go-containerregistry/pkg/v1" ) // Hasher returns a hash function, used in snapshotting to determine if a file has changed @@ -128,14 +125,6 @@ func SHA256(r io.Reader) (string, error) { return hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), nil } -// CurrentPlatform returns the v1.Platform on which the code runs -func currentPlatform() v1.Platform { - return v1.Platform{ - OS: runtime.GOOS, - Architecture: runtime.GOARCH, - } -} - // GetInputFrom returns Reader content func GetInputFrom(r io.Reader) ([]byte, error) { output, err := ioutil.ReadAll(r) From c91d46e75f8c5ed4ac4d93dce9b4a957c1f833b5 Mon Sep 17 00:00:00 2001 From: yw-liu <43599588+yw-liu@users.noreply.github.com> Date: Fri, 8 May 2020 07:20:18 +0800 Subject: [PATCH 11/14] add nsswitch.conf --- deploy/Dockerfile | 1 + deploy/Dockerfile_debug | 1 + deploy/Dockerfile_warmer | 1 + files/nsswitch.conf | 25 +++++++++++++++++++++++++ 4 files changed, 28 insertions(+) create mode 100644 files/nsswitch.conf diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 24b201b1b..842bd1686 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -39,6 +39,7 @@ COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux- COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /kaniko/.docker /kaniko/.docker +COPY files/nsswitch.conf /etc/nsswitch.conf ENV HOME /root ENV USER /root ENV PATH /usr/local/bin:/kaniko diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index 725b0467a..c22cb0e6b 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -50,6 +50,7 @@ COPY --from=1 /distroless/bazel-bin/experimental/busybox/busybox/ /busybox/ VOLUME /busybox COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /kaniko/.docker /kaniko/.docker +COPY files/nsswitch.conf /etc/nsswitch.conf ENV HOME /root ENV USER /root ENV PATH /usr/local/bin:/kaniko:/busybox diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer index dd83c378c..3c7810ee1 100644 --- a/deploy/Dockerfile_warmer +++ b/deploy/Dockerfile_warmer @@ -39,6 +39,7 @@ COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux- COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /kaniko/.docker /kaniko/.docker +COPY files/nsswitch.conf /etc/nsswitch.conf ENV HOME /root ENV USER /root ENV PATH /usr/local/bin:/kaniko diff --git a/files/nsswitch.conf b/files/nsswitch.conf new file mode 100644 index 000000000..9081618a7 --- /dev/null +++ b/files/nsswitch.conf @@ -0,0 +1,25 @@ +# /etc/nsswitch.conf +# +# As described on the web page https://man7.org/linux/man-pages/man3/gethostbyname.3.html, +# without the nsswitch.conf file, the gethostbyname() and gethostbyaddr() domain queries +# will fail to a local name server, thus the /etc/hosts will take no effect. +# +# For example, when hostaliases are specified for a kubernetes pod, without proper settings +# defined in this file, the hostaliases settings will not take effect. +# +# Following contents of this file is from the ubuntu:16.04 docker image. + +passwd: compat +group: compat +shadow: compat +gshadow: files + +hosts: files dns +networks: files + +protocols: db files +services: db files +ethers: db files +rpc: db files + +netgroup: nis \ No newline at end of file From 38c0b194a208d2ab5cc661ce0b20fe16e6627fa2 Mon Sep 17 00:00:00 2001 From: Art Begolli Date: Thu, 21 May 2020 13:06:13 +0100 Subject: [PATCH 12/14] docs: add registry-certificate flag to readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 76b60626a..118d34038 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [--log-format](#--log-format) - [--log-timestamp](#--log-timestamp) - [--no-push](#--no-push) + - [--registry-certificate](#--registry-certificate) - [--registry-mirror](#--registry-mirror) - [--reproducible](#--reproducible) - [--single-snapshot](#--single-snapshot) @@ -519,6 +520,12 @@ Set this flag if you want to pull images from a plain HTTP registry. It is suppo Set this flag if you only want to build the image, without pushing to a registry. +#### --registry-certificate + +Set this flag to provide a certificate for TLS communication with a given registry. + +Expected format is `my.registry.url=/path/to/the/certificate.cert` + #### --registry-mirror Set this flag if you want to use a registry mirror instead of default `index.docker.io`. From 41bc04dc12289599f122f396c77fa8103572f094 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Sat, 23 May 2020 14:29:46 -0700 Subject: [PATCH 13/14] add timings for resolving pahts --- pkg/commands/arg_test.go | 1 + pkg/snapshot/snapshot.go | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 pkg/commands/arg_test.go diff --git a/pkg/commands/arg_test.go b/pkg/commands/arg_test.go new file mode 100644 index 000000000..cdff10da7 --- /dev/null +++ b/pkg/commands/arg_test.go @@ -0,0 +1 @@ +package commands diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 449ae918a..6415af870 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -158,6 +158,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { ) timing.DefaultRun.Stop(timer) + timer = timing.Start("Resolving Paths") resolvedFiles, err := filesystem.ResolvePaths(foundPaths, s.whitelist) if err != nil { return nil, nil, err @@ -207,6 +208,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { } } + timing.DefaultRun.Stop(timer) sort.Strings(filesToAdd) // Add files to the layered map for _, file := range filesToAdd { From dcea89c3176f0f363e0c404182d70093155a403a Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Sat, 23 May 2020 15:27:37 -0700 Subject: [PATCH 14/14] rm file --- pkg/commands/arg_test.go | 1 - 1 file changed, 1 deletion(-) delete mode 100644 pkg/commands/arg_test.go diff --git a/pkg/commands/arg_test.go b/pkg/commands/arg_test.go deleted file mode 100644 index cdff10da7..000000000 --- a/pkg/commands/arg_test.go +++ /dev/null @@ -1 +0,0 @@ -package commands