diff --git a/README.md b/README.md index 1575397a9..118d34038 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # kaniko - Build Images In Kubernetes +`NOTE: Kaniko is not an officially supported Google product` + [![Build Status](https://travis-ci.org/GoogleContainerTools/kaniko.svg?branch=master)](https://travis-ci.org/GoogleContainerTools/kaniko) [![Go Report Card](https://goreportcard.com/badge/github.com/GoogleContainerTools/kaniko)](https://goreportcard.com/report/github.com/GoogleContainerTools/kaniko) ![kaniko logo](logo/Kaniko-Logo.png) @@ -15,7 +17,6 @@ We'd love to hear from you! Join us on [#kaniko Kubernetes Slack](https://kuber :mega: **Please fill out our [quick 5-question survey](https://forms.gle/HhZGEM33x4FUz9Qa6)** so that we can learn how satisfied you are with Kaniko, and what improvements we should make. Thank you! :dancers: -Kaniko is not an officially supported Google project. _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPMENT.md) and [CONTRIBUTING.md](CONTRIBUTING.md)._ @@ -61,6 +62,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [--log-format](#--log-format) - [--log-timestamp](#--log-timestamp) - [--no-push](#--no-push) + - [--registry-certificate](#--registry-certificate) - [--registry-mirror](#--registry-mirror) - [--reproducible](#--reproducible) - [--single-snapshot](#--single-snapshot) @@ -173,6 +175,9 @@ If you are using Azure Blob Storage for context file, you will need to pass [Azu ### Using Private Git Repository You can use `Personal Access Tokens` for Build Contexts from Private Repositories from [GitHub](https://blog.github.com/2012-09-21-easier-builds-and-deployments-using-git-over-https-and-oauth/). +You can either pass this in as part of the git URL (e.g., `git://TOKEN@github.com/acme/myproject.git#refs/heads/mybranch`) +or using the environment variable `GIT_USERNAME`. + ### Using Standard Input If running kaniko and using Standard Input build context, you will need to add the docker or kubernetes `-i, --interactive` flag. Once running, kaniko will then get the data from `STDIN` and create the build context as a compressed tar. @@ -515,6 +520,12 @@ Set this flag if you want to pull images from a plain HTTP registry. It is suppo Set this flag if you only want to build the image, without pushing to a registry. +#### --registry-certificate + +Set this flag to provide a certificate for TLS communication with a given registry. + +Expected format is `my.registry.url=/path/to/the/certificate.cert` + #### --registry-mirror Set this flag if you want to use a registry mirror instead of default `index.docker.io`. diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 24b201b1b..f3277905f 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -18,8 +18,8 @@ FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper -ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.5.0/docker-credential-gcr_linux_amd64-1.5.0.tar.gz /usr/local/bin/ -RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-1.5.0.tar.gz +ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.0.1/docker-credential-gcr_linux_amd64-2.0.1.tar.gz /usr/local/bin/ +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-2.0.1.tar.gz # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 @@ -39,6 +39,7 @@ COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux- COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /kaniko/.docker /kaniko/.docker +COPY files/nsswitch.conf /etc/nsswitch.conf ENV HOME /root ENV USER /root ENV PATH /usr/local/bin:/kaniko diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index 725b0467a..4480cf00c 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -19,11 +19,11 @@ FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper -ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.5.0/docker-credential-gcr_linux_amd64-1.5.0.tar.gz /usr/local/bin/ -RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-1.5.0.tar.gz +ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.0.1/docker-credential-gcr_linux_amd64-2.0.1.tar.gz /usr/local/bin/ +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-2.0.1.tar.gz # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login -RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 +RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 # ACR docker credential helper ADD https://aadacr.blob.core.windows.net/acr-docker-credential-helper/docker-credential-acr-linux-amd64.tar.gz /usr/local/bin RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-acr-linux-amd64.tar.gz @@ -50,6 +50,7 @@ COPY --from=1 /distroless/bazel-bin/experimental/busybox/busybox/ /busybox/ VOLUME /busybox COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /kaniko/.docker /kaniko/.docker +COPY files/nsswitch.conf /etc/nsswitch.conf ENV HOME /root ENV USER /root ENV PATH /usr/local/bin:/kaniko:/busybox diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer index dd83c378c..156fdeb43 100644 --- a/deploy/Dockerfile_warmer +++ b/deploy/Dockerfile_warmer @@ -18,8 +18,8 @@ FROM golang:1.14 ARG GOARCH=amd64 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko # Get GCR credential helper -ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.5.0/docker-credential-gcr_linux_amd64-1.5.0.tar.gz /usr/local/bin/ -RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-1.5.0.tar.gz +ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v2.0.1/docker-credential-gcr_linux_amd64-2.0.1.tar.gz /usr/local/bin/ +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-2.0.1.tar.gz # Get Amazon ECR credential helper RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 @@ -39,6 +39,7 @@ COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux- COPY --from=0 /usr/local/bin/docker-credential-acr-linux /kaniko/docker-credential-acr COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY --from=0 /kaniko/.docker /kaniko/.docker +COPY files/nsswitch.conf /etc/nsswitch.conf ENV HOME /root ENV USER /root ENV PATH /usr/local/bin:/kaniko diff --git a/files/nsswitch.conf b/files/nsswitch.conf new file mode 100644 index 000000000..9081618a7 --- /dev/null +++ b/files/nsswitch.conf @@ -0,0 +1,25 @@ +# /etc/nsswitch.conf +# +# As described on the web page https://man7.org/linux/man-pages/man3/gethostbyname.3.html, +# without the nsswitch.conf file, the gethostbyname() and gethostbyaddr() domain queries +# will fail to a local name server, thus the /etc/hosts will take no effect. +# +# For example, when hostaliases are specified for a kubernetes pod, without proper settings +# defined in this file, the hostaliases settings will not take effect. +# +# Following contents of this file is from the ubuntu:16.04 docker image. + +passwd: compat +group: compat +shadow: compat +gshadow: files + +hosts: files dns +networks: files + +protocols: db files +services: db files +ethers: db files +rpc: db files + +netgroup: nis \ No newline at end of file diff --git a/integration/dockerfiles/Dockerfile_test_dockerignore b/integration/dockerfiles/Dockerfile_test_dockerignore index d646872ef..70667623d 100644 --- a/integration/dockerfiles/Dockerfile_test_dockerignore +++ b/integration/dockerfiles/Dockerfile_test_dockerignore @@ -1,5 +1,11 @@ # This dockerfile makes sure the .dockerignore is working # If so then ignore/foo should copy to /foo # If not, then this image won't build because it will attempt to copy three files to /foo, which is a file not a directory -FROM scratch +FROM scratch as base COPY ignore/* /foo + +From base as first +COPY --from=base /foo ignore/bar + +FROM first +COPY --from=first ignore/* /fooAnother/ \ No newline at end of file diff --git a/pkg/buildcontext/git.go b/pkg/buildcontext/git.go index 7c2dddc6d..18d6e2ded 100644 --- a/pkg/buildcontext/git.go +++ b/pkg/buildcontext/git.go @@ -23,12 +23,17 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/constants" git "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) const ( gitPullMethodEnvKey = "GIT_PULL_METHOD" gitPullMethodHTTPS = "https" gitPullMethodHTTP = "http" + + gitAuthUsernameEnvKey = "GIT_USERNAME" + gitAuthPasswordEnvKey = "GIT_PASSWORD" ) var ( @@ -46,6 +51,7 @@ func (g *Git) UnpackTarFromBuildContext() (string, error) { parts := strings.Split(g.context, "#") options := git.CloneOptions{ URL: getGitPullMethod() + "://" + parts[0], + Auth: getGitAuth(), Progress: os.Stdout, } if len(parts) > 1 { @@ -55,6 +61,18 @@ func (g *Git) UnpackTarFromBuildContext() (string, error) { return directory, err } +func getGitAuth() transport.AuthMethod { + username := os.Getenv(gitAuthUsernameEnvKey) + password := os.Getenv(gitAuthPasswordEnvKey) + if username != "" || password != "" { + return &http.BasicAuth{ + Username: username, + Password: password, + } + } + return nil +} + func getGitPullMethod() string { gitPullMethod := os.Getenv(gitPullMethodEnvKey) if ok := supportedGitPullMethods[gitPullMethod]; !ok { diff --git a/pkg/buildcontext/git_test.go b/pkg/buildcontext/git_test.go index ccc53fcb9..aaa487c92 100644 --- a/pkg/buildcontext/git_test.go +++ b/pkg/buildcontext/git_test.go @@ -21,6 +21,8 @@ import ( "testing" "github.com/GoogleContainerTools/kaniko/testutil" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) func TestGetGitPullMethod(t *testing.T) { @@ -80,3 +82,88 @@ func TestGetGitPullMethod(t *testing.T) { }) } } + +func TestGetGitAuth(t *testing.T) { + tests := []struct { + testName string + setEnv func() (expectedValue transport.AuthMethod) + }{ + { + testName: "noEnv", + setEnv: func() (expectedValue transport.AuthMethod) { + expectedValue = nil + return + }, + }, + { + testName: "emptyUsernameEnv", + setEnv: func() (expectedValue transport.AuthMethod) { + _ = os.Setenv(gitAuthUsernameEnvKey, "") + expectedValue = nil + return + }, + }, + { + testName: "emptyPasswordEnv", + setEnv: func() (expectedValue transport.AuthMethod) { + _ = os.Setenv(gitAuthPasswordEnvKey, "") + expectedValue = nil + return + }, + }, + { + testName: "emptyEnv", + setEnv: func() (expectedValue transport.AuthMethod) { + _ = os.Setenv(gitAuthUsernameEnvKey, "") + _ = os.Setenv(gitAuthPasswordEnvKey, "") + expectedValue = nil + return + }, + }, + { + testName: "withUsername", + setEnv: func() (expectedValue transport.AuthMethod) { + username := "foo" + _ = os.Setenv(gitAuthUsernameEnvKey, username) + expectedValue = &http.BasicAuth{Username: username} + return + }, + }, + { + testName: "withPassword", + setEnv: func() (expectedValue transport.AuthMethod) { + pass := "super-secret-password-1234" + _ = os.Setenv(gitAuthPasswordEnvKey, pass) + expectedValue = &http.BasicAuth{Password: pass} + return + }, + }, + { + testName: "withUsernamePassword", + setEnv: func() (expectedValue transport.AuthMethod) { + username := "foo" + pass := "super-secret-password-1234" + _ = os.Setenv(gitAuthUsernameEnvKey, username) + _ = os.Setenv(gitAuthPasswordEnvKey, pass) + expectedValue = &http.BasicAuth{Username: username, Password: pass} + return + }, + }, + } + + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + // Make sure to unset environment vars to get a clean test each time + defer clearTestAuthEnv() + + expectedValue := tt.setEnv() + testutil.CheckDeepEqual(t, expectedValue, getGitAuth()) + }) + } + +} + +func clearTestAuthEnv() { + _ = os.Unsetenv(gitAuthUsernameEnvKey) + _ = os.Unsetenv(gitAuthPasswordEnvKey) +} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 893b1142d..70fca8b1d 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -17,9 +17,7 @@ limitations under the License. package cache import ( - "crypto/tls" "fmt" - "net/http" "os" "path" "path/filepath" @@ -27,6 +25,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/creds" + "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -67,12 +66,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) { cacheRef.Repository.Registry = newReg } - tr := http.DefaultTransport.(*http.Transport) - if rc.Opts.SkipTLSVerifyRegistries.Contains(registryName) { - tr.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } + tr := util.MakeTransport(rc.Opts, registryName) img, err := remote.Image(cacheRef, remote.WithTransport(tr), remote.WithAuthFromKeychain(creds.GetKeychain())) if err != nil { diff --git a/pkg/commands/run.go b/pkg/commands/run.go index f09c9c79f..e0dd1beef 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -65,7 +65,8 @@ func (r *RunCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui logrus.Infof("args: %s", newCommand[1:]) cmd := exec.Command(newCommand[0], newCommand[1:]...) - cmd.Dir = config.WorkingDir + + cmd.Dir = setWorkDirIfExists(config.WorkingDir) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr replacementEnvs := buildArgs.ReplacementEnvs(config.Env) @@ -236,3 +237,10 @@ func (cr *CachingRunCommand) String() string { func (cr *CachingRunCommand) MetadataOnly() bool { return false } + +func setWorkDirIfExists(workdir string) string { + if _, err := os.Lstat(workdir); err == nil { + return workdir + } + return "" +} diff --git a/pkg/commands/run_test.go b/pkg/commands/run_test.go index 5a7139a6b..81d213b04 100644 --- a/pkg/commands/run_test.go +++ b/pkg/commands/run_test.go @@ -316,3 +316,12 @@ func Test_CachingRunCommand_ExecuteCommand(t *testing.T) { }) } } + +func TestSetWorkDirIfExists(t *testing.T) { + testDir, err := ioutil.TempDir("", "workdir") + if err != nil { + t.Error(err) + } + testutil.CheckDeepEqual(t, testDir, setWorkDirIfExists(testDir)) + testutil.CheckDeepEqual(t, "", setWorkDirIfExists("doesnot-exists")) +} diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 376e11ef4..761c09e72 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -44,6 +44,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + image_util "github.com/GoogleContainerTools/kaniko/pkg/image" "github.com/GoogleContainerTools/kaniko/pkg/snapshot" "github.com/GoogleContainerTools/kaniko/pkg/timing" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -84,7 +85,7 @@ type stageBuilder struct { // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string, dcm map[string]string, sid map[string]string, stageNameToIdx map[string]string) (*stageBuilder, error) { - sourceImage, err := util.RetrieveSourceImage(stage, opts) + sourceImage, err := image_util.RetrieveSourceImage(stage, opts) if err != nil { return nil, err } @@ -517,7 +518,7 @@ func CalculateDependencies(stages []config.KanikoStage, opts *config.KanikoOptio } else if s.Name == constants.NoBaseImage { image = empty.Image } else { - image, err = util.RetrieveSourceImage(s, opts) + image, err = image_util.RetrieveSourceImage(s, opts) if err != nil { return nil, err } @@ -596,6 +597,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { } logrus.Infof("Built cross stage deps: %v", crossStageDependencies) + util.IsFirstStage = true for index, stage := range kanikoStages { sb, err := newStageBuilder(opts, stage, crossStageDependencies, digestToCacheKey, stageIdxToDigest, stageNameToIdx) if err != nil { @@ -604,6 +606,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { if err := sb.build(); err != nil { return nil, errors.Wrap(err, "error building stage") } + util.IsFirstStage = false reviewConfig(stage, &sb.cf.Config) @@ -749,7 +752,7 @@ func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) e // This must be an image name, fetch it. logrus.Debugf("Found extra base image stage %s", c.From) - sourceImage, err := util.RetrieveRemoteImage(c.From, opts) + sourceImage, err := image_util.RetrieveRemoteImage(c.From, opts) if err != nil { return err } diff --git a/pkg/executor/push.go b/pkg/executor/push.go index ff81409e3..623c92525 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -17,8 +17,6 @@ limitations under the License. package executor import ( - "crypto/tls" - "crypto/x509" "encoding/json" "fmt" "io/ioutil" @@ -34,6 +32,7 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/creds" "github.com/GoogleContainerTools/kaniko/pkg/timing" + "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/GoogleContainerTools/kaniko/pkg/version" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -41,6 +40,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -51,6 +51,11 @@ type withUserAgent struct { t http.RoundTripper } +// for testing +var ( + newRetry = transport.NewRetry +) + const ( UpstreamClientUaKey = "UPSTREAM_CLIENT_TYPE" ) @@ -76,41 +81,6 @@ func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) { return w.t.RoundTrip(r) } -type CertPool interface { - value() *x509.CertPool - append(path string) error -} - -type X509CertPool struct { - inner x509.CertPool -} - -func (p *X509CertPool) value() *x509.CertPool { - return &p.inner -} - -func (p *X509CertPool) append(path string) error { - pem, err := ioutil.ReadFile(path) - if err != nil { - return err - } - p.inner.AppendCertsFromPEM(pem) - return nil -} - -type systemCertLoader func() CertPool - -var defaultX509Handler systemCertLoader = func() CertPool { - systemCertPool, err := x509.SystemCertPool() - if err != nil { - logrus.Warn("Failed to load system cert pool. Loading empty one instead.") - systemCertPool = x509.NewCertPool() - } - return &X509CertPool{ - inner: *systemCertPool, - } -} - // for testing var ( fs = afero.NewOsFs() @@ -155,7 +125,7 @@ func CheckPushPermissions(opts *config.KanikoOptions) error { } destRef.Repository.Registry = newReg } - tr := makeTransport(opts, registryName, defaultX509Handler) + tr := newRetry(util.MakeTransport(opts, registryName)) if err := checkRemotePushPermission(destRef, creds.GetKeychain(), tr); err != nil { return errors.Wrapf(err, "checking push permission for %q", destRef) } @@ -252,7 +222,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { return errors.Wrap(err, "resolving pushAuth") } - tr := makeTransport(opts, registryName, defaultX509Handler) + tr := newRetry(util.MakeTransport(opts, registryName)) rt := &withUserAgent{t: tr} if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil { @@ -294,26 +264,6 @@ func writeImageOutputs(image v1.Image, destRefs []name.Tag) error { return nil } -func makeTransport(opts *config.KanikoOptions, registryName string, loader systemCertLoader) http.RoundTripper { - // Create a transport to set our user-agent. - var tr http.RoundTripper = http.DefaultTransport.(*http.Transport).Clone() - if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) { - tr.(*http.Transport).TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } else if certificatePath := opts.RegistriesCertificates[registryName]; certificatePath != "" { - systemCertPool := loader() - if err := systemCertPool.append(certificatePath); err != nil { - logrus.WithError(err).Warnf("Failed to load certificate %s for %s\n", certificatePath, registryName) - } else { - tr.(*http.Transport).TLSClientConfig = &tls.Config{ - RootCAs: systemCertPool.value(), - } - } - } - return tr -} - // pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache // if opts.Cache doesn't exist, infer the cache from the given destination func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error { diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go index 7d919db7b..2fd36f23c 100644 --- a/pkg/executor/push_test.go +++ b/pkg/executor/push_test.go @@ -18,8 +18,6 @@ package executor import ( "bytes" - "crypto/tls" - "crypto/x509" "fmt" "io/ioutil" "net/http" @@ -35,6 +33,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/random" "github.com/google/go-containerregistry/pkg/v1/validate" + "github.com/spf13/afero" ) @@ -270,88 +269,6 @@ func TestImageNameDigestFile(t *testing.T) { } -type mockedCertPool struct { - certificatesPath []string -} - -func (m *mockedCertPool) value() *x509.CertPool { - return &x509.CertPool{} -} - -func (m *mockedCertPool) append(path string) error { - m.certificatesPath = append(m.certificatesPath, path) - return nil -} - -func Test_makeTransport(t *testing.T) { - registryName := "my.registry.name" - - tests := []struct { - name string - opts *config.KanikoOptions - check func(*tls.Config, *mockedCertPool) - }{ - { - name: "SkipTLSVerify set", - opts: &config.KanikoOptions{SkipTLSVerify: true}, - check: func(config *tls.Config, pool *mockedCertPool) { - if !config.InsecureSkipVerify { - t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify not set while SkipTLSVerify set") - } - }, - }, - { - name: "SkipTLSVerifyRegistries set with expected registry", - opts: &config.KanikoOptions{SkipTLSVerifyRegistries: []string{registryName}}, - check: func(config *tls.Config, pool *mockedCertPool) { - if !config.InsecureSkipVerify { - t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify not set while SkipTLSVerifyRegistries set with registry name") - } - }, - }, - { - name: "SkipTLSVerifyRegistries set with other registry", - opts: &config.KanikoOptions{SkipTLSVerifyRegistries: []string{fmt.Sprintf("other.%s", registryName)}}, - check: func(config *tls.Config, pool *mockedCertPool) { - if config.InsecureSkipVerify { - t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify set while SkipTLSVerifyRegistries not set with registry name") - } - }, - }, - { - name: "RegistriesCertificates set for registry", - opts: &config.KanikoOptions{RegistriesCertificates: map[string]string{registryName: "/path/to/the/certificate.cert"}}, - check: func(config *tls.Config, pool *mockedCertPool) { - if len(pool.certificatesPath) != 1 || pool.certificatesPath[0] != "/path/to/the/certificate.cert" { - t.Errorf("makeTransport().RegistriesCertificates certificate not appended to system certificates") - } - }, - }, - { - name: "RegistriesCertificates set for another registry", - opts: &config.KanikoOptions{RegistriesCertificates: map[string]string{fmt.Sprintf("other.%s=", registryName): "/path/to/the/certificate.cert"}}, - check: func(config *tls.Config, pool *mockedCertPool) { - if len(pool.certificatesPath) != 0 { - t.Errorf("makeTransport().RegistriesCertificates certificate appended to system certificates while added for other registry") - } - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var certificatesPath []string - certPool := mockedCertPool{ - certificatesPath: certificatesPath, - } - var mockedSystemCertLoader systemCertLoader = func() CertPool { - return &certPool - } - transport := makeTransport(tt.opts, registryName, mockedSystemCertLoader) - tt.check(transport.(*http.Transport).TLSClientConfig, &certPool) - }) - } -} - var calledExecCommand = false var calledCheckPushPermission = false diff --git a/pkg/image/image.go b/pkg/image/image.go deleted file mode 100644 index dbb5a9e9d..000000000 --- a/pkg/image/image.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package image - -import ( - "os" - "strings" - - v1 "github.com/google/go-containerregistry/pkg/v1" - - "github.com/sirupsen/logrus" -) - -// SetEnvVariables sets environment variables as specified in the image -func SetEnvVariables(img v1.Image) error { - cfg, err := img.ConfigFile() - if err != nil { - return err - } - envVars := cfg.Config.Env - for _, envVar := range envVars { - split := strings.SplitN(envVar, "=", 2) - if err := os.Setenv(split[0], split[1]); err != nil { - return err - } - logrus.Infof("Setting environment variable %s", envVar) - } - return nil -} diff --git a/pkg/util/image_util.go b/pkg/image/image_util.go similarity index 92% rename from pkg/util/image_util.go rename to pkg/image/image_util.go index f241210f9..c4e044235 100644 --- a/pkg/util/image_util.go +++ b/pkg/image/image_util.go @@ -14,30 +14,29 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package image import ( - "crypto/tls" "fmt" - "net/http" "path/filepath" + "runtime" "strconv" "strings" - "github.com/GoogleContainerTools/kaniko/pkg/timing" - + "github.com/GoogleContainerTools/kaniko/pkg/cache" + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/creds" + "github.com/GoogleContainerTools/kaniko/pkg/timing" + "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" - "github.com/sirupsen/logrus" - "github.com/GoogleContainerTools/kaniko/pkg/cache" - "github.com/GoogleContainerTools/kaniko/pkg/config" - "github.com/GoogleContainerTools/kaniko/pkg/constants" + "github.com/sirupsen/logrus" ) var ( @@ -56,7 +55,7 @@ func RetrieveSourceImage(stage config.KanikoStage, opts *config.KanikoOptions) ( buildArgs = append(buildArgs, fmt.Sprintf("%s=%s", arg.Key, arg.ValueString())) } buildArgs = append(buildArgs, opts.BuildArgs...) - currentBaseName, err := ResolveEnvironmentReplacement(stage.BaseName, buildArgs, false) + currentBaseName, err := util.ResolveEnvironmentReplacement(stage.BaseName, buildArgs, false) if err != nil { return nil, err } @@ -154,12 +153,7 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { } func remoteOptions(registryName string, opts *config.KanikoOptions) []remote.Option { - tr := http.DefaultTransport.(*http.Transport) - if opts.SkipTLSVerifyPull || opts.SkipTLSVerifyRegistries.Contains(registryName) { - tr.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } + tr := util.MakeTransport(opts, registryName) // on which v1.Platform is this currently running? platform := currentPlatform() @@ -190,3 +184,11 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { } return cache.LocalSource(&opts.CacheOptions, cacheKey) } + +// CurrentPlatform returns the v1.Platform on which the code runs +func currentPlatform() v1.Platform { + return v1.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + } +} diff --git a/pkg/util/image_util_test.go b/pkg/image/image_util_test.go similarity index 99% rename from pkg/util/image_util_test.go rename to pkg/image/image_util_test.go index 2ce681500..3b9e039cf 100644 --- a/pkg/util/image_util_test.go +++ b/pkg/image/image_util_test.go @@ -14,18 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package image import ( "bytes" "testing" - "github.com/GoogleContainerTools/kaniko/pkg/config" - "github.com/GoogleContainerTools/kaniko/testutil" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/parser" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/testutil" ) var ( diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 449ae918a..6415af870 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -158,6 +158,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { ) timing.DefaultRun.Stop(timer) + timer = timing.Start("Resolving Paths") resolvedFiles, err := filesystem.ResolvePaths(foundPaths, s.whitelist) if err != nil { return nil, nil, err @@ -207,6 +208,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { } } + timing.DefaultRun.Stop(timer) sort.Strings(filesToAdd) // Add files to the layered map for _, file := range filesToAdd { diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index af553f8c9..67a3f1d87 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -73,6 +73,7 @@ var whitelist = initialWhitelist var volumes = []string{} var excluded []string +var IsFirstStage = true type ExtractFunction func(string, *tar.Header, io.Reader) error @@ -678,6 +679,10 @@ func GetExcludedFiles(dockerfilepath string, buildcontext string) error { // ExcludeFile returns true if the .dockerignore specified this file should be ignored func ExcludeFile(path, buildcontext string) bool { + // Apply dockerfile excludes for first stage only + if !IsFirstStage { + return false + } if HasFilepathPrefix(path, buildcontext, false) { var err error path, err = filepath.Rel(buildcontext, path) diff --git a/pkg/util/transport_util.go b/pkg/util/transport_util.go new file mode 100644 index 000000000..a8e3f1e3f --- /dev/null +++ b/pkg/util/transport_util.go @@ -0,0 +1,82 @@ +/* +Copyright 2020 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "crypto/tls" + "crypto/x509" + + "io/ioutil" + "net/http" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/sirupsen/logrus" +) + +type CertPool interface { + value() *x509.CertPool + append(path string) error +} + +type X509CertPool struct { + inner x509.CertPool +} + +func (p *X509CertPool) value() *x509.CertPool { + return &p.inner +} + +func (p *X509CertPool) append(path string) error { + pem, err := ioutil.ReadFile(path) + if err != nil { + return err + } + p.inner.AppendCertsFromPEM(pem) + return nil +} + +var systemCertLoader CertPool + +func init() { + systemCertPool, err := x509.SystemCertPool() + if err != nil { + logrus.Warn("Failed to load system cert pool. Loading empty one instead.") + systemCertPool = x509.NewCertPool() + } + systemCertLoader = &X509CertPool{ + inner: *systemCertPool, + } +} + +func MakeTransport(opts *config.KanikoOptions, registryName string) http.RoundTripper { + // Create a transport to set our user-agent. + var tr http.RoundTripper = http.DefaultTransport.(*http.Transport).Clone() + if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) { + tr.(*http.Transport).TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } else if certificatePath := opts.RegistriesCertificates[registryName]; certificatePath != "" { + if err := systemCertLoader.append(certificatePath); err != nil { + logrus.WithError(err).Warnf("Failed to load certificate %s for %s\n", certificatePath, registryName) + } else { + tr.(*http.Transport).TLSClientConfig = &tls.Config{ + RootCAs: systemCertLoader.value(), + } + } + } + return tr +} diff --git a/pkg/util/transport_util_test.go b/pkg/util/transport_util_test.go new file mode 100644 index 000000000..b10be6995 --- /dev/null +++ b/pkg/util/transport_util_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2020 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "testing" + + "github.com/GoogleContainerTools/kaniko/pkg/config" +) + +type mockedCertPool struct { + certificatesPath []string +} + +func (m *mockedCertPool) value() *x509.CertPool { + return &x509.CertPool{} +} + +func (m *mockedCertPool) append(path string) error { + m.certificatesPath = append(m.certificatesPath, path) + return nil +} + +func Test_makeTransport(t *testing.T) { + registryName := "my.registry.name" + + tests := []struct { + name string + opts *config.KanikoOptions + check func(*tls.Config, *mockedCertPool) + }{ + { + name: "SkipTLSVerify set", + opts: &config.KanikoOptions{SkipTLSVerify: true}, + check: func(config *tls.Config, pool *mockedCertPool) { + if !config.InsecureSkipVerify { + t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify not set while SkipTLSVerify set") + } + }, + }, + { + name: "SkipTLSVerifyRegistries set with expected registry", + opts: &config.KanikoOptions{SkipTLSVerifyRegistries: []string{registryName}}, + check: func(config *tls.Config, pool *mockedCertPool) { + if !config.InsecureSkipVerify { + t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify not set while SkipTLSVerifyRegistries set with registry name") + } + }, + }, + { + name: "SkipTLSVerifyRegistries set with other registry", + opts: &config.KanikoOptions{SkipTLSVerifyRegistries: []string{fmt.Sprintf("other.%s", registryName)}}, + check: func(config *tls.Config, pool *mockedCertPool) { + if config.InsecureSkipVerify { + t.Errorf("makeTransport().TLSClientConfig.InsecureSkipVerify set while SkipTLSVerifyRegistries not set with registry name") + } + }, + }, + { + name: "RegistriesCertificates set for registry", + opts: &config.KanikoOptions{RegistriesCertificates: map[string]string{registryName: "/path/to/the/certificate.cert"}}, + check: func(config *tls.Config, pool *mockedCertPool) { + if len(pool.certificatesPath) != 1 || pool.certificatesPath[0] != "/path/to/the/certificate.cert" { + t.Errorf("makeTransport().RegistriesCertificates certificate not appended to system certificates") + } + }, + }, + { + name: "RegistriesCertificates set for another registry", + opts: &config.KanikoOptions{RegistriesCertificates: map[string]string{fmt.Sprintf("other.%s=", registryName): "/path/to/the/certificate.cert"}}, + check: func(config *tls.Config, pool *mockedCertPool) { + if len(pool.certificatesPath) != 0 { + t.Errorf("makeTransport().RegistriesCertificates certificate appended to system certificates while added for other registry") + } + }, + }, + } + savedSystemCertLoader := systemCertLoader + defer func() { systemCertLoader = savedSystemCertLoader }() + for _, tt := range tests { + var certificatesPath []string + certPool := &mockedCertPool{ + certificatesPath: certificatesPath, + } + systemCertLoader = certPool + t.Run(tt.name, func(t *testing.T) { + tr := MakeTransport(tt.opts, registryName) + tt.check(tr.(*http.Transport).TLSClientConfig, certPool) + }) + + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 27340e413..a1dd77f4f 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -23,14 +23,11 @@ import ( "io" "io/ioutil" "os" - "runtime" "strconv" "sync" "syscall" "github.com/minio/highwayhash" - - v1 "github.com/google/go-containerregistry/pkg/v1" ) // Hasher returns a hash function, used in snapshotting to determine if a file has changed @@ -128,14 +125,6 @@ func SHA256(r io.Reader) (string, error) { return hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), nil } -// CurrentPlatform returns the v1.Platform on which the code runs -func currentPlatform() v1.Platform { - return v1.Platform{ - OS: runtime.GOOS, - Architecture: runtime.GOARCH, - } -} - // GetInputFrom returns Reader content func GetInputFrom(r io.Reader) ([]byte, error) { output, err := ioutil.ReadAll(r)