diff --git a/CHANGELOG.md b/CHANGELOG.md index 94dc5302e..142baa25d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,146 @@ +# v0.12.0 Release - 2019-09/13 + +## New Features +* Added `--oci-layout-path` flag to save image in OCI layout. [#744](https://github.com/GoogleContainerTools/kaniko/pull/744) +* Add support for S3 custom endpoint [#698](https://github.com/GoogleContainerTools/kaniko/pull/698) + +## Bug Fixes +* Setting PATH [#760](https://github.com/GoogleContainerTools/kaniko/pull/760) +* Remove leading slash in layer tarball paths (Closes: #726) [#729](https://github.com/GoogleContainerTools/kaniko/pull/729) + +## Updates and Refactors +* Remove cruft [#635](https://github.com/GoogleContainerTools/kaniko/pull/635) +* Add desc for `--skip-tls-verify-pull` to README [#493](https://github.com/GoogleContainerTools/kaniko/pull/493) + +Huge thank you for this release towards our contributors: +- Carlos Alexandro Becker +- Carlos Sanchez +- chhsia0 +- Deniz Zoeteman +- Luke Wood +- Matthew Dawson +- Niels Denissen +- Priya Wadhwa +- Sharif Elgamal +- Takeaki Matsumoto +- Taylor Barrella +- Tejal Desai +- v.rul +- Warren Seymour +- xanonid +- Xueshan Feng +- Роман Небалуев + + +# v0.11.0 Release - 2019-08-23 + +## Bug Fixes +* fix unpacking archives via ADD [#717](https://github.com/GoogleContainerTools/kaniko/pull/717) +* Reverted not including build args in cache key [#739](https://github.com/GoogleContainerTools/kaniko/pull/739) +* Create cache directory if it doesn't already exist [#452](https://github.com/GoogleContainerTools/kaniko/pull/452) + +## New Features +* add multiple user agents to kaniko if upstream_client_type value is set [#750](https://github.com/GoogleContainerTools/kaniko/pull/750) +* Make container layers captured using FS snapshots reproducible [#714](https://github.com/GoogleContainerTools/kaniko/pull/714) +* Include warmer in debug image [#497](https://github.com/GoogleContainerTools/kaniko/pull/497) +* Bailout when there is not enough input arguments [#735](https://github.com/GoogleContainerTools/kaniko/pull/735) +* Add checking image presence in cache prior to downloading it [#723](https://github.com/GoogleContainerTools/kaniko/pull/723) + +## Additonal PRs +* Document how to build from git reference [#730](https://github.com/GoogleContainerTools/kaniko/pull/730) +* Misc. small changes/refactoring [#712](https://github.com/GoogleContainerTools/kaniko/pull/712) +* Update go-containerregistry [#680](https://github.com/GoogleContainerTools/kaniko/pull/680) +* Update version of go-containerregistry [#724](https://github.com/GoogleContainerTools/kaniko/pull/724) +* feat: support specifying branch for cloning [#703](https://github.com/GoogleContainerTools/kaniko/pull/703) + +Huge thank you for this release towards our contributors: +- Carlos Alexandro Becker +- Carlos Sanchez +- Deniz Zoeteman +- Luke Wood +- Matthew Dawson +- priyawadhwa +- sharifelgamal +- Sharif Elgamal +- Taylor Barrella +- Tejal Desai +- v.rul +- Warren Seymour +- Xueshan Feng +- Роман Небалуе + +# v0.10.0 Release - 2019-06-19 + +## Bug Fixes +* Fix kaniko caching [#639](https://github.com/GoogleContainerTools/kaniko/pull/639) +* chore: fix typo [#665](https://github.com/GoogleContainerTools/kaniko/pull/665) +* Fix file mode bug [#618](https://github.com/GoogleContainerTools/kaniko/pull/618) +* Fix arg handling for multi-stage images in COPY instructions. [#621](https://github.com/GoogleContainerTools/kaniko/pull/621) +* Fix parent directory permissions [#619](https://github.com/GoogleContainerTools/kaniko/pull/619) +* Environment variables should be replaced in URLs in ADD commands. [#580](https://github.com/GoogleContainerTools/kaniko/pull/580) +* Update the cache warmer to also save manifests. [#576](https://github.com/GoogleContainerTools/kaniko/pull/576) +* Fix typo in error message [#569](https://github.com/GoogleContainerTools/kaniko/pull/569) + +## New Features +* Add SkipVerify support to CheckPushPermissions. [#663](https://github.com/GoogleContainerTools/kaniko/pull/663) +* Creating github Build Context [#672](https://github.com/GoogleContainerTools/kaniko/pull/672) +* Add `--digest-file` flag to output built digest to file. [#655](https://github.com/GoogleContainerTools/kaniko/pull/655) +* README.md: update BuildKit/img comparison [#642](https://github.com/GoogleContainerTools/kaniko/pull/642) +* Add documentation for --verbosity flag [#634](https://github.com/GoogleContainerTools/kaniko/pull/634) +* Optimize file copying and stage saving between stages. [#605](https://github.com/GoogleContainerTools/kaniko/pull/605) +* Add an integration test for USER unpacking. [#600](https://github.com/GoogleContainerTools/kaniko/pull/600) +* Added missing documentation for --skip-tls-verify-pull arg [#593](https://github.com/GoogleContainerTools/kaniko/pull/593) +* README.me: update Buildah description [#586](https://github.com/GoogleContainerTools/kaniko/pull/586) +* Add missing tests for bucket util [#565](https://github.com/GoogleContainerTools/kaniko/pull/565) +* Look for manifests in the local cache next to the full images. [#570](https://github.com/GoogleContainerTools/kaniko/pull/570) +* Make the run_in_docker script support caching. [#564](https://github.com/GoogleContainerTools/kaniko/pull/564) +* Refactor snapshotting [#561](https://github.com/GoogleContainerTools/kaniko/pull/561) +* Stop storing a separate cache hash. [#560](https://github.com/GoogleContainerTools/kaniko/pull/560) +* Speed up workdir by always returning an empty filelist (rather than a… [#557](https://github.com/GoogleContainerTools/kaniko/pull/557) +* Refactor whitelist handling. [#559](https://github.com/GoogleContainerTools/kaniko/pull/559) +* Refactor the build loop to fetch stagebuilders earlier. [#558](https://github.com/GoogleContainerTools/kaniko/pull/558) + +## Additonal PRs +* Improve changelog dates [#657](https://github.com/GoogleContainerTools/kaniko/pull/657) +* Change verbose output from info to debug [#640](https://github.com/GoogleContainerTools/kaniko/pull/640) +* Check push permissions before building images [#622](https://github.com/GoogleContainerTools/kaniko/pull/622) +* Bump go-containerregistry to 8c1640add99804503b4126abc718931a4d93c31a [#609](https://github.com/GoogleContainerTools/kaniko/pull/609) +* Update go-containerregistry [#599](https://github.com/GoogleContainerTools/kaniko/pull/599) +* Log "Skipping paths under..." to debug [#571](https://github.com/GoogleContainerTools/kaniko/pull/571) + +Huge thank you for this release towards our contributors: +- Achilleas Pipinellis +- Adrian Duong +- Akihiro Suda +- Andreas Bergmeier +- Andrew Rynhard +- Anthony Weston +- Anurag Goel +- Balint Pato +- Christie Wilson +- Daisuke Taniwaki +- Dan Cecile +- Dirk Gustke +- dlorenc +- Fredrik Lönnegren +- Gijs +- Jake Shadle +- James Rawlings +- Jason Hall +- Johan Hernandez +- Johannes 'fish' Ziemke +- Kartik Verma +- linuxshokunin +- MMeent +- Myers Carpenter +- Nándor István Krácser +- Nao YONASHIRO +- Priya Wadhwa +- Sharif Elgamal +- Shuhei Kitagawa +- Valentin Rothberg +- Vincent Demeester + # v0.9.0 Release - 2019-02-08 ## Bug Fixes diff --git a/Gopkg.lock b/Gopkg.lock index 7b09230e2..57a55af22 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -445,15 +445,18 @@ version = "v0.2.0" [[projects]] - digest = "1:d40a26f0daf07f3b5c916356a3e10fabbf97d5166f77e57aa3983013ab57004c" + digest = "1:5924704ec96f00247784c512cc57f45a595030376a7ff2ff993bf356793a2cb0" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", "pkg/authn/k8schain", + "pkg/internal/retry", + "pkg/logs", "pkg/name", "pkg/v1", "pkg/v1/daemon", "pkg/v1/empty", + "pkg/v1/layout", "pkg/v1/mutate", "pkg/v1/partial", "pkg/v1/random", @@ -465,7 +468,7 @@ "pkg/v1/v1util", ] pruneopts = "NUT" - revision = "8621d738a07bc74b2adeafd175a3c738423577a0" + revision = "31e00cede111067bae48bfc2cbfc522b0b36207f" [[projects]] digest = "1:f4f203acd8b11b8747bdcd91696a01dbc95ccb9e2ca2db6abf81c3a4f5e950ce" @@ -719,6 +722,14 @@ revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" version = "v1.0.2" +[[projects]] + branch = "master" + digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c" + name = "github.com/otiai10/copy" + packages = ["."] + pruneopts = "NUT" + revision = "7e9a647135a142c2669943d4a4d29be015ce9392" + [[projects]] digest = "1:cf254277d898b713195cc6b4a3fac8bf738b9f1121625df27843b52b267eec6c" name = "github.com/pelletier/go-buffruneio" @@ -727,22 +738,6 @@ revision = "c37440a7cf42ac63b919c752ca73a85067e05992" version = "v0.2.0" -[[projects]] - branch = "master" - digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c" - name = "github.com/otiai10/copy" - packages = ["."] - pruneopts = "NUT" - revision = "7e9a647135a142c2669943d4a4d29be015ce9392" - -[[projects]] - branch = "master" - digest = "1:15057fc7395024283a7d2639b8afc61c5b6df3fe260ce06ff5834c8464f16b5c" - name = "github.com/otiai10/copy" - packages = ["."] - pruneopts = "NUT" - revision = "7e9a647135a142c2669943d4a4d29be015ce9392" - [[projects]] branch = "master" digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" @@ -1374,6 +1369,7 @@ "github.com/google/go-containerregistry/pkg/v1", "github.com/google/go-containerregistry/pkg/v1/daemon", "github.com/google/go-containerregistry/pkg/v1/empty", + "github.com/google/go-containerregistry/pkg/v1/layout", "github.com/google/go-containerregistry/pkg/v1/mutate", "github.com/google/go-containerregistry/pkg/v1/partial", "github.com/google/go-containerregistry/pkg/v1/remote", @@ -1392,6 +1388,7 @@ "golang.org/x/oauth2", "golang.org/x/sync/errgroup", "gopkg.in/src-d/go-git.v4", + "gopkg.in/src-d/go-git.v4/plumbing", "k8s.io/client-go/discovery", ] solver-name = "gps-cdcl" diff --git a/Gopkg.toml b/Gopkg.toml index 80db44131..e502f07ff 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,7 +37,7 @@ required = [ [[constraint]] name = "github.com/google/go-containerregistry" - revision = "8621d738a07bc74b2adeafd175a3c738423577a0" + revision = "31e00cede111067bae48bfc2cbfc522b0b36207f" [[override]] name = "k8s.io/apimachinery" diff --git a/Makefile b/Makefile index 1e07ddac6..e5633bf20 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ # Bump these on release VERSION_MAJOR ?= 0 -VERSION_MINOR ?= 9 +VERSION_MINOR ?= 11 VERSION_BUILD ?= 0 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD) diff --git a/README.md b/README.md index 18d1d4e72..d1720da88 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [--insecure](#--insecure) - [--insecure-pull](#--insecure-pull) - [--no-push](#--no-push) + - [--oci-layout-path](#--oci-layout-path) - [--reproducible](#--reproducible) - [--single-snapshot](#--single-snapshot) - [--snapshotMode](#--snapshotmode) @@ -93,6 +94,7 @@ Right now, kaniko supports these storage solutions: - GCS Bucket - S3 Bucket - Local Directory +- Git Repository _Note: the local directory option refers to a directory within the kaniko container. If you wish to use this option, you will need to mount in your build context into the container as a directory._ @@ -114,12 +116,12 @@ gsutil cp context.tar.gz gs:// When running kaniko, use the `--context` flag with the appropriate prefix to specify the location of your build context: -| Source | Prefix | -|---------|---------| -| Local Directory | dir://[path to a directory in the kaniko container] | -| GCS Bucket | gs://[bucket name]/[path to .tar.gz] | -| S3 Bucket | s3://[bucket name]/[path to .tar.gz] | -| Git Repository | git://[repository url] | +| Source | Prefix | Example | +|---------|---------|---------| +| Local Directory | dir://[path to a directory in the kaniko container] | `dir:///workspace` | +| GCS Bucket | gs://[bucket name]/[path to .tar.gz] | `gs://kaniko-bucket/path/to/context.tar.gz` | +| S3 Bucket | s3://[bucket name]/[path to .tar.gz] | `s3://kaniko-bucket/path/to/context.tar.gz` | +| Git Repository | git://[repository url][#reference] | `git://github.com/acme/myproject.git#refs/heads/mybranch` | If you don't specify a prefix, kaniko will assume a local directory. For example, to use a GCS bucket called `kaniko-bucket`, you would pass in `--context=gs://kaniko-bucket/path/to/context.tar.gz`. @@ -373,6 +375,19 @@ will write the digest to that file, which is picked up by Kubernetes automatically as the `{{.state.terminated.message}}` of the container. +#### --oci-layout-path + +Set this flag to specify a directory in the container where the OCI image +layout of a built image will be placed. This can be used to automatically +track the exact image built by Kaniko. + +For example, to surface the image digest built in a +[Tekton task](https://github.com/tektoncd/pipeline/blob/v0.6.0/docs/resources.md#surfacing-the-image-digest-built-in-a-task), +this flag should be set to match the image resource `outputImageDir`. + +_Note: Depending on the built image, the media type of the image manifest might be either +`application/vnd.oci.image.manifest.v1+json` or `application/vnd.docker.distribution.manifest.v2+json``._ + #### --insecure-registry Set this flag to use plain HTTP requests when accessing a registry. It is supposed to be used for testing purposes only and should not be used in production! @@ -415,6 +430,10 @@ Set this flag to skip TLS certificate validation when pushing to a registry. It Set this flag to skip TLS certificate validation when pulling from a registry. It is supposed to be used for testing purposes only and should not be used in production! +#### --skip-tls-verify-pull + +Set this flag to skip TLS certificate validation when pulling from a registry. It is supposed to be used for testing purposes only and should not be used in production! + #### --snapshotMode You can set the `--snapshotMode=` flag to set how kaniko will snapshot the filesystem. diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 1fc2672bc..84fea6e8f 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -129,6 +129,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided") RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.") RootCmd.PersistentFlags().StringVarP(&opts.DigestFile, "digest-file", "", "", "Specify a file to save the digest of the built image to.") + RootCmd.PersistentFlags().StringVarP(&opts.OCILayoutPath, "oci-layout-path", "", "", "Path to save the OCI image layout of the built image.") RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end") RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.") @@ -208,12 +209,12 @@ func resolveSourceContext() error { } if opts.Bucket != "" { if !strings.Contains(opts.Bucket, "://") { + // if no prefix use Google Cloud Storage as default for backwards compatibility opts.SrcContext = constants.GCSBuildContextPrefix + opts.Bucket } else { opts.SrcContext = opts.Bucket } } - // if no prefix use Google Cloud Storage as default for backwards compatibility contextExecutor, err := buildcontext.GetBuildContext(opts.SrcContext) if err != nil { return err diff --git a/cmd/warmer/cmd/root.go b/cmd/warmer/cmd/root.go index 0e4908d2b..207c0519a 100644 --- a/cmd/warmer/cmd/root.go +++ b/cmd/warmer/cmd/root.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "os" + "time" "github.com/GoogleContainerTools/kaniko/pkg/cache" "github.com/GoogleContainerTools/kaniko/pkg/config" @@ -51,6 +52,12 @@ var RootCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { + if _, err := os.Stat(opts.CacheDir); os.IsNotExist(err) { + err = os.MkdirAll(opts.CacheDir, 0755) + if err != nil { + exit(errors.Wrap(err, "Failed to create cache directory")) + } + } if err := cache.WarmCache(opts); err != nil { exit(errors.Wrap(err, "Failed warming cache")) } @@ -61,6 +68,8 @@ var RootCmd = &cobra.Command{ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().VarP(&opts.Images, "image", "i", "Image to cache. Set it repeatedly for multiple images.") RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "c", "/cache", "Directory of the cache.") + RootCmd.PersistentFlags().BoolVarP(&opts.Force, "force", "f", false, "Force cache overwriting.") + RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.") } // addHiddenFlags marks certain flags as hidden from the executor help text diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index 51cf02a17..1feb086b7 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -25,7 +25,7 @@ RUN docker-credential-gcr configure-docker RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 COPY . . -RUN make +RUN make && make out/warmer # Stage 1: Get the busybox shell FROM gcr.io/cloud-builders/bazel:latest @@ -35,7 +35,7 @@ RUN bazel build //experimental/busybox:busybox_tar RUN tar -C /distroless/bazel-genfiles/experimental/busybox/ -xf /distroless/bazel-genfiles/experimental/busybox/busybox.tar FROM scratch -COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/executor /kaniko/executor +COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/* /kaniko/ COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login COPY --from=1 /distroless/bazel-genfiles/experimental/busybox/busybox/ /busybox/ diff --git a/integration/dockerfiles/Dockerfile_git_buildcontext b/integration/dockerfiles/Dockerfile_git_buildcontext new file mode 100644 index 000000000..9636fcd3d --- /dev/null +++ b/integration/dockerfiles/Dockerfile_git_buildcontext @@ -0,0 +1,2 @@ +FROM scratch +COPY LICENSE ./LICENSE diff --git a/integration/integration_test.go b/integration/integration_test.go index c11dd1378..fcd6246f9 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -278,6 +278,49 @@ func TestGitBuildcontext(t *testing.T) { checkContainerDiffOutput(t, diff, expected) } +func TestGitBuildContextWithBranch(t *testing.T) { + repo := "github.com/GoogleContainerTools/kaniko#refs/tags/v0.10.0" + dockerfile := "integration/dockerfiles/Dockerfile_test_run_2" + + // Build with docker + dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_test_git") + dockerCmd := exec.Command("docker", + append([]string{"build", + "-t", dockerImage, + "-f", dockerfile, + repo})...) + out, err := RunCommandWithoutTest(dockerCmd) + if err != nil { + t.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) + } + + // Build with kaniko + kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_test_git") + kanikoCmd := exec.Command("docker", + append([]string{"run", + "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", + ExecutorImage, + "-f", dockerfile, + "-d", kanikoImage, + "-c", fmt.Sprintf("git://%s", repo)})...) + + out, err = RunCommandWithoutTest(kanikoCmd) + if err != nil { + t.Errorf("Failed to build image %s with kaniko command \"%s\": %v %s", dockerImage, kanikoCmd.Args, err, string(out)) + } + + // container-diff + daemonDockerImage := daemonPrefix + dockerImage + containerdiffCmd := exec.Command("container-diff", "diff", "--no-cache", + daemonDockerImage, kanikoImage, + "-q", "--type=file", "--type=metadata", "--json") + diff := RunCommand(containerdiffCmd, t) + t.Logf("diff = %s", string(diff)) + + expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage) + checkContainerDiffOutput(t, diff, expected) +} + func TestLayers(t *testing.T) { offset := map[string]int{ "Dockerfile_test_add": 11, diff --git a/pkg/buildcontext/git.go b/pkg/buildcontext/git.go index 1aa8691be..9908350b1 100644 --- a/pkg/buildcontext/git.go +++ b/pkg/buildcontext/git.go @@ -18,9 +18,11 @@ package buildcontext import ( "os" + "strings" "github.com/GoogleContainerTools/kaniko/pkg/constants" git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" ) // Git unifies calls to download and unpack the build context. @@ -31,9 +33,14 @@ type Git struct { // UnpackTarFromBuildContext will provide the directory where Git Repository is Cloned func (g *Git) UnpackTarFromBuildContext() (string, error) { directory := constants.BuildContextDir - _, err := git.PlainClone(directory, false, &git.CloneOptions{ - URL: "https://" + g.context, + parts := strings.Split(g.context, "#") + options := git.CloneOptions{ + URL: "https://" + parts[0], Progress: os.Stdout, - }) + } + if len(parts) > 1 { + options.ReferenceName = plumbing.ReferenceName(parts[1]) + } + _, err := git.PlainClone(directory, false, &options) return directory, err } diff --git a/pkg/buildcontext/s3.go b/pkg/buildcontext/s3.go index 5b77d9c4a..93d404c45 100644 --- a/pkg/buildcontext/s3.go +++ b/pkg/buildcontext/s3.go @@ -19,6 +19,7 @@ package buildcontext import ( "os" "path/filepath" + "strings" "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -36,9 +37,21 @@ type S3 struct { // UnpackTarFromBuildContext download and untar a file from s3 func (s *S3) UnpackTarFromBuildContext() (string, error) { bucket, item := util.GetBucketAndItem(s.context) - sess, err := session.NewSessionWithOptions(session.Options{ + option := session.Options{ SharedConfigState: session.SharedConfigEnable, - }) + } + endpoint := os.Getenv(constants.S3EndpointEnv) + forcePath := false + if strings.ToLower(os.Getenv(constants.S3ForcePathStyle)) == "true" { + forcePath = true + } + if endpoint != "" { + option.Config = aws.Config{ + Endpoint: aws.String(endpoint), + S3ForcePathStyle: aws.Bool(forcePath), + } + } + sess, err := session.NewSessionWithOptions(option) if err != nil { return bucket, err } diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 33c7cae5b..0953a28f6 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -60,7 +60,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) { registryName := cacheRef.Repository.Registry.Name() if rc.Opts.Insecure || rc.Opts.InsecureRegistries.Contains(registryName) { - newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation) + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return nil, err } @@ -88,7 +88,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) { // Layer is stale, rebuild it. if expiry.Before(time.Now()) { logrus.Infof("Cache entry expired: %s", cache) - return nil, errors.New(fmt.Sprintf("Cache entry expired: %s", cache)) + return nil, fmt.Errorf("Cache entry expired: %s", cache) } // Force the manifest to be populated @@ -114,7 +114,7 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { } // LocalSource retieves a source image from a local cache given cacheKey -func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { +func LocalSource(opts *config.CacheOptions, cacheKey string) (v1.Image, error) { cache := opts.CacheDir if cache == "" { return nil, nil diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go index c03746e0b..abdaec76d 100644 --- a/pkg/cache/warm.go +++ b/pkg/cache/warm.go @@ -29,6 +29,7 @@ import ( "github.com/sirupsen/logrus" ) +// WarmCache populates the cache func WarmCache(opts *config.WarmerOptions) error { cacheDir := opts.CacheDir images := opts.Images @@ -41,7 +42,7 @@ func WarmCache(opts *config.WarmerOptions) error { return errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image)) } img, err := remote.Image(cacheRef) - if err != nil { + if err != nil || img == nil { return errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image)) } @@ -50,6 +51,14 @@ func WarmCache(opts *config.WarmerOptions) error { return errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image)) } cachePath := path.Join(cacheDir, digest.String()) + + if !opts.Force { + _, err := LocalSource(&opts.CacheOptions, digest.String()) + if err == nil { + continue + } + } + err = tarball.WriteToFile(cachePath, cacheRef, img) if err != nil { return errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image)) diff --git a/pkg/commands/add.go b/pkg/commands/add.go index 72f97653c..769d050dd 100644 --- a/pkg/commands/add.go +++ b/pkg/commands/add.go @@ -43,7 +43,7 @@ type AddCommand struct { // - If remote file has HTTP Last-Modified header, we set the mtime of the file to that timestamp // - If dest doesn't end with a slash, the filepath is inferred to be / // 2. If is a local tar archive: -// -If is a local tar archive, it is unpacked at the dest, as 'tar -x' would +// - it is unpacked at the dest, as 'tar -x' would func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { replacementEnvs := buildArgs.ReplacementEnvs(config.Env) @@ -71,8 +71,12 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui } a.snapshotFiles = append(a.snapshotFiles, urlDest) } else if util.IsFileLocalTarArchive(fullPath) { - logrus.Infof("Unpacking local tar archive %s to %s", src, dest) - extractedFiles, err := util.UnpackLocalTarArchive(fullPath, dest) + tarDest, err := util.DestinationFilepath("", dest, config.WorkingDir) + if err != nil { + return err + } + logrus.Infof("Unpacking local tar archive %s to %s", src, tarDest) + extractedFiles, err := util.UnpackLocalTarArchive(fullPath, tarDest) if err != nil { return err } diff --git a/pkg/commands/expose.go b/pkg/commands/expose.go index fa497f17d..9d56ee3ea 100644 --- a/pkg/commands/expose.go +++ b/pkg/commands/expose.go @@ -54,7 +54,7 @@ func (r *ExposeCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile. } protocol := strings.Split(p, "/")[1] if !validProtocol(protocol) { - return fmt.Errorf("Invalid protocol: %s", protocol) + return fmt.Errorf("invalid protocol: %s", protocol) } logrus.Infof("Adding exposed port: %s", p) existingPorts[p] = struct{}{} diff --git a/pkg/commands/volume.go b/pkg/commands/volume.go index 2ec0fcb2b..b6d92bf53 100644 --- a/pkg/commands/volume.go +++ b/pkg/commands/volume.go @@ -54,7 +54,7 @@ func (v *VolumeCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile. if _, err := os.Stat(volume); os.IsNotExist(err) { logrus.Infof("Creating directory %s", volume) if err := os.MkdirAll(volume, 0755); err != nil { - return fmt.Errorf("Could not create directory for volume %s: %s", volume, err) + return fmt.Errorf("could not create directory for volume %s: %s", volume, err) } } } diff --git a/pkg/config/options.go b/pkg/config/options.go index ff4d60a13..44af681ec 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -20,8 +20,15 @@ import ( "time" ) +// CacheOptions are base image cache options that are set by command line arguments +type CacheOptions struct { + CacheDir string + CacheTTL time.Duration +} + // KanikoOptions are options that are set by command line arguments type KanikoOptions struct { + CacheOptions DockerfilePath string SrcContext string SnapshotMode string @@ -29,8 +36,8 @@ type KanikoOptions struct { TarPath string Target string CacheRepo string - CacheDir string DigestFile string + OCILayoutPath string Destinations multiArg BuildArgs multiArg Insecure bool @@ -42,13 +49,13 @@ type KanikoOptions struct { NoPush bool Cache bool Cleanup bool - CacheTTL time.Duration InsecureRegistries multiArg SkipTLSVerifyRegistries multiArg } // WarmerOptions are options that are set by command line arguments to the cache warmer. type WarmerOptions struct { - Images multiArg - CacheDir string + CacheOptions + Images multiArg + Force bool } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 0eb29cb9d..d0d84e3f3 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -70,6 +70,10 @@ const ( // Name of the .dockerignore file Dockerignore = ".dockerignore" + + // S3 Custom endpoint ENV name + S3EndpointEnv = "S3_ENDPOINT" + S3ForcePathStyle = "S3_FORCE_PATH_STYLE" ) // ScratchEnvVars are the default environment variables needed for a scratch image. diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 331219159..8865b17d0 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -111,7 +111,7 @@ func Parse(b []byte) ([]instructions.Stage, []instructions.ArgCommand, error) { if err != nil { return nil, nil, err } - return stages, metaArgs, err + return stages, metaArgs, nil } // targetStage returns the index of the target stage kaniko is trying to build diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 6626577cf..e134e52b5 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -124,7 +124,7 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { return nil, err } - if img == empty.Image { + if imageConfig.Config.Env == nil { imageConfig.Config.Env = constants.ScratchEnvVars } return imageConfig, nil @@ -191,6 +191,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro func (s *stageBuilder) build() error { // Set the initial cache key to be the base image digest, the build args and the SrcContext. compositeKey := NewCompositeCache(s.baseImageDigest) + compositeKey.AddKey(s.opts.BuildArgs...) // Apply optimizations to the instructions. if err := s.optimize(*compositeKey, s.cf.Config); err != nil { diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index c48e4bb26..44f6a1211 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -29,6 +29,8 @@ import ( "github.com/GoogleContainerTools/kaniko/testutil" "github.com/google/go-cmp/cmp" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) @@ -405,3 +407,58 @@ func Test_filesToSave(t *testing.T) { }) } } + +func TestInitializeConfig(t *testing.T) { + tests := []struct { + description string + cfg v1.ConfigFile + expected v1.Config + }{ + { + description: "env is not set in the image", + cfg: v1.ConfigFile{ + Config: v1.Config{ + Image: "test", + }, + }, + expected: v1.Config{ + Image: "test", + Env: []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + }, + }, + { + description: "env is set in the image", + cfg: v1.ConfigFile{ + Config: v1.Config{ + Env: []string{ + "PATH=/usr/local/something", + }, + }, + }, + expected: v1.Config{ + Env: []string{ + "PATH=/usr/local/something", + }, + }, + }, + { + description: "image is empty", + expected: v1.Config{ + Env: []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + }, + }, + } + for _, tt := range tests { + img, err := mutate.ConfigFile(empty.Image, &tt.cfg) + if err != nil { + t.Errorf("error seen when running test %s", err) + t.Fail() + } + actual, _ := initializeConfig(img) + testutil.CheckDeepEqual(t, tt.expected, actual.Config) + } +} diff --git a/pkg/executor/foo b/pkg/executor/foo deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkg/executor/push.go b/pkg/executor/push.go index a02d5b9b9..e0ebf2111 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -21,6 +21,8 @@ import ( "fmt" "io/ioutil" "net/http" + "os" + "strings" "time" "github.com/GoogleContainerTools/kaniko/pkg/cache" @@ -32,6 +34,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" @@ -43,8 +46,16 @@ type withUserAgent struct { t http.RoundTripper } +const ( + UpstreamClientUaKey = "UPSTREAM_CLIENT_TYPE" +) + func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) { - r.Header.Set("User-Agent", fmt.Sprintf("kaniko/%s", version.Version())) + ua := []string{fmt.Sprintf("kaniko/%s", version.Version())} + if upstream := os.Getenv(UpstreamClientUaKey); upstream != "" { + ua = append(ua, upstream) + } + r.Header.Set("User-Agent", strings.Join(ua, ",")) return w.t.RoundTrip(r) } @@ -64,7 +75,10 @@ func CheckPushPermissions(opts *config.KanikoOptions) error { if checked[destRef.Context().RepositoryStr()] { continue } - if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), http.DefaultTransport); err != nil { + + registryName := destRef.Repository.Registry.Name() + tr := makeTransport(opts, registryName) + if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), tr); err != nil { return errors.Wrapf(err, "checking push permission for %q", destRef) } checked[destRef.Context().RepositoryStr()] = true @@ -88,6 +102,16 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { } } + if opts.OCILayoutPath != "" { + path, err := layout.Write(opts.OCILayoutPath, empty.Index) + if err != nil { + return errors.Wrap(err, "writing empty layout") + } + if err := path.AppendImage(image); err != nil { + return errors.Wrap(err, "appending image") + } + } + destRefs := []name.Tag{} for _, destination := range opts.Destinations { destRef, err := name.NewTag(destination, name.WeakValidation) @@ -114,7 +138,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { for _, destRef := range destRefs { registryName := destRef.Repository.Registry.Name() if opts.Insecure || opts.InsecureRegistries.Contains(registryName) { - newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation) + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return errors.Wrap(err, "getting new insecure registry") } @@ -126,16 +150,10 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { return errors.Wrap(err, "resolving pushAuth") } - // Create a transport to set our user-agent. - tr := http.DefaultTransport - if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) { - tr.(*http.Transport).TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } + tr := makeTransport(opts, registryName) rt := &withUserAgent{t: tr} - if err := remote.Write(destRef, image, pushAuth, rt); err != nil { + if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil { return errors.Wrap(err, fmt.Sprintf("failed to push to destination %s", destRef)) } } @@ -143,6 +161,17 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { return nil } +func makeTransport(opts *config.KanikoOptions, registryName string) http.RoundTripper { + // Create a transport to set our user-agent. + tr := http.DefaultTransport + if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) { + tr.(*http.Transport).TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } + return tr +} + // pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache // if opts.Cache doesn't exist, infer the cache from the given destination func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error { diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go new file mode 100644 index 000000000..63857f4f0 --- /dev/null +++ b/pkg/executor/push_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "testing" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/testutil" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/validate" +) + +func TestHeaderAdded(t *testing.T) { + tests := []struct { + name string + upstream string + expected string + }{{ + name: "upstream env variable set", + upstream: "skaffold-v0.25.45", + expected: "kaniko/unset,skaffold-v0.25.45", + }, { + name: "upstream env variable not set", + expected: "kaniko/unset", + }, + } + for _, test := range tests { + + t.Run(test.name, func(t *testing.T) { + rt := &withUserAgent{t: &mockRoundTripper{}} + if test.upstream != "" { + os.Setenv("UPSTREAM_CLIENT_TYPE", test.upstream) + defer func() { os.Unsetenv("UPSTREAM_CLIENT_TYPE") }() + } + req, err := http.NewRequest("GET", "dummy", nil) + if err != nil { + t.Fatalf("culd not create a req due to %s", err) + } + resp, err := rt.RoundTrip(req) + testutil.CheckError(t, false, err) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expected, string(body)) + }) + } + +} + +type mockRoundTripper struct { +} + +func (m *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + ua := r.UserAgent() + return &http.Response{Body: ioutil.NopCloser(bytes.NewBufferString(ua))}, nil +} + +func TestOCILayoutPath(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("could not create temp dir: %s", err) + } + defer os.RemoveAll(tmpDir) + + image, err := random.Image(1024, 4) + if err != nil { + t.Fatalf("could not create image: %s", err) + } + + digest, err := image.Digest() + if err != nil { + t.Fatalf("could not get image digest: %s", err) + } + + want, err := image.Manifest() + if err != nil { + t.Fatalf("could not get image manifest: %s", err) + } + + opts := config.KanikoOptions{ + NoPush: true, + OCILayoutPath: tmpDir, + } + + if err := DoPush(image, &opts); err != nil { + t.Fatalf("could not push image: %s", err) + } + + layoutIndex, err := layout.ImageIndexFromPath(tmpDir) + if err != nil { + t.Fatalf("could not get index from layout: %s", err) + } + testutil.CheckError(t, false, validate.Index(layoutIndex)) + + layoutImage, err := layoutIndex.Image(digest) + if err != nil { + t.Fatalf("could not get image from layout: %s", err) + } + + got, err := layoutImage.Manifest() + testutil.CheckErrorAndDeepEqual(t, false, err, want, got) +} diff --git a/pkg/snapshot/layered_map.go b/pkg/snapshot/layered_map.go index ad2bee4d2..56e8da4f0 100644 --- a/pkg/snapshot/layered_map.go +++ b/pkg/snapshot/layered_map.go @@ -103,13 +103,13 @@ func (l *LayeredMap) Add(s string) error { // Use hash function and add to layers newV, err := l.hasher(s) if err != nil { - return fmt.Errorf("Error creating hash for %s: %v", s, err) + return fmt.Errorf("error creating hash for %s: %v", s, err) } l.layers[len(l.layers)-1][s] = newV return nil } -// CheckFileChange checkes whether a given file changed +// CheckFileChange checks whether a given file changed // from the current layered map by its hashing function. // Returns true if the file is changed. func (l *LayeredMap) CheckFileChange(s string) (bool, error) { diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 90638168d..5577da087 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "path/filepath" + "sort" "syscall" "github.com/GoogleContainerTools/kaniko/pkg/timing" @@ -80,7 +81,7 @@ func (s *Snapshotter) TakeSnapshot(files []string) (string, error) { // Add files to the layered map for _, file := range filesToAdd { if err := s.l.Add(file); err != nil { - return "", fmt.Errorf("Unable to add file %s to layered map: %s", file, err) + return "", fmt.Errorf("unable to add file %s to layered map: %s", file, err) } } @@ -183,13 +184,15 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { } } - // Also add parent directories to keep the permission of them correctly. + // Also add parent directories to keep their permissions correctly. filesToAdd = filesWithParentDirs(filesToAdd) + sort.Strings(filesToAdd) + // Add files to the layered map for _, file := range filesToAdd { if err := s.l.Add(file); err != nil { - return nil, nil, fmt.Errorf("Unable to add file %s to layered map: %s", file, err) + return nil, nil, fmt.Errorf("unable to add file %s to layered map: %s", file, err) } } diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index ea6f4bceb..798ae6c09 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "sort" + "strings" "testing" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -31,6 +32,7 @@ import ( func TestSnapshotFSFileChange(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { t.Fatal(err) @@ -55,16 +57,16 @@ func TestSnapshotFSFileChange(t *testing.T) { } // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles tr := tar.NewReader(f) - fooPath := filepath.Join(testDir, "foo") - batPath := filepath.Join(testDir, "bar/bat") + fooPath := filepath.Join(testDirWithoutLeadingSlash, "foo") + batPath := filepath.Join(testDirWithoutLeadingSlash, "bar/bat") snapshotFiles := map[string]string{ fooPath: "newbaz1", batPath: "baz", } - for _, dir := range util.ParentDirectories(fooPath) { + for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(fooPath) { snapshotFiles[dir] = "" } - for _, dir := range util.ParentDirectories(batPath) { + for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) { snapshotFiles[dir] = "" } numFiles := 0 @@ -87,14 +89,55 @@ func TestSnapshotFSFileChange(t *testing.T) { } } +func TestSnapshotFSIsReproducible(t *testing.T) { + testDir, snapshotter, cleanup, err := setUpTestDir() + defer cleanup() + if err != nil { + t.Fatal(err) + } + // Make some changes to the filesystem + newFiles := map[string]string{ + "foo": "newbaz1", + "bar/bat": "baz", + } + if err := testutil.SetupFiles(testDir, newFiles); err != nil { + t.Fatalf("Error setting up fs: %s", err) + } + // Take another snapshot + tarPath, err := snapshotter.TakeSnapshotFS() + if err != nil { + t.Fatalf("Error taking snapshot of fs: %s", err) + } + + f, err := os.Open(tarPath) + if err != nil { + t.Fatal(err) + } + // Check contents of the snapshot, make sure contents are sorted by name + tr := tar.NewReader(f) + var filesInTar []string + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + filesInTar = append(filesInTar, hdr.Name) + } + if !sort.StringsAreSorted(filesInTar) { + t.Fatalf("Expected the file in the tar archive were sorted, actual list was not sorted: %v", filesInTar) + } +} + func TestSnapshotFSChangePermissions(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { t.Fatal(err) } // Change permissions on a file batPath := filepath.Join(testDir, "bar/bat") + batPathWithoutLeadingSlash := filepath.Join(testDirWithoutLeadingSlash, "bar/bat") if err := os.Chmod(batPath, 0600); err != nil { t.Fatalf("Error changing permissions on %s: %v", batPath, err) } @@ -110,9 +153,9 @@ func TestSnapshotFSChangePermissions(t *testing.T) { // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles tr := tar.NewReader(f) snapshotFiles := map[string]string{ - batPath: "baz2", + batPathWithoutLeadingSlash: "baz2", } - for _, dir := range util.ParentDirectories(batPath) { + for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) { snapshotFiles[dir] = "" } numFiles := 0 @@ -121,6 +164,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) { if err == io.EOF { break } + t.Logf("Info %s in tar", hdr.Name) numFiles++ if _, isFile := snapshotFiles[hdr.Name]; !isFile { t.Fatalf("File %s unexpectedly in tar", hdr.Name) @@ -137,6 +181,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) { func TestSnapshotFiles(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { t.Fatal(err) @@ -158,9 +203,9 @@ func TestSnapshotFiles(t *testing.T) { defer os.Remove(tarPath) expectedFiles := []string{ - filepath.Join(testDir, "foo"), + filepath.Join(testDirWithoutLeadingSlash, "foo"), } - expectedFiles = append(expectedFiles, util.ParentDirectories(filepath.Join(testDir, "foo"))...) + expectedFiles = append(expectedFiles, util.ParentDirectoriesWithoutLeadingSlash(filepath.Join(testDir, "foo"))...) f, err := os.Open(tarPath) if err != nil { diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go index a38972ced..6d9471ebd 100644 --- a/pkg/util/command_util.go +++ b/pkg/util/command_util.go @@ -55,7 +55,7 @@ func ResolveEnvironmentReplacementList(values, envs []string, isFilepath bool) ( // ResolveEnvironmentReplacement resolves replacing env variables in some text from envs // It takes in a string representation of the command, the value to be resolved, and a list of envs (config.Env) -// Ex: fp = $foo/newdir, envs = [foo=/foodir], then this should return /foodir/newdir +// Ex: value = $foo/newdir, envs = [foo=/foodir], then this should return /foodir/newdir // The dockerfile/shell package handles processing env values // It handles escape characters and supports expansion from the config.Env array // Shlex handles some of the following use cases (these and more are tested in integration tests) @@ -325,13 +325,12 @@ func GetUserFromUsername(userStr string, groupStr string) (string, string, error // Lookup by username userObj, err := user.Lookup(userStr) if err != nil { - if _, ok := err.(user.UnknownUserError); ok { - // Lookup by id - userObj, err = user.LookupId(userStr) - if err != nil { - return "", "", err - } - } else { + if _, ok := err.(user.UnknownUserError); !ok { + return "", "", err + } + // Lookup by id + userObj, err = user.LookupId(userStr) + if err != nil { return "", "", err } } @@ -341,12 +340,11 @@ func GetUserFromUsername(userStr string, groupStr string) (string, string, error if groupStr != "" { group, err = user.LookupGroup(groupStr) if err != nil { - if _, ok := err.(user.UnknownGroupError); ok { - group, err = user.LookupGroupId(groupStr) - if err != nil { - return "", "", err - } - } else { + if _, ok := err.(user.UnknownGroupError); !ok { + return "", "", err + } + group, err = user.LookupGroupId(groupStr) + if err != nil { return "", "", err } } diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 5f72c7784..4abfcd4a4 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -188,7 +188,7 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error { switch hdr.Typeflag { case tar.TypeReg: logrus.Debugf("creating file %s", path) - // It's possible a file is in the tar before it's directory. + // It's possible a file is in the tar before its directory. if _, err := os.Stat(dir); os.IsNotExist(err) { logrus.Debugf("base %s for file %s does not exist. Creating.", base, path) if err := os.MkdirAll(dir, 0755); err != nil { @@ -288,12 +288,7 @@ func checkWhitelistRoot(root string) bool { if root == constants.RootDir { return false } - for _, wl := range whitelist { - if HasFilepathPrefix(root, wl.Path, wl.PrefixMatchOnly) { - return true - } - } - return false + return CheckWhitelist(root) } // Get whitelist from roots of mounted files @@ -382,6 +377,24 @@ func ParentDirectories(path string) []string { return paths } +// ParentDirectoriesWithoutLeadingSlash returns a list of paths to all parent directories +// all subdirectories do not contain a leading / +// Ex. /some/temp/dir -> [/, some, some/temp, some/temp/dir] +func ParentDirectoriesWithoutLeadingSlash(path string) []string { + path = filepath.Clean(path) + dirs := strings.Split(path, "/") + dirPath := "" + paths := []string{constants.RootDir} + for index, dir := range dirs { + if dir == "" || index == (len(dirs)-1) { + continue + } + dirPath = filepath.Join(dirPath, dir) + paths = append(paths, dirPath) + } + return paths +} + // FilepathExists returns true if the path exists func FilepathExists(path string) bool { _, err := os.Lstat(path) diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index eff39d5d9..c44908056 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -177,6 +177,38 @@ func Test_ParentDirectories(t *testing.T) { } } +func Test_ParentDirectoriesWithoutLeadingSlash(t *testing.T) { + tests := []struct { + name string + path string + expected []string + }{ + { + name: "regular path", + path: "/path/to/dir", + expected: []string{ + "/", + "path", + "path/to", + }, + }, + { + name: "current directory", + path: ".", + expected: []string{ + "/", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := ParentDirectoriesWithoutLeadingSlash(tt.path) + testutil.CheckErrorAndDeepEqual(t, false, nil, tt.expected, actual) + }) + } +} + func Test_CheckWhitelist(t *testing.T) { type args struct { path string diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 6d7ed8a8d..0978a6fef 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -102,7 +102,7 @@ func remoteImage(image string, opts *config.KanikoOptions) (v1.Image, error) { registryName := ref.Context().RegistryStr() if opts.InsecurePull || opts.InsecureRegistries.Contains(registryName) { - newReg, err := name.NewInsecureRegistry(registryName, name.WeakValidation) + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return nil, err } @@ -149,5 +149,5 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { cacheKey = d.String() } - return cache.LocalSource(opts, cacheKey) + return cache.LocalSource(&opts.CacheOptions, cacheKey) } diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go index bc1cc67a0..213ef68e3 100644 --- a/pkg/util/tar_util.go +++ b/pkg/util/tar_util.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "syscall" "github.com/docker/docker/pkg/archive" @@ -74,7 +75,14 @@ func (t *Tar) AddFileToTar(p string) error { if err != nil { return err } - hdr.Name = p + + if p != "/" { + // Docker uses no leading / in the tarball + hdr.Name = strings.TrimLeft(p, "/") + } else { + // allow entry for / to preserve permission changes etc. (currently ignored anyway by Docker runtime) + hdr.Name = p + } hardlink, linkDst := t.checkHardlink(p, i) if hardlink { @@ -104,7 +112,8 @@ func (t *Tar) Whiteout(p string) error { name := ".wh." + filepath.Base(p) th := &tar.Header{ - Name: filepath.Join(dir, name), + // Docker uses no leading / in the tarball + Name: strings.TrimLeft(filepath.Join(dir, name), "/"), Size: 0, } if err := t.w.WriteHeader(th); err != nil { diff --git a/run_in_docker.sh b/run_in_docker.sh index b3bdd4988..9397f1a71 100755 --- a/run_in_docker.sh +++ b/run_in_docker.sh @@ -15,8 +15,9 @@ #!/bin/bash set -e -if [ $# -lt 3 ]; - then echo "Usage: run_in_docker.sh " +if [ $# -lt 3 ]; then + echo "Usage: run_in_docker.sh " + exit 1 fi dockerfile=$1 diff --git a/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi b/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi deleted file mode 120000 index 5d7eddc7f..000000000 --- a/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi +++ /dev/null @@ -1 +0,0 @@ -../kenobi \ No newline at end of file diff --git a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go index aee1fedb9..8b2ead5db 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go +++ b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go @@ -19,11 +19,11 @@ import ( "errors" "fmt" "io/ioutil" - "log" "os" "path/filepath" "runtime" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" ) @@ -100,19 +100,19 @@ var ( func (dk *defaultKeychain) Resolve(reg name.Registry) (Authenticator, error) { dir, err := configDir() if err != nil { - log.Printf("Unable to determine config dir: %v", err) + logs.Warn.Printf("Unable to determine config dir: %v", err) return Anonymous, nil } file := filepath.Join(dir, "config.json") content, err := ioutil.ReadFile(file) if err != nil { - log.Printf("Unable to read %q: %v", file, err) + logs.Warn.Printf("Unable to read %q: %v", file, err) return Anonymous, nil } var cf cfg if err := json.Unmarshal(content, &cf); err != nil { - log.Printf("Unable to parse %q: %v", file, err) + logs.Warn.Printf("Unable to parse %q: %v", file, err) return Anonymous, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go b/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go new file mode 100644 index 000000000..87f730955 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go @@ -0,0 +1,68 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package retry provides methods for retrying operations. It is a thin wrapper +// around k8s.io/apimachinery/pkg/util/wait to make certain operations easier. +package retry + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/util/wait" +) + +// This is implemented by several errors in the net package as well as our +// transport.Error. +type temporary interface { + Temporary() bool +} + +// IsTemporary returns true if err implements Temporary() and it returns true. +func IsTemporary(err error) bool { + if te, ok := err.(temporary); ok && te.Temporary() { + return true + } + return false +} + +// IsNotNil returns true if err is not nil. +func IsNotNil(err error) bool { + return err != nil +} + +// Predicate determines whether an error should be retried. +type Predicate func(error) (retry bool) + +// Retry retries a given function, f, until a predicate is satisfied, using +// exponential backoff. If the predicate is never satisfied, it will return the +// last error returned by f. +func Retry(f func() error, p Predicate, backoff wait.Backoff) (err error) { + if f == nil { + return fmt.Errorf("nil f passed to retry") + } + if p == nil { + return fmt.Errorf("nil p passed to retry") + } + + condition := func() (bool, error) { + err = f() + if p(err) { + return false, nil + } + return true, err + } + + wait.ExponentialBackoff(backoff, condition) + return +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go b/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go new file mode 100644 index 000000000..af3c1a3b7 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go @@ -0,0 +1,29 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logs exposes the loggers used by this library. +package logs + +import ( + "io/ioutil" + "log" +) + +var ( + // Warn is used to log non-fatal errors. + Warn = log.New(ioutil.Discard, "", log.LstdFlags) + + // Progress is used to log notable, successful events. + Progress = log.New(ioutil.Discard, "", log.LstdFlags) +) diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/check.go b/vendor/github.com/google/go-containerregistry/pkg/name/check.go index 01a25d554..01b03e562 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/check.go @@ -19,15 +19,6 @@ import ( "unicode/utf8" ) -// Strictness defines the level of strictness for name validation. -type Strictness int - -// Enums for CRUD operations. -const ( - StrictValidation Strictness = iota - WeakValidation -) - // stripRunesFn returns a function which returns -1 (i.e. a value which // signals deletion in strings.Map) for runes in 'runes', and the rune otherwise. func stripRunesFn(runes string) func(rune) rune { diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go index dc573ef1d..2dc0f7f37 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package name defines structured types for representing image references. package name import ( @@ -63,8 +62,8 @@ func checkDigest(name string) error { return checkElement("digest", name, digestChars, 7+64, 7+64) } -// NewDigest returns a new Digest representing the given name, according to the given strictness. -func NewDigest(name string, strict Strictness) (Digest, error) { +// NewDigest returns a new Digest representing the given name. +func NewDigest(name string, opts ...Option) (Digest, error) { // Split on "@" parts := strings.Split(name, digestDelim) if len(parts) != 2 { @@ -78,12 +77,12 @@ func NewDigest(name string, strict Strictness) (Digest, error) { return Digest{}, err } - tag, err := NewTag(base, strict) + tag, err := NewTag(base, opts...) if err == nil { base = tag.Repository.Name() } - repo, err := NewRepository(base, strict) + repo, err := NewRepository(base, opts...) if err != nil { return Digest{}, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/doc.go b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go new file mode 100644 index 000000000..b294794dc --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go @@ -0,0 +1,42 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package name defines structured types for representing image references. +// +// What's in a name? For image references, not nearly enough! +// +// Image references look a lot like URLs, but they differ in that they don't +// contain the scheme (http or https), they can end with a :tag or a @digest +// (the latter being validated), and they perform defaulting for missing +// components. +// +// Since image references don't contain the scheme, we do our best to infer +// if we use http or https from the given hostname. We allow http fallback for +// any host that looks like localhost (localhost, 127.0.0.1, ::1), ends in +// ".local", or is in the "private" address space per RFC 1918. For everything +// else, we assume https only. To override this heuristic, use the Insecure +// option. +// +// Image references with a digest signal to us that we should verify the content +// of the image matches the digest. E.g. when pulling a Digest reference, we'll +// calculate the sha256 of the manifest returned by the registry and error out +// if it doesn't match what we asked for. +// +// For defaulting, we interpret "ubuntu" as +// "index.docker.io/library/ubuntu:latest" because we add the missing repo +// "library", the missing registry "index.docker.io", and the missing tag +// "latest". To disable this defaulting, use the StrictValidation option. This +// is useful e.g. to only allow image references that explicitly set a tag or +// digest, so that you don't accidentally pull "latest". +package name diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/options.go b/vendor/github.com/google/go-containerregistry/pkg/name/options.go new file mode 100644 index 000000000..98beaae11 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/name/options.go @@ -0,0 +1,49 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package name + +type options struct { + strict bool // weak by default + insecure bool // secure by default +} + +func makeOptions(opts ...Option) options { + opt := options{} + for _, o := range opts { + o(&opt) + } + return opt +} + +// Option is a functional option for name parsing. +type Option func(*options) + +// StrictValidation is an Option that requires image references to be fully +// specified; i.e. no defaulting for registry (dockerhub), repo (library), +// or tag (latest). +func StrictValidation(opts *options) { + opts.strict = true +} + +// WeakValidation is an Option that sets defaults when parsing names, see +// StrictValidation. +func WeakValidation(opts *options) { + opts.strict = false +} + +// Insecure is an Option that allows image references to be fetched without TLS. +func Insecure(opts *options) { + opts.insecure = true +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go index 58775daa3..cca303405 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/ref.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/ref.go @@ -38,11 +38,11 @@ type Reference interface { } // ParseReference parses the string as a reference, either by tag or digest. -func ParseReference(s string, strict Strictness) (Reference, error) { - if t, err := NewTag(s, strict); err == nil { +func ParseReference(s string, opts ...Option) (Reference, error) { + if t, err := NewTag(s, opts...); err == nil { return t, nil } - if d, err := NewDigest(s, strict); err == nil { + if d, err := NewDigest(s, opts...); err == nil { return d, nil } // TODO: Combine above errors into something more useful? diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go index ab7419308..c12dd46c2 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go @@ -114,8 +114,9 @@ func checkRegistry(name string) error { // NewRegistry returns a Registry based on the given name. // Strict validation requires explicit, valid RFC 3986 URI authorities to be given. -func NewRegistry(name string, strict Strictness) (Registry, error) { - if strict == StrictValidation && len(name) == 0 { +func NewRegistry(name string, opts ...Option) (Registry, error) { + opt := makeOptions(opts...) + if opt.strict && len(name) == 0 { return Registry{}, NewErrBadName("strict validation requires the registry to be explicitly defined") } @@ -129,16 +130,13 @@ func NewRegistry(name string, strict Strictness) (Registry, error) { name = DefaultRegistry } - return Registry{registry: name}, nil + return Registry{registry: name, insecure: opt.insecure}, nil } // NewInsecureRegistry returns an Insecure Registry based on the given name. -// Strict validation requires explicit, valid RFC 3986 URI authorities to be given. -func NewInsecureRegistry(name string, strict Strictness) (Registry, error) { - reg, err := NewRegistry(name, strict) - if err != nil { - return Registry{}, err - } - reg.insecure = true - return reg, nil +// +// Deprecated: Use the Insecure Option with NewRegistry instead. +func NewInsecureRegistry(name string, opts ...Option) (Registry, error) { + opts = append(opts, Insecure) + return NewRegistry(name, opts...) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go index 43cc5b82b..f33377988 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go @@ -68,7 +68,8 @@ func checkRepository(repository string) error { } // NewRepository returns a new Repository representing the given name, according to the given strictness. -func NewRepository(name string, strict Strictness) (Repository, error) { +func NewRepository(name string, opts ...Option) (Repository, error) { + opt := makeOptions(opts...) if len(name) == 0 { return Repository{}, NewErrBadName("a repository name must be specified") } @@ -88,11 +89,11 @@ func NewRepository(name string, strict Strictness) (Repository, error) { return Repository{}, err } - reg, err := NewRegistry(registry, strict) + reg, err := NewRegistry(registry, opts...) if err != nil { return Repository{}, err } - if hasImplicitNamespace(repo, reg) && strict == StrictValidation { + if hasImplicitNamespace(repo, reg) && opt.strict { return Repository{}, NewErrBadName("strict validation requires the full repository path (missing 'library')") } return Repository{reg, repo}, nil diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/tag.go b/vendor/github.com/google/go-containerregistry/pkg/name/tag.go index b8375e1f9..e6cce34db 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/tag.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/tag.go @@ -71,7 +71,8 @@ func checkTag(name string) error { } // NewTag returns a new Tag representing the given name, according to the given strictness. -func NewTag(name string, strict Strictness) (Tag, error) { +func NewTag(name string, opts ...Option) (Tag, error) { + opt := makeOptions(opts...) base := name tag := "" @@ -87,13 +88,13 @@ func NewTag(name string, strict Strictness) (Tag, error) { // even when not being strict. // If we are being strict, we want to validate the tag regardless in case // it's empty. - if tag != "" || strict == StrictValidation { + if tag != "" || opt.strict { if err := checkTag(tag); err != nil { return Tag{}, err } } - repo, err := NewRepository(base, strict) + repo, err := NewRepository(base, opts...) if err != nil { return Tag{}, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/image.go index 17b9839a6..9ef026799 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/image.go @@ -19,6 +19,7 @@ import ( ) // Image defines the interface for interacting with an OCI v1 image. +//go:generate counterfeiter -o fake/image.go . Image type Image interface { // Layers returns the ordered collection of filesystem layers that comprise this image. // The order of the list is oldest/base layer first, and most-recent/top layer last. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/index.go index 604e6de36..b4e84e02a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/index.go @@ -19,6 +19,7 @@ import ( ) // ImageIndex defines the interface for interacting with an OCI image index. +//go:generate counterfeiter -o fake/index.go . ImageIndex type ImageIndex interface { // MediaType of this image's manifest. MediaType() (types.MediaType, error) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go index 8b5091e45..57447d263 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go @@ -16,6 +16,8 @@ package v1 import ( "io" + + "github.com/google/go-containerregistry/pkg/v1/types" ) // Layer is an interface for accessing the properties of a particular layer of a v1.Image @@ -34,4 +36,7 @@ type Layer interface { // Size returns the compressed size of the Layer. Size() (int64, error) + + // MediaType returns the media type of the Layer. + MediaType() (types.MediaType, error) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go new file mode 100644 index 000000000..ba90d4cdb --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go @@ -0,0 +1,38 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" + "io/ioutil" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Blob returns a blob with the given hash from the Path. +func (l Path) Blob(h v1.Hash) (io.ReadCloser, error) { + return os.Open(l.blobPath(h)) +} + +// Bytes is a convenience function to return a blob from the Path as +// a byte slice. +func (l Path) Bytes(h v1.Hash) ([]byte, error) { + return ioutil.ReadFile(l.blobPath(h)) +} + +func (l Path) blobPath(h v1.Hash) string { + return l.path("blobs", h.Algorithm, h.Hex) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go new file mode 100644 index 000000000..d80d27363 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go @@ -0,0 +1,19 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout provides facilities for reading/writing artifacts from/to +// an OCI image layout on disk, see: +// +// https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go new file mode 100644 index 000000000..7c76a10cb --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go @@ -0,0 +1,131 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "fmt" + "io" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type layoutImage struct { + path Path + desc v1.Descriptor + manifestLock sync.Mutex // Protects rawManifest + rawManifest []byte +} + +var _ partial.CompressedImageCore = (*layoutImage)(nil) + +// Image reads a v1.Image with digest h from the Path. +func (l Path) Image(h v1.Hash) (v1.Image, error) { + ii, err := l.ImageIndex() + if err != nil { + return nil, err + } + + return ii.Image(h) +} + +func (li *layoutImage) MediaType() (types.MediaType, error) { + return li.desc.MediaType, nil +} + +// Implements WithManifest for partial.Blobset. +func (li *layoutImage) Manifest() (*v1.Manifest, error) { + return partial.Manifest(li) +} + +func (li *layoutImage) RawManifest() ([]byte, error) { + li.manifestLock.Lock() + defer li.manifestLock.Unlock() + if li.rawManifest != nil { + return li.rawManifest, nil + } + + b, err := li.path.Bytes(li.desc.Digest) + if err != nil { + return nil, err + } + + li.rawManifest = b + return li.rawManifest, nil +} + +func (li *layoutImage) RawConfigFile() ([]byte, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + return li.path.Bytes(manifest.Config.Digest) +} + +func (li *layoutImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + if h == manifest.Config.Digest { + return partial.CompressedLayer(&compressedBlob{ + path: li.path, + desc: manifest.Config, + }), nil + } + + for _, desc := range manifest.Layers { + if h == desc.Digest { + switch desc.MediaType { + case types.OCILayer, types.DockerLayer: + return partial.CompressedToLayer(&compressedBlob{ + path: li.path, + desc: desc, + }) + default: + // TODO: We assume everything is a compressed blob, but that might not be true. + // TODO: Handle foreign layers. + return nil, fmt.Errorf("unexpected media type: %v for layer: %v", desc.MediaType, desc.Digest) + } + } + } + + return nil, fmt.Errorf("could not find layer in image: %s", h) +} + +type compressedBlob struct { + path Path + desc v1.Descriptor +} + +func (b *compressedBlob) Digest() (v1.Hash, error) { + return b.desc.Digest, nil +} + +func (b *compressedBlob) Compressed() (io.ReadCloser, error) { + return b.path.Blob(b.desc.Digest) +} + +func (b *compressedBlob) Size() (int64, error) { + return b.desc.Size, nil +} + +func (b *compressedBlob) MediaType() (types.MediaType, error) { + return b.desc.MediaType, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go new file mode 100644 index 000000000..aba139d9d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go @@ -0,0 +1,146 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var _ v1.ImageIndex = (*layoutIndex)(nil) + +type layoutIndex struct { + path Path + rawIndex []byte +} + +// ImageIndexFromPath is a convenience function which constructs a Path and returns its v1.ImageIndex. +func ImageIndexFromPath(path string) (v1.ImageIndex, error) { + lp, err := FromPath(path) + if err != nil { + return nil, err + } + return lp.ImageIndex() +} + +// ImageIndex returns a v1.ImageIndex for the Path. +func (l Path) ImageIndex() (v1.ImageIndex, error) { + rawIndex, err := ioutil.ReadFile(l.path("index.json")) + if err != nil { + return nil, err + } + + idx := &layoutIndex{ + path: l, + rawIndex: rawIndex, + } + + return idx, nil +} + +func (i *layoutIndex) MediaType() (types.MediaType, error) { + return types.OCIImageIndex, nil +} + +func (i *layoutIndex) Digest() (v1.Hash, error) { + return partial.Digest(i) +} + +func (i *layoutIndex) IndexManifest() (*v1.IndexManifest, error) { + var index v1.IndexManifest + err := json.Unmarshal(i.rawIndex, &index) + return &index, err +} + +func (i *layoutIndex) RawManifest() ([]byte, error) { + return i.rawIndex, nil +} + +func (i *layoutIndex) Image(h v1.Hash) (v1.Image, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIManifestSchema1, types.DockerManifestSchema2) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + img := &layoutImage{ + path: i.path, + desc: *desc, + } + return partial.CompressedToImage(img) +} + +func (i *layoutIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIImageIndex, types.DockerManifestList) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + rawIndex, err := i.path.Bytes(h) + if err != nil { + return nil, err + } + + return &layoutIndex{ + path: i.path, + rawIndex: rawIndex, + }, nil +} + +func (i *layoutIndex) Blob(h v1.Hash) (io.ReadCloser, error) { + return i.path.Blob(h) +} + +func (i *layoutIndex) findDescriptor(h v1.Hash) (*v1.Descriptor, error) { + im, err := i.IndexManifest() + if err != nil { + return nil, err + } + + for _, desc := range im.Manifests { + if desc.Digest == h { + return &desc, nil + } + } + + return nil, fmt.Errorf("could not find descriptor in index: %s", h) +} + +// TODO: Pull this out into methods on types.MediaType? e.g. instead, have: +// * mt.IsIndex() +// * mt.IsImage() +func isExpectedMediaType(mt types.MediaType, expected ...types.MediaType) bool { + for _, allowed := range expected { + if mt == allowed { + return true + } + } + return false +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go new file mode 100644 index 000000000..a031ff5ae --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go @@ -0,0 +1,25 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import "path/filepath" + +// Path represents an OCI image layout rooted in a file system path +type Path string + +func (l Path) path(elem ...string) string { + complete := []string{string(l)} + return filepath.Join(append(complete, elem...)...) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go new file mode 100644 index 000000000..5569e51de --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go @@ -0,0 +1,42 @@ +package layout + +import v1 "github.com/google/go-containerregistry/pkg/v1" + +// Option is a functional option for Layout. +// +// TODO: We'll need to change this signature to support Sparse/Thin images. +// Or, alternatively, wrap it in a sparse.Image that returns an empty list for layers? +type Option func(*v1.Descriptor) error + +// WithAnnotations adds annotations to the artifact descriptor. +func WithAnnotations(annotations map[string]string) Option { + return func(desc *v1.Descriptor) error { + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + for k, v := range annotations { + desc.Annotations[k] = v + } + + return nil + } +} + +// WithURLs adds urls to the artifact descriptor. +func WithURLs(urls []string) Option { + return func(desc *v1.Descriptor) error { + if desc.URLs == nil { + desc.URLs = []string{} + } + desc.URLs = append(desc.URLs, urls...) + return nil + } +} + +// WithPlatform sets the platform of the artifact descriptor. +func WithPlatform(platform v1.Platform) Option { + return func(desc *v1.Descriptor) error { + desc.Platform = &platform + return nil + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go new file mode 100644 index 000000000..796abc7dd --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go @@ -0,0 +1,32 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "os" + "path/filepath" +) + +// FromPath reads an OCI image layout at path and constructs a layout.Path. +func FromPath(path string) (Path, error) { + // TODO: check oci-layout exists + + _, err := os.Stat(filepath.Join(path, "index.json")) + if err != nil { + return "", err + } + + return Path(path), nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go new file mode 100644 index 000000000..2abb9a586 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go @@ -0,0 +1,301 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "golang.org/x/sync/errgroup" +) + +var layoutFile = `{ + "imageLayoutVersion": "1.0.0" +}` + +// AppendImage writes a v1.Image to the Path and updates +// the index.json to reference it. +func (l Path) AppendImage(img v1.Image, options ...Option) error { + if err := l.writeImage(img); err != nil { + return err + } + + mt, err := img.MediaType() + if err != nil { + return err + } + + d, err := img.Digest() + if err != nil { + return err + } + + manifest, err := img.RawManifest() + if err != nil { + return err + } + + desc := v1.Descriptor{ + MediaType: mt, + Size: int64(len(manifest)), + Digest: d, + } + + for _, opt := range options { + if err := opt(&desc); err != nil { + return err + } + } + + return l.AppendDescriptor(desc) +} + +// AppendIndex writes a v1.ImageIndex to the Path and updates +// the index.json to reference it. +func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error { + if err := l.writeIndex(ii); err != nil { + return err + } + + mt, err := ii.MediaType() + if err != nil { + return err + } + + d, err := ii.Digest() + if err != nil { + return err + } + + manifest, err := ii.RawManifest() + if err != nil { + return err + } + + desc := v1.Descriptor{ + MediaType: mt, + Size: int64(len(manifest)), + Digest: d, + } + + for _, opt := range options { + if err := opt(&desc); err != nil { + return err + } + } + + return l.AppendDescriptor(desc) +} + +// AppendDescriptor adds a descriptor to the index.json of the Path. +func (l Path) AppendDescriptor(desc v1.Descriptor) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + index.Manifests = append(index.Manifests, desc) + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.writeFile("index.json", rawIndex) +} + +func (l Path) writeFile(name string, data []byte) error { + if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + return ioutil.WriteFile(l.path(name), data, os.ModePerm) + +} + +// WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at +// blobs/{hash.Algorithm}/{hash.Hex}. +func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error { + dir := l.path("blobs", hash.Algorithm) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + file := filepath.Join(dir, hash.Hex) + if _, err := os.Stat(file); err == nil { + // Blob already exists, that's fine. + return nil + } + w, err := os.Create(file) + if err != nil { + return err + } + defer w.Close() + + _, err = io.Copy(w, r) + return err +} + +// TODO: A streaming version of WriteBlob so we don't have to know the hash +// before we write it. + +// TODO: For streaming layers we should write to a tmp file then Rename to the +// final digest. +func (l Path) writeLayer(layer v1.Layer) error { + d, err := layer.Digest() + if err != nil { + return err + } + + r, err := layer.Compressed() + if err != nil { + return err + } + + return l.WriteBlob(d, r) +} + +func (l Path) writeImage(img v1.Image) error { + layers, err := img.Layers() + if err != nil { + return err + } + + // Write the layers concurrently. + var g errgroup.Group + for _, layer := range layers { + layer := layer + g.Go(func() error { + return l.writeLayer(layer) + }) + } + if err := g.Wait(); err != nil { + return err + } + + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return err + } + cfgBlob, err := img.RawConfigFile() + if err != nil { + return err + } + if err := l.WriteBlob(cfgName, ioutil.NopCloser(bytes.NewReader(cfgBlob))); err != nil { + return err + } + + // Write the img manifest. + d, err := img.Digest() + if err != nil { + return err + } + manifest, err := img.RawManifest() + if err != nil { + return err + } + + return l.WriteBlob(d, ioutil.NopCloser(bytes.NewReader(manifest))) +} + +func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error { + index, err := ii.IndexManifest() + if err != nil { + return err + } + + // Walk the descriptors and write any v1.Image or v1.ImageIndex that we find. + // If we come across something we don't expect, just write it as a blob. + for _, desc := range index.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + ii, err := ii.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := l.writeIndex(ii); err != nil { + return err + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := ii.Image(desc.Digest) + if err != nil { + return err + } + if err := l.writeImage(img); err != nil { + return err + } + default: + // TODO: The layout could reference arbitrary things, which we should + // probably just pass through. + } + } + + rawIndex, err := ii.RawManifest() + if err != nil { + return err + } + + return l.writeFile(indexFile, rawIndex) +} + +func (l Path) writeIndex(ii v1.ImageIndex) error { + // Always just write oci-layout file, since it's small. + if err := l.writeFile("oci-layout", []byte(layoutFile)); err != nil { + return err + } + + h, err := ii.Digest() + if err != nil { + return err + } + + indexFile := filepath.Join("blobs", h.Algorithm, h.Hex) + return l.writeIndexToFile(indexFile, ii) + +} + +// Write constructs a Path at path from an ImageIndex. +// +// The contents are written in the following format: +// At the top level, there is: +// One oci-layout file containing the version of this image-layout. +// One index.json file listing descriptors for the contained images. +// Under blobs/, there is, for each image: +// One file for each layer, named after the layer's SHA. +// One file for each config blob, named after its SHA. +// One file for each manifest blob, named after its SHA. +func Write(path string, ii v1.ImageIndex) (Path, error) { + lp := Path(path) + // Always just write oci-layout file, since it's small. + if err := lp.writeFile("oci-layout", []byte(layoutFile)); err != nil { + return "", err + } + + // TODO create blobs/ in case there is a blobs file which would prevent the directory from being created + + return lp, lp.writeIndexToFile("index.json", ii) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index a327e7594..eafd599b9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -77,11 +77,14 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) { } cf.Config = cfg + // Downstream tooling expects these to match. + cf.ContainerConfig = cfg - return configFile(base, cf) + return ConfigFile(base, cf) } -func configFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { +// ConfigFile mutates the provided v1.Image to have the provided v1.ConfigFile +func ConfigFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) { m, err := base.Manifest() if err != nil { return nil, err @@ -106,7 +109,7 @@ func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) { cfg := cf.DeepCopy() cfg.Created = created - return configFile(base, cfg) + return ConfigFile(base, cfg) } type image struct { @@ -162,11 +165,9 @@ func (i *image) compute() error { manifest := m.DeepCopy() manifestLayers := manifest.Layers for _, add := range i.adds { - d := v1.Descriptor{ - MediaType: types.DockerLayer, - } - + d := v1.Descriptor{} var err error + if d.Size, err = add.Layer.Size(); err != nil { return err } @@ -175,6 +176,10 @@ func (i *image) compute() error { return err } + if d.MediaType, err = add.Layer.MediaType(); err != nil { + return err + } + manifestLayers = append(manifestLayers, d) digestMap[d.Digest] = add.Layer } @@ -467,7 +472,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { // Copy basic config over cfg.Config = ocf.Config - cfg.ContainerConfig = ocf.ContainerConfig + cfg.ContainerConfig = ocf.Config // Downstream tooling expects these to match. // Strip away timestamps from the config file cfg.Created = v1.Time{Time: t} @@ -476,7 +481,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { h.Created = v1.Time{Time: t} } - return configFile(newImage, cfg) + return ConfigFile(newImage, cfg) } func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { @@ -555,5 +560,5 @@ func Canonical(img v1.Image) (v1.Image, error) { cfg.ContainerConfig.Hostname = "" cfg.DockerVersion = "" - return configFile(img, cfg) + return ConfigFile(img, cfg) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go index 497d1af0d..1c631b7aa 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go @@ -18,6 +18,7 @@ import ( "io" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -32,6 +33,9 @@ type CompressedLayer interface { // Size returns the compressed size of the Layer. Size() (int64, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) } // compressedLayerExtender implements v1.Image using the compressed base properties. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go index 9f75723ec..07658be42 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go @@ -32,6 +32,9 @@ type UncompressedLayer interface { // Uncompressed returns an io.ReadCloser for the uncompressed layer contents. Uncompressed() (io.ReadCloser, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) } // uncompressedLayerExtender implements v1.Image using the uncompressed base properties. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go index f724ec8ab..992672048 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go @@ -22,6 +22,7 @@ import ( "io/ioutil" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -80,6 +81,12 @@ func (cl *configLayer) Size() (int64, error) { return int64(len(cl.content)), nil } +func (cl *configLayer) MediaType() (types.MediaType, error) { + // Defaulting this to OCIConfigJSON as it should remain + // backwards compatible with DockerConfigJSON + return types.OCIConfigJSON, nil +} + var _ v1.Layer = (*configLayer)(nil) // ConfigLayer implements v1.Layer from the raw config bytes. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go b/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go index df9b2959e..bb9886433 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go @@ -21,4 +21,5 @@ type Platform struct { OSVersion string `json:"os.version,omitempty"` OSFeatures []string `json:"os.features,omitempty"` Variant string `json:"variant,omitempty"` + Features []string `json:"features,omitempty"` } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go index cc269d6b5..e09984cbf 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go @@ -45,6 +45,14 @@ func (ul *uncompressedLayer) Uncompressed() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewBuffer(ul.content)), nil } +// MediaType returns the media type of the layer +func (ul *uncompressedLayer) MediaType() (types.MediaType, error) { + // Technically the media type should be 'application/tar' but given that our + // v1.Layer doesn't force consumers to care about whether the layer is compressed + // we should be fine returning the DockerLayer media type + return types.DockerLayer, nil +} + var _ partial.UncompressedLayer = (*uncompressedLayer)(nil) // Image returns a pseudo-randomly generated Image. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go index aa574eb8b..da0fa24c3 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go @@ -1,6 +1,7 @@ package remote import ( + "fmt" "net/http" "github.com/google/go-containerregistry/pkg/authn" @@ -18,13 +19,13 @@ import ( func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { auth, err := kc.Resolve(ref.Context().Registry) if err != nil { - return err + return fmt.Errorf("resolving authorization for %v failed: %v", ref.Context().Registry, err) } scopes := []string{ref.Scope(transport.PushScope)} tr, err := transport.New(ref.Context().Registry, auth, t, scopes) if err != nil { - return err + return fmt.Errorf("creating push check transport for %v failed: %v", ref.Context().Registry, err) } // TODO(jasonhall): Against GCR, just doing the token handshake is // enough, but this doesn't extend to Dockerhub diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go index 2032e276e..c034435f9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go @@ -20,15 +20,18 @@ import ( "net/http" "net/url" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) // Delete removes the specified image reference from the remote registry. -func Delete(ref name.Reference, auth authn.Authenticator, t http.RoundTripper) error { +func Delete(ref name.Reference, options ...Option) error { + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } scopes := []string{ref.Scope(transport.DeleteScope)} - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go new file mode 100644 index 000000000..6c3620740 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -0,0 +1,271 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var defaultPlatform = v1.Platform{ + Architecture: "amd64", + OS: "linux", +} + +// ErrSchema1 indicates that we received a schema1 manifest from the registry. +// This library doesn't have plans to support this legacy image format: +// https://github.com/google/go-containerregistry/issues/377 +type ErrSchema1 struct { + schema string +} + +func NewErrSchema1(schema types.MediaType) error { + return &ErrSchema1{ + schema: string(schema), + } +} + +// Error implements error. +func (e *ErrSchema1) Error() string { + return fmt.Sprintf("unsupported MediaType: %q, see https://github.com/google/go-containerregistry/issues/377", e.schema) +} + +// Descriptor provides access to metadata about remote artifact and accessors +// for efficiently converting it into a v1.Image or v1.ImageIndex. +type Descriptor struct { + fetcher + v1.Descriptor + Manifest []byte + + // So we can share this implementation with Image.. + platform v1.Platform +} + +// Get returns a remote.Descriptor for the given reference. The response from +// the registry is left un-interpreted, for the most part. This is useful for +// querying what kind of artifact a reference represents. +func Get(ref name.Reference, options ...Option) (*Descriptor, error) { + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + types.DockerManifestList, + types.OCIImageIndex, + // Just to look at them. + types.DockerManifestSchema1, + types.DockerManifestSchema1Signed, + } + return get(ref, acceptable, options...) +} + +// Handle options and fetch the manifest with the acceptable MediaTypes in the +// Accept header. +func get(ref name.Reference, acceptable []types.MediaType, options ...Option) (*Descriptor, error) { + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return nil, err + } + + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, []string{ref.Scope(transport.PullScope)}) + if err != nil { + return nil, err + } + + f := fetcher{ + Ref: ref, + Client: &http.Client{Transport: tr}, + } + + b, desc, err := f.fetchManifest(ref, acceptable) + if err != nil { + return nil, err + } + + return &Descriptor{ + fetcher: f, + Manifest: b, + Descriptor: *desc, + platform: o.platform, + }, nil +} + +// Image converts the Descriptor into a v1.Image. +// +// If the fetched artifact is already an image, it will just return it. +// +// If the fetched artifact is an index, it will attempt to resolve the index to +// a child image with the appropriate platform. +// +// See WithPlatform to set the desired platform. +func (d *Descriptor) Image() (v1.Image, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, NewErrSchema1(d.MediaType) + case types.OCIImageIndex, types.DockerManifestList: + // We want an image but the registry has an index, resolve it to an image. + return d.remoteIndex().imageByPlatform(d.platform) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // These are expected. Enumerated here to allow a default case. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + + // Wrap the v1.Layers returned by this v1.Image in a hint for downstream + // remote.Write calls to facilitate cross-repo "mounting". + imgCore, err := partial.CompressedToImage(d.remoteImage()) + if err != nil { + return nil, err + } + return &mountableImage{ + Image: imgCore, + Reference: d.Ref, + }, nil +} + +// ImageIndex converts the Descriptor into a v1.ImageIndex. +func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) { + switch d.MediaType { + case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: + // We don't care to support schema 1 images: + // https://github.com/google/go-containerregistry/issues/377 + return nil, NewErrSchema1(d.MediaType) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + // We want an index but the registry has an image, nothing we can do. + return nil, fmt.Errorf("unexpected media type for ImageIndex(): %s; call Image() instead", d.MediaType) + case types.OCIImageIndex, types.DockerManifestList: + // These are expected. + default: + // We could just return an error here, but some registries (e.g. static + // registries) don't set the Content-Type headers correctly, so instead... + // TODO(#390): Log a warning. + } + return d.remoteIndex(), nil +} + +func (d *Descriptor) remoteImage() *remoteImage { + return &remoteImage{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +func (d *Descriptor) remoteIndex() *remoteIndex { + return &remoteIndex{ + fetcher: fetcher{ + Ref: d.Ref, + Client: d.Client, + }, + manifest: d.Manifest, + mediaType: d.MediaType, + } +} + +// fetcher implements methods for reading from a registry. +type fetcher struct { + Ref name.Reference + Client *http.Client +} + +// url returns a url.Url for the specified path in the context of this remote image reference. +func (f *fetcher) url(resource, identifier string) url.URL { + return url.URL{ + Scheme: f.Ref.Context().Registry.Scheme(), + Host: f.Ref.Context().RegistryStr(), + Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), + } +} + +func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { + u := f.url("manifests", ref.Identifier()) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, nil, err + } + accept := []string{} + for _, mt := range acceptable { + accept = append(accept, string(mt)) + } + req.Header.Set("Accept", strings.Join(accept, ",")) + + resp, err := f.Client.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if err := transport.CheckError(resp, http.StatusOK); err != nil { + return nil, nil, err + } + + manifest, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + digest, size, err := v1.SHA256(bytes.NewReader(manifest)) + if err != nil { + return nil, nil, err + } + + mediaType := types.MediaType(resp.Header.Get("Content-Type")) + contentDigest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest")) + if err == nil && mediaType == types.DockerManifestSchema1Signed { + // If we can parse the digest from the header, and it's a signed schema 1 + // manifest, let's use that for the digest to appease older registries. + digest = contentDigest + } + + // Validate the digest matches what we asked for, if pulling by digest. + if dgst, ok := ref.(name.Digest); ok { + if digest.String() != dgst.DigestStr() { + return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) + } + } else { + // Do nothing for tags; I give up. + // + // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, + // but so many registries implement this incorrectly that it's not worth checking. + // + // For reference: + // https://github.com/docker/distribution/issues/2395 + // https://github.com/GoogleContainerTools/kaniko/issues/298 + } + + // Return all this info since we have to calculate it anyway. + desc := v1.Descriptor{ + Digest: digest, + Size: size, + MediaType: mediaType, + } + + return manifest, &desc, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index 1be0ad2ea..752c71608 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -15,16 +15,12 @@ package remote import ( - "bytes" "fmt" "io" "io/ioutil" "net/http" - "net/url" - "strings" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" @@ -33,11 +29,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/v1util" ) -var defaultPlatform = v1.Platform{ - Architecture: "amd64", - OS: "linux", -} - // remoteImage accesses an image from a remote registry type remoteImage struct { fetcher @@ -46,135 +37,26 @@ type remoteImage struct { configLock sync.Mutex // Protects config config []byte mediaType types.MediaType - platform v1.Platform } -// ImageOption is a functional option for Image. -type ImageOption func(*imageOpener) error - var _ partial.CompressedImageCore = (*remoteImage)(nil) -type imageOpener struct { - auth authn.Authenticator - transport http.RoundTripper - ref name.Reference - client *http.Client - platform v1.Platform -} +// Image provides access to a remote image reference. +func Image(ref name.Reference, options ...Option) (v1.Image, error) { + acceptable := []types.MediaType{ + types.DockerManifestSchema2, + types.OCIManifestSchema1, + // We resolve these to images later. + types.DockerManifestList, + types.OCIImageIndex, + } -func (i *imageOpener) Open() (v1.Image, error) { - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + desc, err := get(ref, acceptable, options...) if err != nil { return nil, err } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - platform: i.platform, - } - imgCore, err := partial.CompressedToImage(ri) - if err != nil { - return imgCore, err - } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: i.ref, - }, nil -} -// Image provides access to a remote image reference, applying functional options -// to the underlying imageOpener before resolving the reference into a v1.Image. -func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) { - img := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, - platform: defaultPlatform, - } - - for _, option := range options { - if err := option(img); err != nil { - return nil, err - } - } - return img.Open() -} - -// fetcher implements methods for reading from a remote image. -type fetcher struct { - Ref name.Reference - Client *http.Client -} - -// url returns a url.Url for the specified path in the context of this remote image reference. -func (f *fetcher) url(resource, identifier string) url.URL { - return url.URL{ - Scheme: f.Ref.Context().Registry.Scheme(), - Host: f.Ref.Context().RegistryStr(), - Path: fmt.Sprintf("/v2/%s/%s/%s", f.Ref.Context().RepositoryStr(), resource, identifier), - } -} - -func (f *fetcher) fetchManifest(acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) { - u := f.url("manifests", f.Ref.Identifier()) - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return nil, nil, err - } - accept := []string{} - for _, mt := range acceptable { - accept = append(accept, string(mt)) - } - req.Header.Set("Accept", strings.Join(accept, ",")) - - resp, err := f.Client.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if err := transport.CheckError(resp, http.StatusOK); err != nil { - return nil, nil, err - } - - manifest, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } - - digest, size, err := v1.SHA256(bytes.NewReader(manifest)) - if err != nil { - return nil, nil, err - } - - // Validate the digest matches what we asked for, if pulling by digest. - if dgst, ok := f.Ref.(name.Digest); ok { - if digest.String() != dgst.DigestStr() { - return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) - } - } else { - // Do nothing for tags; I give up. - // - // We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry, - // but so many registries implement this incorrectly that it's not worth checking. - // - // For reference: - // https://github.com/docker/distribution/issues/2395 - // https://github.com/GoogleContainerTools/kaniko/issues/298 - } - - // Return all this info since we have to calculate it anyway. - desc := v1.Descriptor{ - Digest: digest, - Size: size, - MediaType: types.MediaType(resp.Header.Get("Content-Type")), - } - - return manifest, &desc, nil + return desc.Image() } func (r *remoteImage) MediaType() (types.MediaType, error) { @@ -184,7 +66,6 @@ func (r *remoteImage) MediaType() (types.MediaType, error) { return types.DockerManifestSchema2, nil } -// TODO(jonjohnsonjr): Handle manifest lists. func (r *remoteImage) RawManifest() ([]byte, error) { r.manifestLock.Lock() defer r.manifestLock.Unlock() @@ -192,26 +73,18 @@ func (r *remoteImage) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteImage. acceptable := []types.MediaType{ types.DockerManifestSchema2, types.OCIManifestSchema1, - // We'll resolve these to an image based on the platform. - types.DockerManifestList, - types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } - // We want an image but the registry has an index, resolve it to an image. - for desc.MediaType == types.DockerManifestList || desc.MediaType == types.OCIImageIndex { - manifest, desc, err = r.matchImage(manifest) - if err != nil { - return nil, err - } - } - r.mediaType = desc.MediaType r.manifest = manifest return r.manifest, nil @@ -278,6 +151,22 @@ func (rl *remoteLayer) Manifest() (*v1.Manifest, error) { return partial.Manifest(rl.ri) } +// MediaType implements v1.Layer +func (rl *remoteLayer) MediaType() (types.MediaType, error) { + m, err := rl.Manifest() + if err != nil { + return "", err + } + + for _, layer := range m.Layers { + if layer.Digest == rl.digest { + return layer.MediaType, nil + } + } + + return "", fmt.Errorf("unable to find layer with digest: %v", rl.digest) +} + // Size implements partial.CompressedLayer func (rl *remoteLayer) Size() (int64, error) { // Look up the size of this digest in the manifest to avoid a request. @@ -302,36 +191,3 @@ func (r *remoteImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) digest: h, }, nil } - -// This naively matches the first manifest with matching Architecture and OS. -// -// We should probably use this instead: -// github.com/containerd/containerd/platforms -// -// But first we'd need to migrate to: -// github.com/opencontainers/image-spec/specs-go/v1 -func (r *remoteImage) matchImage(rawIndex []byte) ([]byte, *v1.Descriptor, error) { - index, err := v1.ParseIndexManifest(bytes.NewReader(rawIndex)) - if err != nil { - return nil, nil, err - } - for _, childDesc := range index.Manifests { - // If platform is missing from child descriptor, assume it's amd64/linux. - p := defaultPlatform - if childDesc.Platform != nil { - p = *childDesc.Platform - } - if r.platform.Architecture == p.Architecture && r.platform.OS == p.OS { - childRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), childDesc.Digest), name.StrictValidation) - if err != nil { - return nil, nil, err - } - r.fetcher = fetcher{ - Client: r.Client, - Ref: childRef, - } - return r.fetchManifest([]types.MediaType{childDesc.MediaType}) - } - } - return nil, nil, fmt.Errorf("no matching image for %s/%s, index: %s", r.platform.Architecture, r.platform.OS, string(rawIndex)) -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index 03afc481a..043dc83b6 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -17,14 +17,11 @@ package remote import ( "bytes" "fmt" - "net/http" "sync" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -36,30 +33,19 @@ type remoteIndex struct { mediaType types.MediaType } -// Index provides access to a remote index reference, applying functional options -// to the underlying imageOpener before resolving the reference into a v1.ImageIndex. -func Index(ref name.Reference, options ...ImageOption) (v1.ImageIndex, error) { - i := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, +// Index provides access to a remote index reference. +func Index(ref name.Reference, options ...Option) (v1.ImageIndex, error) { + acceptable := []types.MediaType{ + types.DockerManifestList, + types.OCIImageIndex, } - for _, option := range options { - if err := option(i); err != nil { - return nil, err - } - } - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + desc, err := get(ref, acceptable, options...) if err != nil { return nil, err } - return &remoteIndex{ - fetcher: fetcher{ - Ref: i.ref, - Client: &http.Client{Transport: tr}, - }, - }, nil + + return desc.ImageIndex() } func (r *remoteIndex) MediaType() (types.MediaType, error) { @@ -80,11 +66,14 @@ func (r *remoteIndex) RawManifest() ([]byte, error) { return r.manifest, nil } + // NOTE(jonjohnsonjr): We should never get here because the public entrypoints + // do type-checking via remote.Descriptor. I've left this here for tests that + // directly instantiate a remoteIndex. acceptable := []types.MediaType{ types.DockerManifestList, types.OCIImageIndex, } - manifest, desc, err := r.fetchManifest(acceptable) + manifest, desc, err := r.fetchManifest(r.Ref, acceptable) if err != nil { return nil, err } @@ -103,37 +92,139 @@ func (r *remoteIndex) IndexManifest() (*v1.IndexManifest, error) { } func (r *remoteIndex) Image(h v1.Hash) (v1.Image, error) { - imgRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) + desc, err := r.childByHash(h) if err != nil { return nil, err } - ri := &remoteImage{ - fetcher: fetcher{ - Ref: imgRef, - Client: r.Client, - }, - } - imgCore, err := partial.CompressedToImage(ri) - if err != nil { - return imgCore, err - } - // Wrap the v1.Layers returned by this v1.Image in a hint for downstream - // remote.Write calls to facilitate cross-repo "mounting". - return &mountableImage{ - Image: imgCore, - Reference: r.Ref, - }, nil + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() } func (r *remoteIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { - idxRef, err := name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) + desc, err := r.childByHash(h) if err != nil { return nil, err } - return &remoteIndex{ + return desc.ImageIndex() +} + +func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) { + desc, err := r.childByPlatform(platform) + if err != nil { + return nil, err + } + + // Descriptor.Image will handle coercing nested indexes into an Image. + return desc.Image() +} + +// This naively matches the first manifest with matching platform attributes. +// +// We should probably use this instead: +// github.com/containerd/containerd/platforms +// +// But first we'd need to migrate to: +// github.com/opencontainers/image-spec/specs-go/v1 +func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + // If platform is missing from child descriptor, assume it's amd64/linux. + p := defaultPlatform + if childDesc.Platform != nil { + p = *childDesc.Platform + } + + if matchesPlatform(p, platform) { + return r.childDescriptor(childDesc, platform) + } + } + return nil, fmt.Errorf("no child with platform %s/%s in index %s", platform.Architecture, platform.OS, r.Ref) +} + +func (r *remoteIndex) childByHash(h v1.Hash) (*Descriptor, error) { + index, err := r.IndexManifest() + if err != nil { + return nil, err + } + for _, childDesc := range index.Manifests { + if h == childDesc.Digest { + return r.childDescriptor(childDesc, defaultPlatform) + } + } + return nil, fmt.Errorf("no child with digest %s in index %s", h, r.Ref) +} + +func (r *remoteIndex) childRef(h v1.Hash) (name.Reference, error) { + return name.ParseReference(fmt.Sprintf("%s@%s", r.Ref.Context(), h), name.StrictValidation) +} + +// Convert one of this index's child's v1.Descriptor into a remote.Descriptor, with the given platform option. +func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) (*Descriptor, error) { + ref, err := r.childRef(child.Digest) + if err != nil { + return nil, err + } + manifest, desc, err := r.fetchManifest(ref, []types.MediaType{child.MediaType}) + if err != nil { + return nil, err + } + return &Descriptor{ fetcher: fetcher{ - Ref: idxRef, + Ref: ref, Client: r.Client, }, + Manifest: manifest, + Descriptor: *desc, + platform: platform, }, nil } + +// matchesPlatform checks if the given platform matches the required platforms. +// The given platform matches the required platform if +// - architecture and OS are identical. +// - OS version and variant are identical if provided. +// - features and OS features of the required platform are subsets of those of the given platform. +func matchesPlatform(given, required v1.Platform) bool { + // Required fields that must be identical. + if given.Architecture != required.Architecture || given.OS != required.OS { + return false + } + + // Optional fields that may be empty, but must be identical if provided. + if required.OSVersion != "" && given.OSVersion != required.OSVersion { + return false + } + if required.Variant != "" && given.Variant != required.Variant { + return false + } + + // Verify required platform's features are a subset of given platform's features. + if !isSubset(given.OSFeatures, required.OSFeatures) { + return false + } + if !isSubset(given.Features, required.Features) { + return false + } + + return true +} + +// isSubset checks if the required array of strings is a subset of the given lst. +func isSubset(lst, required []string) bool { + set := make(map[string]bool) + for _, value := range lst { + set[value] = true + } + + for _, value := range required { + if _, ok := set[value]; !ok { + return false + } + } + + return true +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go index 1a36d0a4b..4cbdc4f19 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go @@ -20,7 +20,6 @@ import ( "net/http" "net/url" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) @@ -30,10 +29,15 @@ type tags struct { Tags []string `json:"tags"` } -// List calls /tags/list for the given repository. -func List(repo name.Repository, auth authn.Authenticator, t http.RoundTripper) ([]string, error) { +// List calls /tags/list for the given repository, returning the list of tags +// in the "tags" property. +func List(repo name.Repository, options ...Option) ([]string, error) { + o, err := makeOptions(repo.Registry, options...) + if err != nil { + return nil, err + } scopes := []string{repo.Scope(transport.PullScope)} - tr, err := transport.New(repo.Registry, auth, t, scopes) + tr, err := transport.New(repo.Registry, o.auth, o.transport, scopes) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 335e3fe5b..c60344840 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -15,50 +15,96 @@ package remote import ( - "log" "net/http" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) +// Option is a functional option for remote operations. +type Option func(*options) error + +type options struct { + auth authn.Authenticator + keychain authn.Keychain + transport http.RoundTripper + platform v1.Platform +} + +func makeOptions(reg name.Registry, opts ...Option) (*options, error) { + o := &options{ + auth: authn.Anonymous, + transport: http.DefaultTransport, + platform: defaultPlatform, + } + + for _, option := range opts { + if err := option(o); err != nil { + return nil, err + } + } + + if o.keychain != nil { + auth, err := o.keychain.Resolve(reg) + if err != nil { + return nil, err + } + if auth == authn.Anonymous { + logs.Warn.Println("No matching credentials were found, falling back on anonymous") + } + o.auth = auth + } + + // Wrap the transport in something that can retry network flakes. + o.transport = transport.NewRetry(o.transport) + + return o, nil +} + // WithTransport is a functional option for overriding the default transport -// on a remote image -func WithTransport(t http.RoundTripper) ImageOption { - return func(i *imageOpener) error { - i.transport = t +// for remote operations. +// +// The default transport its http.DefaultTransport. +func WithTransport(t http.RoundTripper) Option { + return func(o *options) error { + o.transport = t return nil } } // WithAuth is a functional option for overriding the default authenticator -// on a remote image -func WithAuth(auth authn.Authenticator) ImageOption { - return func(i *imageOpener) error { - i.auth = auth +// for remote operations. +// +// The default authenticator is authn.Anonymous. +func WithAuth(auth authn.Authenticator) Option { + return func(o *options) error { + o.auth = auth return nil } } // WithAuthFromKeychain is a functional option for overriding the default -// authenticator on a remote image using an authn.Keychain -func WithAuthFromKeychain(keys authn.Keychain) ImageOption { - return func(i *imageOpener) error { - auth, err := keys.Resolve(i.ref.Context().Registry) - if err != nil { - return err - } - if auth == authn.Anonymous { - log.Println("No matching credentials were found, falling back on anonymous") - } - i.auth = auth +// authenticator for remote operations, using an authn.Keychain to find +// credentials. +// +// The default authenticator is authn.Anonymous. +func WithAuthFromKeychain(keys authn.Keychain) Option { + return func(o *options) error { + o.keychain = keys return nil } } -func WithPlatform(p v1.Platform) ImageOption { - return func(i *imageOpener) error { - i.platform = p +// WithPlatform is a functional option for overriding the default platform +// that Image and Descriptor.Image use for resolving an index to an image. +// +// The default platform is amd64/linux. +func WithPlatform(p v1.Platform) Option { + return func(o *options) error { + o.platform = p return nil } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go index e77f47f69..b98b4a625 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go @@ -40,7 +40,7 @@ func (bt *basicTransport) RoundTrip(in *http.Request) (*http.Response, error) { // we are redirected, only set it when the authorization header matches // the host with which we are interacting. // In case of redirect http.Client can use an empty Host, check URL too. - if in.Host == bt.target || in.URL.Host == bt.target { + if hdr != "" && (in.Host == bt.target || in.URL.Host == bt.target) { in.Header.Set("Authorization", hdr) } in.Header.Set("User-Agent", transportName) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index f72ab276d..326708b97 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -18,8 +18,10 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/url" + "strings" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" @@ -45,6 +47,11 @@ type bearerTransport struct { var _ http.RoundTripper = (*bearerTransport)(nil) +var portMap = map[string]string{ + "http": "80", + "https": "443", +} + // RoundTrip implements http.RoundTripper func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { sendRequest := func() (*http.Response, error) { @@ -58,12 +65,19 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { // we are redirected, only set it when the authorization header matches // the registry with which we are interacting. // In case of redirect http.Client can use an empty Host, check URL too. - if in.Host == bt.registry.RegistryStr() || in.URL.Host == bt.registry.RegistryStr() { + canonicalHeaderHost := bt.canonicalAddress(in.Host) + canonicalURLHost := bt.canonicalAddress(in.URL.Host) + canonicalRegistryHost := bt.canonicalAddress(bt.registry.RegistryStr()) + if canonicalHeaderHost == canonicalRegistryHost || canonicalURLHost == canonicalRegistryHost { in.Header.Set("Authorization", hdr) + + // When we ping() the registry, we determine whether to use http or https + // based on which scheme was successful. That is only valid for the + // registry server and not e.g. a separate token server or blob storage, + // so we should only override the scheme if the host is the registry. + in.URL.Scheme = bt.scheme } in.Header.Set("User-Agent", transportName) - - in.URL.Scheme = bt.scheme return bt.inner.RoundTrip(in) } @@ -140,3 +154,28 @@ func (bt *bearerTransport) refresh() error { bt.bearer = &bearer return nil } + +func (bt *bearerTransport) canonicalAddress(host string) (address string) { + // The host may be any one of: + // - hostname + // - hostname:port + // - ipv4 + // - ipv4:port + // - ipv6 + // - [ipv6]:port + // As net.SplitHostPort returns an error if the host does not contain a port, we should only attempt + // to call it when we know that the address contains a port + if strings.Count(host, ":") == 1 || (strings.Count(host, ":") >= 2 && strings.Contains(host, "]:")) { + hostname, port, err := net.SplitHostPort(host) + if err != nil { + return host + } + if port == "" { + port = portMap[bt.scheme] + } + + return net.JoinHostPort(hostname, port) + } + + return net.JoinHostPort(host, portMap[bt.scheme]) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 44885effa..35e5d798a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -26,6 +26,10 @@ import ( // https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors type Error struct { Errors []Diagnostic `json:"errors,omitempty"` + // The http status code returned. + StatusCode int + // The raw body if we couldn't understand it. + rawBody string } // Check that Error implements error @@ -35,7 +39,10 @@ var _ error = (*Error)(nil) func (e *Error) Error() string { switch len(e.Errors) { case 0: - return "" + if len(e.rawBody) == 0 { + return fmt.Sprintf("unsupported status code %d", e.StatusCode) + } + return fmt.Sprintf("unsupported status code %d; body: %s", e.StatusCode, e.rawBody) case 1: return e.Errors[0].String() default: @@ -48,6 +55,20 @@ func (e *Error) Error() string { } } +// Temporary returns whether the request that preceded the error is temporary. +func (e *Error) Temporary() bool { + if len(e.Errors) == 0 { + return false + } + for _, d := range e.Errors { + // TODO: Include other error types. + if d.Code != BlobUploadInvalidErrorCode { + return false + } + } + return true +} + // Diagnostic represents a single error returned by a Docker registry interaction. type Diagnostic struct { Code ErrorCode `json:"code"` @@ -101,11 +122,10 @@ func CheckError(resp *http.Response, codes ...int) error { } // https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors - var structuredError Error - if err := json.Unmarshal(b, &structuredError); err != nil { - // If the response isn't an unstructured error, then return some - // reasonable error response containing the response body. - return fmt.Errorf("unsupported status code %d; body: %s", resp.StatusCode, string(b)) + structuredError := &Error{} + if err := json.Unmarshal(b, structuredError); err != nil { + structuredError.rawBody = string(b) } - return &structuredError + structuredError.StatusCode = resp.StatusCode + return structuredError } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go new file mode 100644 index 000000000..b6b2181da --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go @@ -0,0 +1,89 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "net/http" + "time" + + "github.com/google/go-containerregistry/pkg/internal/retry" + "k8s.io/apimachinery/pkg/util/wait" +) + +// Sleep for 0.1, 0.3, 0.9, 2.7 seconds. This should cover networking blips. +var defaultBackoff = wait.Backoff{ + Duration: 100 * time.Millisecond, + Factor: 3.0, + Jitter: 0.1, + Steps: 5, +} + +var _ http.RoundTripper = (*retryTransport)(nil) + +// retryTransport wraps a RoundTripper and retries temporary network errors. +type retryTransport struct { + inner http.RoundTripper + backoff wait.Backoff + predicate retry.Predicate +} + +// Option is a functional option for retryTransport. +type Option func(*options) + +type options struct { + backoff wait.Backoff + predicate retry.Predicate +} + +// WithRetryBackoff sets the backoff for retry operations. +func WithRetryBackoff(backoff wait.Backoff) Option { + return func(o *options) { + o.backoff = backoff + } +} + +// WithRetryPredicate sets the predicate for retry operations. +func WithRetryPredicate(predicate func(error) bool) Option { + return func(o *options) { + o.predicate = predicate + } +} + +// NewRetry returns a transport that retries errors. +func NewRetry(inner http.RoundTripper, opts ...Option) http.RoundTripper { + o := &options{ + backoff: defaultBackoff, + predicate: retry.IsTemporary, + } + + for _, opt := range opts { + opt(o) + } + + return &retryTransport{ + inner: inner, + backoff: o.backoff, + predicate: o.predicate, + } +} + +func (t *retryTransport) RoundTrip(in *http.Request) (out *http.Response, err error) { + roundtrip := func() error { + out, err = t.inner.RoundTrip(in) + return err + } + retry.Retry(roundtrip, t.predicate, t.backoff) + return +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 66f148155..8806aa187 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -19,18 +19,19 @@ import ( "errors" "fmt" "io" - "log" "net/http" "net/url" + "time" - "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/internal/retry" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" - "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/util/wait" ) type manifest interface { @@ -40,14 +41,19 @@ type manifest interface { } // Write pushes the provided img to the specified image reference. -func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.RoundTripper) error { +func Write(ref name.Reference, img v1.Image, options ...Option) error { ls, err := img.Layers() if err != nil { return err } + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } + scopes := scopesForUploadingImage(ref, ls) - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } @@ -57,17 +63,17 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro } // Upload individual layers in goroutines and collect any errors. - // If we can dedupe by the layer digest, try to do so. If the layer is - // a stream.Layer, we can't dedupe and might re-upload. + // If we can dedupe by the layer digest, try to do so. If we can't determine + // the digest for whatever reason, we can't dedupe and might re-upload. var g errgroup.Group uploaded := map[v1.Hash]bool{} for _, l := range ls { l := l - if _, ok := l.(*stream.Layer); !ok { - h, err := l.Digest() - if err != nil { - return err - } + + // Streaming layers calculate their digests while uploading them. Assume + // an error here indicates we need to upload the layer. + h, err := l.Digest() + if err == nil { // If we can determine the layer's digest ahead of // time, use it to dedupe uploads. if uploaded[h] { @@ -81,14 +87,15 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro }) } - if l, err := partial.ConfigLayer(img); err == stream.ErrNotComputed { - // We can't read the ConfigLayer, because of streaming layers, since the - // config hasn't been calculated yet. + if l, err := partial.ConfigLayer(img); err != nil { + // We can't read the ConfigLayer, possibly because of streaming layers, + // since the layer DiffIDs haven't been calculated yet. Attempt to wait + // for the other layers to be uploaded, then try the config again. if err := g.Wait(); err != nil { return err } - // Now that all the layers are uploaded, upload the config file blob. + // Now that all the layers are uploaded, try to upload the config file blob. l, err := partial.ConfigLayer(img) if err != nil { return err @@ -96,9 +103,6 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro if err := w.uploadOne(l); err != nil { return err } - } else if err != nil { - // This is an actual error, not a streaming error, just return it. - return err } else { // We *can* read the ConfigLayer, so upload it concurrently with the layers. g.Go(func() error { @@ -285,25 +289,16 @@ func (w *writer) commitBlob(location, digest string) error { // uploadOne performs a complete upload of a single layer. func (w *writer) uploadOne(l v1.Layer) error { - var from, mount, digest string - if _, ok := l.(*stream.Layer); !ok { - // Layer isn't streamable, we should take advantage of that to - // skip uploading if possible. - // By sending ?digest= in the request, we'll also check that - // our computed digest matches the one computed by the - // registry. - h, err := l.Digest() - if err != nil { - return err - } - digest = h.String() - + var from, mount string + if h, err := l.Digest(); err == nil { + // If we know the digest, this isn't a streaming layer. Do an existence + // check so we can skip uploading the layer if possible. existing, err := w.checkExistingBlob(h) if err != nil { return err } if existing { - log.Printf("existing blob: %v", h) + logs.Progress.Printf("existing blob: %v", h) return nil } @@ -315,38 +310,50 @@ func (w *writer) uploadOne(l v1.Layer) error { } } - location, mounted, err := w.initiateUpload(from, mount) - if err != nil { - return err - } else if mounted { + tryUpload := func() error { + location, mounted, err := w.initiateUpload(from, mount) + if err != nil { + return err + } else if mounted { + h, err := l.Digest() + if err != nil { + return err + } + logs.Progress.Printf("mounted blob: %s", h.String()) + return nil + } + + blob, err := l.Compressed() + if err != nil { + return err + } + location, err = w.streamBlob(blob, location) + if err != nil { + return err + } + h, err := l.Digest() if err != nil { return err } - log.Printf("mounted blob: %s", h.String()) + digest := h.String() + + if err := w.commitBlob(location, digest); err != nil { + return err + } + logs.Progress.Printf("pushed blob: %s", digest) return nil } - blob, err := l.Compressed() - if err != nil { - return err - } - location, err = w.streamBlob(blob, location) - if err != nil { - return err + // Try this three times, waiting 1s after first failure, 3s after second. + backoff := wait.Backoff{ + Duration: 1.0 * time.Second, + Factor: 3.0, + Jitter: 0.1, + Steps: 3, } - h, err := l.Digest() - if err != nil { - return err - } - digest = h.String() - - if err := w.commitBlob(location, digest); err != nil { - return err - } - log.Printf("pushed blob: %s", digest) - return nil + return retry.Retry(tryUpload, retry.IsTemporary, backoff) } // commitImage does a PUT of the image's manifest. @@ -385,7 +392,7 @@ func (w *writer) commitImage(man manifest) error { } // The image was successfully pushed! - log.Printf("%v: digest: %v size: %d", w.ref, digest, len(raw)) + logs.Progress.Printf("%v: digest: %v size: %d", w.ref, digest, len(raw)) return nil } @@ -416,14 +423,18 @@ func scopesForUploadingImage(ref name.Reference, layers []v1.Layer) []string { // WriteIndex pushes the provided ImageIndex to the specified image reference. // WriteIndex will attempt to push all of the referenced manifests before // attempting to push the ImageIndex, to retain referential integrity. -func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, t http.RoundTripper) error { +func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { index, err := ii.IndexManifest() if err != nil { return err } + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } scopes := []string{ref.Scope(transport.PushScope)} - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } @@ -442,7 +453,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, return err } if exists { - log.Printf("existing manifest: %v", desc.Digest) + logs.Progress.Printf("existing manifest: %v", desc.Digest) continue } @@ -453,7 +464,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, return err } - if err := WriteIndex(ref, ii, auth, t); err != nil { + if err := WriteIndex(ref, ii, WithAuth(o.auth), WithTransport(o.transport)); err != nil { return err } case types.OCIManifestSchema1, types.DockerManifestSchema2: @@ -461,7 +472,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, if err != nil { return err } - if err := Write(ref, img, auth, t); err != nil { + if err := Write(ref, img, WithAuth(o.auth), WithTransport(o.transport)); err != nil { return err } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go index f8895a226..aa816359c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go @@ -24,6 +24,7 @@ import ( "sync" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" ) var ( @@ -81,6 +82,13 @@ func (l *Layer) Size() (int64, error) { return l.size, nil } +// MediaType implements v1.Layer +func (l *Layer) MediaType() (types.MediaType, error) { + // We return DockerLayer for now as uncompressed layers + // are unimplemented + return types.DockerLayer, nil +} + // Uncompressed implements v1.Layer. func (l *Layer) Uncompressed() (io.ReadCloser, error) { return nil, errors.New("NYI: stream.Layer.Uncompressed is not implemented") diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go index ced18735c..06ecd9a77 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go @@ -119,7 +119,7 @@ func (td tarDescriptor) findSpecifiedImageDescriptor(tag *name.Tag) (*singleImag } for _, img := range td { for _, tagStr := range img.RepoTags { - repoTag, err := name.NewTag(tagStr, name.WeakValidation) + repoTag, err := name.NewTag(tagStr) if err != nil { return nil, err } @@ -226,6 +226,13 @@ func (ulft *uncompressedLayerFromTarball) Uncompressed() (io.ReadCloser, error) return extractFileFromTar(ulft.opener, ulft.filePath) } +func (ulft *uncompressedLayerFromTarball) MediaType() (types.MediaType, error) { + // Technically the media type should be 'application/tar' but given that our + // v1.Layer doesn't force consumers to care about whether the layer is compressed + // we should be fine returning the DockerLayer media type + return types.DockerLayer, nil +} + func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) { cfg, err := partial.ConfigFile(i) if err != nil { @@ -310,6 +317,11 @@ func (clft *compressedLayerFromTarball) Compressed() (io.ReadCloser, error) { return extractFileFromTar(clft.opener, clft.filePath) } +// MediaType implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + // Size implements partial.CompressedLayer func (clft *compressedLayerFromTarball) Size() (int64, error) { r, err := clft.Compressed() diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index 00256e8f2..85c1b7838 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -15,12 +15,14 @@ package tarball import ( + "bytes" "compress/gzip" "io" "io/ioutil" "os" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -62,6 +64,10 @@ func (l *layer) Size() (int64, error) { return l.size, nil } +func (l *layer) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + // LayerFromFile returns a v1.Layer given a tarball func LayerFromFile(path string) (v1.Layer, error) { opener := func() (io.ReadCloser, error) { @@ -103,6 +109,18 @@ func LayerFromOpener(opener Opener) (v1.Layer, error) { }, nil } +// LayerFromReader returns a v1.Layer given a io.Reader. +func LayerFromReader(reader io.Reader) (v1.Layer, error) { + // Buffering due to Opener requiring multiple calls. + a, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + return LayerFromOpener(func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(a)), nil + }) +} + func computeDigest(opener Opener, compressed bool) (v1.Hash, int64, error) { rc, err := opener() if err != nil { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go new file mode 100644 index 000000000..91ca87a5f --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package validate provides methods for validating image correctness. +package validate diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go new file mode 100644 index 000000000..c71d7d65e --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go @@ -0,0 +1,297 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "strings" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Image validates that img does not violate any invariants of the image format. +func Image(img v1.Image) error { + errs := []string{} + if err := validateLayers(img); err != nil { + errs = append(errs, fmt.Sprintf("validating layers: %v", err)) + } + + if err := validateConfig(img); err != nil { + errs = append(errs, fmt.Sprintf("validating config: %v", err)) + } + + if err := validateManifest(img); err != nil { + errs = append(errs, fmt.Sprintf("validating manifest: %v", err)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n\n")) + } + return nil +} + +func validateConfig(img v1.Image) error { + cn, err := img.ConfigName() + if err != nil { + return err + } + + rc, err := img.RawConfigFile() + if err != nil { + return err + } + + hash, size, err := v1.SHA256(bytes.NewReader(rc)) + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + cf, err := img.ConfigFile() + if err != nil { + return err + } + + pcf, err := v1.ParseConfigFile(bytes.NewReader(rc)) + if err != nil { + return err + } + + errs := []string{} + if cn != hash { + errs = append(errs, fmt.Sprintf("mismatched config digest: ConfigName()=%s, SHA256(RawConfigFile())=%s", cn, hash)) + } + + if want, got := m.Config.Size, size; want != got { + errs = append(errs, fmt.Sprintf("mismatched config size: Manifest.Config.Size()=%d, len(RawConfigFile())=%d", want, got)) + } + + if diff := cmp.Diff(pcf, cf); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched config content: (-ParseConfigFile(RawConfigFile()) +ConfigFile()) %s", diff)) + } + + if cf.RootFS.Type != "layers" { + errs = append(errs, fmt.Sprintf("invalid ConfigFile.RootFS.Type: %q != %q", cf.RootFS.Type, "layers")) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateLayers(img v1.Image) error { + layers, err := img.Layers() + if err != nil { + return err + } + + digests := []v1.Hash{} + diffids := []v1.Hash{} + sizes := []int64{} + for _, layer := range layers { + // TODO: Test layer.Uncompressed. + compressed, err := layer.Compressed() + if err != nil { + return err + } + + // Keep track of compressed digest. + digester := sha256.New() + // Everything read from compressed is written to digester to compute digest. + hashCompressed := io.TeeReader(compressed, digester) + + // Call io.Copy to write from the layer Reader through to the tarReader on + // the other side of the pipe. + pr, pw := io.Pipe() + var size int64 + go func() { + n, err := io.Copy(pw, hashCompressed) + if err != nil { + pw.CloseWithError(err) + return + } + size = n + + // Now close the compressed reader, to flush the gzip stream + // and calculate digest/diffID/size. This will cause pr to + // return EOF which will cause readers of the Compressed stream + // to finish reading. + pw.CloseWithError(compressed.Close()) + }() + + // Read the bytes through gzip.Reader to compute the DiffID. + uncompressed, err := gzip.NewReader(pr) + if err != nil { + return err + } + diffider := sha256.New() + hashUncompressed := io.TeeReader(uncompressed, diffider) + + // Ensure there aren't duplicate file paths. + tarReader := tar.NewReader(hashUncompressed) + files := make(map[string]struct{}) + for { + hdr, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if _, ok := files[hdr.Name]; ok { + return fmt.Errorf("duplicate file path: %s", hdr.Name) + } + files[hdr.Name] = struct{}{} + } + + // Discard any trailing padding that the tar.Reader doesn't consume. + if _, err := io.Copy(ioutil.Discard, hashUncompressed); err != nil { + return err + } + + if err := uncompressed.Close(); err != nil { + return err + } + + digest := v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(digester.Sum(make([]byte, 0, digester.Size()))), + } + + diffid := v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(diffider.Sum(make([]byte, 0, diffider.Size()))), + } + + // Compute all of these first before we call Config() and Manifest() to allow + // for lazy access e.g. for stream.Layer. + digests = append(digests, digest) + diffids = append(diffids, diffid) + sizes = append(sizes, size) + } + + cf, err := img.ConfigFile() + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + errs := []string{} + for i, layer := range layers { + digest, err := layer.Digest() + if err != nil { + return err + } + diffid, err := layer.DiffID() + if err != nil { + return err + } + size, err := layer.Size() + if err != nil { + return err + } + + if digest != digests[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Digest()=%s, SHA256(Compressed())=%s", i, digest, digests[i])) + } + + if m.Layers[i].Digest != digests[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Manifest.Layers[%d].Digest=%s, SHA256(Compressed())=%s", i, i, m.Layers[i].Digest, digests[i])) + } + + if diffid != diffids[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: DiffID()=%s, SHA256(Gunzip(Compressed()))=%s", i, diffid, diffids[i])) + } + + if cf.RootFS.DiffIDs[i] != diffids[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: ConfigFile.RootFS.DiffIDs[%d]=%s, SHA256(Gunzip(Compressed()))=%s", i, i, cf.RootFS.DiffIDs[i], diffids[i])) + } + + if size != sizes[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Size()=%d, len(Compressed())=%d", i, size, sizes[i])) + } + + if m.Layers[i].Size != sizes[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Manifest.Layers[%d].Size=%d, len(Compressed())=%d", i, i, m.Layers[i].Size, sizes[i])) + } + + } + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateManifest(img v1.Image) error { + digest, err := img.Digest() + if err != nil { + return err + } + + rm, err := img.RawManifest() + if err != nil { + return err + } + + hash, _, err := v1.SHA256(bytes.NewReader(rm)) + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + pm, err := v1.ParseManifest(bytes.NewReader(rm)) + if err != nil { + return err + } + + errs := []string{} + if digest != hash { + errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash)) + } + + if diff := cmp.Diff(pm, m); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseManifest(RawManifest()) +Manifest()) %s", diff)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go new file mode 100644 index 000000000..871e24153 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go @@ -0,0 +1,123 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Index validates that idx does not violate any invariants of the index format. +func Index(idx v1.ImageIndex) error { + errs := []string{} + + if err := validateChildren(idx); err != nil { + errs = append(errs, fmt.Sprintf("validating children: %v", err)) + } + + if err := validateIndexManifest(idx); err != nil { + errs = append(errs, fmt.Sprintf("validating index manifest: %v", err)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n\n")) + } + return nil +} + +func validateChildren(idx v1.ImageIndex) error { + manifest, err := idx.IndexManifest() + if err != nil { + return err + } + + errs := []string{} + for i, desc := range manifest.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + idx, err := idx.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := Index(idx); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate index Manifests[%d](%s): %v", i, desc.Digest, err)) + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := idx.Image(desc.Digest) + if err != nil { + return err + } + if err := Image(img); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate image Manifests[%d](%s): %v", i, desc.Digest, err)) + } + default: + return fmt.Errorf("todo: validate index Blob()") + } + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateIndexManifest(idx v1.ImageIndex) error { + digest, err := idx.Digest() + if err != nil { + return err + } + + rm, err := idx.RawManifest() + if err != nil { + return err + } + + hash, _, err := v1.SHA256(bytes.NewReader(rm)) + if err != nil { + return err + } + + m, err := idx.IndexManifest() + if err != nil { + return err + } + + pm, err := v1.ParseIndexManifest(bytes.NewReader(rm)) + if err != nil { + return err + } + + errs := []string{} + if digest != hash { + errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash)) + } + + if diff := cmp.Diff(pm, m); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseIndexManifest(RawManifest()) +Manifest()) %s", diff)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +}