diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index f25773d76..b0d8784b8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,6 +7,7 @@ about: Report a bug in kaniko **Actual behavior** A clear and concise description of what the bug is. + **Expected behavior** A clear and concise description of what you expected to happen. @@ -21,3 +22,16 @@ Steps to reproduce the behavior: - Build Context Please provide or clearly describe any files needed to build the Dockerfile (ADD/COPY commands) - Kaniko Image (fully qualified with digest) + + **Triage Notes for the Maintainers** + + + + | **Description** | **Yes/No** | + |----------------|---------------| + | Please check if this a new feature you are proposing | | + | Please check if the build works in docker but not in kaniko | | + | Please check if this error is seen when you use `--cache` flag | | + | Please check if your dockerfile is a multistage dockerfile | | + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..35e1abd67 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,41 @@ + + + +Fixes `#`. _in case of a bug fix, this should point to a bug and any other related issue(s)_ + +**Description** + + + +**Submitter Checklist** + +These are the criteria that every PR should meet, please check them off as you +review them: + +- [ ] Includes [unit tests](../DEVELOPMENT.md#creating-a-pr) +- [ ] Adds integration tests if needed. + +_See [the contribution guide](../CONTRIBUTING.md) for more details._ + + +**Reviewer Notes** + +- [ ] The code flow looks good. +- [ ] Unit tests and or integration tests added. + + +**Release Notes** + +Describe any changes here so maintainer can include it in the release notes, or delete this block. + +``` +Examples of user facing changes: +- Skaffold config changes like + e.g. "Add buildArgs to `Kustomize` deployer skaffold config." +- Bug fixes + e.g. "Improve skaffold init behaviour when tags are used in manifests" +- Any changes in skaffold behavior + e.g. "Artiface cachine is turned on by default." + +``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 353250455..142baa25d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,37 @@ +# v0.12.0 Release - 2019-09/13 + +## New Features +* Added `--oci-layout-path` flag to save image in OCI layout. [#744](https://github.com/GoogleContainerTools/kaniko/pull/744) +* Add support for S3 custom endpoint [#698](https://github.com/GoogleContainerTools/kaniko/pull/698) + +## Bug Fixes +* Setting PATH [#760](https://github.com/GoogleContainerTools/kaniko/pull/760) +* Remove leading slash in layer tarball paths (Closes: #726) [#729](https://github.com/GoogleContainerTools/kaniko/pull/729) + +## Updates and Refactors +* Remove cruft [#635](https://github.com/GoogleContainerTools/kaniko/pull/635) +* Add desc for `--skip-tls-verify-pull` to README [#493](https://github.com/GoogleContainerTools/kaniko/pull/493) + +Huge thank you for this release towards our contributors: +- Carlos Alexandro Becker +- Carlos Sanchez +- chhsia0 +- Deniz Zoeteman +- Luke Wood +- Matthew Dawson +- Niels Denissen +- Priya Wadhwa +- Sharif Elgamal +- Takeaki Matsumoto +- Taylor Barrella +- Tejal Desai +- v.rul +- Warren Seymour +- xanonid +- Xueshan Feng +- Π ΠΎΠΌΠ°Π½ НСбалуСв + + # v0.11.0 Release - 2019-08-23 ## Bug Fixes diff --git a/Gopkg.lock b/Gopkg.lock index bd23f0578..57a55af22 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -445,7 +445,7 @@ version = "v0.2.0" [[projects]] - digest = "1:16c8837e951303ef6388132bc875337660a48ea2dedf1c941ca118ea92d2a3d2" + digest = "1:5924704ec96f00247784c512cc57f45a595030376a7ff2ff993bf356793a2cb0" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", @@ -456,6 +456,7 @@ "pkg/v1", "pkg/v1/daemon", "pkg/v1/empty", + "pkg/v1/layout", "pkg/v1/mutate", "pkg/v1/partial", "pkg/v1/random", @@ -467,7 +468,7 @@ "pkg/v1/v1util", ] pruneopts = "NUT" - revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74" + revision = "31e00cede111067bae48bfc2cbfc522b0b36207f" [[projects]] digest = "1:f4f203acd8b11b8747bdcd91696a01dbc95ccb9e2ca2db6abf81c3a4f5e950ce" @@ -1368,6 +1369,7 @@ "github.com/google/go-containerregistry/pkg/v1", "github.com/google/go-containerregistry/pkg/v1/daemon", "github.com/google/go-containerregistry/pkg/v1/empty", + "github.com/google/go-containerregistry/pkg/v1/layout", "github.com/google/go-containerregistry/pkg/v1/mutate", "github.com/google/go-containerregistry/pkg/v1/partial", "github.com/google/go-containerregistry/pkg/v1/remote", diff --git a/Gopkg.toml b/Gopkg.toml index 64d3b5b7b..e502f07ff 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,7 +37,7 @@ required = [ [[constraint]] name = "github.com/google/go-containerregistry" - revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74" + revision = "31e00cede111067bae48bfc2cbfc522b0b36207f" [[override]] name = "k8s.io/apimachinery" diff --git a/Makefile b/Makefile index 881c55a5d..f3295afeb 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ # Bump these on release VERSION_MAJOR ?= 0 -VERSION_MINOR ?= 11 +VERSION_MINOR ?= 12 VERSION_BUILD ?= 0 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD) diff --git a/README.md b/README.md index 466dd4ab5..38a4a63a3 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME - [--insecure](#--insecure) - [--insecure-pull](#--insecure-pull) - [--no-push](#--no-push) + - [--oci-layout-path](#--oci-layout-path) - [--reproducible](#--reproducible) - [--single-snapshot](#--single-snapshot) - [--snapshotMode](#--snapshotmode) @@ -374,6 +375,19 @@ will write the digest to that file, which is picked up by Kubernetes automatically as the `{{.state.terminated.message}}` of the container. +#### --oci-layout-path + +Set this flag to specify a directory in the container where the OCI image +layout of a built image will be placed. This can be used to automatically +track the exact image built by Kaniko. + +For example, to surface the image digest built in a +[Tekton task](https://github.com/tektoncd/pipeline/blob/v0.6.0/docs/resources.md#surfacing-the-image-digest-built-in-a-task), +this flag should be set to match the image resource `outputImageDir`. + +_Note: Depending on the built image, the media type of the image manifest might be either +`application/vnd.oci.image.manifest.v1+json` or `application/vnd.docker.distribution.manifest.v2+json``._ + #### --insecure-registry Set this flag to use plain HTTP requests when accessing a registry. It is supposed to be used for testing purposes only and should not be used in production! diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 69a0addc3..29ec4162d 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -131,6 +131,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided") RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.") RootCmd.PersistentFlags().StringVarP(&opts.DigestFile, "digest-file", "", "", "Specify a file to save the digest of the built image to.") + RootCmd.PersistentFlags().StringVarP(&opts.OCILayoutPath, "oci-layout-path", "", "", "Path to save the OCI image layout of the built image.") RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end") RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.") diff --git a/pkg/buildcontext/s3.go b/pkg/buildcontext/s3.go index 5b77d9c4a..93d404c45 100644 --- a/pkg/buildcontext/s3.go +++ b/pkg/buildcontext/s3.go @@ -19,6 +19,7 @@ package buildcontext import ( "os" "path/filepath" + "strings" "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -36,9 +37,21 @@ type S3 struct { // UnpackTarFromBuildContext download and untar a file from s3 func (s *S3) UnpackTarFromBuildContext() (string, error) { bucket, item := util.GetBucketAndItem(s.context) - sess, err := session.NewSessionWithOptions(session.Options{ + option := session.Options{ SharedConfigState: session.SharedConfigEnable, - }) + } + endpoint := os.Getenv(constants.S3EndpointEnv) + forcePath := false + if strings.ToLower(os.Getenv(constants.S3ForcePathStyle)) == "true" { + forcePath = true + } + if endpoint != "" { + option.Config = aws.Config{ + Endpoint: aws.String(endpoint), + S3ForcePathStyle: aws.Bool(forcePath), + } + } + sess, err := session.NewSessionWithOptions(option) if err != nil { return bucket, err } diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 102411c5d..0953a28f6 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -59,7 +59,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) { } registryName := cacheRef.Repository.Registry.Name() - if rc.Opts.InsecureRegistries.Contains(registryName) { + if rc.Opts.Insecure || rc.Opts.InsecureRegistries.Contains(registryName) { newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) if err != nil { return nil, err diff --git a/pkg/config/options.go b/pkg/config/options.go index 627a51531..44af681ec 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -37,6 +37,7 @@ type KanikoOptions struct { Target string CacheRepo string DigestFile string + OCILayoutPath string Destinations multiArg BuildArgs multiArg Insecure bool diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 0eb29cb9d..d0d84e3f3 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -70,6 +70,10 @@ const ( // Name of the .dockerignore file Dockerignore = ".dockerignore" + + // S3 Custom endpoint ENV name + S3EndpointEnv = "S3_ENDPOINT" + S3ForcePathStyle = "S3_FORCE_PATH_STYLE" ) // ScratchEnvVars are the default environment variables needed for a scratch image. diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 13f9db92d..e134e52b5 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -124,7 +124,7 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { return nil, err } - if img == empty.Image { + if imageConfig.Config.Env == nil { imageConfig.Config.Env = constants.ScratchEnvVars } return imageConfig, nil diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index c48e4bb26..44f6a1211 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -29,6 +29,8 @@ import ( "github.com/GoogleContainerTools/kaniko/testutil" "github.com/google/go-cmp/cmp" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) @@ -405,3 +407,58 @@ func Test_filesToSave(t *testing.T) { }) } } + +func TestInitializeConfig(t *testing.T) { + tests := []struct { + description string + cfg v1.ConfigFile + expected v1.Config + }{ + { + description: "env is not set in the image", + cfg: v1.ConfigFile{ + Config: v1.Config{ + Image: "test", + }, + }, + expected: v1.Config{ + Image: "test", + Env: []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + }, + }, + { + description: "env is set in the image", + cfg: v1.ConfigFile{ + Config: v1.Config{ + Env: []string{ + "PATH=/usr/local/something", + }, + }, + }, + expected: v1.Config{ + Env: []string{ + "PATH=/usr/local/something", + }, + }, + }, + { + description: "image is empty", + expected: v1.Config{ + Env: []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + }, + }, + } + for _, tt := range tests { + img, err := mutate.ConfigFile(empty.Image, &tt.cfg) + if err != nil { + t.Errorf("error seen when running test %s", err) + t.Fail() + } + actual, _ := initializeConfig(img) + testutil.CheckDeepEqual(t, tt.expected, actual.Config) + } +} diff --git a/pkg/executor/foo b/pkg/executor/foo deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 486bcd32a..4ee4b0d2b 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -34,6 +34,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" @@ -58,7 +59,7 @@ func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) { return w.t.RoundTrip(r) } -// CheckPushPermissionos checks that the configured credentials can be used to +// CheckPushPermissions checks that the configured credentials can be used to // push to every specified destination. func CheckPushPermissions(opts *config.KanikoOptions) error { if opts.NoPush { @@ -76,6 +77,13 @@ func CheckPushPermissions(opts *config.KanikoOptions) error { } registryName := destRef.Repository.Registry.Name() + if opts.Insecure || opts.InsecureRegistries.Contains(registryName) { + newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure) + if err != nil { + return errors.Wrap(err, "getting new insecure registry") + } + destRef.Repository.Registry = newReg + } tr := makeTransport(opts, registryName) if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), tr); err != nil { return errors.Wrapf(err, "checking push permission for %q", destRef) @@ -101,6 +109,16 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { } } + if opts.OCILayoutPath != "" { + path, err := layout.Write(opts.OCILayoutPath, empty.Index) + if err != nil { + return errors.Wrap(err, "writing empty layout") + } + if err := path.AppendImage(image); err != nil { + return errors.Wrap(err, "appending image") + } + } + destRefs := []name.Tag{} for _, destination := range opts.Destinations { destRef, err := name.NewTag(destination, name.WeakValidation) diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go index 2f9729960..63857f4f0 100644 --- a/pkg/executor/push_test.go +++ b/pkg/executor/push_test.go @@ -23,7 +23,11 @@ import ( "os" "testing" + "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/testutil" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/validate" ) func TestHeaderAdded(t *testing.T) { @@ -69,3 +73,49 @@ func (m *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { ua := r.UserAgent() return &http.Response{Body: ioutil.NopCloser(bytes.NewBufferString(ua))}, nil } + +func TestOCILayoutPath(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("could not create temp dir: %s", err) + } + defer os.RemoveAll(tmpDir) + + image, err := random.Image(1024, 4) + if err != nil { + t.Fatalf("could not create image: %s", err) + } + + digest, err := image.Digest() + if err != nil { + t.Fatalf("could not get image digest: %s", err) + } + + want, err := image.Manifest() + if err != nil { + t.Fatalf("could not get image manifest: %s", err) + } + + opts := config.KanikoOptions{ + NoPush: true, + OCILayoutPath: tmpDir, + } + + if err := DoPush(image, &opts); err != nil { + t.Fatalf("could not push image: %s", err) + } + + layoutIndex, err := layout.ImageIndexFromPath(tmpDir) + if err != nil { + t.Fatalf("could not get index from layout: %s", err) + } + testutil.CheckError(t, false, validate.Index(layoutIndex)) + + layoutImage, err := layoutIndex.Image(digest) + if err != nil { + t.Fatalf("could not get image from layout: %s", err) + } + + got, err := layoutImage.Manifest() + testutil.CheckErrorAndDeepEqual(t, false, err, want, got) +} diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index bfa445f58..798ae6c09 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "sort" + "strings" "testing" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -31,6 +32,7 @@ import ( func TestSnapshotFSFileChange(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { t.Fatal(err) @@ -55,16 +57,16 @@ func TestSnapshotFSFileChange(t *testing.T) { } // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles tr := tar.NewReader(f) - fooPath := filepath.Join(testDir, "foo") - batPath := filepath.Join(testDir, "bar/bat") + fooPath := filepath.Join(testDirWithoutLeadingSlash, "foo") + batPath := filepath.Join(testDirWithoutLeadingSlash, "bar/bat") snapshotFiles := map[string]string{ fooPath: "newbaz1", batPath: "baz", } - for _, dir := range util.ParentDirectories(fooPath) { + for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(fooPath) { snapshotFiles[dir] = "" } - for _, dir := range util.ParentDirectories(batPath) { + for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) { snapshotFiles[dir] = "" } numFiles := 0 @@ -128,12 +130,14 @@ func TestSnapshotFSIsReproducible(t *testing.T) { func TestSnapshotFSChangePermissions(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { t.Fatal(err) } // Change permissions on a file batPath := filepath.Join(testDir, "bar/bat") + batPathWithoutLeadingSlash := filepath.Join(testDirWithoutLeadingSlash, "bar/bat") if err := os.Chmod(batPath, 0600); err != nil { t.Fatalf("Error changing permissions on %s: %v", batPath, err) } @@ -149,9 +153,9 @@ func TestSnapshotFSChangePermissions(t *testing.T) { // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles tr := tar.NewReader(f) snapshotFiles := map[string]string{ - batPath: "baz2", + batPathWithoutLeadingSlash: "baz2", } - for _, dir := range util.ParentDirectories(batPath) { + for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) { snapshotFiles[dir] = "" } numFiles := 0 @@ -160,6 +164,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) { if err == io.EOF { break } + t.Logf("Info %s in tar", hdr.Name) numFiles++ if _, isFile := snapshotFiles[hdr.Name]; !isFile { t.Fatalf("File %s unexpectedly in tar", hdr.Name) @@ -176,6 +181,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) { func TestSnapshotFiles(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() + testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { t.Fatal(err) @@ -197,9 +203,9 @@ func TestSnapshotFiles(t *testing.T) { defer os.Remove(tarPath) expectedFiles := []string{ - filepath.Join(testDir, "foo"), + filepath.Join(testDirWithoutLeadingSlash, "foo"), } - expectedFiles = append(expectedFiles, util.ParentDirectories(filepath.Join(testDir, "foo"))...) + expectedFiles = append(expectedFiles, util.ParentDirectoriesWithoutLeadingSlash(filepath.Join(testDir, "foo"))...) f, err := os.Open(tarPath) if err != nil { diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index d22dcb4aa..338caa02a 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -121,6 +121,10 @@ func DeleteFilesystem() error { logrus.Info("Deleting filesystem...") return filepath.Walk(constants.RootDir, func(path string, info os.FileInfo, _ error) error { if CheckWhitelist(path) { + if !isExist(path) { + logrus.Debugf("Path %s whitelisted, but not exists", path) + return nil + } if info.IsDir() { return filepath.SkipDir } @@ -138,6 +142,14 @@ func DeleteFilesystem() error { }) } +// isExists returns true if path exists +func isExist(path string) bool { + if _, err := os.Stat(path); err == nil { + return true + } + return false +} + // ChildDirInWhitelist returns true if there is a child file or directory of the path in the whitelist func childDirInWhitelist(path string) bool { for _, d := range whitelist { @@ -377,6 +389,24 @@ func ParentDirectories(path string) []string { return paths } +// ParentDirectoriesWithoutLeadingSlash returns a list of paths to all parent directories +// all subdirectories do not contain a leading / +// Ex. /some/temp/dir -> [/, some, some/temp, some/temp/dir] +func ParentDirectoriesWithoutLeadingSlash(path string) []string { + path = filepath.Clean(path) + dirs := strings.Split(path, "/") + dirPath := "" + paths := []string{constants.RootDir} + for index, dir := range dirs { + if dir == "" || index == (len(dirs)-1) { + continue + } + dirPath = filepath.Join(dirPath, dir) + paths = append(paths, dirPath) + } + return paths +} + // FilepathExists returns true if the path exists func FilepathExists(path string) bool { _, err := os.Lstat(path) diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index eff39d5d9..c44908056 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -177,6 +177,38 @@ func Test_ParentDirectories(t *testing.T) { } } +func Test_ParentDirectoriesWithoutLeadingSlash(t *testing.T) { + tests := []struct { + name string + path string + expected []string + }{ + { + name: "regular path", + path: "/path/to/dir", + expected: []string{ + "/", + "path", + "path/to", + }, + }, + { + name: "current directory", + path: ".", + expected: []string{ + "/", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := ParentDirectoriesWithoutLeadingSlash(tt.path) + testutil.CheckErrorAndDeepEqual(t, false, nil, tt.expected, actual) + }) + } +} + func Test_CheckWhitelist(t *testing.T) { type args struct { path string diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go index bc1cc67a0..213ef68e3 100644 --- a/pkg/util/tar_util.go +++ b/pkg/util/tar_util.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "syscall" "github.com/docker/docker/pkg/archive" @@ -74,7 +75,14 @@ func (t *Tar) AddFileToTar(p string) error { if err != nil { return err } - hdr.Name = p + + if p != "/" { + // Docker uses no leading / in the tarball + hdr.Name = strings.TrimLeft(p, "/") + } else { + // allow entry for / to preserve permission changes etc. (currently ignored anyway by Docker runtime) + hdr.Name = p + } hardlink, linkDst := t.checkHardlink(p, i) if hardlink { @@ -104,7 +112,8 @@ func (t *Tar) Whiteout(p string) error { name := ".wh." + filepath.Base(p) th := &tar.Header{ - Name: filepath.Join(dir, name), + // Docker uses no leading / in the tarball + Name: strings.TrimLeft(filepath.Join(dir, name), "/"), Size: 0, } if err := t.w.WriteHeader(th); err != nil { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go new file mode 100644 index 000000000..ba90d4cdb --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go @@ -0,0 +1,38 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" + "io/ioutil" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Blob returns a blob with the given hash from the Path. +func (l Path) Blob(h v1.Hash) (io.ReadCloser, error) { + return os.Open(l.blobPath(h)) +} + +// Bytes is a convenience function to return a blob from the Path as +// a byte slice. +func (l Path) Bytes(h v1.Hash) ([]byte, error) { + return ioutil.ReadFile(l.blobPath(h)) +} + +func (l Path) blobPath(h v1.Hash) string { + return l.path("blobs", h.Algorithm, h.Hex) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go new file mode 100644 index 000000000..d80d27363 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go @@ -0,0 +1,19 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout provides facilities for reading/writing artifacts from/to +// an OCI image layout on disk, see: +// +// https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go new file mode 100644 index 000000000..7c76a10cb --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go @@ -0,0 +1,131 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "fmt" + "io" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type layoutImage struct { + path Path + desc v1.Descriptor + manifestLock sync.Mutex // Protects rawManifest + rawManifest []byte +} + +var _ partial.CompressedImageCore = (*layoutImage)(nil) + +// Image reads a v1.Image with digest h from the Path. +func (l Path) Image(h v1.Hash) (v1.Image, error) { + ii, err := l.ImageIndex() + if err != nil { + return nil, err + } + + return ii.Image(h) +} + +func (li *layoutImage) MediaType() (types.MediaType, error) { + return li.desc.MediaType, nil +} + +// Implements WithManifest for partial.Blobset. +func (li *layoutImage) Manifest() (*v1.Manifest, error) { + return partial.Manifest(li) +} + +func (li *layoutImage) RawManifest() ([]byte, error) { + li.manifestLock.Lock() + defer li.manifestLock.Unlock() + if li.rawManifest != nil { + return li.rawManifest, nil + } + + b, err := li.path.Bytes(li.desc.Digest) + if err != nil { + return nil, err + } + + li.rawManifest = b + return li.rawManifest, nil +} + +func (li *layoutImage) RawConfigFile() ([]byte, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + return li.path.Bytes(manifest.Config.Digest) +} + +func (li *layoutImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + if h == manifest.Config.Digest { + return partial.CompressedLayer(&compressedBlob{ + path: li.path, + desc: manifest.Config, + }), nil + } + + for _, desc := range manifest.Layers { + if h == desc.Digest { + switch desc.MediaType { + case types.OCILayer, types.DockerLayer: + return partial.CompressedToLayer(&compressedBlob{ + path: li.path, + desc: desc, + }) + default: + // TODO: We assume everything is a compressed blob, but that might not be true. + // TODO: Handle foreign layers. + return nil, fmt.Errorf("unexpected media type: %v for layer: %v", desc.MediaType, desc.Digest) + } + } + } + + return nil, fmt.Errorf("could not find layer in image: %s", h) +} + +type compressedBlob struct { + path Path + desc v1.Descriptor +} + +func (b *compressedBlob) Digest() (v1.Hash, error) { + return b.desc.Digest, nil +} + +func (b *compressedBlob) Compressed() (io.ReadCloser, error) { + return b.path.Blob(b.desc.Digest) +} + +func (b *compressedBlob) Size() (int64, error) { + return b.desc.Size, nil +} + +func (b *compressedBlob) MediaType() (types.MediaType, error) { + return b.desc.MediaType, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go new file mode 100644 index 000000000..aba139d9d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go @@ -0,0 +1,146 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var _ v1.ImageIndex = (*layoutIndex)(nil) + +type layoutIndex struct { + path Path + rawIndex []byte +} + +// ImageIndexFromPath is a convenience function which constructs a Path and returns its v1.ImageIndex. +func ImageIndexFromPath(path string) (v1.ImageIndex, error) { + lp, err := FromPath(path) + if err != nil { + return nil, err + } + return lp.ImageIndex() +} + +// ImageIndex returns a v1.ImageIndex for the Path. +func (l Path) ImageIndex() (v1.ImageIndex, error) { + rawIndex, err := ioutil.ReadFile(l.path("index.json")) + if err != nil { + return nil, err + } + + idx := &layoutIndex{ + path: l, + rawIndex: rawIndex, + } + + return idx, nil +} + +func (i *layoutIndex) MediaType() (types.MediaType, error) { + return types.OCIImageIndex, nil +} + +func (i *layoutIndex) Digest() (v1.Hash, error) { + return partial.Digest(i) +} + +func (i *layoutIndex) IndexManifest() (*v1.IndexManifest, error) { + var index v1.IndexManifest + err := json.Unmarshal(i.rawIndex, &index) + return &index, err +} + +func (i *layoutIndex) RawManifest() ([]byte, error) { + return i.rawIndex, nil +} + +func (i *layoutIndex) Image(h v1.Hash) (v1.Image, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIManifestSchema1, types.DockerManifestSchema2) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + img := &layoutImage{ + path: i.path, + desc: *desc, + } + return partial.CompressedToImage(img) +} + +func (i *layoutIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIImageIndex, types.DockerManifestList) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + rawIndex, err := i.path.Bytes(h) + if err != nil { + return nil, err + } + + return &layoutIndex{ + path: i.path, + rawIndex: rawIndex, + }, nil +} + +func (i *layoutIndex) Blob(h v1.Hash) (io.ReadCloser, error) { + return i.path.Blob(h) +} + +func (i *layoutIndex) findDescriptor(h v1.Hash) (*v1.Descriptor, error) { + im, err := i.IndexManifest() + if err != nil { + return nil, err + } + + for _, desc := range im.Manifests { + if desc.Digest == h { + return &desc, nil + } + } + + return nil, fmt.Errorf("could not find descriptor in index: %s", h) +} + +// TODO: Pull this out into methods on types.MediaType? e.g. instead, have: +// * mt.IsIndex() +// * mt.IsImage() +func isExpectedMediaType(mt types.MediaType, expected ...types.MediaType) bool { + for _, allowed := range expected { + if mt == allowed { + return true + } + } + return false +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go new file mode 100644 index 000000000..a031ff5ae --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go @@ -0,0 +1,25 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import "path/filepath" + +// Path represents an OCI image layout rooted in a file system path +type Path string + +func (l Path) path(elem ...string) string { + complete := []string{string(l)} + return filepath.Join(append(complete, elem...)...) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go new file mode 100644 index 000000000..5569e51de --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go @@ -0,0 +1,42 @@ +package layout + +import v1 "github.com/google/go-containerregistry/pkg/v1" + +// Option is a functional option for Layout. +// +// TODO: We'll need to change this signature to support Sparse/Thin images. +// Or, alternatively, wrap it in a sparse.Image that returns an empty list for layers? +type Option func(*v1.Descriptor) error + +// WithAnnotations adds annotations to the artifact descriptor. +func WithAnnotations(annotations map[string]string) Option { + return func(desc *v1.Descriptor) error { + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + for k, v := range annotations { + desc.Annotations[k] = v + } + + return nil + } +} + +// WithURLs adds urls to the artifact descriptor. +func WithURLs(urls []string) Option { + return func(desc *v1.Descriptor) error { + if desc.URLs == nil { + desc.URLs = []string{} + } + desc.URLs = append(desc.URLs, urls...) + return nil + } +} + +// WithPlatform sets the platform of the artifact descriptor. +func WithPlatform(platform v1.Platform) Option { + return func(desc *v1.Descriptor) error { + desc.Platform = &platform + return nil + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go new file mode 100644 index 000000000..796abc7dd --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go @@ -0,0 +1,32 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "os" + "path/filepath" +) + +// FromPath reads an OCI image layout at path and constructs a layout.Path. +func FromPath(path string) (Path, error) { + // TODO: check oci-layout exists + + _, err := os.Stat(filepath.Join(path, "index.json")) + if err != nil { + return "", err + } + + return Path(path), nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go new file mode 100644 index 000000000..2abb9a586 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go @@ -0,0 +1,301 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "golang.org/x/sync/errgroup" +) + +var layoutFile = `{ + "imageLayoutVersion": "1.0.0" +}` + +// AppendImage writes a v1.Image to the Path and updates +// the index.json to reference it. +func (l Path) AppendImage(img v1.Image, options ...Option) error { + if err := l.writeImage(img); err != nil { + return err + } + + mt, err := img.MediaType() + if err != nil { + return err + } + + d, err := img.Digest() + if err != nil { + return err + } + + manifest, err := img.RawManifest() + if err != nil { + return err + } + + desc := v1.Descriptor{ + MediaType: mt, + Size: int64(len(manifest)), + Digest: d, + } + + for _, opt := range options { + if err := opt(&desc); err != nil { + return err + } + } + + return l.AppendDescriptor(desc) +} + +// AppendIndex writes a v1.ImageIndex to the Path and updates +// the index.json to reference it. +func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error { + if err := l.writeIndex(ii); err != nil { + return err + } + + mt, err := ii.MediaType() + if err != nil { + return err + } + + d, err := ii.Digest() + if err != nil { + return err + } + + manifest, err := ii.RawManifest() + if err != nil { + return err + } + + desc := v1.Descriptor{ + MediaType: mt, + Size: int64(len(manifest)), + Digest: d, + } + + for _, opt := range options { + if err := opt(&desc); err != nil { + return err + } + } + + return l.AppendDescriptor(desc) +} + +// AppendDescriptor adds a descriptor to the index.json of the Path. +func (l Path) AppendDescriptor(desc v1.Descriptor) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + index.Manifests = append(index.Manifests, desc) + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.writeFile("index.json", rawIndex) +} + +func (l Path) writeFile(name string, data []byte) error { + if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + return ioutil.WriteFile(l.path(name), data, os.ModePerm) + +} + +// WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at +// blobs/{hash.Algorithm}/{hash.Hex}. +func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error { + dir := l.path("blobs", hash.Algorithm) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + file := filepath.Join(dir, hash.Hex) + if _, err := os.Stat(file); err == nil { + // Blob already exists, that's fine. + return nil + } + w, err := os.Create(file) + if err != nil { + return err + } + defer w.Close() + + _, err = io.Copy(w, r) + return err +} + +// TODO: A streaming version of WriteBlob so we don't have to know the hash +// before we write it. + +// TODO: For streaming layers we should write to a tmp file then Rename to the +// final digest. +func (l Path) writeLayer(layer v1.Layer) error { + d, err := layer.Digest() + if err != nil { + return err + } + + r, err := layer.Compressed() + if err != nil { + return err + } + + return l.WriteBlob(d, r) +} + +func (l Path) writeImage(img v1.Image) error { + layers, err := img.Layers() + if err != nil { + return err + } + + // Write the layers concurrently. + var g errgroup.Group + for _, layer := range layers { + layer := layer + g.Go(func() error { + return l.writeLayer(layer) + }) + } + if err := g.Wait(); err != nil { + return err + } + + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return err + } + cfgBlob, err := img.RawConfigFile() + if err != nil { + return err + } + if err := l.WriteBlob(cfgName, ioutil.NopCloser(bytes.NewReader(cfgBlob))); err != nil { + return err + } + + // Write the img manifest. + d, err := img.Digest() + if err != nil { + return err + } + manifest, err := img.RawManifest() + if err != nil { + return err + } + + return l.WriteBlob(d, ioutil.NopCloser(bytes.NewReader(manifest))) +} + +func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error { + index, err := ii.IndexManifest() + if err != nil { + return err + } + + // Walk the descriptors and write any v1.Image or v1.ImageIndex that we find. + // If we come across something we don't expect, just write it as a blob. + for _, desc := range index.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + ii, err := ii.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := l.writeIndex(ii); err != nil { + return err + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := ii.Image(desc.Digest) + if err != nil { + return err + } + if err := l.writeImage(img); err != nil { + return err + } + default: + // TODO: The layout could reference arbitrary things, which we should + // probably just pass through. + } + } + + rawIndex, err := ii.RawManifest() + if err != nil { + return err + } + + return l.writeFile(indexFile, rawIndex) +} + +func (l Path) writeIndex(ii v1.ImageIndex) error { + // Always just write oci-layout file, since it's small. + if err := l.writeFile("oci-layout", []byte(layoutFile)); err != nil { + return err + } + + h, err := ii.Digest() + if err != nil { + return err + } + + indexFile := filepath.Join("blobs", h.Algorithm, h.Hex) + return l.writeIndexToFile(indexFile, ii) + +} + +// Write constructs a Path at path from an ImageIndex. +// +// The contents are written in the following format: +// At the top level, there is: +// One oci-layout file containing the version of this image-layout. +// One index.json file listing descriptors for the contained images. +// Under blobs/, there is, for each image: +// One file for each layer, named after the layer's SHA. +// One file for each config blob, named after its SHA. +// One file for each manifest blob, named after its SHA. +func Write(path string, ii v1.ImageIndex) (Path, error) { + lp := Path(path) + // Always just write oci-layout file, since it's small. + if err := lp.writeFile("oci-layout", []byte(layoutFile)); err != nil { + return "", err + } + + // TODO create blobs/ in case there is a blobs file which would prevent the directory from being created + + return lp, lp.writeIndexToFile("index.json", ii) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index 813205dad..eafd599b9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -165,11 +165,9 @@ func (i *image) compute() error { manifest := m.DeepCopy() manifestLayers := manifest.Layers for _, add := range i.adds { - d := v1.Descriptor{ - MediaType: types.DockerLayer, - } - + d := v1.Descriptor{} var err error + if d.Size, err = add.Layer.Size(); err != nil { return err } @@ -178,6 +176,10 @@ func (i *image) compute() error { return err } + if d.MediaType, err = add.Layer.MediaType(); err != nil { + return err + } + manifestLayers = append(manifestLayers, d) digestMap[d.Digest] = add.Layer } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go index 144b99ecc..6c3620740 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -16,7 +16,6 @@ package remote import ( "bytes" - "errors" "fmt" "io/ioutil" "net/http" @@ -38,7 +37,20 @@ var defaultPlatform = v1.Platform{ // ErrSchema1 indicates that we received a schema1 manifest from the registry. // This library doesn't have plans to support this legacy image format: // https://github.com/google/go-containerregistry/issues/377 -var ErrSchema1 = errors.New("unsupported MediaType: https://github.com/google/go-containerregistry/issues/377") +type ErrSchema1 struct { + schema string +} + +func NewErrSchema1(schema types.MediaType) error { + return &ErrSchema1{ + schema: string(schema), + } +} + +// Error implements error. +func (e *ErrSchema1) Error() string { + return fmt.Sprintf("unsupported MediaType: %q, see https://github.com/google/go-containerregistry/issues/377", e.schema) +} // Descriptor provides access to metadata about remote artifact and accessors // for efficiently converting it into a v1.Image or v1.ImageIndex. @@ -111,7 +123,7 @@ func (d *Descriptor) Image() (v1.Image, error) { case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: // We don't care to support schema 1 images: // https://github.com/google/go-containerregistry/issues/377 - return nil, ErrSchema1 + return nil, NewErrSchema1(d.MediaType) case types.OCIImageIndex, types.DockerManifestList: // We want an image but the registry has an index, resolve it to an image. return d.remoteIndex().imageByPlatform(d.platform) @@ -141,7 +153,7 @@ func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) { case types.DockerManifestSchema1, types.DockerManifestSchema1Signed: // We don't care to support schema 1 images: // https://github.com/google/go-containerregistry/issues/377 - return nil, ErrSchema1 + return nil, NewErrSchema1(d.MediaType) case types.OCIManifestSchema1, types.DockerManifestSchema2: // We want an index but the registry has an image, nothing we can do. return nil, fmt.Errorf("unexpected media type for ImageIndex(): %s; call Image() instead", d.MediaType) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 3673a341b..35e5d798a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -26,6 +26,10 @@ import ( // https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors type Error struct { Errors []Diagnostic `json:"errors,omitempty"` + // The http status code returned. + StatusCode int + // The raw body if we couldn't understand it. + rawBody string } // Check that Error implements error @@ -35,7 +39,10 @@ var _ error = (*Error)(nil) func (e *Error) Error() string { switch len(e.Errors) { case 0: - return "" + if len(e.rawBody) == 0 { + return fmt.Sprintf("unsupported status code %d", e.StatusCode) + } + return fmt.Sprintf("unsupported status code %d; body: %s", e.StatusCode, e.rawBody) case 1: return e.Errors[0].String() default: @@ -115,11 +122,10 @@ func CheckError(resp *http.Response, codes ...int) error { } // https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors - var structuredError Error - if err := json.Unmarshal(b, &structuredError); err != nil { - // If the response isn't an unstructured error, then return some - // reasonable error response containing the response body. - return fmt.Errorf("unsupported status code %d; body: %s", resp.StatusCode, string(b)) + structuredError := &Error{} + if err := json.Unmarshal(b, structuredError); err != nil { + structuredError.rawBody = string(b) } - return &structuredError + structuredError.StatusCode = resp.StatusCode + return structuredError } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go new file mode 100644 index 000000000..91ca87a5f --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package validate provides methods for validating image correctness. +package validate diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go new file mode 100644 index 000000000..c71d7d65e --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go @@ -0,0 +1,297 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "strings" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Image validates that img does not violate any invariants of the image format. +func Image(img v1.Image) error { + errs := []string{} + if err := validateLayers(img); err != nil { + errs = append(errs, fmt.Sprintf("validating layers: %v", err)) + } + + if err := validateConfig(img); err != nil { + errs = append(errs, fmt.Sprintf("validating config: %v", err)) + } + + if err := validateManifest(img); err != nil { + errs = append(errs, fmt.Sprintf("validating manifest: %v", err)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n\n")) + } + return nil +} + +func validateConfig(img v1.Image) error { + cn, err := img.ConfigName() + if err != nil { + return err + } + + rc, err := img.RawConfigFile() + if err != nil { + return err + } + + hash, size, err := v1.SHA256(bytes.NewReader(rc)) + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + cf, err := img.ConfigFile() + if err != nil { + return err + } + + pcf, err := v1.ParseConfigFile(bytes.NewReader(rc)) + if err != nil { + return err + } + + errs := []string{} + if cn != hash { + errs = append(errs, fmt.Sprintf("mismatched config digest: ConfigName()=%s, SHA256(RawConfigFile())=%s", cn, hash)) + } + + if want, got := m.Config.Size, size; want != got { + errs = append(errs, fmt.Sprintf("mismatched config size: Manifest.Config.Size()=%d, len(RawConfigFile())=%d", want, got)) + } + + if diff := cmp.Diff(pcf, cf); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched config content: (-ParseConfigFile(RawConfigFile()) +ConfigFile()) %s", diff)) + } + + if cf.RootFS.Type != "layers" { + errs = append(errs, fmt.Sprintf("invalid ConfigFile.RootFS.Type: %q != %q", cf.RootFS.Type, "layers")) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateLayers(img v1.Image) error { + layers, err := img.Layers() + if err != nil { + return err + } + + digests := []v1.Hash{} + diffids := []v1.Hash{} + sizes := []int64{} + for _, layer := range layers { + // TODO: Test layer.Uncompressed. + compressed, err := layer.Compressed() + if err != nil { + return err + } + + // Keep track of compressed digest. + digester := sha256.New() + // Everything read from compressed is written to digester to compute digest. + hashCompressed := io.TeeReader(compressed, digester) + + // Call io.Copy to write from the layer Reader through to the tarReader on + // the other side of the pipe. + pr, pw := io.Pipe() + var size int64 + go func() { + n, err := io.Copy(pw, hashCompressed) + if err != nil { + pw.CloseWithError(err) + return + } + size = n + + // Now close the compressed reader, to flush the gzip stream + // and calculate digest/diffID/size. This will cause pr to + // return EOF which will cause readers of the Compressed stream + // to finish reading. + pw.CloseWithError(compressed.Close()) + }() + + // Read the bytes through gzip.Reader to compute the DiffID. + uncompressed, err := gzip.NewReader(pr) + if err != nil { + return err + } + diffider := sha256.New() + hashUncompressed := io.TeeReader(uncompressed, diffider) + + // Ensure there aren't duplicate file paths. + tarReader := tar.NewReader(hashUncompressed) + files := make(map[string]struct{}) + for { + hdr, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if _, ok := files[hdr.Name]; ok { + return fmt.Errorf("duplicate file path: %s", hdr.Name) + } + files[hdr.Name] = struct{}{} + } + + // Discard any trailing padding that the tar.Reader doesn't consume. + if _, err := io.Copy(ioutil.Discard, hashUncompressed); err != nil { + return err + } + + if err := uncompressed.Close(); err != nil { + return err + } + + digest := v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(digester.Sum(make([]byte, 0, digester.Size()))), + } + + diffid := v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(diffider.Sum(make([]byte, 0, diffider.Size()))), + } + + // Compute all of these first before we call Config() and Manifest() to allow + // for lazy access e.g. for stream.Layer. + digests = append(digests, digest) + diffids = append(diffids, diffid) + sizes = append(sizes, size) + } + + cf, err := img.ConfigFile() + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + errs := []string{} + for i, layer := range layers { + digest, err := layer.Digest() + if err != nil { + return err + } + diffid, err := layer.DiffID() + if err != nil { + return err + } + size, err := layer.Size() + if err != nil { + return err + } + + if digest != digests[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Digest()=%s, SHA256(Compressed())=%s", i, digest, digests[i])) + } + + if m.Layers[i].Digest != digests[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Manifest.Layers[%d].Digest=%s, SHA256(Compressed())=%s", i, i, m.Layers[i].Digest, digests[i])) + } + + if diffid != diffids[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: DiffID()=%s, SHA256(Gunzip(Compressed()))=%s", i, diffid, diffids[i])) + } + + if cf.RootFS.DiffIDs[i] != diffids[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: ConfigFile.RootFS.DiffIDs[%d]=%s, SHA256(Gunzip(Compressed()))=%s", i, i, cf.RootFS.DiffIDs[i], diffids[i])) + } + + if size != sizes[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Size()=%d, len(Compressed())=%d", i, size, sizes[i])) + } + + if m.Layers[i].Size != sizes[i] { + errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Manifest.Layers[%d].Size=%d, len(Compressed())=%d", i, i, m.Layers[i].Size, sizes[i])) + } + + } + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateManifest(img v1.Image) error { + digest, err := img.Digest() + if err != nil { + return err + } + + rm, err := img.RawManifest() + if err != nil { + return err + } + + hash, _, err := v1.SHA256(bytes.NewReader(rm)) + if err != nil { + return err + } + + m, err := img.Manifest() + if err != nil { + return err + } + + pm, err := v1.ParseManifest(bytes.NewReader(rm)) + if err != nil { + return err + } + + errs := []string{} + if digest != hash { + errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash)) + } + + if diff := cmp.Diff(pm, m); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseManifest(RawManifest()) +Manifest()) %s", diff)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go new file mode 100644 index 000000000..871e24153 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/index.go @@ -0,0 +1,123 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validate + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// Index validates that idx does not violate any invariants of the index format. +func Index(idx v1.ImageIndex) error { + errs := []string{} + + if err := validateChildren(idx); err != nil { + errs = append(errs, fmt.Sprintf("validating children: %v", err)) + } + + if err := validateIndexManifest(idx); err != nil { + errs = append(errs, fmt.Sprintf("validating index manifest: %v", err)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n\n")) + } + return nil +} + +func validateChildren(idx v1.ImageIndex) error { + manifest, err := idx.IndexManifest() + if err != nil { + return err + } + + errs := []string{} + for i, desc := range manifest.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + idx, err := idx.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := Index(idx); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate index Manifests[%d](%s): %v", i, desc.Digest, err)) + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := idx.Image(desc.Digest) + if err != nil { + return err + } + if err := Image(img); err != nil { + errs = append(errs, fmt.Sprintf("failed to validate image Manifests[%d](%s): %v", i, desc.Digest, err)) + } + default: + return fmt.Errorf("todo: validate index Blob()") + } + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +func validateIndexManifest(idx v1.ImageIndex) error { + digest, err := idx.Digest() + if err != nil { + return err + } + + rm, err := idx.RawManifest() + if err != nil { + return err + } + + hash, _, err := v1.SHA256(bytes.NewReader(rm)) + if err != nil { + return err + } + + m, err := idx.IndexManifest() + if err != nil { + return err + } + + pm, err := v1.ParseIndexManifest(bytes.NewReader(rm)) + if err != nil { + return err + } + + errs := []string{} + if digest != hash { + errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash)) + } + + if diff := cmp.Diff(pm, m); diff != "" { + errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseIndexManifest(RawManifest()) +Manifest()) %s", diff)) + } + + if len(errs) != 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +}