Merge branch 'master' into version

This commit is contained in:
Prashant 2019-10-02 21:39:32 +05:30
commit 24c2dcc867
35 changed files with 1550 additions and 34 deletions

View File

@ -7,6 +7,7 @@ about: Report a bug in kaniko
**Actual behavior**
A clear and concise description of what the bug is.
**Expected behavior**
A clear and concise description of what you expected to happen.
@ -21,3 +22,16 @@ Steps to reproduce the behavior:
- Build Context
Please provide or clearly describe any files needed to build the Dockerfile (ADD/COPY commands)
- Kaniko Image (fully qualified with digest)
**Triage Notes for the Maintainers**
<!-- 🎉🎉🎉 Thank you for an opening an issue !!! 🎉🎉🎉
We are doing our best to get to this. Please help us by helping us prioritize your issue by filling the section below -->
| **Description** | **Yes/No** |
|----------------|---------------|
| Please check if this a new feature you are proposing | <ul><li>- [ ] </li></ul>|
| Please check if the build works in docker but not in kaniko | <ul><li>- [ ] </li></ul>|
| Please check if this error is seen when you use `--cache` flag | <ul><li>- [ ] </li></ul>|
| Please check if your dockerfile is a multistage dockerfile | <ul><li>- [ ] </li></ul>|

41
.github/pull_request_template.md vendored Normal file
View File

@ -0,0 +1,41 @@
<!-- 🎉🎉🎉 Thank you for the PR!!! 🎉🎉🎉 -->
Fixes `#<issue number>`. _in case of a bug fix, this should point to a bug and any other related issue(s)_
**Description**
<!-- Describe your changes here- ideally you can get that description straight from
your descriptive commit message(s)! -->
**Submitter Checklist**
These are the criteria that every PR should meet, please check them off as you
review them:
- [ ] Includes [unit tests](../DEVELOPMENT.md#creating-a-pr)
- [ ] Adds integration tests if needed.
_See [the contribution guide](../CONTRIBUTING.md) for more details._
**Reviewer Notes**
- [ ] The code flow looks good.
- [ ] Unit tests and or integration tests added.
**Release Notes**
Describe any changes here so maintainer can include it in the release notes, or delete this block.
```
Examples of user facing changes:
- Skaffold config changes like
e.g. "Add buildArgs to `Kustomize` deployer skaffold config."
- Bug fixes
e.g. "Improve skaffold init behaviour when tags are used in manifests"
- Any changes in skaffold behavior
e.g. "Artiface cachine is turned on by default."
```

View File

@ -1,3 +1,37 @@
# v0.12.0 Release - 2019-09/13
## New Features
* Added `--oci-layout-path` flag to save image in OCI layout. [#744](https://github.com/GoogleContainerTools/kaniko/pull/744)
* Add support for S3 custom endpoint [#698](https://github.com/GoogleContainerTools/kaniko/pull/698)
## Bug Fixes
* Setting PATH [#760](https://github.com/GoogleContainerTools/kaniko/pull/760)
* Remove leading slash in layer tarball paths (Closes: #726) [#729](https://github.com/GoogleContainerTools/kaniko/pull/729)
## Updates and Refactors
* Remove cruft [#635](https://github.com/GoogleContainerTools/kaniko/pull/635)
* Add desc for `--skip-tls-verify-pull` to README [#493](https://github.com/GoogleContainerTools/kaniko/pull/493)
Huge thank you for this release towards our contributors:
- Carlos Alexandro Becker
- Carlos Sanchez
- chhsia0
- Deniz Zoeteman
- Luke Wood
- Matthew Dawson
- Niels Denissen
- Priya Wadhwa
- Sharif Elgamal
- Takeaki Matsumoto
- Taylor Barrella
- Tejal Desai
- v.rul
- Warren Seymour
- xanonid
- Xueshan Feng
- Роман Небалуев
# v0.11.0 Release - 2019-08-23
## Bug Fixes

6
Gopkg.lock generated
View File

@ -445,7 +445,7 @@
version = "v0.2.0"
[[projects]]
digest = "1:16c8837e951303ef6388132bc875337660a48ea2dedf1c941ca118ea92d2a3d2"
digest = "1:5924704ec96f00247784c512cc57f45a595030376a7ff2ff993bf356793a2cb0"
name = "github.com/google/go-containerregistry"
packages = [
"pkg/authn",
@ -456,6 +456,7 @@
"pkg/v1",
"pkg/v1/daemon",
"pkg/v1/empty",
"pkg/v1/layout",
"pkg/v1/mutate",
"pkg/v1/partial",
"pkg/v1/random",
@ -467,7 +468,7 @@
"pkg/v1/v1util",
]
pruneopts = "NUT"
revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74"
revision = "31e00cede111067bae48bfc2cbfc522b0b36207f"
[[projects]]
digest = "1:f4f203acd8b11b8747bdcd91696a01dbc95ccb9e2ca2db6abf81c3a4f5e950ce"
@ -1368,6 +1369,7 @@
"github.com/google/go-containerregistry/pkg/v1",
"github.com/google/go-containerregistry/pkg/v1/daemon",
"github.com/google/go-containerregistry/pkg/v1/empty",
"github.com/google/go-containerregistry/pkg/v1/layout",
"github.com/google/go-containerregistry/pkg/v1/mutate",
"github.com/google/go-containerregistry/pkg/v1/partial",
"github.com/google/go-containerregistry/pkg/v1/remote",

View File

@ -37,7 +37,7 @@ required = [
[[constraint]]
name = "github.com/google/go-containerregistry"
revision = "273af77a08b28b49cc2cff2dd8ae50a5094dac74"
revision = "31e00cede111067bae48bfc2cbfc522b0b36207f"
[[override]]
name = "k8s.io/apimachinery"

View File

@ -14,7 +14,7 @@
# Bump these on release
VERSION_MAJOR ?= 0
VERSION_MINOR ?= 11
VERSION_MINOR ?= 12
VERSION_BUILD ?= 0
VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)

View File

@ -44,6 +44,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME
- [--insecure](#--insecure)
- [--insecure-pull](#--insecure-pull)
- [--no-push](#--no-push)
- [--oci-layout-path](#--oci-layout-path)
- [--reproducible](#--reproducible)
- [--single-snapshot](#--single-snapshot)
- [--snapshotMode](#--snapshotmode)
@ -374,6 +375,19 @@ will write the digest to that file, which is picked up by
Kubernetes automatically as the `{{.state.terminated.message}}`
of the container.
#### --oci-layout-path
Set this flag to specify a directory in the container where the OCI image
layout of a built image will be placed. This can be used to automatically
track the exact image built by Kaniko.
For example, to surface the image digest built in a
[Tekton task](https://github.com/tektoncd/pipeline/blob/v0.6.0/docs/resources.md#surfacing-the-image-digest-built-in-a-task),
this flag should be set to match the image resource `outputImageDir`.
_Note: Depending on the built image, the media type of the image manifest might be either
`application/vnd.oci.image.manifest.v1+json` or `application/vnd.docker.distribution.manifest.v2+json``._
#### --insecure-registry
Set this flag to use plain HTTP requests when accessing a registry. It is supposed to be used for testing purposes only and should not be used in production!

View File

@ -131,6 +131,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) {
RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided")
RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.")
RootCmd.PersistentFlags().StringVarP(&opts.DigestFile, "digest-file", "", "", "Specify a file to save the digest of the built image to.")
RootCmd.PersistentFlags().StringVarP(&opts.OCILayoutPath, "oci-layout-path", "", "", "Path to save the OCI image layout of the built image.")
RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image")
RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end")
RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.")

View File

@ -19,6 +19,7 @@ package buildcontext
import (
"os"
"path/filepath"
"strings"
"github.com/GoogleContainerTools/kaniko/pkg/constants"
"github.com/GoogleContainerTools/kaniko/pkg/util"
@ -36,9 +37,21 @@ type S3 struct {
// UnpackTarFromBuildContext download and untar a file from s3
func (s *S3) UnpackTarFromBuildContext() (string, error) {
bucket, item := util.GetBucketAndItem(s.context)
sess, err := session.NewSessionWithOptions(session.Options{
option := session.Options{
SharedConfigState: session.SharedConfigEnable,
})
}
endpoint := os.Getenv(constants.S3EndpointEnv)
forcePath := false
if strings.ToLower(os.Getenv(constants.S3ForcePathStyle)) == "true" {
forcePath = true
}
if endpoint != "" {
option.Config = aws.Config{
Endpoint: aws.String(endpoint),
S3ForcePathStyle: aws.Bool(forcePath),
}
}
sess, err := session.NewSessionWithOptions(option)
if err != nil {
return bucket, err
}

2
pkg/cache/cache.go vendored
View File

@ -59,7 +59,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) {
}
registryName := cacheRef.Repository.Registry.Name()
if rc.Opts.InsecureRegistries.Contains(registryName) {
if rc.Opts.Insecure || rc.Opts.InsecureRegistries.Contains(registryName) {
newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
if err != nil {
return nil, err

View File

@ -37,6 +37,7 @@ type KanikoOptions struct {
Target string
CacheRepo string
DigestFile string
OCILayoutPath string
Destinations multiArg
BuildArgs multiArg
Insecure bool

View File

@ -70,6 +70,10 @@ const (
// Name of the .dockerignore file
Dockerignore = ".dockerignore"
// S3 Custom endpoint ENV name
S3EndpointEnv = "S3_ENDPOINT"
S3ForcePathStyle = "S3_FORCE_PATH_STYLE"
)
// ScratchEnvVars are the default environment variables needed for a scratch image.

View File

@ -124,7 +124,7 @@ func initializeConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) {
return nil, err
}
if img == empty.Image {
if imageConfig.Config.Env == nil {
imageConfig.Config.Env = constants.ScratchEnvVars
}
return imageConfig, nil

View File

@ -29,6 +29,8 @@ import (
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/google/go-cmp/cmp"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
)
@ -405,3 +407,58 @@ func Test_filesToSave(t *testing.T) {
})
}
}
func TestInitializeConfig(t *testing.T) {
tests := []struct {
description string
cfg v1.ConfigFile
expected v1.Config
}{
{
description: "env is not set in the image",
cfg: v1.ConfigFile{
Config: v1.Config{
Image: "test",
},
},
expected: v1.Config{
Image: "test",
Env: []string{
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
},
},
},
{
description: "env is set in the image",
cfg: v1.ConfigFile{
Config: v1.Config{
Env: []string{
"PATH=/usr/local/something",
},
},
},
expected: v1.Config{
Env: []string{
"PATH=/usr/local/something",
},
},
},
{
description: "image is empty",
expected: v1.Config{
Env: []string{
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
},
},
},
}
for _, tt := range tests {
img, err := mutate.ConfigFile(empty.Image, &tt.cfg)
if err != nil {
t.Errorf("error seen when running test %s", err)
t.Fail()
}
actual, _ := initializeConfig(img)
testutil.CheckDeepEqual(t, tt.expected, actual.Config)
}
}

View File

View File

@ -34,6 +34,7 @@ import (
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/tarball"
@ -58,7 +59,7 @@ func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) {
return w.t.RoundTrip(r)
}
// CheckPushPermissionos checks that the configured credentials can be used to
// CheckPushPermissions checks that the configured credentials can be used to
// push to every specified destination.
func CheckPushPermissions(opts *config.KanikoOptions) error {
if opts.NoPush {
@ -76,6 +77,13 @@ func CheckPushPermissions(opts *config.KanikoOptions) error {
}
registryName := destRef.Repository.Registry.Name()
if opts.Insecure || opts.InsecureRegistries.Contains(registryName) {
newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
if err != nil {
return errors.Wrap(err, "getting new insecure registry")
}
destRef.Repository.Registry = newReg
}
tr := makeTransport(opts, registryName)
if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), tr); err != nil {
return errors.Wrapf(err, "checking push permission for %q", destRef)
@ -101,6 +109,16 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error {
}
}
if opts.OCILayoutPath != "" {
path, err := layout.Write(opts.OCILayoutPath, empty.Index)
if err != nil {
return errors.Wrap(err, "writing empty layout")
}
if err := path.AppendImage(image); err != nil {
return errors.Wrap(err, "appending image")
}
}
destRefs := []name.Tag{}
for _, destination := range opts.Destinations {
destRef, err := name.NewTag(destination, name.WeakValidation)

View File

@ -23,7 +23,11 @@ import (
"os"
"testing"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/random"
"github.com/google/go-containerregistry/pkg/v1/validate"
)
func TestHeaderAdded(t *testing.T) {
@ -69,3 +73,49 @@ func (m *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
ua := r.UserAgent()
return &http.Response{Body: ioutil.NopCloser(bytes.NewBufferString(ua))}, nil
}
func TestOCILayoutPath(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("could not create temp dir: %s", err)
}
defer os.RemoveAll(tmpDir)
image, err := random.Image(1024, 4)
if err != nil {
t.Fatalf("could not create image: %s", err)
}
digest, err := image.Digest()
if err != nil {
t.Fatalf("could not get image digest: %s", err)
}
want, err := image.Manifest()
if err != nil {
t.Fatalf("could not get image manifest: %s", err)
}
opts := config.KanikoOptions{
NoPush: true,
OCILayoutPath: tmpDir,
}
if err := DoPush(image, &opts); err != nil {
t.Fatalf("could not push image: %s", err)
}
layoutIndex, err := layout.ImageIndexFromPath(tmpDir)
if err != nil {
t.Fatalf("could not get index from layout: %s", err)
}
testutil.CheckError(t, false, validate.Index(layoutIndex))
layoutImage, err := layoutIndex.Image(digest)
if err != nil {
t.Fatalf("could not get image from layout: %s", err)
}
got, err := layoutImage.Manifest()
testutil.CheckErrorAndDeepEqual(t, false, err, want, got)
}

View File

@ -22,6 +22,7 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/GoogleContainerTools/kaniko/pkg/util"
@ -31,6 +32,7 @@ import (
func TestSnapshotFSFileChange(t *testing.T) {
testDir, snapshotter, cleanup, err := setUpTestDir()
testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/")
defer cleanup()
if err != nil {
t.Fatal(err)
@ -55,16 +57,16 @@ func TestSnapshotFSFileChange(t *testing.T) {
}
// Check contents of the snapshot, make sure contents is equivalent to snapshotFiles
tr := tar.NewReader(f)
fooPath := filepath.Join(testDir, "foo")
batPath := filepath.Join(testDir, "bar/bat")
fooPath := filepath.Join(testDirWithoutLeadingSlash, "foo")
batPath := filepath.Join(testDirWithoutLeadingSlash, "bar/bat")
snapshotFiles := map[string]string{
fooPath: "newbaz1",
batPath: "baz",
}
for _, dir := range util.ParentDirectories(fooPath) {
for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(fooPath) {
snapshotFiles[dir] = ""
}
for _, dir := range util.ParentDirectories(batPath) {
for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) {
snapshotFiles[dir] = ""
}
numFiles := 0
@ -128,12 +130,14 @@ func TestSnapshotFSIsReproducible(t *testing.T) {
func TestSnapshotFSChangePermissions(t *testing.T) {
testDir, snapshotter, cleanup, err := setUpTestDir()
testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/")
defer cleanup()
if err != nil {
t.Fatal(err)
}
// Change permissions on a file
batPath := filepath.Join(testDir, "bar/bat")
batPathWithoutLeadingSlash := filepath.Join(testDirWithoutLeadingSlash, "bar/bat")
if err := os.Chmod(batPath, 0600); err != nil {
t.Fatalf("Error changing permissions on %s: %v", batPath, err)
}
@ -149,9 +153,9 @@ func TestSnapshotFSChangePermissions(t *testing.T) {
// Check contents of the snapshot, make sure contents is equivalent to snapshotFiles
tr := tar.NewReader(f)
snapshotFiles := map[string]string{
batPath: "baz2",
batPathWithoutLeadingSlash: "baz2",
}
for _, dir := range util.ParentDirectories(batPath) {
for _, dir := range util.ParentDirectoriesWithoutLeadingSlash(batPath) {
snapshotFiles[dir] = ""
}
numFiles := 0
@ -160,6 +164,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) {
if err == io.EOF {
break
}
t.Logf("Info %s in tar", hdr.Name)
numFiles++
if _, isFile := snapshotFiles[hdr.Name]; !isFile {
t.Fatalf("File %s unexpectedly in tar", hdr.Name)
@ -176,6 +181,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) {
func TestSnapshotFiles(t *testing.T) {
testDir, snapshotter, cleanup, err := setUpTestDir()
testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/")
defer cleanup()
if err != nil {
t.Fatal(err)
@ -197,9 +203,9 @@ func TestSnapshotFiles(t *testing.T) {
defer os.Remove(tarPath)
expectedFiles := []string{
filepath.Join(testDir, "foo"),
filepath.Join(testDirWithoutLeadingSlash, "foo"),
}
expectedFiles = append(expectedFiles, util.ParentDirectories(filepath.Join(testDir, "foo"))...)
expectedFiles = append(expectedFiles, util.ParentDirectoriesWithoutLeadingSlash(filepath.Join(testDir, "foo"))...)
f, err := os.Open(tarPath)
if err != nil {

View File

@ -121,6 +121,10 @@ func DeleteFilesystem() error {
logrus.Info("Deleting filesystem...")
return filepath.Walk(constants.RootDir, func(path string, info os.FileInfo, _ error) error {
if CheckWhitelist(path) {
if !isExist(path) {
logrus.Debugf("Path %s whitelisted, but not exists", path)
return nil
}
if info.IsDir() {
return filepath.SkipDir
}
@ -138,6 +142,14 @@ func DeleteFilesystem() error {
})
}
// isExists returns true if path exists
func isExist(path string) bool {
if _, err := os.Stat(path); err == nil {
return true
}
return false
}
// ChildDirInWhitelist returns true if there is a child file or directory of the path in the whitelist
func childDirInWhitelist(path string) bool {
for _, d := range whitelist {
@ -377,6 +389,24 @@ func ParentDirectories(path string) []string {
return paths
}
// ParentDirectoriesWithoutLeadingSlash returns a list of paths to all parent directories
// all subdirectories do not contain a leading /
// Ex. /some/temp/dir -> [/, some, some/temp, some/temp/dir]
func ParentDirectoriesWithoutLeadingSlash(path string) []string {
path = filepath.Clean(path)
dirs := strings.Split(path, "/")
dirPath := ""
paths := []string{constants.RootDir}
for index, dir := range dirs {
if dir == "" || index == (len(dirs)-1) {
continue
}
dirPath = filepath.Join(dirPath, dir)
paths = append(paths, dirPath)
}
return paths
}
// FilepathExists returns true if the path exists
func FilepathExists(path string) bool {
_, err := os.Lstat(path)

View File

@ -177,6 +177,38 @@ func Test_ParentDirectories(t *testing.T) {
}
}
func Test_ParentDirectoriesWithoutLeadingSlash(t *testing.T) {
tests := []struct {
name string
path string
expected []string
}{
{
name: "regular path",
path: "/path/to/dir",
expected: []string{
"/",
"path",
"path/to",
},
},
{
name: "current directory",
path: ".",
expected: []string{
"/",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual := ParentDirectoriesWithoutLeadingSlash(tt.path)
testutil.CheckErrorAndDeepEqual(t, false, nil, tt.expected, actual)
})
}
}
func Test_CheckWhitelist(t *testing.T) {
type args struct {
path string

View File

@ -25,6 +25,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/docker/docker/pkg/archive"
@ -74,7 +75,14 @@ func (t *Tar) AddFileToTar(p string) error {
if err != nil {
return err
}
hdr.Name = p
if p != "/" {
// Docker uses no leading / in the tarball
hdr.Name = strings.TrimLeft(p, "/")
} else {
// allow entry for / to preserve permission changes etc. (currently ignored anyway by Docker runtime)
hdr.Name = p
}
hardlink, linkDst := t.checkHardlink(p, i)
if hardlink {
@ -104,7 +112,8 @@ func (t *Tar) Whiteout(p string) error {
name := ".wh." + filepath.Base(p)
th := &tar.Header{
Name: filepath.Join(dir, name),
// Docker uses no leading / in the tarball
Name: strings.TrimLeft(filepath.Join(dir, name), "/"),
Size: 0,
}
if err := t.w.WriteHeader(th); err != nil {

View File

@ -0,0 +1,38 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import (
"io"
"io/ioutil"
"os"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// Blob returns a blob with the given hash from the Path.
func (l Path) Blob(h v1.Hash) (io.ReadCloser, error) {
return os.Open(l.blobPath(h))
}
// Bytes is a convenience function to return a blob from the Path as
// a byte slice.
func (l Path) Bytes(h v1.Hash) ([]byte, error) {
return ioutil.ReadFile(l.blobPath(h))
}
func (l Path) blobPath(h v1.Hash) string {
return l.path("blobs", h.Algorithm, h.Hex)
}

View File

@ -0,0 +1,19 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package layout provides facilities for reading/writing artifacts from/to
// an OCI image layout on disk, see:
//
// https://github.com/opencontainers/image-spec/blob/master/image-layout.md
package layout

View File

@ -0,0 +1,131 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import (
"fmt"
"io"
"sync"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
)
type layoutImage struct {
path Path
desc v1.Descriptor
manifestLock sync.Mutex // Protects rawManifest
rawManifest []byte
}
var _ partial.CompressedImageCore = (*layoutImage)(nil)
// Image reads a v1.Image with digest h from the Path.
func (l Path) Image(h v1.Hash) (v1.Image, error) {
ii, err := l.ImageIndex()
if err != nil {
return nil, err
}
return ii.Image(h)
}
func (li *layoutImage) MediaType() (types.MediaType, error) {
return li.desc.MediaType, nil
}
// Implements WithManifest for partial.Blobset.
func (li *layoutImage) Manifest() (*v1.Manifest, error) {
return partial.Manifest(li)
}
func (li *layoutImage) RawManifest() ([]byte, error) {
li.manifestLock.Lock()
defer li.manifestLock.Unlock()
if li.rawManifest != nil {
return li.rawManifest, nil
}
b, err := li.path.Bytes(li.desc.Digest)
if err != nil {
return nil, err
}
li.rawManifest = b
return li.rawManifest, nil
}
func (li *layoutImage) RawConfigFile() ([]byte, error) {
manifest, err := li.Manifest()
if err != nil {
return nil, err
}
return li.path.Bytes(manifest.Config.Digest)
}
func (li *layoutImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) {
manifest, err := li.Manifest()
if err != nil {
return nil, err
}
if h == manifest.Config.Digest {
return partial.CompressedLayer(&compressedBlob{
path: li.path,
desc: manifest.Config,
}), nil
}
for _, desc := range manifest.Layers {
if h == desc.Digest {
switch desc.MediaType {
case types.OCILayer, types.DockerLayer:
return partial.CompressedToLayer(&compressedBlob{
path: li.path,
desc: desc,
})
default:
// TODO: We assume everything is a compressed blob, but that might not be true.
// TODO: Handle foreign layers.
return nil, fmt.Errorf("unexpected media type: %v for layer: %v", desc.MediaType, desc.Digest)
}
}
}
return nil, fmt.Errorf("could not find layer in image: %s", h)
}
type compressedBlob struct {
path Path
desc v1.Descriptor
}
func (b *compressedBlob) Digest() (v1.Hash, error) {
return b.desc.Digest, nil
}
func (b *compressedBlob) Compressed() (io.ReadCloser, error) {
return b.path.Blob(b.desc.Digest)
}
func (b *compressedBlob) Size() (int64, error) {
return b.desc.Size, nil
}
func (b *compressedBlob) MediaType() (types.MediaType, error) {
return b.desc.MediaType, nil
}

View File

@ -0,0 +1,146 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
)
var _ v1.ImageIndex = (*layoutIndex)(nil)
type layoutIndex struct {
path Path
rawIndex []byte
}
// ImageIndexFromPath is a convenience function which constructs a Path and returns its v1.ImageIndex.
func ImageIndexFromPath(path string) (v1.ImageIndex, error) {
lp, err := FromPath(path)
if err != nil {
return nil, err
}
return lp.ImageIndex()
}
// ImageIndex returns a v1.ImageIndex for the Path.
func (l Path) ImageIndex() (v1.ImageIndex, error) {
rawIndex, err := ioutil.ReadFile(l.path("index.json"))
if err != nil {
return nil, err
}
idx := &layoutIndex{
path: l,
rawIndex: rawIndex,
}
return idx, nil
}
func (i *layoutIndex) MediaType() (types.MediaType, error) {
return types.OCIImageIndex, nil
}
func (i *layoutIndex) Digest() (v1.Hash, error) {
return partial.Digest(i)
}
func (i *layoutIndex) IndexManifest() (*v1.IndexManifest, error) {
var index v1.IndexManifest
err := json.Unmarshal(i.rawIndex, &index)
return &index, err
}
func (i *layoutIndex) RawManifest() ([]byte, error) {
return i.rawIndex, nil
}
func (i *layoutIndex) Image(h v1.Hash) (v1.Image, error) {
// Look up the digest in our manifest first to return a better error.
desc, err := i.findDescriptor(h)
if err != nil {
return nil, err
}
if !isExpectedMediaType(desc.MediaType, types.OCIManifestSchema1, types.DockerManifestSchema2) {
return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType)
}
img := &layoutImage{
path: i.path,
desc: *desc,
}
return partial.CompressedToImage(img)
}
func (i *layoutIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) {
// Look up the digest in our manifest first to return a better error.
desc, err := i.findDescriptor(h)
if err != nil {
return nil, err
}
if !isExpectedMediaType(desc.MediaType, types.OCIImageIndex, types.DockerManifestList) {
return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType)
}
rawIndex, err := i.path.Bytes(h)
if err != nil {
return nil, err
}
return &layoutIndex{
path: i.path,
rawIndex: rawIndex,
}, nil
}
func (i *layoutIndex) Blob(h v1.Hash) (io.ReadCloser, error) {
return i.path.Blob(h)
}
func (i *layoutIndex) findDescriptor(h v1.Hash) (*v1.Descriptor, error) {
im, err := i.IndexManifest()
if err != nil {
return nil, err
}
for _, desc := range im.Manifests {
if desc.Digest == h {
return &desc, nil
}
}
return nil, fmt.Errorf("could not find descriptor in index: %s", h)
}
// TODO: Pull this out into methods on types.MediaType? e.g. instead, have:
// * mt.IsIndex()
// * mt.IsImage()
func isExpectedMediaType(mt types.MediaType, expected ...types.MediaType) bool {
for _, allowed := range expected {
if mt == allowed {
return true
}
}
return false
}

View File

@ -0,0 +1,25 @@
// Copyright 2019 The original author or authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import "path/filepath"
// Path represents an OCI image layout rooted in a file system path
type Path string
func (l Path) path(elem ...string) string {
complete := []string{string(l)}
return filepath.Join(append(complete, elem...)...)
}

View File

@ -0,0 +1,42 @@
package layout
import v1 "github.com/google/go-containerregistry/pkg/v1"
// Option is a functional option for Layout.
//
// TODO: We'll need to change this signature to support Sparse/Thin images.
// Or, alternatively, wrap it in a sparse.Image that returns an empty list for layers?
type Option func(*v1.Descriptor) error
// WithAnnotations adds annotations to the artifact descriptor.
func WithAnnotations(annotations map[string]string) Option {
return func(desc *v1.Descriptor) error {
if desc.Annotations == nil {
desc.Annotations = make(map[string]string)
}
for k, v := range annotations {
desc.Annotations[k] = v
}
return nil
}
}
// WithURLs adds urls to the artifact descriptor.
func WithURLs(urls []string) Option {
return func(desc *v1.Descriptor) error {
if desc.URLs == nil {
desc.URLs = []string{}
}
desc.URLs = append(desc.URLs, urls...)
return nil
}
}
// WithPlatform sets the platform of the artifact descriptor.
func WithPlatform(platform v1.Platform) Option {
return func(desc *v1.Descriptor) error {
desc.Platform = &platform
return nil
}
}

View File

@ -0,0 +1,32 @@
// Copyright 2019 The original author or authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import (
"os"
"path/filepath"
)
// FromPath reads an OCI image layout at path and constructs a layout.Path.
func FromPath(path string) (Path, error) {
// TODO: check oci-layout exists
_, err := os.Stat(filepath.Join(path, "index.json"))
if err != nil {
return "", err
}
return Path(path), nil
}

View File

@ -0,0 +1,301 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package layout
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"os"
"path/filepath"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"golang.org/x/sync/errgroup"
)
var layoutFile = `{
"imageLayoutVersion": "1.0.0"
}`
// AppendImage writes a v1.Image to the Path and updates
// the index.json to reference it.
func (l Path) AppendImage(img v1.Image, options ...Option) error {
if err := l.writeImage(img); err != nil {
return err
}
mt, err := img.MediaType()
if err != nil {
return err
}
d, err := img.Digest()
if err != nil {
return err
}
manifest, err := img.RawManifest()
if err != nil {
return err
}
desc := v1.Descriptor{
MediaType: mt,
Size: int64(len(manifest)),
Digest: d,
}
for _, opt := range options {
if err := opt(&desc); err != nil {
return err
}
}
return l.AppendDescriptor(desc)
}
// AppendIndex writes a v1.ImageIndex to the Path and updates
// the index.json to reference it.
func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error {
if err := l.writeIndex(ii); err != nil {
return err
}
mt, err := ii.MediaType()
if err != nil {
return err
}
d, err := ii.Digest()
if err != nil {
return err
}
manifest, err := ii.RawManifest()
if err != nil {
return err
}
desc := v1.Descriptor{
MediaType: mt,
Size: int64(len(manifest)),
Digest: d,
}
for _, opt := range options {
if err := opt(&desc); err != nil {
return err
}
}
return l.AppendDescriptor(desc)
}
// AppendDescriptor adds a descriptor to the index.json of the Path.
func (l Path) AppendDescriptor(desc v1.Descriptor) error {
ii, err := l.ImageIndex()
if err != nil {
return err
}
index, err := ii.IndexManifest()
if err != nil {
return err
}
index.Manifests = append(index.Manifests, desc)
rawIndex, err := json.MarshalIndent(index, "", " ")
if err != nil {
return err
}
return l.writeFile("index.json", rawIndex)
}
func (l Path) writeFile(name string, data []byte) error {
if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) {
return err
}
return ioutil.WriteFile(l.path(name), data, os.ModePerm)
}
// WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at
// blobs/{hash.Algorithm}/{hash.Hex}.
func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error {
dir := l.path("blobs", hash.Algorithm)
if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
return err
}
file := filepath.Join(dir, hash.Hex)
if _, err := os.Stat(file); err == nil {
// Blob already exists, that's fine.
return nil
}
w, err := os.Create(file)
if err != nil {
return err
}
defer w.Close()
_, err = io.Copy(w, r)
return err
}
// TODO: A streaming version of WriteBlob so we don't have to know the hash
// before we write it.
// TODO: For streaming layers we should write to a tmp file then Rename to the
// final digest.
func (l Path) writeLayer(layer v1.Layer) error {
d, err := layer.Digest()
if err != nil {
return err
}
r, err := layer.Compressed()
if err != nil {
return err
}
return l.WriteBlob(d, r)
}
func (l Path) writeImage(img v1.Image) error {
layers, err := img.Layers()
if err != nil {
return err
}
// Write the layers concurrently.
var g errgroup.Group
for _, layer := range layers {
layer := layer
g.Go(func() error {
return l.writeLayer(layer)
})
}
if err := g.Wait(); err != nil {
return err
}
// Write the config.
cfgName, err := img.ConfigName()
if err != nil {
return err
}
cfgBlob, err := img.RawConfigFile()
if err != nil {
return err
}
if err := l.WriteBlob(cfgName, ioutil.NopCloser(bytes.NewReader(cfgBlob))); err != nil {
return err
}
// Write the img manifest.
d, err := img.Digest()
if err != nil {
return err
}
manifest, err := img.RawManifest()
if err != nil {
return err
}
return l.WriteBlob(d, ioutil.NopCloser(bytes.NewReader(manifest)))
}
func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error {
index, err := ii.IndexManifest()
if err != nil {
return err
}
// Walk the descriptors and write any v1.Image or v1.ImageIndex that we find.
// If we come across something we don't expect, just write it as a blob.
for _, desc := range index.Manifests {
switch desc.MediaType {
case types.OCIImageIndex, types.DockerManifestList:
ii, err := ii.ImageIndex(desc.Digest)
if err != nil {
return err
}
if err := l.writeIndex(ii); err != nil {
return err
}
case types.OCIManifestSchema1, types.DockerManifestSchema2:
img, err := ii.Image(desc.Digest)
if err != nil {
return err
}
if err := l.writeImage(img); err != nil {
return err
}
default:
// TODO: The layout could reference arbitrary things, which we should
// probably just pass through.
}
}
rawIndex, err := ii.RawManifest()
if err != nil {
return err
}
return l.writeFile(indexFile, rawIndex)
}
func (l Path) writeIndex(ii v1.ImageIndex) error {
// Always just write oci-layout file, since it's small.
if err := l.writeFile("oci-layout", []byte(layoutFile)); err != nil {
return err
}
h, err := ii.Digest()
if err != nil {
return err
}
indexFile := filepath.Join("blobs", h.Algorithm, h.Hex)
return l.writeIndexToFile(indexFile, ii)
}
// Write constructs a Path at path from an ImageIndex.
//
// The contents are written in the following format:
// At the top level, there is:
// One oci-layout file containing the version of this image-layout.
// One index.json file listing descriptors for the contained images.
// Under blobs/, there is, for each image:
// One file for each layer, named after the layer's SHA.
// One file for each config blob, named after its SHA.
// One file for each manifest blob, named after its SHA.
func Write(path string, ii v1.ImageIndex) (Path, error) {
lp := Path(path)
// Always just write oci-layout file, since it's small.
if err := lp.writeFile("oci-layout", []byte(layoutFile)); err != nil {
return "", err
}
// TODO create blobs/ in case there is a blobs file which would prevent the directory from being created
return lp, lp.writeIndexToFile("index.json", ii)
}

View File

@ -165,11 +165,9 @@ func (i *image) compute() error {
manifest := m.DeepCopy()
manifestLayers := manifest.Layers
for _, add := range i.adds {
d := v1.Descriptor{
MediaType: types.DockerLayer,
}
d := v1.Descriptor{}
var err error
if d.Size, err = add.Layer.Size(); err != nil {
return err
}
@ -178,6 +176,10 @@ func (i *image) compute() error {
return err
}
if d.MediaType, err = add.Layer.MediaType(); err != nil {
return err
}
manifestLayers = append(manifestLayers, d)
digestMap[d.Digest] = add.Layer
}

View File

@ -16,7 +16,6 @@ package remote
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"net/http"
@ -38,7 +37,20 @@ var defaultPlatform = v1.Platform{
// ErrSchema1 indicates that we received a schema1 manifest from the registry.
// This library doesn't have plans to support this legacy image format:
// https://github.com/google/go-containerregistry/issues/377
var ErrSchema1 = errors.New("unsupported MediaType: https://github.com/google/go-containerregistry/issues/377")
type ErrSchema1 struct {
schema string
}
func NewErrSchema1(schema types.MediaType) error {
return &ErrSchema1{
schema: string(schema),
}
}
// Error implements error.
func (e *ErrSchema1) Error() string {
return fmt.Sprintf("unsupported MediaType: %q, see https://github.com/google/go-containerregistry/issues/377", e.schema)
}
// Descriptor provides access to metadata about remote artifact and accessors
// for efficiently converting it into a v1.Image or v1.ImageIndex.
@ -111,7 +123,7 @@ func (d *Descriptor) Image() (v1.Image, error) {
case types.DockerManifestSchema1, types.DockerManifestSchema1Signed:
// We don't care to support schema 1 images:
// https://github.com/google/go-containerregistry/issues/377
return nil, ErrSchema1
return nil, NewErrSchema1(d.MediaType)
case types.OCIImageIndex, types.DockerManifestList:
// We want an image but the registry has an index, resolve it to an image.
return d.remoteIndex().imageByPlatform(d.platform)
@ -141,7 +153,7 @@ func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) {
case types.DockerManifestSchema1, types.DockerManifestSchema1Signed:
// We don't care to support schema 1 images:
// https://github.com/google/go-containerregistry/issues/377
return nil, ErrSchema1
return nil, NewErrSchema1(d.MediaType)
case types.OCIManifestSchema1, types.DockerManifestSchema2:
// We want an index but the registry has an image, nothing we can do.
return nil, fmt.Errorf("unexpected media type for ImageIndex(): %s; call Image() instead", d.MediaType)

View File

@ -26,6 +26,10 @@ import (
// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors
type Error struct {
Errors []Diagnostic `json:"errors,omitempty"`
// The http status code returned.
StatusCode int
// The raw body if we couldn't understand it.
rawBody string
}
// Check that Error implements error
@ -35,7 +39,10 @@ var _ error = (*Error)(nil)
func (e *Error) Error() string {
switch len(e.Errors) {
case 0:
return "<empty transport.Error response>"
if len(e.rawBody) == 0 {
return fmt.Sprintf("unsupported status code %d", e.StatusCode)
}
return fmt.Sprintf("unsupported status code %d; body: %s", e.StatusCode, e.rawBody)
case 1:
return e.Errors[0].String()
default:
@ -115,11 +122,10 @@ func CheckError(resp *http.Response, codes ...int) error {
}
// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors
var structuredError Error
if err := json.Unmarshal(b, &structuredError); err != nil {
// If the response isn't an unstructured error, then return some
// reasonable error response containing the response body.
return fmt.Errorf("unsupported status code %d; body: %s", resp.StatusCode, string(b))
structuredError := &Error{}
if err := json.Unmarshal(b, structuredError); err != nil {
structuredError.rawBody = string(b)
}
return &structuredError
structuredError.StatusCode = resp.StatusCode
return structuredError
}

View File

@ -0,0 +1,16 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package validate provides methods for validating image correctness.
package validate

View File

@ -0,0 +1,297 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package validate
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"strings"
"github.com/google/go-cmp/cmp"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// Image validates that img does not violate any invariants of the image format.
func Image(img v1.Image) error {
errs := []string{}
if err := validateLayers(img); err != nil {
errs = append(errs, fmt.Sprintf("validating layers: %v", err))
}
if err := validateConfig(img); err != nil {
errs = append(errs, fmt.Sprintf("validating config: %v", err))
}
if err := validateManifest(img); err != nil {
errs = append(errs, fmt.Sprintf("validating manifest: %v", err))
}
if len(errs) != 0 {
return errors.New(strings.Join(errs, "\n\n"))
}
return nil
}
func validateConfig(img v1.Image) error {
cn, err := img.ConfigName()
if err != nil {
return err
}
rc, err := img.RawConfigFile()
if err != nil {
return err
}
hash, size, err := v1.SHA256(bytes.NewReader(rc))
if err != nil {
return err
}
m, err := img.Manifest()
if err != nil {
return err
}
cf, err := img.ConfigFile()
if err != nil {
return err
}
pcf, err := v1.ParseConfigFile(bytes.NewReader(rc))
if err != nil {
return err
}
errs := []string{}
if cn != hash {
errs = append(errs, fmt.Sprintf("mismatched config digest: ConfigName()=%s, SHA256(RawConfigFile())=%s", cn, hash))
}
if want, got := m.Config.Size, size; want != got {
errs = append(errs, fmt.Sprintf("mismatched config size: Manifest.Config.Size()=%d, len(RawConfigFile())=%d", want, got))
}
if diff := cmp.Diff(pcf, cf); diff != "" {
errs = append(errs, fmt.Sprintf("mismatched config content: (-ParseConfigFile(RawConfigFile()) +ConfigFile()) %s", diff))
}
if cf.RootFS.Type != "layers" {
errs = append(errs, fmt.Sprintf("invalid ConfigFile.RootFS.Type: %q != %q", cf.RootFS.Type, "layers"))
}
if len(errs) != 0 {
return errors.New(strings.Join(errs, "\n"))
}
return nil
}
func validateLayers(img v1.Image) error {
layers, err := img.Layers()
if err != nil {
return err
}
digests := []v1.Hash{}
diffids := []v1.Hash{}
sizes := []int64{}
for _, layer := range layers {
// TODO: Test layer.Uncompressed.
compressed, err := layer.Compressed()
if err != nil {
return err
}
// Keep track of compressed digest.
digester := sha256.New()
// Everything read from compressed is written to digester to compute digest.
hashCompressed := io.TeeReader(compressed, digester)
// Call io.Copy to write from the layer Reader through to the tarReader on
// the other side of the pipe.
pr, pw := io.Pipe()
var size int64
go func() {
n, err := io.Copy(pw, hashCompressed)
if err != nil {
pw.CloseWithError(err)
return
}
size = n
// Now close the compressed reader, to flush the gzip stream
// and calculate digest/diffID/size. This will cause pr to
// return EOF which will cause readers of the Compressed stream
// to finish reading.
pw.CloseWithError(compressed.Close())
}()
// Read the bytes through gzip.Reader to compute the DiffID.
uncompressed, err := gzip.NewReader(pr)
if err != nil {
return err
}
diffider := sha256.New()
hashUncompressed := io.TeeReader(uncompressed, diffider)
// Ensure there aren't duplicate file paths.
tarReader := tar.NewReader(hashUncompressed)
files := make(map[string]struct{})
for {
hdr, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if _, ok := files[hdr.Name]; ok {
return fmt.Errorf("duplicate file path: %s", hdr.Name)
}
files[hdr.Name] = struct{}{}
}
// Discard any trailing padding that the tar.Reader doesn't consume.
if _, err := io.Copy(ioutil.Discard, hashUncompressed); err != nil {
return err
}
if err := uncompressed.Close(); err != nil {
return err
}
digest := v1.Hash{
Algorithm: "sha256",
Hex: hex.EncodeToString(digester.Sum(make([]byte, 0, digester.Size()))),
}
diffid := v1.Hash{
Algorithm: "sha256",
Hex: hex.EncodeToString(diffider.Sum(make([]byte, 0, diffider.Size()))),
}
// Compute all of these first before we call Config() and Manifest() to allow
// for lazy access e.g. for stream.Layer.
digests = append(digests, digest)
diffids = append(diffids, diffid)
sizes = append(sizes, size)
}
cf, err := img.ConfigFile()
if err != nil {
return err
}
m, err := img.Manifest()
if err != nil {
return err
}
errs := []string{}
for i, layer := range layers {
digest, err := layer.Digest()
if err != nil {
return err
}
diffid, err := layer.DiffID()
if err != nil {
return err
}
size, err := layer.Size()
if err != nil {
return err
}
if digest != digests[i] {
errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Digest()=%s, SHA256(Compressed())=%s", i, digest, digests[i]))
}
if m.Layers[i].Digest != digests[i] {
errs = append(errs, fmt.Sprintf("mismatched layer[%d] digest: Manifest.Layers[%d].Digest=%s, SHA256(Compressed())=%s", i, i, m.Layers[i].Digest, digests[i]))
}
if diffid != diffids[i] {
errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: DiffID()=%s, SHA256(Gunzip(Compressed()))=%s", i, diffid, diffids[i]))
}
if cf.RootFS.DiffIDs[i] != diffids[i] {
errs = append(errs, fmt.Sprintf("mismatched layer[%d] diffid: ConfigFile.RootFS.DiffIDs[%d]=%s, SHA256(Gunzip(Compressed()))=%s", i, i, cf.RootFS.DiffIDs[i], diffids[i]))
}
if size != sizes[i] {
errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Size()=%d, len(Compressed())=%d", i, size, sizes[i]))
}
if m.Layers[i].Size != sizes[i] {
errs = append(errs, fmt.Sprintf("mismatched layer[%d] size: Manifest.Layers[%d].Size=%d, len(Compressed())=%d", i, i, m.Layers[i].Size, sizes[i]))
}
}
if len(errs) != 0 {
return errors.New(strings.Join(errs, "\n"))
}
return nil
}
func validateManifest(img v1.Image) error {
digest, err := img.Digest()
if err != nil {
return err
}
rm, err := img.RawManifest()
if err != nil {
return err
}
hash, _, err := v1.SHA256(bytes.NewReader(rm))
if err != nil {
return err
}
m, err := img.Manifest()
if err != nil {
return err
}
pm, err := v1.ParseManifest(bytes.NewReader(rm))
if err != nil {
return err
}
errs := []string{}
if digest != hash {
errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash))
}
if diff := cmp.Diff(pm, m); diff != "" {
errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseManifest(RawManifest()) +Manifest()) %s", diff))
}
if len(errs) != 0 {
return errors.New(strings.Join(errs, "\n"))
}
return nil
}

View File

@ -0,0 +1,123 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package validate
import (
"bytes"
"errors"
"fmt"
"strings"
"github.com/google/go-cmp/cmp"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// Index validates that idx does not violate any invariants of the index format.
func Index(idx v1.ImageIndex) error {
errs := []string{}
if err := validateChildren(idx); err != nil {
errs = append(errs, fmt.Sprintf("validating children: %v", err))
}
if err := validateIndexManifest(idx); err != nil {
errs = append(errs, fmt.Sprintf("validating index manifest: %v", err))
}
if len(errs) != 0 {
return errors.New(strings.Join(errs, "\n\n"))
}
return nil
}
func validateChildren(idx v1.ImageIndex) error {
manifest, err := idx.IndexManifest()
if err != nil {
return err
}
errs := []string{}
for i, desc := range manifest.Manifests {
switch desc.MediaType {
case types.OCIImageIndex, types.DockerManifestList:
idx, err := idx.ImageIndex(desc.Digest)
if err != nil {
return err
}
if err := Index(idx); err != nil {
errs = append(errs, fmt.Sprintf("failed to validate index Manifests[%d](%s): %v", i, desc.Digest, err))
}
case types.OCIManifestSchema1, types.DockerManifestSchema2:
img, err := idx.Image(desc.Digest)
if err != nil {
return err
}
if err := Image(img); err != nil {
errs = append(errs, fmt.Sprintf("failed to validate image Manifests[%d](%s): %v", i, desc.Digest, err))
}
default:
return fmt.Errorf("todo: validate index Blob()")
}
}
if len(errs) != 0 {
return errors.New(strings.Join(errs, "\n"))
}
return nil
}
func validateIndexManifest(idx v1.ImageIndex) error {
digest, err := idx.Digest()
if err != nil {
return err
}
rm, err := idx.RawManifest()
if err != nil {
return err
}
hash, _, err := v1.SHA256(bytes.NewReader(rm))
if err != nil {
return err
}
m, err := idx.IndexManifest()
if err != nil {
return err
}
pm, err := v1.ParseIndexManifest(bytes.NewReader(rm))
if err != nil {
return err
}
errs := []string{}
if digest != hash {
errs = append(errs, fmt.Sprintf("mismatched manifest digest: Digest()=%s, SHA256(RawManifest())=%s", digest, hash))
}
if diff := cmp.Diff(pm, m); diff != "" {
errs = append(errs, fmt.Sprintf("mismatched manifest content: (-ParseIndexManifest(RawManifest()) +Manifest()) %s", diff))
}
if len(errs) != 0 {
return errors.New(strings.Join(errs, "\n"))
}
return nil
}