public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

adding reproducible flag (#205) 

* adding reproducible test

* newer version of go-containerregistry

* new ImageOptions

* switch reproducible flag to default to false

* small fixes

* update dep
This commit is contained in:
Sharif Elgamal 2018-06-22 12:00:44 -07:00 committed by GitHub
parent 4198901540
commit a7c82cf6f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 296 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Gopkg.lock generated
View File

@ -399,7 +399,7 @@
"pkg/v1/types",
"pkg/v1/v1util"
]
revision = "5e2bd1f4bf61add62944828d54e239d352daaabf"
revision = "3f6471078a9661a9a439bd5e71a371aff429566a"
[[projects]]
name = "github.com/googleapis/gax-go"

3
cmd/executor/cmd/root.go
View File

@ -41,6 +41,7 @@ var (
buildArgs multiArg
tarPath string
singleSnapshot bool
reproducible bool
)
func init() {
@ -56,6 +57,7 @@ func init() {
RootCmd.PersistentFlags().BoolVarP(&force, "force", "", false, "Force building outside of a container")
RootCmd.PersistentFlags().StringVarP(&tarPath, "tarPath", "", "", "Path to save the image in as a tarball instead of pushing")
RootCmd.PersistentFlags().BoolVarP(&singleSnapshot, "single-snapshot", "", false, "Set this flag to take a single snapshot at the end of the build.")
RootCmd.PersistentFlags().BoolVarP(&reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible")
}
var RootCmd = &cobra.Command{
@ -87,6 +89,7 @@ var RootCmd = &cobra.Command{
SnapshotMode: snapshotMode,
Args: buildArgs,
SingleSnapshot: singleSnapshot,
Reproducible: reproducible,
})
if err != nil {
logrus.Error(err)

20
integration/dockerfiles/Dockerfile_test_copy_reproducible Normal file
View File

@ -0,0 +1,20 @@
FROM alpine:3.7
COPY context/foo foo
COPY context/foo /foodir/
COPY context/bar/b* bar/
COPY context/fo? /foo2
COPY context/bar/doesnotexist* context/foo hello
COPY ./context/empty /empty
COPY ./ dir/
COPY . newdir
COPY context/bar /baz/
COPY ["context/foo", "/tmp/foo" ]
COPY context/b* /baz/
COPY context/foo context/bar/ba? /test/
COPY context/arr[[]0].txt /mydir/
COPY context/bar/bat .
ENV contextenv ./context
COPY ${contextenv}/foo /tmp/foo2
COPY $contextenv/foo /tmp/foo3
COPY $contextenv/* /tmp/${contextenv}/

13
integration/integration_test.go
View File

@ -119,7 +119,10 @@ func TestRun(t *testing.T) {
"Dockerfile_test_scratch": {"--single-snapshot"},
}
// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
testsToIgnore := []string{"Dockerfile_test_user_run"}
bucketContextTests := []string{"Dockerfile_test_copy_bucket"}
reproducibleTests := []string{"Dockerfile_test_env"}
_, ex, _, _ := runtime.Caller(0)
cwd := filepath.Dir(ex)
@ -161,6 +164,14 @@ func TestRun(t *testing.T) {
}
}
reproducibleFlag := ""
for _, d := range reproducibleTests {
if d == dockerfile {
reproducibleFlag = "--reproducible"
break
}
}
// build kaniko image
additionalFlags := append(buildArgs, additionalFlagsMap[dockerfile]...)
kanikoImage := strings.ToLower(testRepo + kanikoPrefix + dockerfile)
@ -170,7 +181,7 @@ func TestRun(t *testing.T) {
"-v", cwd + ":/workspace",
executorImage,
"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
"-d", kanikoImage,
"-d", kanikoImage, reproducibleFlag,
contextFlag, contextPath},
additionalFlags...)...,
)

2
pkg/dockerfile/dockerfile.go
View File

@ -108,7 +108,7 @@ func Dependencies(index int, stages []instructions.Stage, buildArgs *BuildArgs)
if err != nil {
return nil, err
}
sourceImage, err = remote.Image(ref, auth, http.DefaultTransport)
sourceImage, err = remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport))
if err != nil {
return nil, err
}

11
pkg/executor/executor.go
View File

@ -52,6 +52,7 @@ type KanikoBuildArgs struct {
SnapshotMode string
Args []string
SingleSnapshot bool
Reproducible bool
}
func DoBuild(k KanikoBuildArgs) (name.Reference, v1.Image, error) {
@ -94,7 +95,7 @@ func DoBuild(k KanikoBuildArgs) (name.Reference, v1.Image, error) {
if err != nil {
return nil, nil, err
}
sourceImage, err = remote.Image(ref, auth, http.DefaultTransport)
sourceImage, err = remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport))
if err != nil {
return nil, nil, err
}
@ -174,6 +175,14 @@ func DoBuild(k KanikoBuildArgs) (name.Reference, v1.Image, error) {
if err != nil {
return nil, nil, err
}
if k.Reproducible {
sourceImage, err = mutate.Canonical(sourceImage)
if err != nil {
return nil, nil, err
}
}
return ref, sourceImage, nil
}
if err := saveStageDependencies(index, stages, buildArgs.Clone()); err != nil {

167
vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go generated vendored
View File

@ -21,12 +21,17 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"path/filepath"
"strings"
"time"
"github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/google/go-containerregistry/pkg/v1/v1util"
)
const whiteoutPrefix = ".wh."
@ -128,11 +133,6 @@ func Append(base v1.Image, adds ...Addendum) (v1.Image, error) {
// Config mutates the provided v1.Image to have the provided v1.Config
func Config(base v1.Image, cfg v1.Config) (v1.Image, error) {
m, err := base.Manifest()
if err != nil {
return nil, err
}
cf, err := base.ConfigFile()
if err != nil {
return nil, err
@ -140,10 +140,19 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) {
cf.Config = cfg
return configFile(base, cf)
}
func configFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) {
m, err := base.Manifest()
if err != nil {
return nil, err
}
image := &image{
Image: base,
manifest: m.DeepCopy(),
configFile: cf.DeepCopy(),
configFile: cfg,
digestMap: make(map[v1.Hash]v1.Layer),
}
@ -160,13 +169,8 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) {
return image, nil
}
// Created mutates the provided v1.Image to have the provided v1.Time
// CreatedAt mutates the provided v1.Image to have the provided v1.Time
func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) {
m, err := base.Manifest()
if err != nil {
return nil, err
}
cf, err := base.ConfigFile()
if err != nil {
return nil, err
@ -175,18 +179,7 @@ func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) {
cfg := cf.DeepCopy()
cfg.Created = created
image := &image{
Image: base,
manifest: m.DeepCopy(),
configFile: cfg,
digestMap: make(map[v1.Hash]v1.Layer),
}
image.manifest.Config.Digest, err = image.ConfigName()
if err != nil {
return nil, err
}
return image, nil
return configFile(base, cfg)
}
type image struct {
@ -392,3 +385,129 @@ func inWhiteoutDir(fileMap map[string]bool, file string) bool {
}
return false
}
// Time sets all timestamps in an image to the given timestamp.
func Time(img v1.Image, t time.Time) (v1.Image, error) {
newImage := empty.Image
layers, err := img.Layers()
if err != nil {
return nil, fmt.Errorf("Error getting image layers: %v", err)
}
// Strip away all timestamps from layers
var newLayers []v1.Layer
for _, layer := range layers {
newLayer, err := layerTime(layer, t)
if err != nil {
return nil, fmt.Errorf("Error setting layer times: %v", err)
}
newLayers = append(newLayers, newLayer)
}
newImage, err = AppendLayers(newImage, newLayers...)
if err != nil {
return nil, fmt.Errorf("Error appending layers: %v", err)
}
ocf, err := img.ConfigFile()
if err != nil {
return nil, fmt.Errorf("Error getting original config file: %v", err)
}
cf, err := newImage.ConfigFile()
if err != nil {
return nil, fmt.Errorf("Error setting config file: %v", err)
}
cfg := cf.DeepCopy()
// Copy basic config over
cfg.Config = ocf.Config
cfg.ContainerConfig = ocf.ContainerConfig
// Strip away timestamps from the config file
cfg.Created = v1.Time{Time: t}
for _, h := range cfg.History {
h.Created = v1.Time{Time: t}
}
return configFile(newImage, cfg)
}
func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) {
layerReader, err := layer.Uncompressed()
if err != nil {
return nil, fmt.Errorf("Error getting layer: %v", err)
}
w := new(bytes.Buffer)
tarWriter := tar.NewWriter(w)
defer tarWriter.Close()
tarReader := tar.NewReader(layerReader)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("Error reading layer: %v", err)
}
header.ModTime = t
if err := tarWriter.WriteHeader(header); err != nil {
return nil, fmt.Errorf("Error writing tar header: %v", err)
}
if header.Typeflag == tar.TypeReg {
if _, err = io.Copy(tarWriter, tarReader); err != nil {
return nil, fmt.Errorf("Error writing layer file: %v", err)
}
}
}
b := w.Bytes()
// gzip the contents, then create the layer
opener := func() (io.ReadCloser, error) {
g, err := v1util.GzipReadCloser(ioutil.NopCloser(bytes.NewReader(b)))
if err != nil {
return nil, fmt.Errorf("Error compressing layer: %v", err)
}
return g, nil
}
layer, err = tarball.LayerFromOpener(opener)
if err != nil {
return nil, fmt.Errorf("Error creating layer: %v", err)
}
return layer, nil
}
// Canonical is a helper function to combine Time and configFile
// to remove any randomness during a docker build.
func Canonical(img v1.Image) (v1.Image, error) {
// Set all timestamps to 0
created := time.Time{}
img, err := Time(img, created)
if err != nil {
return nil, err
}
cf, err := img.ConfigFile()
if err != nil {
return nil, err
}
// Get rid of host-dependent random config
cfg := cf.DeepCopy()
cfg.Container = ""
cfg.Config.Hostname = ""
cfg.ContainerConfig.Hostname = ""
cfg.DockerVersion = ""
return configFile(img, cfg)
}

45
vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go generated vendored
View File

@ -42,30 +42,55 @@ type remoteImage struct {
config []byte
}
type ImageOption func(*imageOpener) error
var _ partial.CompressedImageCore = (*remoteImage)(nil)
// Image accesses a given image reference over the provided transport, with the provided authentication.
func Image(ref name.Reference, auth authn.Authenticator, t http.RoundTripper) (v1.Image, error) {
scopes := []string{ref.Scope(transport.PullScope)}
tr, err := transport.New(ref.Context().Registry, auth, t, scopes)
type imageOpener struct {
auth authn.Authenticator
transport http.RoundTripper
ref name.Reference
client *http.Client
}
func (i *imageOpener) Open() (v1.Image, error) {
tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)})
if err != nil {
return nil, err
}
img, err := partial.CompressedToImage(&remoteImage{
ref: ref,
ri := &remoteImage{
ref: i.ref,
client: &http.Client{Transport: tr},
})
}
imgCore, err := partial.CompressedToImage(ri)
if err != nil {
return nil, err
return imgCore, err
}
// Wrap the v1.Layers returned by this v1.Image in a hint for downstream
// remote.Write calls to facilitate cross-repo "mounting".
return &mountableImage{
Image: img,
Repository: ref.Context(),
Image: imgCore,
Reference: i.ref,
}, nil
}
// Image provides access to a remote image reference, applying functional options
// to the underlying imageOpener before resolving the reference into a v1.Image.
func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) {
img := &imageOpener{
auth: authn.Anonymous,
transport: http.DefaultTransport,
ref: ref,
}
for _, option := range options {
if err := option(img); err != nil {
return nil, err
}
}
return img.Open()
}
func (r *remoteImage) url(resource, identifier string) url.URL {
return url.URL{
Scheme: transport.Scheme(r.ref.Context().Registry),

16
vendor/github.com/google/go-containerregistry/pkg/v1/remote/mount.go generated vendored
View File

@ -24,7 +24,7 @@ import (
type MountableLayer struct {
v1.Layer
Repository name.Repository
Reference name.Reference
}
// mountableImage wraps the v1.Layer references returned by the embedded v1.Image
@ -33,7 +33,7 @@ type MountableLayer struct {
type mountableImage struct {
v1.Image
Repository name.Repository
Reference name.Reference
}
// Layers implements v1.Image
@ -45,8 +45,8 @@ func (mi *mountableImage) Layers() ([]v1.Layer, error) {
mls := make([]v1.Layer, 0, len(ls))
for _, l := range ls {
mls = append(mls, &MountableLayer{
Layer: l,
Repository: mi.Repository,
Layer: l,
Reference: mi.Reference,
})
}
return mls, nil
@ -59,8 +59,8 @@ func (mi *mountableImage) LayerByDigest(d v1.Hash) (v1.Layer, error) {
return nil, err
}
return &MountableLayer{
Layer: l,
Repository: mi.Repository,
Layer: l,
Reference: mi.Reference,
}, nil
}
@ -71,7 +71,7 @@ func (mi *mountableImage) LayerByDiffID(d v1.Hash) (v1.Layer, error) {
return nil, err
}
return &MountableLayer{
Layer: l,
Repository: mi.Repository,
Layer: l,
Reference: mi.Reference,
}, nil
}

61
vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go generated vendored Normal file
View File

@ -0,0 +1,61 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"net/http"
"github.com/google/go-containerregistry/pkg/authn"
)
// WithTransport is a functional option for overriding the default transport
// on a remote image
func WithTransport(t http.RoundTripper) ImageOption {
return func(i *imageOpener) error {
return i.setTransport(t)
}
}
// WithAuth is a functional option for overriding the default authenticator
// on a remote image
func WithAuth(auth authn.Authenticator) ImageOption {
return func(i *imageOpener) error {
return i.setAuth(auth)
}
}
// WithAuthFromKeychain is a functional option for overriding the default
// authenticator on a remote image using an authn.Keychain
func WithAuthFromKeychain(keys authn.Keychain) ImageOption {
return func(i *imageOpener) error {
auth, err := keys.Resolve(i.ref.Context().Registry)
if err != nil {
return err
}
return i.setAuth(auth)
}
}
// Set client on image using provided transport, and the default authenticator
func (i *imageOpener) setTransport(t http.RoundTripper) error {
i.transport = t
return nil
}
// Set client on image using provided authenticator, and the default transport
func (i *imageOpener) setAuth(auth authn.Authenticator) error {
i.auth = auth
return nil
}

4
vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go generated vendored
View File

@ -45,7 +45,7 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro
scopes := []string{ref.Scope(transport.PushScope)}
for _, l := range ls {
if ml, ok := l.(*MountableLayer); ok {
scopes = append(scopes, ml.Repository.Scope(transport.PullScope))
scopes = append(scopes, ml.Reference.Context().Scope(transport.PullScope))
}
}
@ -145,7 +145,7 @@ func (w *writer) initiateUpload(h v1.Hash) (location string, mounted bool, err e
// if "mount" is specified, even if no "from" sources are specified. If this turns out
// to not be broadly applicable then we should replace mounts without "from"s with a HEAD.
if ml, ok := l.(*MountableLayer); ok {
uv["from"] = []string{ml.Repository.RepositoryStr()}
uv["from"] = []string{ml.Reference.Context().RepositoryStr()}
}
u.RawQuery = uv.Encode()