adding reproducible flag (#205)

* adding reproducible test

* newer version of go-containerregistry

* new ImageOptions

* switch reproducible flag to default to false

* small fixes

* update dep

Gopkg.lock

@@ -399,7 +399,7 @@
     "pkg/v1/types",
     "pkg/v1/v1util"
   ]
-  revision = "5e2bd1f4bf61add62944828d54e239d352daaabf"
+  revision = "3f6471078a9661a9a439bd5e71a371aff429566a"
 
 [[projects]]
   name = "github.com/googleapis/gax-go"

cmd/executor/cmd/root.go

@@ -41,6 +41,7 @@ var (
 	buildArgs                   multiArg
 	tarPath                     string
 	singleSnapshot              bool
+	reproducible                bool
 )
 
 func init() {
@@ -56,6 +57,7 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&force, "force", "", false, "Force building outside of a container")
 	RootCmd.PersistentFlags().StringVarP(&tarPath, "tarPath", "", "", "Path to save the image in as a tarball instead of pushing")
 	RootCmd.PersistentFlags().BoolVarP(&singleSnapshot, "single-snapshot", "", false, "Set this flag to take a single snapshot at the end of the build.")
+	RootCmd.PersistentFlags().BoolVarP(&reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible")
 }
 
 var RootCmd = &cobra.Command{
@@ -87,6 +89,7 @@ var RootCmd = &cobra.Command{
 			SnapshotMode:   snapshotMode,
 			Args:           buildArgs,
 			SingleSnapshot: singleSnapshot,
+			Reproducible:   reproducible,
 		})
 		if err != nil {
 			logrus.Error(err)

integration/dockerfiles/Dockerfile_test_copy_reproducible

@@ -0,0 +1,20 @@
+FROM alpine:3.7
+COPY context/foo foo
+COPY context/foo /foodir/
+COPY context/bar/b* bar/
+COPY context/fo? /foo2
+COPY context/bar/doesnotexist* context/foo hello
+COPY ./context/empty /empty
+COPY ./ dir/
+COPY . newdir
+COPY context/bar /baz/
+COPY ["context/foo", "/tmp/foo" ]
+COPY context/b* /baz/
+COPY context/foo context/bar/ba? /test/
+COPY context/arr[[]0].txt /mydir/
+COPY context/bar/bat .
+
+ENV contextenv ./context
+COPY ${contextenv}/foo /tmp/foo2
+COPY $contextenv/foo /tmp/foo3
+COPY $contextenv/* /tmp/${contextenv}/

integration/integration_test.go

@@ -119,7 +119,10 @@ func TestRun(t *testing.T) {
 		"Dockerfile_test_scratch": {"--single-snapshot"},
 	}
 
+	// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
+	testsToIgnore := []string{"Dockerfile_test_user_run"}
 	bucketContextTests := []string{"Dockerfile_test_copy_bucket"}
+	reproducibleTests := []string{"Dockerfile_test_env"}
 
 	_, ex, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(ex)
@@ -161,6 +164,14 @@ func TestRun(t *testing.T) {
 				}
 			}
 
+			reproducibleFlag := ""
+			for _, d := range reproducibleTests {
+				if d == dockerfile {
+					reproducibleFlag = "--reproducible"
+					break
+				}
+			}
+
 			// build kaniko image
 			additionalFlags := append(buildArgs, additionalFlagsMap[dockerfile]...)
 			kanikoImage := strings.ToLower(testRepo + kanikoPrefix + dockerfile)
@@ -170,7 +181,7 @@ func TestRun(t *testing.T) {
 					"-v", cwd + ":/workspace",
 					executorImage,
 					"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
-					"-d", kanikoImage,
+					"-d", kanikoImage, reproducibleFlag,
 					contextFlag, contextPath},
 					additionalFlags...)...,
 			)

pkg/dockerfile/dockerfile.go

@@ -108,7 +108,7 @@ func Dependencies(index int, stages []instructions.Stage, buildArgs *BuildArgs)
 			if err != nil {
 				return nil, err
 			}
-			sourceImage, err = remote.Image(ref, auth, http.DefaultTransport)
+			sourceImage, err = remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport))
 			if err != nil {
 				return nil, err
 			}

pkg/executor/executor.go

@@ -52,6 +52,7 @@ type KanikoBuildArgs struct {
 	SnapshotMode   string
 	Args           []string
 	SingleSnapshot bool
+	Reproducible   bool
 }
 
 func DoBuild(k KanikoBuildArgs) (name.Reference, v1.Image, error) {
@@ -94,7 +95,7 @@ func DoBuild(k KanikoBuildArgs) (name.Reference, v1.Image, error) {
 			if err != nil {
 				return nil, nil, err
 			}
-			sourceImage, err = remote.Image(ref, auth, http.DefaultTransport)
+			sourceImage, err = remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport))
 			if err != nil {
 				return nil, nil, err
 			}
@@ -174,6 +175,14 @@ func DoBuild(k KanikoBuildArgs) (name.Reference, v1.Image, error) {
 			if err != nil {
 				return nil, nil, err
 			}
+
+			if k.Reproducible {
+				sourceImage, err = mutate.Canonical(sourceImage)
+				if err != nil {
+					return nil, nil, err
+				}
+			}
+
 			return ref, sourceImage, nil
 		}
 		if err := saveStageDependencies(index, stages, buildArgs.Clone()); err != nil {

vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go

@@ -21,12 +21,17 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/google/go-containerregistry/pkg/v1/types"
+	"github.com/google/go-containerregistry/pkg/v1/v1util"
 )
 
 const whiteoutPrefix = ".wh."
@@ -128,11 +133,6 @@ func Append(base v1.Image, adds ...Addendum) (v1.Image, error) {
 
 // Config mutates the provided v1.Image to have the provided v1.Config
 func Config(base v1.Image, cfg v1.Config) (v1.Image, error) {
-	m, err := base.Manifest()
-	if err != nil {
-		return nil, err
-	}
-
 	cf, err := base.ConfigFile()
 	if err != nil {
 		return nil, err
@@ -140,10 +140,19 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) {
 
 	cf.Config = cfg
 
+	return configFile(base, cf)
+}
+
+func configFile(base v1.Image, cfg *v1.ConfigFile) (v1.Image, error) {
+	m, err := base.Manifest()
+	if err != nil {
+		return nil, err
+	}
+
 	image := &image{
 		Image:      base,
 		manifest:   m.DeepCopy(),
-		configFile: cf.DeepCopy(),
+		configFile: cfg,
 		digestMap:  make(map[v1.Hash]v1.Layer),
 	}
 
@@ -160,13 +169,8 @@ func Config(base v1.Image, cfg v1.Config) (v1.Image, error) {
 	return image, nil
 }
 
-// Created mutates the provided v1.Image to have the provided v1.Time
+// CreatedAt mutates the provided v1.Image to have the provided v1.Time
 func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) {
-	m, err := base.Manifest()
-	if err != nil {
-		return nil, err
-	}
-
 	cf, err := base.ConfigFile()
 	if err != nil {
 		return nil, err
@@ -175,18 +179,7 @@ func CreatedAt(base v1.Image, created v1.Time) (v1.Image, error) {
 	cfg := cf.DeepCopy()
 	cfg.Created = created
 
-	image := &image{
+	return configFile(base, cfg)
-		Image:      base,
-		manifest:   m.DeepCopy(),
-		configFile: cfg,
-		digestMap:  make(map[v1.Hash]v1.Layer),
-	}
-
-	image.manifest.Config.Digest, err = image.ConfigName()
-	if err != nil {
-		return nil, err
-	}
-	return image, nil
 }
 
 type image struct {
@@ -392,3 +385,129 @@ func inWhiteoutDir(fileMap map[string]bool, file string) bool {
 	}
 	return false
 }
+
+// Time sets all timestamps in an image to the given timestamp.
+func Time(img v1.Image, t time.Time) (v1.Image, error) {
+	newImage := empty.Image
+
+	layers, err := img.Layers()
+	if err != nil {
+
+		return nil, fmt.Errorf("Error getting image layers: %v", err)
+	}
+
+	// Strip away all timestamps from layers
+	var newLayers []v1.Layer
+	for _, layer := range layers {
+		newLayer, err := layerTime(layer, t)
+		if err != nil {
+			return nil, fmt.Errorf("Error setting layer times: %v", err)
+		}
+		newLayers = append(newLayers, newLayer)
+	}
+
+	newImage, err = AppendLayers(newImage, newLayers...)
+	if err != nil {
+		return nil, fmt.Errorf("Error appending layers: %v", err)
+	}
+
+	ocf, err := img.ConfigFile()
+	if err != nil {
+		return nil, fmt.Errorf("Error getting original config file: %v", err)
+	}
+
+	cf, err := newImage.ConfigFile()
+	if err != nil {
+		return nil, fmt.Errorf("Error setting config file: %v", err)
+	}
+
+	cfg := cf.DeepCopy()
+
+	// Copy basic config over
+	cfg.Config = ocf.Config
+	cfg.ContainerConfig = ocf.ContainerConfig
+
+	// Strip away timestamps from the config file
+	cfg.Created = v1.Time{Time: t}
+
+	for _, h := range cfg.History {
+		h.Created = v1.Time{Time: t}
+	}
+
+	return configFile(newImage, cfg)
+}
+
+func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) {
+	layerReader, err := layer.Uncompressed()
+	if err != nil {
+		return nil, fmt.Errorf("Error getting layer: %v", err)
+	}
+	w := new(bytes.Buffer)
+	tarWriter := tar.NewWriter(w)
+	defer tarWriter.Close()
+
+	tarReader := tar.NewReader(layerReader)
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("Error reading layer: %v", err)
+		}
+
+		header.ModTime = t
+		if err := tarWriter.WriteHeader(header); err != nil {
+			return nil, fmt.Errorf("Error writing tar header: %v", err)
+		}
+
+		if header.Typeflag == tar.TypeReg {
+			if _, err = io.Copy(tarWriter, tarReader); err != nil {
+				return nil, fmt.Errorf("Error writing layer file: %v", err)
+			}
+		}
+	}
+
+	b := w.Bytes()
+	// gzip the contents, then create the layer
+	opener := func() (io.ReadCloser, error) {
+		g, err := v1util.GzipReadCloser(ioutil.NopCloser(bytes.NewReader(b)))
+		if err != nil {
+			return nil, fmt.Errorf("Error compressing layer: %v", err)
+		}
+
+		return g, nil
+	}
+	layer, err = tarball.LayerFromOpener(opener)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating layer: %v", err)
+	}
+
+	return layer, nil
+}
+
+// Canonical is a helper function to combine Time and configFile
+// to remove any randomness during a docker build.
+func Canonical(img v1.Image) (v1.Image, error) {
+	// Set all timestamps to 0
+	created := time.Time{}
+	img, err := Time(img, created)
+	if err != nil {
+		return nil, err
+	}
+
+	cf, err := img.ConfigFile()
+	if err != nil {
+		return nil, err
+	}
+
+	// Get rid of host-dependent random config
+	cfg := cf.DeepCopy()
+
+	cfg.Container = ""
+	cfg.Config.Hostname = ""
+	cfg.ContainerConfig.Hostname = ""
+	cfg.DockerVersion = ""
+
+	return configFile(img, cfg)
+}

vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go

@@ -42,30 +42,55 @@ type remoteImage struct {
 	config       []byte
 }
 
+type ImageOption func(*imageOpener) error
+
 var _ partial.CompressedImageCore = (*remoteImage)(nil)
 
-// Image accesses a given image reference over the provided transport, with the provided authentication.
+type imageOpener struct {
-func Image(ref name.Reference, auth authn.Authenticator, t http.RoundTripper) (v1.Image, error) {
+	auth      authn.Authenticator
-	scopes := []string{ref.Scope(transport.PullScope)}
+	transport http.RoundTripper
-	tr, err := transport.New(ref.Context().Registry, auth, t, scopes)
+	ref       name.Reference
+	client    *http.Client
+}
+
+func (i *imageOpener) Open() (v1.Image, error) {
+	tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)})
 	if err != nil {
 		return nil, err
 	}
-	img, err := partial.CompressedToImage(&remoteImage{
+	ri := &remoteImage{
-		ref:    ref,
+		ref:    i.ref,
 		client: &http.Client{Transport: tr},
-	})
+	}
+	imgCore, err := partial.CompressedToImage(ri)
 	if err != nil {
-		return nil, err
+		return imgCore, err
 	}
 	// Wrap the v1.Layers returned by this v1.Image in a hint for downstream
 	// remote.Write calls to facilitate cross-repo "mounting".
 	return &mountableImage{
-		Image:      img,
+		Image:     imgCore,
-		Repository: ref.Context(),
+		Reference: i.ref,
 	}, nil
 }
 
+// Image provides access to a remote image reference, applying functional options
+// to the underlying imageOpener before resolving the reference into a v1.Image.
+func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) {
+	img := &imageOpener{
+		auth:      authn.Anonymous,
+		transport: http.DefaultTransport,
+		ref:       ref,
+	}
+
+	for _, option := range options {
+		if err := option(img); err != nil {
+			return nil, err
+		}
+	}
+	return img.Open()
+}
+
 func (r *remoteImage) url(resource, identifier string) url.URL {
 	return url.URL{
 		Scheme: transport.Scheme(r.ref.Context().Registry),

vendor/github.com/google/go-containerregistry/pkg/v1/remote/mount.go

@@ -24,7 +24,7 @@ import (
 type MountableLayer struct {
 	v1.Layer
 
-	Repository name.Repository
+	Reference name.Reference
 }
 
 // mountableImage wraps the v1.Layer references returned by the embedded v1.Image
@@ -33,7 +33,7 @@ type MountableLayer struct {
 type mountableImage struct {
 	v1.Image
 
-	Repository name.Repository
+	Reference name.Reference
 }
 
 // Layers implements v1.Image
@@ -46,7 +46,7 @@ func (mi *mountableImage) Layers() ([]v1.Layer, error) {
 	for _, l := range ls {
 		mls = append(mls, &MountableLayer{
 			Layer:     l,
-			Repository: mi.Repository,
+			Reference: mi.Reference,
 		})
 	}
 	return mls, nil
@@ -60,7 +60,7 @@ func (mi *mountableImage) LayerByDigest(d v1.Hash) (v1.Layer, error) {
 	}
 	return &MountableLayer{
 		Layer:     l,
-		Repository: mi.Repository,
+		Reference: mi.Reference,
 	}, nil
 }
 
@@ -72,6 +72,6 @@ func (mi *mountableImage) LayerByDiffID(d v1.Hash) (v1.Layer, error) {
 	}
 	return &MountableLayer{
 		Layer:     l,
-		Repository: mi.Repository,
+		Reference: mi.Reference,
 	}, nil
 }

vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go

@@ -0,0 +1,61 @@
+// Copyright 2018 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remote
+
+import (
+	"net/http"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+)
+
+// WithTransport is a functional option for overriding the default transport
+// on a remote image
+func WithTransport(t http.RoundTripper) ImageOption {
+	return func(i *imageOpener) error {
+		return i.setTransport(t)
+	}
+}
+
+// WithAuth is a functional option for overriding the default authenticator
+// on a remote image
+func WithAuth(auth authn.Authenticator) ImageOption {
+	return func(i *imageOpener) error {
+		return i.setAuth(auth)
+	}
+}
+
+// WithAuthFromKeychain is a functional option for overriding the default
+// authenticator on a remote image using an authn.Keychain
+func WithAuthFromKeychain(keys authn.Keychain) ImageOption {
+	return func(i *imageOpener) error {
+		auth, err := keys.Resolve(i.ref.Context().Registry)
+		if err != nil {
+			return err
+		}
+		return i.setAuth(auth)
+	}
+}
+
+// Set client on image using provided transport, and the default authenticator
+func (i *imageOpener) setTransport(t http.RoundTripper) error {
+	i.transport = t
+	return nil
+}
+
+// Set client on image using provided authenticator, and the default transport
+func (i *imageOpener) setAuth(auth authn.Authenticator) error {
+	i.auth = auth
+	return nil
+}

vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go

@@ -45,7 +45,7 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro
 	scopes := []string{ref.Scope(transport.PushScope)}
 	for _, l := range ls {
 		if ml, ok := l.(*MountableLayer); ok {
-			scopes = append(scopes, ml.Repository.Scope(transport.PullScope))
+			scopes = append(scopes, ml.Reference.Context().Scope(transport.PullScope))
 		}
 	}
 
@@ -145,7 +145,7 @@ func (w *writer) initiateUpload(h v1.Hash) (location string, mounted bool, err e
 	// if "mount" is specified, even if no "from" sources are specified.  If this turns out
 	// to not be broadly applicable then we should replace mounts without "from"s with a HEAD.
 	if ml, ok := l.(*MountableLayer); ok {
-		uv["from"] = []string{ml.Repository.RepositoryStr()}
+		uv["from"] = []string{ml.Reference.Context().RepositoryStr()}
 	}
 	u.RawQuery = uv.Encode()