Merge remote-tracking branch 'upstream/master'

CHANGELOG.md

@@ -1,3 +1,40 @@
+# v0.11.0 Release - 2019-08-23
+
+## Bug Fixes
+* fix unpacking archives via ADD [#717](https://github.com/GoogleContainerTools/kaniko/pull/717)
+* Reverted not including build args in cache key [#739](https://github.com/GoogleContainerTools/kaniko/pull/739)
+* Create cache directory if it doesn't already exist [#452](https://github.com/GoogleContainerTools/kaniko/pull/452)
+
+## New Features
+* add multiple user agents to kaniko if upstream_client_type value  is set [#750](https://github.com/GoogleContainerTools/kaniko/pull/750)
+* Make container layers captured using FS snapshots reproducible [#714](https://github.com/GoogleContainerTools/kaniko/pull/714)
+* Include warmer in debug image [#497](https://github.com/GoogleContainerTools/kaniko/pull/497)
+* Bailout when there is not enough input arguments [#735](https://github.com/GoogleContainerTools/kaniko/pull/735)
+* Add checking image presence in cache prior to downloading it [#723](https://github.com/GoogleContainerTools/kaniko/pull/723)
+
+## Additonal PRs
+* Document how to build from git reference [#730](https://github.com/GoogleContainerTools/kaniko/pull/730)
+* Misc. small changes/refactoring [#712](https://github.com/GoogleContainerTools/kaniko/pull/712)
+* Update go-containerregistry [#680](https://github.com/GoogleContainerTools/kaniko/pull/680)
+* Update version of go-containerregistry [#724](https://github.com/GoogleContainerTools/kaniko/pull/724)
+* feat: support specifying branch for cloning [#703](https://github.com/GoogleContainerTools/kaniko/pull/703)
+
+Huge thank you for this release towards our contributors: 
+- Carlos Alexandro Becker
+- Carlos Sanchez
+- Deniz Zoeteman
+- Luke Wood
+- Matthew Dawson
+- priyawadhwa
+- sharifelgamal
+- Sharif Elgamal
+- Taylor Barrella
+- Tejal Desai
+- v.rul
+- Warren Seymour
+- Xueshan Feng
+- Роман Небалуе
+
 # v0.10.0 Release - 2019-06-19
 
 ## Bug Fixes

Makefile

@@ -14,7 +14,7 @@
 
 # Bump these on release
 VERSION_MAJOR ?= 0
-VERSION_MINOR ?= 10
+VERSION_MINOR ?= 11
 VERSION_BUILD ?= 0
 
 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)

README.md

@@ -93,6 +93,7 @@ Right now, kaniko supports these storage solutions:
 - GCS Bucket
 - S3 Bucket
 - Local Directory
+- Git Repository
 
 _Note: the local directory option refers to a directory within the kaniko container.
 If you wish to use this option, you will need to mount in your build context into the container as a directory._
@@ -114,12 +115,12 @@ gsutil cp context.tar.gz gs://<bucket name>
 
 When running kaniko, use the `--context` flag with the appropriate prefix to specify the location of your build context:
 
-|  Source | Prefix  |
-|---------|---------|
-| Local Directory  | dir://[path to a directory in the kaniko container]  |
-| GCS Bucket       | gs://[bucket name]/[path to .tar.gz]     |
-| S3 Bucket        | s3://[bucket name]/[path to .tar.gz]     |
-| Git Repository   | git://[repository url]     |
+|  Source | Prefix  | Example |
+|---------|---------|---------|
+| Local Directory  | dir://[path to a directory in the kaniko container] | `dir:///workspace` |
+| GCS Bucket       | gs://[bucket name]/[path to .tar.gz]                | `gs://kaniko-bucket/path/to/context.tar.gz` |
+| S3 Bucket        | s3://[bucket name]/[path to .tar.gz]                | `s3://kaniko-bucket/path/to/context.tar.gz` |
+| Git Repository   | git://[repository url][#reference]                  | `git://github.com/acme/myproject.git#refs/heads/mybranch` |
 
 If you don't specify a prefix, kaniko will assume a local directory.
 For example, to use a GCS bucket called `kaniko-bucket`, you would pass in `--context=gs://kaniko-bucket/path/to/context.tar.gz`.

cmd/warmer/cmd/root.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/cache"
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
@@ -51,6 +52,12 @@ var RootCmd = &cobra.Command{
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
+		if _, err := os.Stat(opts.CacheDir); os.IsNotExist(err) {
+			err = os.MkdirAll(opts.CacheDir, 0755)
+			if err != nil {
+				exit(errors.Wrap(err, "Failed to create cache directory"))
+			}
+		}
 		if err := cache.WarmCache(opts); err != nil {
 			exit(errors.Wrap(err, "Failed warming cache"))
 		}
@@ -61,6 +68,8 @@ var RootCmd = &cobra.Command{
 func addKanikoOptionsFlags(cmd *cobra.Command) {
 	RootCmd.PersistentFlags().VarP(&opts.Images, "image", "i", "Image to cache. Set it repeatedly for multiple images.")
 	RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "c", "/cache", "Directory of the cache.")
+	RootCmd.PersistentFlags().BoolVarP(&opts.Force, "force", "f", false, "Force cache overwriting.")
+	RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.")
 }
 
 // addHiddenFlags marks certain flags as hidden from the executor help text

deploy/Dockerfile_debug

@@ -25,7 +25,7 @@ RUN docker-credential-gcr configure-docker
 RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login
 RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 
 COPY . .
-RUN make
+RUN make && make out/warmer
 
 # Stage 1: Get the busybox shell
 FROM gcr.io/cloud-builders/bazel:latest
@@ -35,7 +35,7 @@ RUN bazel build //experimental/busybox:busybox_tar
 RUN tar -C /distroless/bazel-genfiles/experimental/busybox/ -xf /distroless/bazel-genfiles/experimental/busybox/busybox.tar
 
 FROM scratch
-COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/executor /kaniko/executor
+COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/* /kaniko/
 COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr
 COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login
 COPY --from=1 /distroless/bazel-genfiles/experimental/busybox/busybox/ /busybox/

integration/dockerfiles/Dockerfile_git_buildcontext

@@ -0,0 +1,2 @@
+FROM scratch
+COPY LICENSE ./LICENSE

pkg/cache/cache.go

@@ -88,7 +88,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) {
 	// Layer is stale, rebuild it.
 	if expiry.Before(time.Now()) {
 		logrus.Infof("Cache entry expired: %s", cache)
-		return nil, errors.New(fmt.Sprintf("Cache entry expired: %s", cache))
+		return nil, fmt.Errorf("Cache entry expired: %s", cache)
 	}
 
 	// Force the manifest to be populated
@@ -114,7 +114,7 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) {
 }
 
 // LocalSource retieves a source image from a local cache given cacheKey
-func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) {
+func LocalSource(opts *config.CacheOptions, cacheKey string) (v1.Image, error) {
 	cache := opts.CacheDir
 	if cache == "" {
 		return nil, nil

pkg/cache/warm.go

@@ -29,6 +29,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// WarmCache populates the cache
 func WarmCache(opts *config.WarmerOptions) error {
 	cacheDir := opts.CacheDir
 	images := opts.Images
@@ -41,7 +42,7 @@ func WarmCache(opts *config.WarmerOptions) error {
 			return errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image))
 		}
 		img, err := remote.Image(cacheRef)
-		if err != nil {
+		if err != nil || img == nil {
 			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image))
 		}
 
@@ -50,6 +51,14 @@ func WarmCache(opts *config.WarmerOptions) error {
 			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image))
 		}
 		cachePath := path.Join(cacheDir, digest.String())
+
+		if !opts.Force {
+			_, err := LocalSource(&opts.CacheOptions, digest.String())
+			if err == nil {
+				continue
+			}
+		}
+
 		err = tarball.WriteToFile(cachePath, cacheRef, img)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image))

pkg/commands/add.go

@@ -71,8 +71,12 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
 			}
 			a.snapshotFiles = append(a.snapshotFiles, urlDest)
 		} else if util.IsFileLocalTarArchive(fullPath) {
-			logrus.Infof("Unpacking local tar archive %s to %s", src, dest)
-			extractedFiles, err := util.UnpackLocalTarArchive(fullPath, dest)
+			tarDest, err := util.DestinationFilepath("", dest, config.WorkingDir)
+			if err != nil {
+				return err
+			}
+			logrus.Infof("Unpacking local tar archive %s to %s", src, tarDest)
+			extractedFiles, err := util.UnpackLocalTarArchive(fullPath, tarDest)
 			if err != nil {
 				return err
 			}

pkg/config/options.go

@@ -20,8 +20,15 @@ import (
 	"time"
 )
 
+// CacheOptions are base image cache options that are set by command line arguments
+type CacheOptions struct {
+	CacheDir string
+	CacheTTL time.Duration
+}
+
 // KanikoOptions are options that are set by command line arguments
 type KanikoOptions struct {
+	CacheOptions
 	DockerfilePath          string
 	SrcContext              string
 	SnapshotMode            string
@@ -29,7 +36,6 @@ type KanikoOptions struct {
 	TarPath                 string
 	Target                  string
 	CacheRepo               string
-	CacheDir                string
 	DigestFile              string
 	Destinations            multiArg
 	BuildArgs               multiArg
@@ -42,13 +48,13 @@ type KanikoOptions struct {
 	NoPush                  bool
 	Cache                   bool
 	Cleanup                 bool
-	CacheTTL                time.Duration
 	InsecureRegistries      multiArg
 	SkipTLSVerifyRegistries multiArg
 }
 
 // WarmerOptions are options that are set by command line arguments to the cache warmer.
 type WarmerOptions struct {
-	Images   multiArg
-	CacheDir string
+	CacheOptions
+	Images multiArg
+	Force  bool
 }

pkg/executor/build.go

@@ -191,6 +191,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro
 func (s *stageBuilder) build() error {
 	// Set the initial cache key to be the base image digest, the build args and the SrcContext.
 	compositeKey := NewCompositeCache(s.baseImageDigest)
+	compositeKey.AddKey(s.opts.BuildArgs...)
 
 	// Apply optimizations to the instructions.
 	if err := s.optimize(*compositeKey, s.cf.Config); err != nil {

pkg/executor/push.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/cache"
@@ -43,8 +45,16 @@ type withUserAgent struct {
 	t http.RoundTripper
 }
 
+const (
+	UpstreamClientUaKey = "UPSTREAM_CLIENT_TYPE"
+)
+
 func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) {
-	r.Header.Set("User-Agent", fmt.Sprintf("kaniko/%s", version.Version()))
+	ua := []string{fmt.Sprintf("kaniko/%s", version.Version())}
+	if upstream := os.Getenv(UpstreamClientUaKey); upstream != "" {
+		ua = append(ua, upstream)
+	}
+	r.Header.Set("User-Agent", strings.Join(ua, ","))
 	return w.t.RoundTrip(r)
 }
 

pkg/executor/push_test.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package executor
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"testing"
+
+	"github.com/GoogleContainerTools/kaniko/testutil"
+)
+
+func TestHeaderAdded(t *testing.T) {
+	tests := []struct {
+		name     string
+		upstream string
+		expected string
+	}{{
+		name:     "upstream env variable set",
+		upstream: "skaffold-v0.25.45",
+		expected: "kaniko/unset,skaffold-v0.25.45",
+	}, {
+		name:     "upstream env variable not set",
+		expected: "kaniko/unset",
+	},
+	}
+	for _, test := range tests {
+
+		t.Run(test.name, func(t *testing.T) {
+			rt := &withUserAgent{t: &mockRoundTripper{}}
+			if test.upstream != "" {
+				os.Setenv("UPSTREAM_CLIENT_TYPE", test.upstream)
+				defer func() { os.Unsetenv("UPSTREAM_CLIENT_TYPE") }()
+			}
+			req, err := http.NewRequest("GET", "dummy", nil)
+			if err != nil {
+				t.Fatalf("culd not create a req due to %s", err)
+			}
+			resp, err := rt.RoundTrip(req)
+			testutil.CheckError(t, false, err)
+			defer resp.Body.Close()
+			body, err := ioutil.ReadAll(resp.Body)
+			testutil.CheckErrorAndDeepEqual(t, false, err, test.expected, string(body))
+		})
+	}
+
+}
+
+type mockRoundTripper struct {
+}
+
+func (m *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	ua := r.UserAgent()
+	return &http.Response{Body: ioutil.NopCloser(bytes.NewBufferString(ua))}, nil
+}

pkg/snapshot/snapshot.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
+	"sort"
 	"syscall"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/timing"
@@ -186,6 +187,8 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
 	// Also add parent directories to keep their permissions correctly.
 	filesToAdd = filesWithParentDirs(filesToAdd)
 
+	sort.Strings(filesToAdd)
+
 	// Add files to the layered map
 	for _, file := range filesToAdd {
 		if err := s.l.Add(file); err != nil {

pkg/snapshot/snapshot_test.go

@@ -87,6 +87,45 @@ func TestSnapshotFSFileChange(t *testing.T) {
 	}
 }
 
+func TestSnapshotFSIsReproducible(t *testing.T) {
+	testDir, snapshotter, cleanup, err := setUpTestDir()
+	defer cleanup()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Make some changes to the filesystem
+	newFiles := map[string]string{
+		"foo":     "newbaz1",
+		"bar/bat": "baz",
+	}
+	if err := testutil.SetupFiles(testDir, newFiles); err != nil {
+		t.Fatalf("Error setting up fs: %s", err)
+	}
+	// Take another snapshot
+	tarPath, err := snapshotter.TakeSnapshotFS()
+	if err != nil {
+		t.Fatalf("Error taking snapshot of fs: %s", err)
+	}
+
+	f, err := os.Open(tarPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Check contents of the snapshot, make sure contents are sorted by name
+	tr := tar.NewReader(f)
+	var filesInTar []string
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		filesInTar = append(filesInTar, hdr.Name)
+	}
+	if !sort.StringsAreSorted(filesInTar) {
+		t.Fatalf("Expected the file in the tar archive were sorted, actual list was not sorted: %v", filesInTar)
+	}
+}
+
 func TestSnapshotFSChangePermissions(t *testing.T) {
 	testDir, snapshotter, cleanup, err := setUpTestDir()
 	defer cleanup()

pkg/util/image_util.go

@@ -149,5 +149,5 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) {
 		cacheKey = d.String()
 	}
 
-	return cache.LocalSource(opts, cacheKey)
+	return cache.LocalSource(&opts.CacheOptions, cacheKey)
 }

run_in_docker.sh

@@ -15,8 +15,9 @@
 #!/bin/bash
 set -e
 
-if [ $# -lt 3 ];
-    then echo "Usage: run_in_docker.sh <path to Dockerfile> <context directory> <image tag> <cache>"
+if [ $# -lt 3 ]; then 
+    echo "Usage: run_in_docker.sh <path to Dockerfile> <context directory> <image tag> <cache>"
+    exit 1
 fi
 
 dockerfile=$1