diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bc746dc5..353250455 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,40 @@ +# v0.11.0 Release - 2019-08-23 + +## Bug Fixes +* fix unpacking archives via ADD [#717](https://github.com/GoogleContainerTools/kaniko/pull/717) +* Reverted not including build args in cache key [#739](https://github.com/GoogleContainerTools/kaniko/pull/739) +* Create cache directory if it doesn't already exist [#452](https://github.com/GoogleContainerTools/kaniko/pull/452) + +## New Features +* add multiple user agents to kaniko if upstream_client_type value is set [#750](https://github.com/GoogleContainerTools/kaniko/pull/750) +* Make container layers captured using FS snapshots reproducible [#714](https://github.com/GoogleContainerTools/kaniko/pull/714) +* Include warmer in debug image [#497](https://github.com/GoogleContainerTools/kaniko/pull/497) +* Bailout when there is not enough input arguments [#735](https://github.com/GoogleContainerTools/kaniko/pull/735) +* Add checking image presence in cache prior to downloading it [#723](https://github.com/GoogleContainerTools/kaniko/pull/723) + +## Additonal PRs +* Document how to build from git reference [#730](https://github.com/GoogleContainerTools/kaniko/pull/730) +* Misc. small changes/refactoring [#712](https://github.com/GoogleContainerTools/kaniko/pull/712) +* Update go-containerregistry [#680](https://github.com/GoogleContainerTools/kaniko/pull/680) +* Update version of go-containerregistry [#724](https://github.com/GoogleContainerTools/kaniko/pull/724) +* feat: support specifying branch for cloning [#703](https://github.com/GoogleContainerTools/kaniko/pull/703) + +Huge thank you for this release towards our contributors: +- Carlos Alexandro Becker +- Carlos Sanchez +- Deniz Zoeteman +- Luke Wood +- Matthew Dawson +- priyawadhwa +- sharifelgamal +- Sharif Elgamal +- Taylor Barrella +- Tejal Desai +- v.rul +- Warren Seymour +- Xueshan Feng +- Роман Небалуе + # v0.10.0 Release - 2019-06-19 ## Bug Fixes diff --git a/Makefile b/Makefile index 036f1e017..e5633bf20 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ # Bump these on release VERSION_MAJOR ?= 0 -VERSION_MINOR ?= 10 +VERSION_MINOR ?= 11 VERSION_BUILD ?= 0 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD) diff --git a/README.md b/README.md index 18d1d4e72..466dd4ab5 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Right now, kaniko supports these storage solutions: - GCS Bucket - S3 Bucket - Local Directory +- Git Repository _Note: the local directory option refers to a directory within the kaniko container. If you wish to use this option, you will need to mount in your build context into the container as a directory._ @@ -114,12 +115,12 @@ gsutil cp context.tar.gz gs:// When running kaniko, use the `--context` flag with the appropriate prefix to specify the location of your build context: -| Source | Prefix | -|---------|---------| -| Local Directory | dir://[path to a directory in the kaniko container] | -| GCS Bucket | gs://[bucket name]/[path to .tar.gz] | -| S3 Bucket | s3://[bucket name]/[path to .tar.gz] | -| Git Repository | git://[repository url] | +| Source | Prefix | Example | +|---------|---------|---------| +| Local Directory | dir://[path to a directory in the kaniko container] | `dir:///workspace` | +| GCS Bucket | gs://[bucket name]/[path to .tar.gz] | `gs://kaniko-bucket/path/to/context.tar.gz` | +| S3 Bucket | s3://[bucket name]/[path to .tar.gz] | `s3://kaniko-bucket/path/to/context.tar.gz` | +| Git Repository | git://[repository url][#reference] | `git://github.com/acme/myproject.git#refs/heads/mybranch` | If you don't specify a prefix, kaniko will assume a local directory. For example, to use a GCS bucket called `kaniko-bucket`, you would pass in `--context=gs://kaniko-bucket/path/to/context.tar.gz`. diff --git a/cmd/warmer/cmd/root.go b/cmd/warmer/cmd/root.go index 0e4908d2b..207c0519a 100644 --- a/cmd/warmer/cmd/root.go +++ b/cmd/warmer/cmd/root.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "os" + "time" "github.com/GoogleContainerTools/kaniko/pkg/cache" "github.com/GoogleContainerTools/kaniko/pkg/config" @@ -51,6 +52,12 @@ var RootCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { + if _, err := os.Stat(opts.CacheDir); os.IsNotExist(err) { + err = os.MkdirAll(opts.CacheDir, 0755) + if err != nil { + exit(errors.Wrap(err, "Failed to create cache directory")) + } + } if err := cache.WarmCache(opts); err != nil { exit(errors.Wrap(err, "Failed warming cache")) } @@ -61,6 +68,8 @@ var RootCmd = &cobra.Command{ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().VarP(&opts.Images, "image", "i", "Image to cache. Set it repeatedly for multiple images.") RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "c", "/cache", "Directory of the cache.") + RootCmd.PersistentFlags().BoolVarP(&opts.Force, "force", "f", false, "Force cache overwriting.") + RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.") } // addHiddenFlags marks certain flags as hidden from the executor help text diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index 51cf02a17..1feb086b7 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -25,7 +25,7 @@ RUN docker-credential-gcr configure-docker RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 COPY . . -RUN make +RUN make && make out/warmer # Stage 1: Get the busybox shell FROM gcr.io/cloud-builders/bazel:latest @@ -35,7 +35,7 @@ RUN bazel build //experimental/busybox:busybox_tar RUN tar -C /distroless/bazel-genfiles/experimental/busybox/ -xf /distroless/bazel-genfiles/experimental/busybox/busybox.tar FROM scratch -COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/executor /kaniko/executor +COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/* /kaniko/ COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login COPY --from=1 /distroless/bazel-genfiles/experimental/busybox/busybox/ /busybox/ diff --git a/integration/dockerfiles/Dockerfile_git_buildcontext b/integration/dockerfiles/Dockerfile_git_buildcontext new file mode 100644 index 000000000..9636fcd3d --- /dev/null +++ b/integration/dockerfiles/Dockerfile_git_buildcontext @@ -0,0 +1,2 @@ +FROM scratch +COPY LICENSE ./LICENSE diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 6ec9d0ce7..102411c5d 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -88,7 +88,7 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) { // Layer is stale, rebuild it. if expiry.Before(time.Now()) { logrus.Infof("Cache entry expired: %s", cache) - return nil, errors.New(fmt.Sprintf("Cache entry expired: %s", cache)) + return nil, fmt.Errorf("Cache entry expired: %s", cache) } // Force the manifest to be populated @@ -114,7 +114,7 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { } // LocalSource retieves a source image from a local cache given cacheKey -func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { +func LocalSource(opts *config.CacheOptions, cacheKey string) (v1.Image, error) { cache := opts.CacheDir if cache == "" { return nil, nil diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go index c03746e0b..abdaec76d 100644 --- a/pkg/cache/warm.go +++ b/pkg/cache/warm.go @@ -29,6 +29,7 @@ import ( "github.com/sirupsen/logrus" ) +// WarmCache populates the cache func WarmCache(opts *config.WarmerOptions) error { cacheDir := opts.CacheDir images := opts.Images @@ -41,7 +42,7 @@ func WarmCache(opts *config.WarmerOptions) error { return errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image)) } img, err := remote.Image(cacheRef) - if err != nil { + if err != nil || img == nil { return errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image)) } @@ -50,6 +51,14 @@ func WarmCache(opts *config.WarmerOptions) error { return errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image)) } cachePath := path.Join(cacheDir, digest.String()) + + if !opts.Force { + _, err := LocalSource(&opts.CacheOptions, digest.String()) + if err == nil { + continue + } + } + err = tarball.WriteToFile(cachePath, cacheRef, img) if err != nil { return errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image)) diff --git a/pkg/commands/add.go b/pkg/commands/add.go index 27d666646..769d050dd 100644 --- a/pkg/commands/add.go +++ b/pkg/commands/add.go @@ -71,8 +71,12 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui } a.snapshotFiles = append(a.snapshotFiles, urlDest) } else if util.IsFileLocalTarArchive(fullPath) { - logrus.Infof("Unpacking local tar archive %s to %s", src, dest) - extractedFiles, err := util.UnpackLocalTarArchive(fullPath, dest) + tarDest, err := util.DestinationFilepath("", dest, config.WorkingDir) + if err != nil { + return err + } + logrus.Infof("Unpacking local tar archive %s to %s", src, tarDest) + extractedFiles, err := util.UnpackLocalTarArchive(fullPath, tarDest) if err != nil { return err } diff --git a/pkg/config/options.go b/pkg/config/options.go index ff4d60a13..627a51531 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -20,8 +20,15 @@ import ( "time" ) +// CacheOptions are base image cache options that are set by command line arguments +type CacheOptions struct { + CacheDir string + CacheTTL time.Duration +} + // KanikoOptions are options that are set by command line arguments type KanikoOptions struct { + CacheOptions DockerfilePath string SrcContext string SnapshotMode string @@ -29,7 +36,6 @@ type KanikoOptions struct { TarPath string Target string CacheRepo string - CacheDir string DigestFile string Destinations multiArg BuildArgs multiArg @@ -42,13 +48,13 @@ type KanikoOptions struct { NoPush bool Cache bool Cleanup bool - CacheTTL time.Duration InsecureRegistries multiArg SkipTLSVerifyRegistries multiArg } // WarmerOptions are options that are set by command line arguments to the cache warmer. type WarmerOptions struct { - Images multiArg - CacheDir string + CacheOptions + Images multiArg + Force bool } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 6626577cf..13f9db92d 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -191,6 +191,7 @@ func (s *stageBuilder) optimize(compositeKey CompositeCache, cfg v1.Config) erro func (s *stageBuilder) build() error { // Set the initial cache key to be the base image digest, the build args and the SrcContext. compositeKey := NewCompositeCache(s.baseImageDigest) + compositeKey.AddKey(s.opts.BuildArgs...) // Apply optimizations to the instructions. if err := s.optimize(*compositeKey, s.cf.Config); err != nil { diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 59d275d69..486bcd32a 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -21,6 +21,8 @@ import ( "fmt" "io/ioutil" "net/http" + "os" + "strings" "time" "github.com/GoogleContainerTools/kaniko/pkg/cache" @@ -43,8 +45,16 @@ type withUserAgent struct { t http.RoundTripper } +const ( + UpstreamClientUaKey = "UPSTREAM_CLIENT_TYPE" +) + func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) { - r.Header.Set("User-Agent", fmt.Sprintf("kaniko/%s", version.Version())) + ua := []string{fmt.Sprintf("kaniko/%s", version.Version())} + if upstream := os.Getenv(UpstreamClientUaKey); upstream != "" { + ua = append(ua, upstream) + } + r.Header.Set("User-Agent", strings.Join(ua, ",")) return w.t.RoundTrip(r) } diff --git a/pkg/executor/push_test.go b/pkg/executor/push_test.go new file mode 100644 index 000000000..2f9729960 --- /dev/null +++ b/pkg/executor/push_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "testing" + + "github.com/GoogleContainerTools/kaniko/testutil" +) + +func TestHeaderAdded(t *testing.T) { + tests := []struct { + name string + upstream string + expected string + }{{ + name: "upstream env variable set", + upstream: "skaffold-v0.25.45", + expected: "kaniko/unset,skaffold-v0.25.45", + }, { + name: "upstream env variable not set", + expected: "kaniko/unset", + }, + } + for _, test := range tests { + + t.Run(test.name, func(t *testing.T) { + rt := &withUserAgent{t: &mockRoundTripper{}} + if test.upstream != "" { + os.Setenv("UPSTREAM_CLIENT_TYPE", test.upstream) + defer func() { os.Unsetenv("UPSTREAM_CLIENT_TYPE") }() + } + req, err := http.NewRequest("GET", "dummy", nil) + if err != nil { + t.Fatalf("culd not create a req due to %s", err) + } + resp, err := rt.RoundTrip(req) + testutil.CheckError(t, false, err) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + testutil.CheckErrorAndDeepEqual(t, false, err, test.expected, string(body)) + }) + } + +} + +type mockRoundTripper struct { +} + +func (m *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + ua := r.UserAgent() + return &http.Response{Body: ioutil.NopCloser(bytes.NewBufferString(ua))}, nil +} diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 02f5ffcc7..5577da087 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "path/filepath" + "sort" "syscall" "github.com/GoogleContainerTools/kaniko/pkg/timing" @@ -186,6 +187,8 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { // Also add parent directories to keep their permissions correctly. filesToAdd = filesWithParentDirs(filesToAdd) + sort.Strings(filesToAdd) + // Add files to the layered map for _, file := range filesToAdd { if err := s.l.Add(file); err != nil { diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index ea6f4bceb..bfa445f58 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -87,6 +87,45 @@ func TestSnapshotFSFileChange(t *testing.T) { } } +func TestSnapshotFSIsReproducible(t *testing.T) { + testDir, snapshotter, cleanup, err := setUpTestDir() + defer cleanup() + if err != nil { + t.Fatal(err) + } + // Make some changes to the filesystem + newFiles := map[string]string{ + "foo": "newbaz1", + "bar/bat": "baz", + } + if err := testutil.SetupFiles(testDir, newFiles); err != nil { + t.Fatalf("Error setting up fs: %s", err) + } + // Take another snapshot + tarPath, err := snapshotter.TakeSnapshotFS() + if err != nil { + t.Fatalf("Error taking snapshot of fs: %s", err) + } + + f, err := os.Open(tarPath) + if err != nil { + t.Fatal(err) + } + // Check contents of the snapshot, make sure contents are sorted by name + tr := tar.NewReader(f) + var filesInTar []string + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + filesInTar = append(filesInTar, hdr.Name) + } + if !sort.StringsAreSorted(filesInTar) { + t.Fatalf("Expected the file in the tar archive were sorted, actual list was not sorted: %v", filesInTar) + } +} + func TestSnapshotFSChangePermissions(t *testing.T) { testDir, snapshotter, cleanup, err := setUpTestDir() defer cleanup() diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index dcc09ada8..0978a6fef 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -149,5 +149,5 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { cacheKey = d.String() } - return cache.LocalSource(opts, cacheKey) + return cache.LocalSource(&opts.CacheOptions, cacheKey) } diff --git a/run_in_docker.sh b/run_in_docker.sh index b3bdd4988..9397f1a71 100755 --- a/run_in_docker.sh +++ b/run_in_docker.sh @@ -15,8 +15,9 @@ #!/bin/bash set -e -if [ $# -lt 3 ]; - then echo "Usage: run_in_docker.sh " +if [ $# -lt 3 ]; then + echo "Usage: run_in_docker.sh " + exit 1 fi dockerfile=$1