diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 841fce661..97511e092 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -86,7 +86,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().StringVarP(&opts.SnapshotMode, "snapshotMode", "", "full", "Change the file attributes inspected during snapshotting") RootCmd.PersistentFlags().VarP(&opts.BuildArgs, "build-arg", "", "This flag allows you to pass in ARG values at build time. Set it repeatedly for multiple values.") RootCmd.PersistentFlags().BoolVarP(&opts.InsecurePush, "insecure", "", false, "Push to insecure registry using plain HTTP") - RootCmd.PersistentFlags().BoolVarP(&opts.SkipTlsVerify, "skip-tls-verify", "", false, "Push to insecure registry ignoring TLS verify") + RootCmd.PersistentFlags().BoolVarP(&opts.SkipTLSVerify, "skip-tls-verify", "", false, "Push to insecure registry ignoring TLS verify") RootCmd.PersistentFlags().StringVarP(&opts.TarPath, "tarPath", "", "", "Path to save the image in as a tarball instead of pushing") RootCmd.PersistentFlags().BoolVarP(&opts.SingleSnapshot, "single-snapshot", "", false, "Take a single snapshot at the end of the build.") RootCmd.PersistentFlags().BoolVarP(&opts.Reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible") @@ -145,7 +145,7 @@ func resolveSourceContext() error { opts.SrcContext = opts.Bucket } } - // if no prefix use Google Cloud Storage as default for backwards compability + // if no prefix use Google Cloud Storage as default for backwards compatibility contextExecutor, err := buildcontext.GetBuildContext(opts.SrcContext) if err != nil { return err diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py index bcc4b1c8f..83e6b1b3e 100644 --- a/hack/boilerplate/boilerplate.py +++ b/hack/boilerplate/boilerplate.py @@ -18,12 +18,14 @@ from __future__ import print_function import argparse import glob -import json -import mmap import os import re import sys + +SKIPPED_DIRS = ["Godeps", "third_party", ".git", "vendor", "examples", "testdata"] +SKIPPED_FILES = ["install_golint.sh"] + parser = argparse.ArgumentParser() parser.add_argument("filenames", help="list of files to check, all files if unspecified", nargs='*') @@ -71,7 +73,7 @@ def file_passes(filename, refs, regexs): (data, found) = p.subn("", data, 1) # remove shebang from the top of shell files - if extension == "sh": + elif extension == "sh": p = regexs["shebang"] (data, found) = p.subn("", data, 1) @@ -105,17 +107,11 @@ def file_passes(filename, refs, regexs): def file_extension(filename): return os.path.splitext(filename)[1].split(".")[-1].lower() -skipped_dirs = ['Godeps', 'third_party', '.git', "vendor", "differs/testDirs/pipTests"] - def normalize_files(files): newfiles = [] - for pathname in files: - if any(x in pathname for x in skipped_dirs): - continue - newfiles.append(pathname) - for i, pathname in enumerate(newfiles): + for i, pathname in enumerate(files): if not os.path.isabs(pathname): - newfiles[i] = os.path.join(args.rootdir, pathname) + newfiles.append(os.path.join(args.rootdir, pathname)) return newfiles def get_files(extensions): @@ -124,17 +120,14 @@ def get_files(extensions): files = args.filenames else: for root, dirs, walkfiles in os.walk(args.rootdir): - # don't visit certain dirs. This is just a performance improvement - # as we would prune these later in normalize_files(). But doing it - # cuts down the amount of filesystem walking we do and cuts down - # the size of the file list - for d in skipped_dirs: + for d in SKIPPED_DIRS: if d in dirs: dirs.remove(d) for name in walkfiles: - pathname = os.path.join(root, name) - files.append(pathname) + if name not in SKIPPED_FILES: + pathname = os.path.join(root, name) + files.append(pathname) files = normalize_files(files) outfiles = [] diff --git a/hack/gometalinter.json b/hack/gometalinter.json deleted file mode 100644 index 857b558e1..000000000 --- a/hack/gometalinter.json +++ /dev/null @@ -1,17 +0,0 @@ - -{ - "Vendor": true, - "EnableGC": true, - "Debug": false, - "Sort": ["linter", "severity", "path"], - "Enable": [ - "deadcode", - "gofmt", - "golint", - "gosimple", - "ineffassign", - "vet" - ], - - "LineLength": 200 -} diff --git a/hack/install_golint.sh b/hack/install_golint.sh new file mode 100755 index 000000000..6010f3b27 --- /dev/null +++ b/hack/install_golint.sh @@ -0,0 +1,388 @@ +#!/bin/sh +set -e +# Code generated by godownloader on 2018-06-05T12:04:55Z. DO NOT EDIT. +# + +usage() { + this=$1 + cat </dev/null +} +echoerr() { + echo "$@" 1>&2 +} +log_prefix() { + echo "$0" +} +_logp=6 +log_set_priority() { + _logp="$1" +} +log_priority() { + if test -z "$1"; then + echo "$_logp" + return + fi + [ "$1" -le "$_logp" ] +} +log_tag() { + case $1 in + 0) echo "emerg" ;; + 1) echo "alert" ;; + 2) echo "crit" ;; + 3) echo "err" ;; + 4) echo "warning" ;; + 5) echo "notice" ;; + 6) echo "info" ;; + 7) echo "debug" ;; + *) echo "$1" ;; + esac +} +log_debug() { + log_priority 7 || return 0 + echoerr "$(log_prefix)" "$(log_tag 7)" "$@" +} +log_info() { + log_priority 6 || return 0 + echoerr "$(log_prefix)" "$(log_tag 6)" "$@" +} +log_err() { + log_priority 3 || return 0 + echoerr "$(log_prefix)" "$(log_tag 3)" "$@" +} +log_crit() { + log_priority 2 || return 0 + echoerr "$(log_prefix)" "$(log_tag 2)" "$@" +} +uname_os() { + os=$(uname -s | tr '[:upper:]' '[:lower:]') + case "$os" in + msys_nt) os="windows" ;; + esac + echo "$os" +} +uname_arch() { + arch=$(uname -m) + case $arch in + x86_64) arch="amd64" ;; + x86) arch="386" ;; + i686) arch="386" ;; + i386) arch="386" ;; + aarch64) arch="arm64" ;; + armv5*) arch="armv5" ;; + armv6*) arch="armv6" ;; + armv7*) arch="armv7" ;; + esac + echo ${arch} +} +uname_os_check() { + os=$(uname_os) + case "$os" in + darwin) return 0 ;; + dragonfly) return 0 ;; + freebsd) return 0 ;; + linux) return 0 ;; + android) return 0 ;; + nacl) return 0 ;; + netbsd) return 0 ;; + openbsd) return 0 ;; + plan9) return 0 ;; + solaris) return 0 ;; + windows) return 0 ;; + esac + log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib" + return 1 +} +uname_arch_check() { + arch=$(uname_arch) + case "$arch" in + 386) return 0 ;; + amd64) return 0 ;; + arm64) return 0 ;; + armv5) return 0 ;; + armv6) return 0 ;; + armv7) return 0 ;; + ppc64) return 0 ;; + ppc64le) return 0 ;; + mips) return 0 ;; + mipsle) return 0 ;; + mips64) return 0 ;; + mips64le) return 0 ;; + s390x) return 0 ;; + amd64p32) return 0 ;; + esac + log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib" + return 1 +} +untar() { + tarball=$1 + case "${tarball}" in + *.tar.gz | *.tgz) tar -xzf "${tarball}" ;; + *.tar) tar -xf "${tarball}" ;; + *.zip) unzip "${tarball}" ;; + *) + log_err "untar unknown archive format for ${tarball}" + return 1 + ;; + esac +} +mktmpdir() { + test -z "$TMPDIR" && TMPDIR="$(mktemp -d)" + mkdir -p "${TMPDIR}" + echo "${TMPDIR}" +} +http_download_curl() { + local_file=$1 + source_url=$2 + header=$3 + if [ -z "$header" ]; then + code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url") + else + code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url") + fi + if [ "$code" != "200" ]; then + log_debug "http_download_curl received HTTP status $code" + return 1 + fi + return 0 +} +http_download_wget() { + local_file=$1 + source_url=$2 + header=$3 + if [ -z "$header" ]; then + wget -q -O "$local_file" "$source_url" + else + wget -q --header "$header" -O "$local_file" "$source_url" + fi +} +http_download() { + log_debug "http_download $2" + if is_command curl; then + http_download_curl "$@" + return + elif is_command wget; then + http_download_wget "$@" + return + fi + log_crit "http_download unable to find wget or curl" + return 1 +} +http_copy() { + tmp=$(mktemp) + http_download "${tmp}" "$1" "$2" || return 1 + body=$(cat "$tmp") + rm -f "${tmp}" + echo "$body" +} +github_release() { + owner_repo=$1 + version=$2 + test -z "$version" && version="latest" + giturl="https://github.com/${owner_repo}/releases/${version}" + json=$(http_copy "$giturl" "Accept:application/json") + test -z "$json" && return 1 + version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') + test -z "$version" && return 1 + echo "$version" +} +hash_sha256() { + TARGET=${1:-/dev/stdin} + if is_command gsha256sum; then + hash=$(gsha256sum "$TARGET") || return 1 + echo "$hash" | cut -d ' ' -f 1 + elif is_command sha256sum; then + hash=$(sha256sum "$TARGET") || return 1 + echo "$hash" | cut -d ' ' -f 1 + elif is_command shasum; then + hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1 + echo "$hash" | cut -d ' ' -f 1 + elif is_command openssl; then + hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1 + echo "$hash" | cut -d ' ' -f a + else + log_crit "hash_sha256 unable to find command to compute sha-256 hash" + return 1 + fi +} +hash_sha256_verify() { + TARGET=$1 + checksums=$2 + if [ -z "$checksums" ]; then + log_err "hash_sha256_verify checksum file not specified in arg2" + return 1 + fi + BASENAME=${TARGET##*/} + want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1) + if [ -z "$want" ]; then + log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'" + return 1 + fi + got=$(hash_sha256 "$TARGET") + if [ "$want" != "$got" ]; then + log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got" + return 1 + fi +} +cat /dev/null < currentStage { break diff --git a/pkg/dockerfile/dockerfile_test.go b/pkg/dockerfile/dockerfile_test.go index bd09c26ba..cd83c79ff 100644 --- a/pkg/dockerfile/dockerfile_test.go +++ b/pkg/dockerfile/dockerfile_test.go @@ -20,7 +20,6 @@ import ( "strconv" "testing" - "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/testutil" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) @@ -184,7 +183,7 @@ func Test_baseImageIndex(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - actual := baseImageIndex(&config.KanikoOptions{}, test.currentStage, stages) + actual := baseImageIndex(test.currentStage, stages) if actual != test.expected { t.Fatalf("unexpected result, expected %d got %d", test.expected, actual) } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 06a9e7f23..9bcbfda1e 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -43,10 +43,10 @@ import ( // stageBuilder contains all fields necessary to build one stage of a Dockerfile type stageBuilder struct { - stage config.KanikoStage - image v1.Image - cf *v1.ConfigFile - *snapshot.Snapshotter + stage config.KanikoStage + image v1.Image + cf *v1.ConfigFile + snapshotter *snapshot.Snapshotter baseImageDigest string } @@ -67,7 +67,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*sta if err != nil { return nil, err } - l := snapshot.NewLayeredMap(hasher) + l := snapshot.NewLayeredMap(hasher, util.CacheHasher()) snapshotter := snapshot.NewSnapshotter(l, constants.RootDir) digest, err := sourceImage.Digest() @@ -78,7 +78,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*sta stage: stage, image: sourceImage, cf: imageConfig, - Snapshotter: snapshotter, + snapshotter: snapshotter, baseImageDigest: digest.String(), }, nil } @@ -101,7 +101,7 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { return err } // Take initial snapshot - if err := s.Snapshotter.Init(); err != nil { + if err := s.snapshotter.Init(); err != nil { return err } args := dockerfile.NewBuildArgs(opts.BuildArgs) @@ -114,6 +114,9 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { if command == nil { continue } + if err := util.GetFSFromImage(constants.RootDir, s.image); err != nil { + return err + } logrus.Info(command.String()) if err := command.ExecuteCommand(&s.cf.Config, args); err != nil { return err @@ -126,23 +129,23 @@ func (s *stageBuilder) build(opts *config.KanikoOptions) error { // by previous commands. if !s.stage.Final { if finalCmd { - contents, err = s.Snapshotter.TakeSnapshotFS() + contents, err = s.snapshotter.TakeSnapshotFS() } } else { // If we are in single snapshot mode, we only take a snapshot once, after all // commands have completed. if opts.SingleSnapshot { if finalCmd { - contents, err = s.Snapshotter.TakeSnapshotFS() + contents, err = s.snapshotter.TakeSnapshotFS() } } else { // Otherwise, in the final stage we take a snapshot at each command. If we know // the files that were changed, we'll snapshot those explicitly, otherwise we'll // check if anything in the filesystem changed. if files != nil { - contents, err = s.Snapshotter.TakeSnapshot(files) + contents, err = s.snapshotter.TakeSnapshot(files) } else { - contents, err = s.Snapshotter.TakeSnapshotFS() + contents, err = s.snapshotter.TakeSnapshotFS() } } } diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 836663ff6..53799b71a 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -87,7 +87,7 @@ func DoPush(image v1.Image, opts *config.KanikoOptions) error { // Create a transport to set our user-agent. tr := http.DefaultTransport - if opts.SkipTlsVerify { + if opts.SkipTLSVerify { tr.(*http.Transport).TLSClientConfig = &tls.Config{ InsecureSkipVerify: true, } diff --git a/pkg/snapshot/layered_map.go b/pkg/snapshot/layered_map.go index 0d382d766..9a356b0e2 100644 --- a/pkg/snapshot/layered_map.go +++ b/pkg/snapshot/layered_map.go @@ -17,20 +17,28 @@ limitations under the License. package snapshot import ( + "bytes" + "encoding/json" "fmt" "path/filepath" "strings" + + "github.com/GoogleContainerTools/kaniko/pkg/util" ) type LayeredMap struct { layers []map[string]string whiteouts []map[string]string + added []map[string]string hasher func(string) (string, error) + // cacheHasher doesn't include mtime in it's hash so that filesystem cache keys are stable + cacheHasher func(string) (string, error) } -func NewLayeredMap(h func(string) (string, error)) *LayeredMap { +func NewLayeredMap(h func(string) (string, error), c func(string) (string, error)) *LayeredMap { l := LayeredMap{ - hasher: h, + hasher: h, + cacheHasher: c, } l.layers = []map[string]string{} return &l @@ -39,8 +47,18 @@ func NewLayeredMap(h func(string) (string, error)) *LayeredMap { func (l *LayeredMap) Snapshot() { l.whiteouts = append(l.whiteouts, map[string]string{}) l.layers = append(l.layers, map[string]string{}) + l.added = append(l.added, map[string]string{}) } +// Key returns a hash for added files +func (l *LayeredMap) Key() (string, error) { + c := bytes.NewBuffer([]byte{}) + enc := json.NewEncoder(c) + enc.Encode(l.added) + return util.SHA256(c) +} + +// GetFlattenedPathsForWhiteOut returns all paths in the current FS func (l *LayeredMap) GetFlattenedPathsForWhiteOut() map[string]struct{} { paths := map[string]struct{}{} for _, l := range l.layers { @@ -85,11 +103,18 @@ func (l *LayeredMap) MaybeAddWhiteout(s string) (bool, error) { // Add will add the specified file s to the layered map. func (l *LayeredMap) Add(s string) error { + // Use hash function and add to layers newV, err := l.hasher(s) if err != nil { - return fmt.Errorf("Error creating hash for %s: %s", s, err) + return fmt.Errorf("Error creating hash for %s: %v", s, err) } l.layers[len(l.layers)-1][s] = newV + // Use cache hash function and add to added + cacheV, err := l.cacheHasher(s) + if err != nil { + return fmt.Errorf("Error creating cache hash for %s: %v", s, err) + } + l.added[len(l.added)-1][s] = cacheV return nil } diff --git a/pkg/snapshot/layered_map_test.go b/pkg/snapshot/layered_map_test.go new file mode 100644 index 000000000..6bfff81b3 --- /dev/null +++ b/pkg/snapshot/layered_map_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package snapshot + +import ( + "testing" +) + +func Test_CacheKey(t *testing.T) { + tests := []struct { + name string + map1 map[string]string + map2 map[string]string + equal bool + }{ + { + name: "maps are the same", + map1: map[string]string{ + "a": "apple", + "b": "bat", + "c": "cat", + "d": "dog", + "e": "egg", + }, + map2: map[string]string{ + "c": "cat", + "d": "dog", + "b": "bat", + "a": "apple", + "e": "egg", + }, + equal: true, + }, + { + name: "maps are different", + map1: map[string]string{ + "a": "apple", + "b": "bat", + "c": "cat", + }, + map2: map[string]string{ + "c": "", + "b": "bat", + "a": "apple", + }, + equal: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + lm1 := LayeredMap{added: []map[string]string{test.map1}} + lm2 := LayeredMap{added: []map[string]string{test.map2}} + k1, err := lm1.Key() + if err != nil { + t.Fatalf("error getting key for map 1: %v", err) + } + k2, err := lm2.Key() + if err != nil { + t.Fatalf("error getting key for map 2: %v", err) + } + if test.equal != (k1 == k2) { + t.Fatalf("unexpected result: \nExpected\n%s\nActual\n%s\n", k1, k2) + } + }) + } +} diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 4f01441dc..55da45d65 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -49,6 +49,11 @@ func (s *Snapshotter) Init() error { return nil } +// Key returns a string based on the current state of the file system +func (s *Snapshotter) Key() (string, error) { + return s.l.Key() +} + // TakeSnapshot takes a snapshot of the specified files, avoiding directories in the whitelist, and creates // a tarball of the changed files. Return contents of the tarball, and whether or not any files were changed func (s *Snapshotter) TakeSnapshot(files []string) ([]byte, error) { @@ -102,7 +107,8 @@ func (s *Snapshotter) snapshotFiles(f io.Writer, files []string) (bool, error) { logrus.Info("No files changed in this command, skipping snapshotting.") return false, nil } - logrus.Infof("Taking snapshot of files %v...", files) + logrus.Info("Taking snapshot of files...") + logrus.Debugf("Taking snapshot of files %v", files) snapshottedFiles := make(map[string]bool) filesAdded := false diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index 72b6a750a..95daa88f6 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -198,7 +198,7 @@ func setUpTestDir() (string, *Snapshotter, error) { } // Take the initial snapshot - l := NewLayeredMap(util.Hasher()) + l := NewLayeredMap(util.Hasher(), util.CacheHasher()) snapshotter := NewSnapshotter(l, testDir) if err := snapshotter.Init(); err != nil { return testDir, nil, errors.Wrap(err, "initializing snapshotter") diff --git a/pkg/util/bucket_util.go b/pkg/util/bucket_util.go index 105430fb8..e2f4dea6c 100644 --- a/pkg/util/bucket_util.go +++ b/pkg/util/bucket_util.go @@ -17,8 +17,9 @@ limitations under the License. package util import ( - "github.com/GoogleContainerTools/kaniko/pkg/constants" "strings" + + "github.com/GoogleContainerTools/kaniko/pkg/constants" ) func GetBucketAndItem(context string) (string, string) { diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go index c8bb50709..b6574faa8 100644 --- a/pkg/util/command_util.go +++ b/pkg/util/command_util.go @@ -228,10 +228,7 @@ func IsSrcRemoteFileURL(rawurl string) bool { return false } _, err = http.Get(rawurl) - if err != nil { - return false - } - return true + return err == nil } func UpdateConfigEnv(newEnvs []instructions.KeyValuePair, config *v1.Config, replacementEnvs []string) error { diff --git a/pkg/util/command_util_test.go b/pkg/util/command_util_test.go index c83c0b216..dd4ddbf21 100644 --- a/pkg/util/command_util_test.go +++ b/pkg/util/command_util_test.go @@ -23,7 +23,7 @@ import ( "github.com/GoogleContainerTools/kaniko/testutil" ) -var testUrl = "https://github.com/GoogleContainerTools/runtimes-common/blob/master/LICENSE" +var testURL = "https://github.com/GoogleContainerTools/runtimes-common/blob/master/LICENSE" var testEnvReplacement = []struct { path string @@ -220,7 +220,7 @@ var matchSourcesTests = []struct { srcs: []string{ "pkg/*", "/root/dir?", - testUrl, + testURL, }, files: []string{ "pkg/a", @@ -234,7 +234,7 @@ var matchSourcesTests = []struct { "/root/dir1", "pkg/a", "pkg/b", - testUrl, + testURL, }, }, } @@ -344,11 +344,11 @@ var isSrcValidTests = []struct { }, { srcsAndDest: []string{ - testUrl, + testURL, "dest", }, resolvedSources: []string{ - testUrl, + testURL, }, shouldErr: false, }, @@ -369,13 +369,13 @@ var testResolveSources = []struct { srcsAndDest: []string{ "context/foo", "context/b*", - testUrl, + testURL, "dest/", }, expectedList: []string{ "context/foo", "context/bar", - testUrl, + testURL, }, }, } diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 4c6dd9fff..d1136d7bc 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -40,6 +40,9 @@ var whitelist = []string{ // which leads to a special mount on the /var/run/docker.sock file itself, but the directory to exist // in the image with no way to tell if it came from the base image or not. "/var/run", + // similarly, we whitelist /etc/mtab, since there is no way to know if the file was mounted or came + // from the base image + "/etc/mtab", } var volumeWhitelist = []string{} @@ -109,7 +112,7 @@ func GetFSFromImage(root string, img v1.Image) error { // DeleteFilesystem deletes the extracted image file system func DeleteFilesystem() error { logrus.Info("Deleting filesystem...") - err := filepath.Walk(constants.RootDir, func(path string, info os.FileInfo, err error) error { + return filepath.Walk(constants.RootDir, func(path string, info os.FileInfo, _ error) error { whitelisted, err := CheckWhitelist(path) if err != nil { return err @@ -123,7 +126,6 @@ func DeleteFilesystem() error { } return os.RemoveAll(path) }) - return err } // ChildDirInWhitelist returns true if there is a child file or directory of the path in the whitelist @@ -195,7 +197,6 @@ func extractFile(dest string, hdr *tar.Header, tr io.Reader) error { return err } currFile.Close() - case tar.TypeDir: logrus.Debugf("creating dir %s", path) if err := os.MkdirAll(path, mode); err != nil { @@ -310,6 +311,9 @@ func RelativeFiles(fp string, root string) ([]string, error) { fullPath := filepath.Join(root, fp) logrus.Debugf("Getting files and contents at root %s", fullPath) err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } whitelisted, err := CheckWhitelist(path) if err != nil { return err diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index 5bfb1a403..99202e1bb 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -50,7 +50,7 @@ func Test_fileSystemWhitelist(t *testing.T) { } actualWhitelist, err := fileSystemWhitelist(path) - expectedWhitelist := []string{"/kaniko", "/proc", "/dev", "/dev/pts", "/sys", "/var/run"} + expectedWhitelist := []string{"/kaniko", "/proc", "/dev", "/dev/pts", "/sys", "/var/run", "/etc/mtab"} sort.Strings(actualWhitelist) sort.Strings(expectedWhitelist) testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist) @@ -343,8 +343,8 @@ func filesAreHardlinks(first, second string) checker { if err != nil { t.Fatalf("error getting file %s", second) } - stat1 := getSyscallStat_t(fi1) - stat2 := getSyscallStat_t(fi2) + stat1 := getSyscallStatT(fi1) + stat2 := getSyscallStatT(fi2) if stat1.Ino != stat2.Ino { t.Errorf("%s and %s aren't hardlinks as they dont' have the same inode", first, second) } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 61a19b5d3..d19e684f6 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -25,6 +25,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/sirupsen/logrus" @@ -61,7 +62,7 @@ func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string) (v1.Image } // RetrieveConfigFile returns the config file for an image -func RetrieveConfigFile(sourceImage v1.Image) (*v1.ConfigFile, error) { +func RetrieveConfigFile(sourceImage partial.WithConfigFile) (*v1.ConfigFile, error) { imageConfig, err := sourceImage.ConfigFile() if err != nil { return nil, err diff --git a/pkg/util/tar_util.go b/pkg/util/tar_util.go index 5f662d962..bc1cc67a0 100644 --- a/pkg/util/tar_util.go +++ b/pkg/util/tar_util.go @@ -118,7 +118,7 @@ func (t *Tar) Whiteout(p string) error { func (t *Tar) checkHardlink(p string, i os.FileInfo) (bool, string) { hardlink := false linkDst := "" - stat := getSyscallStat_t(i) + stat := getSyscallStatT(i) if stat != nil { nlinks := stat.Nlink if nlinks > 1 { @@ -135,7 +135,7 @@ func (t *Tar) checkHardlink(p string, i os.FileInfo) (bool, string) { return hardlink, linkDst } -func getSyscallStat_t(i os.FileInfo) *syscall.Stat_t { +func getSyscallStatT(i os.FileInfo) *syscall.Stat_t { if sys := i.Sys(); sys != nil { if stat, ok := sys.(*syscall.Stat_t); ok { return stat @@ -195,10 +195,10 @@ func fileIsCompressedTar(src string) (bool, archive.Compression) { func fileIsUncompressedTar(src string) bool { r, err := os.Open(src) - defer r.Close() if err != nil { return false } + defer r.Close() fi, err := os.Lstat(src) if err != nil { return false @@ -210,13 +210,8 @@ func fileIsUncompressedTar(src string) bool { if tr == nil { return false } - for { - _, err := tr.Next() - if err != nil { - return false - } - return true - } + _, err = tr.Next() + return err == nil } // UnpackCompressedTar unpacks the compressed tar at path to dir diff --git a/pkg/util/util.go b/pkg/util/util.go index 617298a7b..873cbae20 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -18,6 +18,7 @@ package util import ( "crypto/md5" + "crypto/sha256" "encoding/hex" "io" "os" @@ -72,6 +73,36 @@ func Hasher() func(string) (string, error) { return hasher } +// CacheHasher takes into account everything the regular hasher does except for mtime +func CacheHasher() func(string) (string, error) { + hasher := func(p string) (string, error) { + h := md5.New() + fi, err := os.Lstat(p) + if err != nil { + return "", err + } + h.Write([]byte(fi.Mode().String())) + + h.Write([]byte(strconv.FormatUint(uint64(fi.Sys().(*syscall.Stat_t).Uid), 36))) + h.Write([]byte(",")) + h.Write([]byte(strconv.FormatUint(uint64(fi.Sys().(*syscall.Stat_t).Gid), 36))) + + if fi.Mode().IsRegular() { + f, err := os.Open(p) + if err != nil { + return "", err + } + defer f.Close() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + } + + return hex.EncodeToString(h.Sum(nil)), nil + } + return hasher +} + // MtimeHasher returns a hash function, which only looks at mtime to determine if a file has changed. // Note that the mtime can lag, so it's possible that a file will have changed but the mtime may look the same. func MtimeHasher() func(string) (string, error) { @@ -86,3 +117,13 @@ func MtimeHasher() func(string) (string, error) { } return hasher } + +// SHA256 returns the shasum of the contents of r +func SHA256(r io.Reader) (string, error) { + hasher := sha256.New() + _, err := io.Copy(hasher, r) + if err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), nil +} diff --git a/test.sh b/test.sh index c332c8f29..c86b974cd 100755 --- a/test.sh +++ b/test.sh @@ -31,7 +31,7 @@ echo "Running validation scripts..." scripts=( "hack/boilerplate.sh" "hack/gofmt.sh" - "hack/gometalinter.sh" + "hack/linter.sh" "hack/dep.sh" ) fail=0