public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #854 from WanzenBug/ignored-files-invalidate-cache 

Fix caching to respect .dockerignore
This commit is contained in:
Tejal Desai 2020-01-29 11:32:43 -08:00 committed by GitHub
parent 65cd9124d1 bab1c170e8
commit 8bdcb4f544
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 479 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/executor/build.go
View File

@ -156,8 +156,10 @@ func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string
compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey)
}
srcCtx := s.opts.SrcContext
for _, f := range files {
if err := compositeKey.AddPath(f); err != nil {
if err := compositeKey.AddPath(f, srcCtx); err != nil {
return compositeKey, err
}
}

18
pkg/executor/build_test.go
View File

@ -538,7 +538,7 @@ func Test_stageBuilder_build(t *testing.T) {
filePath := filepath.Join(dir, file)
ch := NewCompositeCache("", "meow")
ch.AddPath(filePath)
ch.AddPath(filePath, "")
hash, err := ch.Hash()
if err != nil {
t.Errorf("couldn't create hash %v", err)
@ -570,7 +570,7 @@ func Test_stageBuilder_build(t *testing.T) {
filePath := filepath.Join(dir, file)
ch := NewCompositeCache("", "meow")
ch.AddPath(filePath)
ch.AddPath(filePath, "")
hash, err := ch.Hash()
if err != nil {
t.Errorf("couldn't create hash %v", err)
@ -618,7 +618,7 @@ func Test_stageBuilder_build(t *testing.T) {
tarContent := generateTar(t, dir, filename)
ch := NewCompositeCache("", "")
ch.AddPath(filepath)
ch.AddPath(filepath, "")
hash, err := ch.Hash()
if err != nil {
@ -662,7 +662,7 @@ func Test_stageBuilder_build(t *testing.T) {
}
filePath := filepath.Join(dir, filename)
ch := NewCompositeCache("", "")
ch.AddPath(filePath)
ch.AddPath(filePath, "")
hash, err := ch.Hash()
if err != nil {
@ -713,7 +713,7 @@ func Test_stageBuilder_build(t *testing.T) {
}
ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
ch.AddPath(filePath)
ch.AddPath(filePath, "")
hash2, err := ch.Hash()
if err != nil {
@ -721,7 +721,7 @@ func Test_stageBuilder_build(t *testing.T) {
}
ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename))
ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
ch.AddPath(filePath)
ch.AddPath(filePath, "")
image := fakeImage{
ImageLayers: []v1.Layer{
@ -777,14 +777,14 @@ COPY %s bar.txt
}
filePath := filepath.Join(dir, filename)
ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename))
ch.AddPath(filePath)
ch.AddPath(filePath, "")
hash1, err := ch.Hash()
if err != nil {
t.Errorf("couldn't create hash %v", err)
}
ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
ch.AddPath(filePath)
ch.AddPath(filePath, "")
hash2, err := ch.Hash()
if err != nil {
@ -792,7 +792,7 @@ COPY %s bar.txt
}
ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename))
ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
ch.AddPath(filePath)
ch.AddPath(filePath, "")
image := fakeImage{
ImageLayers: []v1.Layer{

29
pkg/executor/composite_cache.go
View File

@ -54,18 +54,28 @@ func (s *CompositeCache) Hash() (string, error) {
return util.SHA256(strings.NewReader(s.Key()))
}
func (s *CompositeCache) AddPath(p string) error {
func (s *CompositeCache) AddPath(p, context string) error {
sha := sha256.New()
fi, err := os.Lstat(p)
if err != nil {
return err
}
if fi.Mode().IsDir() {
k, err := HashDir(p)
empty, k, err := hashDir(p, context)
if err != nil {
return err
}
s.keys = append(s.keys, k)
// Only add the hash of this directory to the key
// if there is any whitelisted content.
if !empty || !util.ExcludeFile(p, context) {
s.keys = append(s.keys, k)
}
return nil
}
if util.ExcludeFile(p, context) {
return nil
}
fh, err := util.CacheHasher()(p)
@ -81,12 +91,18 @@ func (s *CompositeCache) AddPath(p string) error {
}
// HashDir returns a hash of the directory.
func HashDir(p string) (string, error) {
func hashDir(p, context string) (bool, string, error) {
sha := sha256.New()
empty := true
if err := filepath.Walk(p, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
exclude := util.ExcludeFile(path, context)
if exclude {
return nil
}
fileHash, err := util.CacheHasher()(path)
if err != nil {
return err
@ -94,10 +110,11 @@ func HashDir(p string) (string, error) {
if _, err := sha.Write([]byte(fileHash)); err != nil {
return err
}
empty = false
return nil
}); err != nil {
return "", err
return false, "", err
}
return fmt.Sprintf("%x", sha.Sum(nil)), nil
return empty, fmt.Sprintf("%x", sha.Sum(nil)), nil
}

437
pkg/executor/composite_cache_test.go
View File

@ -19,9 +19,12 @@ package executor
import (
"io/ioutil"
"os"
"path"
"path/filepath"
"reflect"
"testing"
"github.com/GoogleContainerTools/kaniko/pkg/util"
)
func Test_NewCompositeCache(t *testing.T) {
@ -77,7 +80,7 @@ func Test_CompositeCache_AddPath_dir(t *testing.T) {
fn := func() string {
r := NewCompositeCache()
if err := r.AddPath(tmpDir); err != nil {
if err := r.AddPath(tmpDir, ""); err != nil {
t.Errorf("expected error to be nil but was %v", err)
}
@ -115,7 +118,7 @@ func Test_CompositeCache_AddPath_file(t *testing.T) {
p := tmpfile.Name()
fn := func() string {
r := NewCompositeCache()
if err := r.AddPath(p); err != nil {
if err := r.AddPath(p, ""); err != nil {
t.Errorf("expected error to be nil but was %v", err)
}
@ -135,3 +138,433 @@ func Test_CompositeCache_AddPath_file(t *testing.T) {
t.Errorf("expected hash %v to equal hash %v", hash1, hash2)
}
}
func createFilesystemStructure(root string, directories, files []string) error {
for _, d := range directories {
dirPath := path.Join(root, d)
if err := os.MkdirAll(dirPath, 0755); err != nil {
return err
}
}
for _, fileName := range files {
filePath := path.Join(root, fileName)
err := ioutil.WriteFile(filePath, []byte(fileName), 0644)
if err != nil {
return err
}
}
return nil
}
func setIgnoreContext(content string) error {
dockerIgnoreDir, err := ioutil.TempDir("", "")
if err != nil {
return err
}
defer os.RemoveAll(dockerIgnoreDir)
err = ioutil.WriteFile(dockerIgnoreDir+".dockerignore", []byte(content), 0644)
if err != nil {
return err
}
err = util.GetExcludedFiles(dockerIgnoreDir, "")
if err != nil {
return err
}
return nil
}
func hashDirectory(dirpath string) (string, error) {
cache1 := NewCompositeCache()
err := cache1.AddPath(dirpath, dirpath)
if err != nil {
return "", err
}
hash, err := cache1.Hash()
if err != nil {
return "", err
}
return hash, nil
}
func Test_CompositeKey_AddPath_Works(t *testing.T) {
tests := []struct {
name string
directories []string
files []string
}{
{
name: "empty",
directories: []string{},
files: []string{},
},
{
name: "dirs",
directories: []string{"foo", "bar", "foobar", "f/o/o"},
files: []string{},
},
{
name: "files",
directories: []string{},
files: []string{"foo", "bar", "foobar"},
},
{
name: "all",
directories: []string{"foo", "bar"},
files: []string{"foo/bar", "bar/baz", "foobar"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testDir1, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir1)
err = createFilesystemStructure(testDir1, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
testDir2, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir2)
err = createFilesystemStructure(testDir2, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
if hash1 != hash2 {
t.Errorf("Expected equal hashes, got: %s and %s", hash1, hash2)
}
})
}
}
func Test_CompositeKey_AddPath_WithExtraFile_Works(t *testing.T) {
tests := []struct {
name string
directories []string
files []string
extraFile string
}{
{
name: "empty",
directories: []string{},
files: []string{},
extraFile: "file",
},
{
name: "dirs",
directories: []string{"foo", "bar", "foobar", "f/o/o"},
files: []string{},
extraFile: "f/o/o/extra",
},
{
name: "files",
directories: []string{},
files: []string{"foo", "bar", "foobar"},
extraFile: "foo.extra",
},
{
name: "all",
directories: []string{"foo", "bar"},
files: []string{"foo/bar", "bar/baz", "foobar"},
extraFile: "bar/extra",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testDir1, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir1)
err = createFilesystemStructure(testDir1, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
testDir2, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir2)
err = createFilesystemStructure(testDir2, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
extraPath := path.Join(testDir2, test.extraFile)
err = ioutil.WriteFile(extraPath, []byte(test.extraFile), 0644)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
if hash1 == hash2 {
t.Errorf("Expected different hashes, got: %s and %s", hash1, hash2)
}
})
}
}
func Test_CompositeKey_AddPath_WithExtraDir_Works(t *testing.T) {
tests := []struct {
name string
directories []string
files []string
extraDir string
}{
{
name: "empty",
directories: []string{},
files: []string{},
extraDir: "extra",
},
{
name: "dirs",
directories: []string{"foo", "bar", "foobar", "f/o/o"},
files: []string{},
extraDir: "f/o/o/extra",
},
{
name: "files",
directories: []string{},
files: []string{"foo", "bar", "foobar"},
extraDir: "foo.extra",
},
{
name: "all",
directories: []string{"foo", "bar"},
files: []string{"foo/bar", "bar/baz", "foobar"},
extraDir: "bar/extra",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testDir1, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir1)
err = createFilesystemStructure(testDir1, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
testDir2, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir2)
err = createFilesystemStructure(testDir2, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
extraPath := path.Join(testDir2, test.extraDir)
err = os.MkdirAll(extraPath, 0644)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
if hash1 == hash2 {
t.Errorf("Expected different hashes, got: %s and %s", hash1, hash2)
}
})
}
}
func Test_CompositeKey_AddPath_WithExtraFilIgnored_Works(t *testing.T) {
tests := []struct {
name string
directories []string
files []string
extraFile string
}{
{
name: "empty",
directories: []string{},
files: []string{},
extraFile: "extra",
},
{
name: "dirs",
directories: []string{"foo", "bar", "foobar", "f/o/o"},
files: []string{},
extraFile: "f/o/o/extra",
},
{
name: "files",
directories: []string{},
files: []string{"foo", "bar", "foobar"},
extraFile: "extra",
},
{
name: "all",
directories: []string{"foo", "bar"},
files: []string{"foo/bar", "bar/baz", "foobar"},
extraFile: "bar/extra",
},
}
err := setIgnoreContext("**/extra")
if err != nil {
t.Fatalf("Error setting exlusion context: %s", err)
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testDir1, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir1)
err = createFilesystemStructure(testDir1, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
testDir2, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir2)
err = createFilesystemStructure(testDir2, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
extraPath := path.Join(testDir2, test.extraFile)
err = ioutil.WriteFile(extraPath, []byte(test.extraFile), 0644)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
if hash1 != hash2 {
t.Errorf("Expected equal hashes, got: %s and %s", hash1, hash2)
}
})
}
}
func Test_CompositeKey_AddPath_WithExtraDirIgnored_Works(t *testing.T) {
tests := []struct {
name string
directories []string
files []string
extraDir string
}{
{
name: "empty",
directories: []string{},
files: []string{},
extraDir: "extra",
},
{
name: "dirs",
directories: []string{"foo", "bar", "foobar", "f/o/o"},
files: []string{},
extraDir: "f/o/o/extra",
},
{
name: "files",
directories: []string{},
files: []string{"foo", "bar", "foobar"},
extraDir: "extra",
},
{
name: "all",
directories: []string{"foo", "bar"},
files: []string{"foo/bar", "bar/baz", "foobar"},
extraDir: "bar/extra",
},
}
err := setIgnoreContext("**/extra")
if err != nil {
t.Fatalf("Error setting exlusion context: %s", err)
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testDir1, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir1)
err = createFilesystemStructure(testDir1, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
testDir2, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Error creating tempdir: %s", err)
}
defer os.RemoveAll(testDir2)
err = createFilesystemStructure(testDir2, test.directories, test.files)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
extraPath := path.Join(testDir2, test.extraDir)
err = os.MkdirAll(extraPath, 0644)
if err != nil {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
if hash1 != hash2 {
t.Errorf("Expected equal hashes, got: %s and %s", hash1, hash2)
}
})
}
}

4
pkg/util/command_util.go
View File

@ -213,7 +213,7 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri
if !ContainsWildcards(srcs) {
totalSrcs := 0
for _, src := range srcs {
if excludeFile(src, root) {
if ExcludeFile(src, root) {
continue
}
totalSrcs++
@ -250,7 +250,7 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri
return errors.Wrap(err, "failed to get relative files")
}
for _, file := range files {
if excludeFile(file, root) {
if ExcludeFile(file, root) {
continue
}
totalFiles++

10
pkg/util/fs_util.go
View File

@ -547,7 +547,7 @@ func CopyDir(src, dest, buildcontext string) ([]string, error) {
fmt.Println(" i am returning from here this", err)
return nil, err
}
if excludeFile(fullPath, buildcontext) {
if ExcludeFile(fullPath, buildcontext) {
logrus.Debugf("%s found in .dockerignore, ignoring", src)
continue
}
@ -580,7 +580,7 @@ func CopyDir(src, dest, buildcontext string) ([]string, error) {
// CopySymlink copies the symlink at src to dest
func CopySymlink(src, dest, buildcontext string) (bool, error) {
if excludeFile(src, buildcontext) {
if ExcludeFile(src, buildcontext) {
logrus.Debugf("%s found in .dockerignore, ignoring", src)
return true, nil
}
@ -601,7 +601,7 @@ func CopySymlink(src, dest, buildcontext string) (bool, error) {
// CopyFile copies the file at src to dest
func CopyFile(src, dest, buildcontext string) (bool, error) {
if excludeFile(src, buildcontext) {
if ExcludeFile(src, buildcontext) {
logrus.Debugf("%s found in .dockerignore, ignoring", src)
return true, nil
}
@ -645,8 +645,8 @@ func GetExcludedFiles(dockerfilepath string, buildcontext string) error {
return err
}
// excludeFile returns true if the .dockerignore specified this file should be ignored
func excludeFile(path, buildcontext string) bool {
// ExcludeFile returns true if the .dockerignore specified this file should be ignored
func ExcludeFile(path, buildcontext string) bool {
if HasFilepathPrefix(path, buildcontext, false) {
var err error
path, err = filepath.Rel(buildcontext, path)

4
pkg/util/fs_util_test.go
View File

@ -921,14 +921,14 @@ func Test_correctDockerignoreFileIsUsed(t *testing.T) {
}
for _, excl := range tt.args.excluded {
t.Run(tt.name+" to exclude "+excl, func(t *testing.T) {
if !excludeFile(excl, tt.args.buildcontext) {
if !ExcludeFile(excl, tt.args.buildcontext) {
t.Errorf("'%v' not excluded", excl)
}
})
}
for _, incl := range tt.args.included {
t.Run(tt.name+" to include "+incl, func(t *testing.T) {
if excludeFile(incl, tt.args.buildcontext) {
if ExcludeFile(incl, tt.args.buildcontext) {
t.Errorf("'%v' not included", incl)
}
})