Merge pull request #854 from WanzenBug/ignored-files-invalidate-cache

Fix caching to respect .dockerignore
2026-10-01 07:31:58 +02:00 · 2020-01-29 11:32:43 -08:00
parent 65cd9124d1 bab1c170e8
commit 8bdcb4f544
7 changed files with 479 additions and 27 deletions
@@ -156,8 +156,10 @@ func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string
 		compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey)
 	}

+	srcCtx := s.opts.SrcContext
+
 	for _, f := range files {
-		if err := compositeKey.AddPath(f); err != nil {
+		if err := compositeKey.AddPath(f, srcCtx); err != nil {
 			return compositeKey, err
 		}
 	}
@@ -538,7 +538,7 @@ func Test_stageBuilder_build(t *testing.T) {
 			filePath := filepath.Join(dir, file)
 			ch := NewCompositeCache("", "meow")

-			ch.AddPath(filePath)
+			ch.AddPath(filePath, "")
 			hash, err := ch.Hash()
 			if err != nil {
 				t.Errorf("couldn't create hash %v", err)
@@ -570,7 +570,7 @@ func Test_stageBuilder_build(t *testing.T) {
 			filePath := filepath.Join(dir, file)
 			ch := NewCompositeCache("", "meow")

-			ch.AddPath(filePath)
+			ch.AddPath(filePath, "")
 			hash, err := ch.Hash()
 			if err != nil {
 				t.Errorf("couldn't create hash %v", err)
@@ -618,7 +618,7 @@ func Test_stageBuilder_build(t *testing.T) {
 			tarContent := generateTar(t, dir, filename)

 			ch := NewCompositeCache("", "")
-			ch.AddPath(filepath)
+			ch.AddPath(filepath, "")

 			hash, err := ch.Hash()
 			if err != nil {
@@ -662,7 +662,7 @@ func Test_stageBuilder_build(t *testing.T) {
 			}
 			filePath := filepath.Join(dir, filename)
 			ch := NewCompositeCache("", "")
-			ch.AddPath(filePath)
+			ch.AddPath(filePath, "")

 			hash, err := ch.Hash()
 			if err != nil {
@@ -713,7 +713,7 @@ func Test_stageBuilder_build(t *testing.T) {
 			}

 			ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
-			ch.AddPath(filePath)
+			ch.AddPath(filePath, "")

 			hash2, err := ch.Hash()
 			if err != nil {
@@ -721,7 +721,7 @@ func Test_stageBuilder_build(t *testing.T) {
 			}
 			ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename))
 			ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
-			ch.AddPath(filePath)
+			ch.AddPath(filePath, "")

 			image := fakeImage{
 				ImageLayers: []v1.Layer{
@@ -777,14 +777,14 @@ COPY %s bar.txt
 			}
 			filePath := filepath.Join(dir, filename)
 			ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename))
-			ch.AddPath(filePath)
+			ch.AddPath(filePath, "")

 			hash1, err := ch.Hash()
 			if err != nil {
 				t.Errorf("couldn't create hash %v", err)
 			}
 			ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
-			ch.AddPath(filePath)
+			ch.AddPath(filePath, "")

 			hash2, err := ch.Hash()
 			if err != nil {
@@ -792,7 +792,7 @@ COPY %s bar.txt
 			}
 			ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename))
 			ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename))
-			ch.AddPath(filePath)
+			ch.AddPath(filePath, "")

 			image := fakeImage{
 				ImageLayers: []v1.Layer{
@@ -54,18 +54,28 @@ func (s *CompositeCache) Hash() (string, error) {
 	return util.SHA256(strings.NewReader(s.Key()))
 }

-func (s *CompositeCache) AddPath(p string) error {
+func (s *CompositeCache) AddPath(p, context string) error {
 	sha := sha256.New()
 	fi, err := os.Lstat(p)
 	if err != nil {
 		return err
 	}
+
 	if fi.Mode().IsDir() {
-		k, err := HashDir(p)
+		empty, k, err := hashDir(p, context)
 		if err != nil {
 			return err
 		}
-		s.keys = append(s.keys, k)
+
+		// Only add the hash of this directory to the key
+		// if there is any whitelisted content.
+		if !empty || !util.ExcludeFile(p, context) {
+			s.keys = append(s.keys, k)
+		}
+		return nil
+	}
+
+	if util.ExcludeFile(p, context) {
 		return nil
 	}
 	fh, err := util.CacheHasher()(p)
@@ -81,12 +91,18 @@ func (s *CompositeCache) AddPath(p string) error {
 }

 // HashDir returns a hash of the directory.
-func HashDir(p string) (string, error) {
+func hashDir(p, context string) (bool, string, error) {
 	sha := sha256.New()
+	empty := true
 	if err := filepath.Walk(p, func(path string, fi os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
+		exclude := util.ExcludeFile(path, context)
+		if exclude {
+			return nil
+		}
+
 		fileHash, err := util.CacheHasher()(path)
 		if err != nil {
 			return err
@@ -94,10 +110,11 @@ func HashDir(p string) (string, error) {
 		if _, err := sha.Write([]byte(fileHash)); err != nil {
 			return err
 		}
+		empty = false
 		return nil
 	}); err != nil {
-		return "", err
+		return false, "", err
 	}

-	return fmt.Sprintf("%x", sha.Sum(nil)), nil
+	return empty, fmt.Sprintf("%x", sha.Sum(nil)), nil
 }
@@ -19,9 +19,12 @@ package executor
 import (
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"reflect"
 	"testing"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
 )

 func Test_NewCompositeCache(t *testing.T) {
@@ -77,7 +80,7 @@ func Test_CompositeCache_AddPath_dir(t *testing.T) {

 	fn := func() string {
 		r := NewCompositeCache()
-		if err := r.AddPath(tmpDir); err != nil {
+		if err := r.AddPath(tmpDir, ""); err != nil {
 			t.Errorf("expected error to be nil but was %v", err)
 		}

@@ -115,7 +118,7 @@ func Test_CompositeCache_AddPath_file(t *testing.T) {
 	p := tmpfile.Name()
 	fn := func() string {
 		r := NewCompositeCache()
-		if err := r.AddPath(p); err != nil {
+		if err := r.AddPath(p, ""); err != nil {
 			t.Errorf("expected error to be nil but was %v", err)
 		}

@@ -135,3 +138,433 @@ func Test_CompositeCache_AddPath_file(t *testing.T) {
 		t.Errorf("expected hash %v to equal hash %v", hash1, hash2)
 	}
 }
+
+func createFilesystemStructure(root string, directories, files []string) error {
+	for _, d := range directories {
+		dirPath := path.Join(root, d)
+		if err := os.MkdirAll(dirPath, 0755); err != nil {
+			return err
+		}
+	}
+
+	for _, fileName := range files {
+		filePath := path.Join(root, fileName)
+		err := ioutil.WriteFile(filePath, []byte(fileName), 0644)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func setIgnoreContext(content string) error {
+	dockerIgnoreDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(dockerIgnoreDir)
+	err = ioutil.WriteFile(dockerIgnoreDir+".dockerignore", []byte(content), 0644)
+	if err != nil {
+		return err
+	}
+	err = util.GetExcludedFiles(dockerIgnoreDir, "")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func hashDirectory(dirpath string) (string, error) {
+	cache1 := NewCompositeCache()
+	err := cache1.AddPath(dirpath, dirpath)
+	if err != nil {
+		return "", err
+	}
+
+	hash, err := cache1.Hash()
+	if err != nil {
+		return "", err
+	}
+	return hash, nil
+}
+
+func Test_CompositeKey_AddPath_Works(t *testing.T) {
+	tests := []struct {
+		name        string
+		directories []string
+		files       []string
+	}{
+		{
+			name:        "empty",
+			directories: []string{},
+			files:       []string{},
+		},
+		{
+			name:        "dirs",
+			directories: []string{"foo", "bar", "foobar", "f/o/o"},
+			files:       []string{},
+		},
+		{
+			name:        "files",
+			directories: []string{},
+			files:       []string{"foo", "bar", "foobar"},
+		},
+		{
+			name:        "all",
+			directories: []string{"foo", "bar"},
+			files:       []string{"foo/bar", "bar/baz", "foobar"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testDir1, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir1)
+			err = createFilesystemStructure(testDir1, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			testDir2, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir2)
+			err = createFilesystemStructure(testDir2, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			hash1, err := hashDirectory(testDir1)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+			hash2, err := hashDirectory(testDir2)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+
+			if hash1 != hash2 {
+				t.Errorf("Expected equal hashes, got: %s and %s", hash1, hash2)
+			}
+		})
+	}
+}
+
+func Test_CompositeKey_AddPath_WithExtraFile_Works(t *testing.T) {
+	tests := []struct {
+		name        string
+		directories []string
+		files       []string
+		extraFile   string
+	}{
+		{
+			name:        "empty",
+			directories: []string{},
+			files:       []string{},
+			extraFile:   "file",
+		},
+		{
+			name:        "dirs",
+			directories: []string{"foo", "bar", "foobar", "f/o/o"},
+			files:       []string{},
+			extraFile:   "f/o/o/extra",
+		},
+		{
+			name:        "files",
+			directories: []string{},
+			files:       []string{"foo", "bar", "foobar"},
+			extraFile:   "foo.extra",
+		},
+		{
+			name:        "all",
+			directories: []string{"foo", "bar"},
+			files:       []string{"foo/bar", "bar/baz", "foobar"},
+			extraFile:   "bar/extra",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testDir1, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir1)
+			err = createFilesystemStructure(testDir1, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			testDir2, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir2)
+			err = createFilesystemStructure(testDir2, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+			extraPath := path.Join(testDir2, test.extraFile)
+			err = ioutil.WriteFile(extraPath, []byte(test.extraFile), 0644)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			hash1, err := hashDirectory(testDir1)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+			hash2, err := hashDirectory(testDir2)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+
+			if hash1 == hash2 {
+				t.Errorf("Expected different hashes, got: %s and %s", hash1, hash2)
+			}
+		})
+	}
+}
+
+func Test_CompositeKey_AddPath_WithExtraDir_Works(t *testing.T) {
+	tests := []struct {
+		name        string
+		directories []string
+		files       []string
+		extraDir    string
+	}{
+		{
+			name:        "empty",
+			directories: []string{},
+			files:       []string{},
+			extraDir:    "extra",
+		},
+		{
+			name:        "dirs",
+			directories: []string{"foo", "bar", "foobar", "f/o/o"},
+			files:       []string{},
+			extraDir:    "f/o/o/extra",
+		},
+		{
+			name:        "files",
+			directories: []string{},
+			files:       []string{"foo", "bar", "foobar"},
+			extraDir:    "foo.extra",
+		},
+		{
+			name:        "all",
+			directories: []string{"foo", "bar"},
+			files:       []string{"foo/bar", "bar/baz", "foobar"},
+			extraDir:    "bar/extra",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testDir1, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir1)
+			err = createFilesystemStructure(testDir1, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			testDir2, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir2)
+			err = createFilesystemStructure(testDir2, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+			extraPath := path.Join(testDir2, test.extraDir)
+			err = os.MkdirAll(extraPath, 0644)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			hash1, err := hashDirectory(testDir1)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+			hash2, err := hashDirectory(testDir2)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+
+			if hash1 == hash2 {
+				t.Errorf("Expected different hashes, got: %s and %s", hash1, hash2)
+			}
+		})
+	}
+}
+
+func Test_CompositeKey_AddPath_WithExtraFilIgnored_Works(t *testing.T) {
+	tests := []struct {
+		name        string
+		directories []string
+		files       []string
+		extraFile   string
+	}{
+		{
+			name:        "empty",
+			directories: []string{},
+			files:       []string{},
+			extraFile:   "extra",
+		},
+		{
+			name:        "dirs",
+			directories: []string{"foo", "bar", "foobar", "f/o/o"},
+			files:       []string{},
+			extraFile:   "f/o/o/extra",
+		},
+		{
+			name:        "files",
+			directories: []string{},
+			files:       []string{"foo", "bar", "foobar"},
+			extraFile:   "extra",
+		},
+		{
+			name:        "all",
+			directories: []string{"foo", "bar"},
+			files:       []string{"foo/bar", "bar/baz", "foobar"},
+			extraFile:   "bar/extra",
+		},
+	}
+
+	err := setIgnoreContext("**/extra")
+	if err != nil {
+		t.Fatalf("Error setting exlusion context: %s", err)
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testDir1, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir1)
+			err = createFilesystemStructure(testDir1, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			testDir2, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir2)
+			err = createFilesystemStructure(testDir2, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+			extraPath := path.Join(testDir2, test.extraFile)
+			err = ioutil.WriteFile(extraPath, []byte(test.extraFile), 0644)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			hash1, err := hashDirectory(testDir1)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+			hash2, err := hashDirectory(testDir2)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+
+			if hash1 != hash2 {
+				t.Errorf("Expected equal hashes, got: %s and %s", hash1, hash2)
+			}
+		})
+	}
+}
+
+func Test_CompositeKey_AddPath_WithExtraDirIgnored_Works(t *testing.T) {
+	tests := []struct {
+		name        string
+		directories []string
+		files       []string
+		extraDir    string
+	}{
+		{
+			name:        "empty",
+			directories: []string{},
+			files:       []string{},
+			extraDir:    "extra",
+		},
+		{
+			name:        "dirs",
+			directories: []string{"foo", "bar", "foobar", "f/o/o"},
+			files:       []string{},
+			extraDir:    "f/o/o/extra",
+		},
+		{
+			name:        "files",
+			directories: []string{},
+			files:       []string{"foo", "bar", "foobar"},
+			extraDir:    "extra",
+		},
+		{
+			name:        "all",
+			directories: []string{"foo", "bar"},
+			files:       []string{"foo/bar", "bar/baz", "foobar"},
+			extraDir:    "bar/extra",
+		},
+	}
+
+	err := setIgnoreContext("**/extra")
+	if err != nil {
+		t.Fatalf("Error setting exlusion context: %s", err)
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testDir1, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir1)
+			err = createFilesystemStructure(testDir1, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			testDir2, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			defer os.RemoveAll(testDir2)
+			err = createFilesystemStructure(testDir2, test.directories, test.files)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+			extraPath := path.Join(testDir2, test.extraDir)
+			err = os.MkdirAll(extraPath, 0644)
+			if err != nil {
+				t.Fatalf("Error creating filesytem structure: %s", err)
+			}
+
+			hash1, err := hashDirectory(testDir1)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+			hash2, err := hashDirectory(testDir2)
+			if err != nil {
+				t.Fatalf("Failed to calculate hash: %s", err)
+			}
+
+			if hash1 != hash2 {
+				t.Errorf("Expected equal hashes, got: %s and %s", hash1, hash2)
+			}
+		})
+	}
+}