public_git/kaniko
1
0
Fork 0
mirror of https://github.com/GoogleContainerTools/kaniko synced 2026-10-18 03:08:27 +02:00
Code Issues Releases Wiki Activity

Fix .dockerignore for build context copies in later stages (#1447)

Browse Source 
* Extend .dockerignore integration test with copies in later stages

.dockerignore should continue to apply when copying from the build context in later stages, but it currently doesn't

* Replace excluded global with passed along FileContext struct

This new FileContext struct allows much cleaner handling of context specific file exclusions.
The global excluded file state is no longer needed.

Additionally this also fixes the issue where excluded files aren't being applied for build context copies in later build stages.
This commit is contained in:
Tinjo Schöni
2020-10-08 12:47:14 -07:00
committed by GitHub
parent 0ef3a6b525
commit 5f4e2f1366
14 changed files with 150 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/executor/build.go
+8 -9
View File
@@ -73,6 +73,7 @@ type stageBuilder struct {
baseImageDigest string
finalCacheKey string
opts *config.KanikoOptions
fileContext util.FileContext
cmds []commands.DockerCommand
args *dockerfile.BuildArgs
crossStageDeps map[int][]string
@@ -84,7 +85,7 @@ type stageBuilder struct {
}
// newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage
func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string, dcm map[string]string, sid map[string]string, stageNameToIdx map[string]string) (*stageBuilder, error) {
func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string, dcm map[string]string, sid map[string]string, stageNameToIdx map[string]string, fileContext util.FileContext) (*stageBuilder, error) {
sourceImage, err := image_util.RetrieveSourceImage(stage, opts)
if err != nil {
return nil, err
@@ -117,6 +118,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross
snapshotter: snapshotter,
baseImageDigest: digest.String(),
opts: opts,
fileContext: fileContext,
crossStageDeps: crossStageDeps,
digestToCacheKey: dcm,
stageIdxToDigest: sid,
@@ -127,7 +129,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross
}
for _, cmd := range s.stage.Commands {
command, err := commands.GetCommand(cmd, opts.SrcContext, opts.RunV2)
command, err := commands.GetCommand(cmd, fileContext, opts.RunV2)
if err != nil {
return nil, err
}
@@ -187,10 +189,8 @@ func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string
compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey)
}
srcCtx := s.opts.SrcContext
for _, f := range files {
if err := compositeKey.AddPath(f, srcCtx); err != nil {
if err := compositeKey.AddPath(f, s.fileContext); err != nil {
return compositeKey, err
}
}
@@ -572,7 +572,8 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
}
stageNameToIdx := ResolveCrossStageInstructions(kanikoStages)
if err := util.GetExcludedFiles(opts.DockerfilePath, opts.SrcContext); err != nil {
fileContext, err := util.NewFileContextFromDockerfile(opts.DockerfilePath, opts.SrcContext)
if err != nil {
return nil, err
}
@@ -586,16 +587,14 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
}
logrus.Infof("Built cross stage deps: %v", crossStageDependencies)
util.IsFirstStage = true
for index, stage := range kanikoStages {
sb, err := newStageBuilder(opts, stage, crossStageDependencies, digestToCacheKey, stageIdxToDigest, stageNameToIdx)
sb, err := newStageBuilder(opts, stage, crossStageDependencies, digestToCacheKey, stageIdxToDigest, stageNameToIdx, fileContext)
if err != nil {
return nil, err
}
if err := sb.build(); err != nil {
return nil, errors.Wrap(err, "error building stage")
}
util.IsFirstStage = false
reviewConfig(stage, &sb.cf.Config)
pkg/executor/build_test.go
+9 -8
View File
@@ -31,6 +31,7 @@ import (
"github.com/GoogleContainerTools/kaniko/pkg/commands"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/google/go-cmp/cmp"
v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -679,7 +680,7 @@ func Test_stageBuilder_populateCompositeKey(t *testing.T) {
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
sb := &stageBuilder{opts: &config.KanikoOptions{SrcContext: "workspace"}}
sb := &stageBuilder{fileContext: util.FileContext{Root: "workspace"}}
ck := CompositeCache{}
ck1, err := sb.populateCompositeKey(tc.cmd1.command, []string{}, ck, tc.cmd1.args, tc.cmd1.env)
@@ -720,7 +721,7 @@ func Test_stageBuilder_build(t *testing.T) {
filePath := filepath.Join(dir, file)
ch := NewCompositeCache("", "meow")
ch.AddPath(filePath, "")
ch.AddPath(filePath, util.FileContext{})
hash, err := ch.Hash()
if err != nil {
t.Errorf("couldn't create hash %v", err)
@@ -753,7 +754,7 @@ func Test_stageBuilder_build(t *testing.T) {
filePath := filepath.Join(dir, file)
ch := NewCompositeCache("", "meow")
ch.AddPath(filePath, "")
ch.AddPath(filePath, util.FileContext{})
hash, err := ch.Hash()
if err != nil {
t.Errorf("couldn't create hash %v", err)
@@ -856,7 +857,7 @@ COPY %s bar.txt
// hash1 is the read cachekey for the first layer
expectedCacheKeys: []string{hash1},
pushedCacheKeys: []string{},
commands: getCommands(dir, cmds),
commands: getCommands(util.FileContext{Root: dir}, cmds),
}
}(),
func() testcase {
@@ -872,7 +873,7 @@ COPY %s bar.txt
filePath := filepath.Join(dir, filename)
ch := NewCompositeCache("", fmt.Sprintf("COPY %s bar.txt", filename))
ch.AddPath(filePath, "")
ch.AddPath(filePath, util.FileContext{})
// copy hash
_, err = ch.Hash()
@@ -932,7 +933,7 @@ RUN foobar
image: image,
expectedCacheKeys: []string{runHash},
pushedCacheKeys: []string{},
commands: getCommands(dir, cmds),
commands: getCommands(util.FileContext{Root: dir}, cmds),
}
}(),
func() testcase {
@@ -1139,12 +1140,12 @@ func assertCacheKeys(t *testing.T, expectedCacheKeys, actualCacheKeys []string,
}
}
func getCommands(dir string, cmds []instructions.Command) []commands.DockerCommand {
func getCommands(fileContext util.FileContext, cmds []instructions.Command) []commands.DockerCommand {
outCommands := make([]commands.DockerCommand, 0)
for _, c := range cmds {
cmd, err := commands.GetCommand(
c,
dir,
fileContext,
false,
)
if err != nil {
pkg/executor/composite_cache.go
+5 -5
View File
@@ -55,7 +55,7 @@ func (s *CompositeCache) Hash() (string, error) {
return util.SHA256(strings.NewReader(s.Key()))
}
func (s *CompositeCache) AddPath(p, context string) error {
func (s *CompositeCache) AddPath(p string, context util.FileContext) error {
sha := sha256.New()
fi, err := os.Lstat(p)
if err != nil {
@@ -70,13 +70,13 @@ func (s *CompositeCache) AddPath(p, context string) error {
// Only add the hash of this directory to the key
// if there is any ignored content.
if !empty || !util.ExcludeFile(p, context) {
if !empty || !context.ExcludesFile(p) {
s.keys = append(s.keys, k)
}
return nil
}
if util.ExcludeFile(p, context) {
if context.ExcludesFile(p) {
return nil
}
fh, err := util.CacheHasher()(p)
@@ -92,14 +92,14 @@ func (s *CompositeCache) AddPath(p, context string) error {
}
// HashDir returns a hash of the directory.
func hashDir(p, context string) (bool, string, error) {
func hashDir(p string, context util.FileContext) (bool, string, error) {
sha := sha256.New()
empty := true
if err := filepath.Walk(p, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
exclude := util.ExcludeFile(path, context)
exclude := context.ExcludesFile(path)
if exclude {
return nil
}
pkg/executor/composite_cache_test.go
+27 -24
View File
@@ -80,7 +80,7 @@ func Test_CompositeCache_AddPath_dir(t *testing.T) {
fn := func() string {
r := NewCompositeCache()
if err := r.AddPath(tmpDir, ""); err != nil {
if err := r.AddPath(tmpDir, util.FileContext{}); err != nil {
t.Errorf("expected error to be nil but was %v", err)
}
@@ -118,7 +118,7 @@ func Test_CompositeCache_AddPath_file(t *testing.T) {
p := tmpfile.Name()
fn := func() string {
r := NewCompositeCache()
if err := r.AddPath(p, ""); err != nil {
if err := r.AddPath(p, util.FileContext{}); err != nil {
t.Errorf("expected error to be nil but was %v", err)
}
@@ -158,26 +158,23 @@ func createFilesystemStructure(root string, directories, files []string) error {
return nil
}
func setIgnoreContext(content string) error {
func setIgnoreContext(content string) (util.FileContext, error) {
var fileContext util.FileContext
dockerIgnoreDir, err := ioutil.TempDir("", "")
if err != nil {
return err
return fileContext, err
}
defer os.RemoveAll(dockerIgnoreDir)
err = ioutil.WriteFile(dockerIgnoreDir+".dockerignore", []byte(content), 0644)
if err != nil {
return err
return fileContext, err
}
err = util.GetExcludedFiles(dockerIgnoreDir, "")
if err != nil {
return err
}
return nil
return util.NewFileContextFromDockerfile(dockerIgnoreDir, "")
}
func hashDirectory(dirpath string) (string, error) {
func hashDirectory(dirpath string, fileContext util.FileContext) (string, error) {
cache1 := NewCompositeCache()
err := cache1.AddPath(dirpath, dirpath)
err := cache1.AddPath(dirpath, fileContext)
if err != nil {
return "", err
}
@@ -217,6 +214,8 @@ func Test_CompositeKey_AddPath_Works(t *testing.T) {
},
}
fileContext := util.FileContext{}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testDir1, err := ioutil.TempDir("", "")
@@ -239,11 +238,11 @@ func Test_CompositeKey_AddPath_Works(t *testing.T) {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
hash1, err := hashDirectory(testDir1, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
hash2, err := hashDirectory(testDir2, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
@@ -288,6 +287,8 @@ func Test_CompositeKey_AddPath_WithExtraFile_Works(t *testing.T) {
},
}
fileContext := util.FileContext{}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testDir1, err := ioutil.TempDir("", "")
@@ -315,11 +316,11 @@ func Test_CompositeKey_AddPath_WithExtraFile_Works(t *testing.T) {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
hash1, err := hashDirectory(testDir1, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
hash2, err := hashDirectory(testDir2, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
@@ -364,6 +365,8 @@ func Test_CompositeKey_AddPath_WithExtraDir_Works(t *testing.T) {
},
}
fileContext := util.FileContext{}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testDir1, err := ioutil.TempDir("", "")
@@ -391,11 +394,11 @@ func Test_CompositeKey_AddPath_WithExtraDir_Works(t *testing.T) {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
hash1, err := hashDirectory(testDir1, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
hash2, err := hashDirectory(testDir2, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
@@ -440,7 +443,7 @@ func Test_CompositeKey_AddPath_WithExtraFilIgnored_Works(t *testing.T) {
},
}
err := setIgnoreContext("**/extra")
fileContext, err := setIgnoreContext("**/extra")
if err != nil {
t.Fatalf("Error setting exlusion context: %s", err)
}
@@ -472,11 +475,11 @@ func Test_CompositeKey_AddPath_WithExtraFilIgnored_Works(t *testing.T) {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
hash1, err := hashDirectory(testDir1, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
hash2, err := hashDirectory(testDir2, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
@@ -521,7 +524,7 @@ func Test_CompositeKey_AddPath_WithExtraDirIgnored_Works(t *testing.T) {
},
}
err := setIgnoreContext("**/extra")
fileContext, err := setIgnoreContext("**/extra")
if err != nil {
t.Fatalf("Error setting exlusion context: %s", err)
}
@@ -553,11 +556,11 @@ func Test_CompositeKey_AddPath_WithExtraDirIgnored_Works(t *testing.T) {
t.Fatalf("Error creating filesytem structure: %s", err)
}
hash1, err := hashDirectory(testDir1)
hash1, err := hashDirectory(testDir1, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}
hash2, err := hashDirectory(testDir2)
hash2, err := hashDirectory(testDir2, fileContext)
if err != nil {
t.Fatalf("Failed to calculate hash: %s", err)
}