public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

Use only the necessary files in the cache keys. (#387)

This commit is contained in:
dlorenc 2018-10-15 08:56:34 -05:00 committed by GitHub
parent 72e088fda5
commit 5ac29a9773
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 114 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
pkg/commands/add.go
View File

@ -19,12 +19,13 @@ package commands
import (
"path/filepath"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/google/go-containerregistry/pkg/v1"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/sirupsen/logrus"
)
@ -44,18 +45,13 @@ type AddCommand struct {
// 2. If <src> is a local tar archive:
// -If <src> is a local tar archive, it is unpacked at the dest, as 'tar -x' would
func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
// First, resolve any environment replacement
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
resolvedEnvs, err := util.ResolveEnvironmentReplacementList(a.cmd.SourcesAndDest, replacementEnvs, true)
if err != nil {
return err
}
dest := resolvedEnvs[len(resolvedEnvs)-1]
// Resolve wildcards and get a list of resolved sources
srcs, err := util.ResolveSources(resolvedEnvs, a.buildcontext)
srcs, dest, err := resolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
if err != nil {
return err
}
var unresolvedSrcs []string
// If any of the sources are local tar archives:
// 1. Unpack them to the specified destination
@ -94,6 +90,7 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
},
buildcontext: a.buildcontext,
}
if err := copyCmd.ExecuteCommand(config, buildArgs); err != nil {
return err
}
@ -111,6 +108,26 @@ func (a *AddCommand) String() string {
return a.cmd.String()
}
func (a *AddCommand) UsesContext() bool {
return true
func (a *AddCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
srcs, _, err := resolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
if err != nil {
return nil, err
}
files := []string{}
for _, src := range srcs {
if util.IsSrcRemoteFileURL(src) {
continue
}
if util.IsFileLocalTarArchive(src) {
continue
}
fullPath := filepath.Join(a.buildcontext, src)
files = append(files, fullPath)
}
logrus.Infof("Using files from context: %v", files)
return files, nil
}

16
pkg/commands/base_command.go
View File

@ -16,19 +16,23 @@ limitations under the License.
package commands
import (
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/google/go-containerregistry/pkg/v1"
)
type BaseCommand struct {
cache bool
usesContext bool
cache bool
}
func (b *BaseCommand) CacheCommand() bool {
return b.cache
}
func (b *BaseCommand) UsesContext() bool {
return b.usesContext
}
func (b *BaseCommand) FilesToSnapshot() []string {
return []string{}
}
func (b *BaseCommand) FilesUsedFromContext(_ *v1.Config, _ *dockerfile.BuildArgs) ([]string, error) {
return []string{}, nil
}

2
pkg/commands/commands.go
View File

@ -39,7 +39,7 @@ type DockerCommand interface {
CacheCommand() bool
// Return true if this command depends on the build context.
UsesContext() bool
FilesUsedFromContext(*v1.Config, *dockerfile.BuildArgs) ([]string, error)
}
func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) {

49
pkg/commands/copy.go
View File

@ -20,12 +20,14 @@ import (
"os"
"path/filepath"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/sirupsen/logrus"
"github.com/GoogleContainerTools/kaniko/pkg/constants"
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/google/go-containerregistry/pkg/v1"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
)
type CopyCommand struct {
@ -40,18 +42,14 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
if c.cmd.From != "" {
c.buildcontext = filepath.Join(constants.KanikoDir, c.cmd.From)
}
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
// First, resolve any environment replacement
resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.cmd.SourcesAndDest, replacementEnvs, true)
if err != nil {
return err
}
dest := resolvedEnvs[len(resolvedEnvs)-1]
// Resolve wildcards and get a list of resolved sources
srcs, err := util.ResolveSources(resolvedEnvs, c.buildcontext)
srcs, dest, err := resolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
if err != nil {
return err
}
// For each source, iterate through and copy it over
for _, src := range srcs {
fullPath := filepath.Join(c.buildcontext, src)
@ -94,6 +92,18 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
return nil
}
func resolveEnvAndWildcards(sd instructions.SourcesAndDest, buildcontext string, envs []string) ([]string, string, error) {
// First, resolve any environment replacement
resolvedEnvs, err := util.ResolveEnvironmentReplacementList(sd, envs, true)
if err != nil {
return nil, "", err
}
dest := resolvedEnvs[len(resolvedEnvs)-1]
// Resolve wildcards and get a list of resolved sources
srcs, err := util.ResolveSources(resolvedEnvs, buildcontext)
return srcs, dest, err
}
// FilesToSnapshot should return an empty array if still nil; no files were changed
func (c *CopyCommand) FilesToSnapshot() []string {
return c.snapshotFiles
@ -104,6 +114,23 @@ func (c *CopyCommand) String() string {
return c.cmd.String()
}
func (c *CopyCommand) UsesContext() bool {
return true
func (c *CopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
// We don't use the context if we're performing a copy --from.
if c.cmd.From != "" {
return nil, nil
}
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
srcs, _, err := resolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
if err != nil {
return nil, err
}
files := []string{}
for _, src := range srcs {
fullPath := filepath.Join(c.buildcontext, src)
files = append(files, fullPath)
}
logrus.Infof("Using files from context: %v", files)
return files, nil
}

18
pkg/executor/build.go
View File

@ -125,10 +125,6 @@ func (s *stageBuilder) build() error {
// Set the initial cache key to be the base image digest, the build args and the SrcContext.
compositeKey := NewCompositeCache(s.baseImageDigest)
contextHash, err := HashDir(s.opts.SrcContext)
if err != nil {
return err
}
compositeKey.AddKey(s.opts.BuildArgs...)
cmds := []commands.DockerCommand{}
@ -148,8 +144,16 @@ func (s *stageBuilder) build() error {
// Add the next command to the cache key.
compositeKey.AddKey(command.String())
if command.UsesContext() {
compositeKey.AddKey(contextHash)
// If the command uses files from the context, add them.
files, err := command.FilesUsedFromContext(&s.cf.Config, args)
if err != nil {
return err
}
for _, f := range files {
if err := compositeKey.AddPath(f); err != nil {
return err
}
}
logrus.Info(command.String())
@ -172,7 +176,7 @@ func (s *stageBuilder) build() error {
if err := command.ExecuteCommand(&s.cf.Config, args); err != nil {
return err
}
files := command.FilesToSnapshot()
files = command.FilesToSnapshot()
var contents []byte
if !s.shouldTakeSnapshot(index, files) {

26
pkg/executor/composite_cache.go
View File

@ -53,6 +53,32 @@ func (s *CompositeCache) Hash() (string, error) {
return util.SHA256(strings.NewReader(s.Key()))
}
func (s *CompositeCache) AddPath(p string) error {
sha := sha256.New()
fi, err := os.Lstat(p)
if err != nil {
return err
}
if fi.Mode().IsDir() {
k, err := HashDir(p)
if err != nil {
return err
}
s.keys = append(s.keys, k)
return nil
}
fh, err := util.CacheHasher()(p)
if err != nil {
return err
}
if _, err := sha.Write([]byte(fh)); err != nil {
return err
}
s.keys = append(s.keys, string(sha.Sum(nil)))
return nil
}
// HashDir returns a hash of the directory.
func HashDir(p string) (string, error) {
sha := sha256.New()