public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1300 from tejal29/create_new_run 

add a new run command along with a new flag
This commit is contained in:
Tejal Desai 2020-06-13 09:24:15 -07:00 committed by GitHub
parent a8c317e43a d14b0b8f85
commit aeaea502e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 253 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/executor/cmd/root.go
View File

@ -174,6 +174,7 @@ func addKanikoOptionsFlags() {
RootCmd.PersistentFlags().BoolVarP(&opts.IgnoreVarRun, "whitelist-var-run", "", true, "Ignore /var/run directory when taking image snapshot. Set it to false to preserve /var/run/ in destination image. (Default true).")
RootCmd.PersistentFlags().VarP(&opts.Labels, "label", "", "Set metadata for an image. Set it repeatedly for multiple labels.")
RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile")
RootCmd.PersistentFlags().BoolVarP(&opts.RunV2, "use-new-run", "", false, "Experimental run command to detect file system changes. This new run command does no rely on snapshotting to detect changes.")
}
// addHiddenFlags marks certain flags as hidden from the executor help text

26
integration/dockerfiles/Dockerfile_test_run_new Normal file
View File

@ -0,0 +1,26 @@
# Copyright 2020 Google, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM debian:9.11
RUN echo "hey" > /etc/foo
RUN echo "baz" > /etc/baz
RUN cp /etc/baz /etc/bar
RUN rm /etc/baz
# Test with ARG
ARG file
RUN echo "run" > $file
RUN echo "test home" > $HOME/file
COPY context/foo $HOME/foo

2
integration/images.go
View File

@ -48,6 +48,7 @@ const (
// Arguments to build Dockerfiles with, used for both docker and kaniko builds
var argsMap = map[string][]string{
"Dockerfile_test_run": {"file=/file"},
"Dockerfile_test_run_new": {"file=/file"},
"Dockerfile_test_run_redo": {"file=/file"},
"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
"Dockerfile_test_add": {"file=context/foo"},
@ -75,6 +76,7 @@ var additionalDockerFlagsMap = map[string][]string{
// Arguments to build Dockerfiles with when building with kaniko
var additionalKanikoFlagsMap = map[string][]string{
"Dockerfile_test_add": {"--single-snapshot"},
"Dockerfile_test_run_new": {"--use-new-run=true"},
"Dockerfile_test_run_redo": {"--snapshotMode=redo"},
"Dockerfile_test_scratch": {"--single-snapshot"},
"Dockerfile_test_maintainer": {"--single-snapshot"},

4
pkg/commands/base_command.go
View File

@ -51,3 +51,7 @@ func (b *BaseCommand) RequiresUnpackedFS() bool {
func (b *BaseCommand) ShouldCacheOutput() bool {
return false
}
func (b *BaseCommand) ShouldDetectDeletedFiles() bool {
return false
}

8
pkg/commands/commands.go
View File

@ -52,11 +52,17 @@ type DockerCommand interface {
RequiresUnpackedFS() bool
ShouldCacheOutput() bool
// ShouldDetectDeletedFiles returns true if the command could delete files.
ShouldDetectDeletedFiles() bool
}
func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) {
func GetCommand(cmd instructions.Command, buildcontext string, useNewRun bool) (DockerCommand, error) {
switch c := cmd.(type) {
case *instructions.RunCommand:
if useNewRun {
return &RunMarkerCommand{cmd: c}, nil
}
return &RunCommand{cmd: c}, nil
case *instructions.CopyCommand:
return &CopyCommand{cmd: c, buildcontext: buildcontext}, nil

11
pkg/commands/run.go
View File

@ -46,8 +46,12 @@ var (
)
func (r *RunCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
return runCommandInExec(config, buildArgs, r.cmd)
}
func runCommandInExec(config *v1.Config, buildArgs *dockerfile.BuildArgs, cmdRun *instructions.RunCommand) error {
var newCommand []string
if r.cmd.PrependShell {
if cmdRun.PrependShell {
// This is the default shell on Linux
var shell []string
if len(config.Shell) > 0 {
@ -56,9 +60,9 @@ func (r *RunCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
shell = append(shell, "/bin/sh", "-c")
}
newCommand = append(shell, strings.Join(r.cmd.CmdLine, " "))
newCommand = append(shell, strings.Join(cmdRun.CmdLine, " "))
} else {
newCommand = r.cmd.CmdLine
newCommand = cmdRun.CmdLine
}
logrus.Infof("cmd: %s", newCommand[0])
@ -111,7 +115,6 @@ func (r *RunCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil && err.Error() != "no such process" {
return err
}
return nil
}

108
pkg/commands/run_marker.go Normal file
View File

@ -0,0 +1,108 @@
/*
Copyright 2018 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package commands
import (
"fmt"
"io/ioutil"
"os"
"time"
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/GoogleContainerTools/kaniko/pkg/util"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/sirupsen/logrus"
)
type RunMarkerCommand struct {
BaseCommand
cmd *instructions.RunCommand
Files []string
}
func (r *RunMarkerCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
// run command `touch filemarker`
logrus.Debugf("using new RunMarker command")
markerFile, err := ioutil.TempFile("", "marker")
if err != nil {
return fmt.Errorf("could not place a marker file")
}
defer func() {
os.Remove(markerFile.Name())
}()
markerInfo, err := os.Stat(markerFile.Name())
if err != nil {
return fmt.Errorf("could not place a marker file")
}
// introduce a delay
time.Sleep(time.Second)
if err := runCommandInExec(config, buildArgs, r.cmd); err != nil {
return err
}
// run command find to find all new files generate
isNewer := func(p string) (bool, error) {
fi, err := os.Stat(p)
if err != nil {
return false, err
}
return fi.ModTime().After(markerInfo.ModTime()), nil
}
r.Files, _ = util.WalkFS("/", map[string]struct{}{}, isNewer)
logrus.Debugf("files changed %s", r.Files)
return nil
}
// String returns some information about the command for the image config
func (r *RunMarkerCommand) String() string {
return r.cmd.String()
}
func (r *RunMarkerCommand) FilesToSnapshot() []string {
return r.Files
}
func (r *RunMarkerCommand) ProvidesFilesToSnapshot() bool {
return true
}
// CacheCommand returns true since this command should be cached
func (r *RunMarkerCommand) CacheCommand(img v1.Image) DockerCommand {
return &CachingRunCommand{
img: img,
cmd: r.cmd,
extractFn: util.ExtractFile,
}
}
func (r *RunMarkerCommand) MetadataOnly() bool {
return false
}
func (r *RunMarkerCommand) RequiresUnpackedFS() bool {
return true
}
func (r *RunMarkerCommand) ShouldCacheOutput() bool {
return true
}
func (r *RunMarkerCommand) ShouldDetectDeletedFiles() bool {
return true
}

1
pkg/config/options.go
View File

@ -57,6 +57,7 @@ type KanikoOptions struct {
Cleanup bool
IgnoreVarRun bool
SkipUnusedStages bool
RunV2 bool
}
// WarmerOptions are options that are set by command line arguments to the cache warmer.

29
pkg/executor/build.go
View File

@ -62,7 +62,7 @@ type cachePusher func(*config.KanikoOptions, string, string, string) error
type snapShotter interface {
Init() error
TakeSnapshotFS() (string, error)
TakeSnapshot([]string) (string, error)
TakeSnapshot([]string, bool) (string, error)
}
// stageBuilder contains all fields necessary to build one stage of a Dockerfile
@ -127,7 +127,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross
}
for _, cmd := range s.stage.Commands {
command, err := commands.GetCommand(cmd, opts.SrcContext)
command, err := commands.GetCommand(cmd, opts.SrcContext, opts.RunV2)
if err != nil {
return nil, err
}
@ -319,7 +319,7 @@ func (s *stageBuilder) build() error {
}
initSnapshotTaken := false
if s.opts.SingleSnapshot {
if s.opts.SingleSnapshot || s.opts.RunV2 {
if err := s.initSnapshotWithTimings(); err != nil {
return err
}
@ -372,7 +372,7 @@ func (s *stageBuilder) build() error {
files = command.FilesToSnapshot()
timing.DefaultRun.Stop(t)
if !s.shouldTakeSnapshot(index, files, command.ProvidesFilesToSnapshot()) {
if !s.shouldTakeSnapshot(index, command.MetadataOnly()) {
continue
}
if isCacheCommand {
@ -382,7 +382,7 @@ func (s *stageBuilder) build() error {
return errors.Wrap(err, "failed to save layer")
}
} else {
tarPath, err := s.takeSnapshot(files)
tarPath, err := s.takeSnapshot(files, command.ShouldDetectDeletedFiles())
if err != nil {
return errors.Wrap(err, "failed to take snapshot")
}
@ -416,7 +416,7 @@ func (s *stageBuilder) build() error {
return nil
}
func (s *stageBuilder) takeSnapshot(files []string) (string, error) {
func (s *stageBuilder) takeSnapshot(files []string, shdDelete bool) (string, error) {
var snapshot string
var err error
@ -426,13 +426,13 @@ func (s *stageBuilder) takeSnapshot(files []string) (string, error) {
} else {
// Volumes are very weird. They get snapshotted in the next command.
files = append(files, util.Volumes()...)
snapshot, err = s.snapshotter.TakeSnapshot(files)
snapshot, err = s.snapshotter.TakeSnapshot(files, shdDelete)
}
timing.DefaultRun.Stop(t)
return snapshot, err
}
func (s *stageBuilder) shouldTakeSnapshot(index int, files []string, provideFiles bool) bool {
func (s *stageBuilder) shouldTakeSnapshot(index int, isMetadatCmd bool) bool {
isLastCommand := index == len(s.cmds)-1
// We only snapshot the very end with single snapshot mode on.
@ -445,17 +445,8 @@ func (s *stageBuilder) shouldTakeSnapshot(index int, files []string, provideFile
return true
}
// if command does not provide files, snapshot everything.
if !provideFiles {
return true
}
// Don't snapshot an empty list.
if len(files) == 0 {
return false
}
return true
// if command is a metadata command, do not snapshot.
return !isMetadatCmd
}
func (s *stageBuilder) saveSnapshotToImage(createdBy string, tarPath string) error {

18
pkg/executor/build_test.go
View File

@ -103,9 +103,8 @@ func Test_stageBuilder_shouldTakeSnapshot(t *testing.T) {
cmds []commands.DockerCommand
}
type args struct {
index int
files []string
hasFiles bool
index int
metadataOnly bool
}
tests := []struct {
name string
@ -158,9 +157,8 @@ func Test_stageBuilder_shouldTakeSnapshot(t *testing.T) {
stage: config.KanikoStage{},
},
args: args{
index: 0,
files: []string{},
hasFiles: true,
index: 0,
metadataOnly: true,
},
want: false,
},
@ -172,9 +170,8 @@ func Test_stageBuilder_shouldTakeSnapshot(t *testing.T) {
},
},
args: args{
index: 0,
files: nil,
hasFiles: false,
index: 0,
metadataOnly: false,
},
want: true,
},
@ -204,7 +201,7 @@ func Test_stageBuilder_shouldTakeSnapshot(t *testing.T) {
opts: tt.fields.opts,
cmds: tt.fields.cmds,
}
if got := s.shouldTakeSnapshot(tt.args.index, tt.args.files, tt.args.hasFiles); got != tt.want {
if got := s.shouldTakeSnapshot(tt.args.index, tt.args.metadataOnly); got != tt.want {
t.Errorf("stageBuilder.shouldTakeSnapshot() = %v, want %v", got, tt.want)
}
})
@ -1246,6 +1243,7 @@ func getCommands(dir string, cmds []instructions.Command) []commands.DockerComma
cmd, err := commands.GetCommand(
c,
dir,
false,
)
if err != nil {
panic(err)

8
pkg/executor/fakes.go
View File

@ -38,7 +38,7 @@ func (f fakeSnapShotter) Init() error { return nil }
func (f fakeSnapShotter) TakeSnapshotFS() (string, error) {
return f.tarPath, nil
}
func (f fakeSnapShotter) TakeSnapshot(_ []string) (string, error) {
func (f fakeSnapShotter) TakeSnapshot(_ []string, _ bool) (string, error) {
return f.tarPath, nil
}
@ -73,6 +73,9 @@ func (m MockDockerCommand) RequiresUnpackedFS() bool {
func (m MockDockerCommand) ShouldCacheOutput() bool {
return true
}
func (m MockDockerCommand) ShouldDetectDeletedFiles() bool {
return false
}
type MockCachedDockerCommand struct {
contextFiles []string
@ -93,6 +96,9 @@ func (m MockCachedDockerCommand) ProvidesFilesToSnapshot() bool {
func (m MockCachedDockerCommand) CacheCommand(image v1.Image) commands.DockerCommand {
return nil
}
func (m MockCachedDockerCommand) ShouldDetectDeletedFiles() bool {
return false
}
func (m MockCachedDockerCommand) FilesUsedFromContext(c *v1.Config, args *dockerfile.BuildArgs) ([]string, error) {
return m.contextFiles, nil
}

18
pkg/snapshot/layered_map.go
View File

@ -31,7 +31,7 @@ import (
type LayeredMap struct {
layers []map[string]string
whiteouts []map[string]string
whiteouts []map[string]struct{}
layerHashCache map[string]string
hasher func(string) (string, error)
// cacheHasher doesn't include mtime in it's hash so that filesystem cache keys are stable
@ -49,7 +49,7 @@ func NewLayeredMap(h func(string) (string, error), c func(string) (string, error
}
func (l *LayeredMap) Snapshot() {
l.whiteouts = append(l.whiteouts, map[string]string{})
l.whiteouts = append(l.whiteouts, map[string]struct{}{})
l.layers = append(l.layers, map[string]string{})
}
@ -84,21 +84,21 @@ func (l *LayeredMap) Get(s string) (string, bool) {
return "", false
}
func (l *LayeredMap) GetWhiteout(s string) (string, bool) {
func (l *LayeredMap) GetWhiteout(s string) bool {
for i := len(l.whiteouts) - 1; i >= 0; i-- {
if v, ok := l.whiteouts[i][s]; ok {
return v, ok
if _, ok := l.whiteouts[i][s]; ok {
return ok
}
}
return "", false
return false
}
func (l *LayeredMap) MaybeAddWhiteout(s string) bool {
whiteout, ok := l.GetWhiteout(s)
if ok && whiteout == s {
ok := l.GetWhiteout(s)
if ok {
return false
}
l.whiteouts[len(l.whiteouts)-1][s] = s
l.whiteouts[len(l.whiteouts)-1][s] = struct{}{}
return true
}

66
pkg/snapshot/snapshot.go
View File

@ -24,13 +24,11 @@ import (
"sort"
"syscall"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/filesystem"
"github.com/GoogleContainerTools/kaniko/pkg/timing"
"github.com/karrick/godirwalk"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/sirupsen/logrus"
)
@ -62,7 +60,7 @@ func (s *Snapshotter) Key() (string, error) {
// TakeSnapshot takes a snapshot of the specified files, avoiding directories in the ignorelist, and creates
// a tarball of the changed files. Return contents of the tarball, and whether or not any files were changed
func (s *Snapshotter) TakeSnapshot(files []string) (string, error) {
func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string, error) {
f, err := ioutil.TempFile(config.KanikoDir, "")
if err != nil {
return "", err
@ -81,7 +79,7 @@ func (s *Snapshotter) TakeSnapshot(files []string) (string, error) {
}
logrus.Info("Taking snapshot of files...")
logrus.Debugf("Taking snapshot of files %v", files)
logrus.Debugf("Taking snapshot of files %v", filesToAdd)
sort.Strings(filesToAdd)
@ -92,9 +90,27 @@ func (s *Snapshotter) TakeSnapshot(files []string) (string, error) {
}
}
// Get whiteout paths
filesToWhiteout := []string{}
if shdCheckDelete {
_, deletedFiles := util.WalkFS(s.directory, s.l.getFlattenedPathsForWhiteOut(), func(s string) (bool, error) {
return true, nil
})
// The paths left here are the ones that have been deleted in this layer.
for path := range deletedFiles {
// Only add the whiteout if the directory for the file still exists.
dir := filepath.Dir(path)
if _, ok := deletedFiles[dir]; !ok {
if s.l.MaybeAddWhiteout(path) {
logrus.Debugf("Adding whiteout for %s", path)
filesToWhiteout = append(filesToWhiteout, path)
}
}
}
}
t := util.NewTar(f)
defer t.Close()
if err := writeToTar(t, filesToAdd, nil); err != nil {
if err := writeToTar(t, filesToAdd, filesToWhiteout); err != nil {
return "", err
}
return f.Name(), nil
@ -133,36 +149,8 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
s.l.Snapshot()
timer := timing.Start("Walking filesystem")
changedPaths := make([]string, 0)
// Get a list of all the files that existed before this layer
existingPaths := s.l.getFlattenedPathsForWhiteOut()
godirwalk.Walk(s.directory, &godirwalk.Options{
Callback: func(path string, ent *godirwalk.Dirent) error {
if util.IsInIgnoreList(path) {
if util.IsDestDir(path) {
logrus.Tracef("Skipping paths under %s, as it is a ignored directory", path)
return filepath.SkipDir
}
return nil
}
if ok, err := s.l.CheckFileChange(path); err != nil {
return err
} else if ok {
changedPaths = append(changedPaths, path)
}
delete(existingPaths, path)
return nil
},
Unsorted: true,
},
)
timing.DefaultRun.Stop(timer)
timer = timing.Start("Resolving Paths")
changedPaths, deletedPaths := util.WalkFS(s.directory, s.l.getFlattenedPathsForWhiteOut(), s.l.CheckFileChange)
timer := timing.Start("Resolving Paths")
filesToAdd := []string{}
resolvedFiles, err := filesystem.ResolvePaths(changedPaths, s.ignorelist)
@ -179,10 +167,10 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
// The paths left here are the ones that have been deleted in this layer.
filesToWhiteOut := []string{}
for path := range existingPaths {
for path := range deletedPaths {
// Only add the whiteout if the directory for the file still exists.
dir := filepath.Dir(path)
if _, ok := existingPaths[dir]; !ok {
if _, ok := deletedPaths[dir]; !ok {
if s.l.MaybeAddWhiteout(path) {
logrus.Debugf("Adding whiteout for %s", path)
filesToWhiteOut = append(filesToWhiteOut, path)

4
pkg/snapshot/snapshot_test.go
View File

@ -214,7 +214,7 @@ func TestSnapshotFiles(t *testing.T) {
filesToSnapshot := []string{
filepath.Join(testDir, "foo"),
}
tarPath, err := snapshotter.TakeSnapshot(filesToSnapshot)
tarPath, err := snapshotter.TakeSnapshot(filesToSnapshot, false)
if err != nil {
t.Fatal(err)
}
@ -361,7 +361,7 @@ func TestSnasphotPreservesFileOrder(t *testing.T) {
}
// Take a snapshot
tarPath, err := snapshotter.TakeSnapshot(filesToSnapshot)
tarPath, err := snapshotter.TakeSnapshot(filesToSnapshot, false)
if err != nil {
t.Fatalf("Error taking snapshot of fs: %s", err)

36
pkg/util/fs_util.go
View File

@ -31,14 +31,16 @@ import (
"syscall"
"time"
otiai10Cpy "github.com/otiai10/copy"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/docker/docker/builder/dockerignore"
"github.com/docker/docker/pkg/fileutils"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/karrick/godirwalk"
otiai10Cpy "github.com/otiai10/copy"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/timing"
)
const DoNotChangeUID = -1
@ -875,3 +877,31 @@ func UpdateInitialIgnoreList(ignoreVarRun bool) {
PrefixMatchOnly: false,
})
}
func WalkFS(dir string, existingPaths map[string]struct{}, f func(string) (bool, error)) ([]string, map[string]struct{}) {
foundPaths := make([]string, 0)
timer := timing.Start("Walking filesystem")
godirwalk.Walk(dir, &godirwalk.Options{
Callback: func(path string, ent *godirwalk.Dirent) error {
if IsInIgnoreList(path) {
if IsDestDir(path) {
logrus.Tracef("Skipping paths under %s, as it is a ignored directory", path)
return filepath.SkipDir
}
return nil
}
delete(existingPaths, path)
if t, err := f(path); err != nil {
return err
} else if t {
foundPaths = append(foundPaths, path)
}
return nil
},
Unsorted: true,
},
)
timing.DefaultRun.Stop(timer)
return foundPaths, existingPaths
}

2
scripts/misc-integration-test.sh
View File

@ -19,7 +19,7 @@ set -e
TESTS=$(./scripts/integration-test.sh -list=Test -mod=vendor)
TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers' | grep -v 'TestK8s')
TESTS=$(echo $TESTS | tr ' ' '\n' | grep 'Test'| grep -v 'TestRun' | grep -v 'TestLayers' | grep -v 'TestK8s' | grep -v 'TestSnapshotBenchmark')
RUN_ARG=''
count=0