public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

better improvements

This commit is contained in:
Tejal Desai 2020-06-05 23:44:17 -07:00
parent 43338d4b2f
commit 58276274e8
10 changed files with 103 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -88,9 +88,9 @@ integration-test-misc:
.PHONY: images
images:
docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:latest -f deploy/Dockerfile .
docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug .
docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/warmer:latest -f deploy/Dockerfile_warmer .
docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:perf-latest -f deploy/Dockerfile .
#docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug .
#docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/warmer:latest -f deploy/Dockerfile_warmer .
.PHONY: push
push:

4
integration/benchmark_fs/cloudbuild.yaml
View File

@ -1,9 +1,9 @@
steps:
- name: 'gcr.io/kaniko-project/executor:latest'
- name: 'gcr.io/kaniko-project/executor:perf-latest'
args:
- --build-arg=NUM=${_COUNT}
- --no-push
- --snapshotMode=redo
- --use-new-run=true
env:
- 'BENCHMARK_FILE=gs://$PROJECT_ID/gcb/benchmark_file_${_COUNT}'
timeout: 2400s

2
integration/benchmark_test.go
View File

@ -125,7 +125,7 @@ func TestSnapshotBenchmarkGcloud(t *testing.T) {
}
contextDir := filepath.Join(cwd, "benchmark_fs")
nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000}
nums := []int{10000} //, 50000, 100000, 200000, 300000, 500000, 700000}
var wg sync.WaitGroup
fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")

8
integration/images.go
View File

@ -76,6 +76,7 @@ var additionalDockerFlagsMap = map[string][]string{
var additionalKanikoFlagsMap = map[string][]string{
"Dockerfile_test_add": {"--single-snapshot"},
"Dockerfile_test_run_new": {"--use-new-run=true"},
"Dockerfile_test_run": {"-v=debug"},
"Dockerfile_test_scratch": {"--single-snapshot"},
"Dockerfile_test_maintainer": {"--single-snapshot"},
"Dockerfile_test_target": {"--target=second"},
@ -272,11 +273,13 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
kanikoImage := GetKanikoImage(imageRepo, dockerfile)
timer = timing.Start(dockerfile + "_kaniko")
if _, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
contextDir, gcsBucket, serviceAccount, true); err != nil {
out, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
contextDir, gcsBucket, serviceAccount, true)
if err != nil {
return err
}
timing.DefaultRun.Stop(timer)
fmt.Println(out)
d.filesBuilt[dockerfile] = struct{}{}
@ -451,5 +454,6 @@ func buildKanikoImage(
return "", fmt.Errorf("Output check failed for image %s with kaniko command : %s %s", kanikoImage, err, string(out))
}
}
fmt.Println("kaniko command\n", string(out))
return benchmarkDir, nil
}

41
integration/integration_test.go
View File

@ -142,26 +142,26 @@ func buildRequiredImages() error {
name: "Building kaniko image",
command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."},
},
{
name: "Building cache warmer image",
command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."},
},
{
name: "Building onbuild base image",
command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."},
},
{
name: "Pushing onbuild base image",
command: []string{"docker", "push", config.onbuildBaseImage},
},
{
name: "Building hardlink base image",
command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."},
},
{
name: "Pushing hardlink base image",
command: []string{"docker", "push", config.hardlinkBaseImage},
},
//{
// name: "Building cache warmer image",
// command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."},
//},
//{
// name: "Building onbuild base image",
// command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."},
//},
//{
// name: "Pushing onbuild base image",
// command: []string{"docker", "push", config.onbuildBaseImage},
//},
//{
// name: "Building hardlink base image",
// command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."},
//},
//{
// name: "Pushing hardlink base image",
// command: []string{"docker", "push", config.hardlinkBaseImage},
//},
}
for _, setupCmd := range setupCommands {
@ -294,6 +294,7 @@ func TestGitBuildcontextSubPath(t *testing.T) {
kanikoCmd := exec.Command("docker", dockerRunFlags...)
out, err = RunCommandWithoutTest(kanikoCmd)
fmt.Println(out)
if err != nil {
t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
}

43
pkg/commands/run_marker.go
View File

@ -17,16 +17,15 @@ limitations under the License.
package commands
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/GoogleContainerTools/kaniko/pkg/util"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/sirupsen/logrus"
)
type RunMarkerCommand struct {
@ -37,32 +36,32 @@ type RunMarkerCommand struct {
func (r *RunMarkerCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
// run command `touch filemarker`
logrus.Debugf("using new RunMarker command")
markerFile, err := ioutil.TempFile("", "marker")
if err != nil {
return fmt.Errorf("could not place a marker file")
}
defer func() {
os.Remove(markerFile.Name())
}()
markerInfo, err := os.Stat(markerFile.Name())
if err != nil {
return fmt.Errorf("could not place a marker file")
}
if err := runCommandInExec(config, buildArgs, r.cmd); err != nil {
return err
}
// run command find to find all new files generated
find := exec.Command("find", "/", "-newer", markerFile.Name())
out, err := find.Output()
if err != nil {
r.Files = []string{}
return nil
}
r.Files = []string{}
s := strings.Split(string(out), "\n")
for _, path := range s {
path = filepath.Clean(path)
if util.IsDestDir(path) || util.CheckIgnoreList(path) {
continue
// run command find to find all new files generate
isNewer := func(p string) (bool, error) {
fi, err := os.Stat(p)
if err != nil {
return false, err
}
r.Files = append(r.Files, path)
return fi.ModTime().After(markerInfo.ModTime()), nil
}
r.Files = util.WalkFS("/", isNewer)
logrus.Debugf("files changed %s", r.Files)
return nil
}
@ -72,11 +71,11 @@ func (r *RunMarkerCommand) String() string {
}
func (r *RunMarkerCommand) FilesToSnapshot() []string {
return nil
return r.Files
}
func (r *RunMarkerCommand) ProvidesFilesToSnapshot() bool {
return false
return true
}
// CacheCommand returns true since this command should be cached
@ -101,6 +100,6 @@ func (r *RunMarkerCommand) ShouldCacheOutput() bool {
return true
}
func (b *BaseCommand) ShouldDetectDelete() bool {
func (r *RunMarkerCommand) ShouldDetectDeletedFiles() bool {
return true
}

2
pkg/executor/build.go
View File

@ -319,7 +319,7 @@ func (s *stageBuilder) build() error {
}
initSnapshotTaken := false
if s.opts.SingleSnapshot {
if s.opts.SingleSnapshot || s.opts.RunV2 {
if err := s.initSnapshotWithTimings(); err != nil {
return err
}

18
pkg/snapshot/layered_map.go
View File

@ -31,7 +31,7 @@ import (
type LayeredMap struct {
layers []map[string]string
whiteouts []map[string]string
whiteouts []map[string]struct{}
hasher func(string) (string, error)
// cacheHasher doesn't include mtime in it's hash so that filesystem cache keys are stable
cacheHasher func(string) (string, error)
@ -47,7 +47,7 @@ func NewLayeredMap(h func(string) (string, error), c func(string) (string, error
}
func (l *LayeredMap) Snapshot() {
l.whiteouts = append(l.whiteouts, map[string]string{})
l.whiteouts = append(l.whiteouts, map[string]struct{}{})
l.layers = append(l.layers, map[string]string{})
}
@ -82,21 +82,21 @@ func (l *LayeredMap) Get(s string) (string, bool) {
return "", false
}
func (l *LayeredMap) GetWhiteout(s string) (string, bool) {
func (l *LayeredMap) GetWhiteout(s string) bool {
for i := len(l.whiteouts) - 1; i >= 0; i-- {
if v, ok := l.whiteouts[i][s]; ok {
return v, ok
if _, ok := l.whiteouts[i][s]; ok {
return ok
}
}
return "", false
return false
}
func (l *LayeredMap) MaybeAddWhiteout(s string) bool {
whiteout, ok := l.GetWhiteout(s)
if ok && whiteout == s {
ok := l.GetWhiteout(s)
if ok {
return false
}
l.whiteouts[len(l.whiteouts)-1][s] = s
l.whiteouts[len(l.whiteouts)-1][s] = struct{}{}
return true
}

41
pkg/snapshot/snapshot.go
View File

@ -24,11 +24,9 @@ import (
"sort"
"syscall"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/filesystem"
"github.com/GoogleContainerTools/kaniko/pkg/timing"
"github.com/karrick/godirwalk"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/sirupsen/logrus"
@ -36,6 +34,7 @@ import (
// For testing
var snapshotPathPrefix = config.KanikoDir
var allPass = func(s string) (bool, error) { return true, nil }
// Snapshotter holds the root directory from which to take snapshots, and a list of snapshots taken
type Snapshotter struct {
@ -81,7 +80,7 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string,
}
logrus.Info("Taking snapshot of files...")
logrus.Debugf("Taking snapshot of files %v", files)
logrus.Debugf("Taking snapshot of files %v", filesToAdd)
sort.Strings(filesToAdd)
@ -96,23 +95,23 @@ func (s *Snapshotter) TakeSnapshot(files []string, shdCheckDelete bool) (string,
filesToWhiteout := []string{}
if shdCheckDelete {
existingPaths := s.l.getFlattenedPathsForWhiteOut()
foundFiles := walkFS(s.directory)
foundFiles := util.WalkFS(s.directory, allPass)
for _, file := range foundFiles {
delete(existingPaths, file)
}
// The paths left here are the ones that have been deleted in this layer.
filesToWhiteOut := []string{}
for path := range existingPaths {
// Only add the whiteout if the directory for the file still exists.
dir := filepath.Dir(path)
if _, ok := existingPaths[dir]; !ok {
if s.l.MaybeAddWhiteout(path) {
logrus.Debugf("Adding whiteout for %s", path)
filesToWhiteOut = append(filesToWhiteOut, path)
filesToWhiteout = append(filesToWhiteout, path)
}
}
}
}
logrus.Infof("whiteouts %s", filesToWhiteout)
t := util.NewTar(f)
defer t.Close()
if err := writeToTar(t, filesToAdd, filesToWhiteout); err != nil {
@ -154,7 +153,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
s.l.Snapshot()
foundPaths := walkFS(s.directory)
foundPaths := util.WalkFS(s.directory, allPass)
timer := timing.Start("Resolving Paths")
// First handle whiteouts
// Get a list of all the files that existed before this layer
@ -265,29 +264,3 @@ func filesWithLinks(path string) ([]string, error) {
}
return []string{path, link}, nil
}
func walkFS(dir string) []string {
foundPaths := make([]string, 0)
timer := timing.Start("Walking filesystem")
godirwalk.Walk(dir, &godirwalk.Options{
Callback: func(path string, ent *godirwalk.Dirent) error {
if util.IsInIgnoreList(path) {
if util.IsDestDir(path) {
logrus.Tracef("Skipping paths under %s, as it is a ignored directory", path)
return filepath.SkipDir
}
return nil
}
foundPaths = append(foundPaths, path)
return nil
},
Unsorted: true,
},
)
timing.DefaultRun.Stop(timer)
return foundPaths
}

35
pkg/util/fs_util.go
View File

@ -31,14 +31,16 @@ import (
"syscall"
"time"
otiai10Cpy "github.com/otiai10/copy"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/docker/docker/builder/dockerignore"
"github.com/docker/docker/pkg/fileutils"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/karrick/godirwalk"
otiai10Cpy "github.com/otiai10/copy"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/timing"
)
const DoNotChangeUID = -1
@ -875,3 +877,30 @@ func UpdateInitialIgnoreList(ignoreVarRun bool) {
PrefixMatchOnly: false,
})
}
func WalkFS(dir string, f func(string) (bool, error)) []string {
foundPaths := make([]string, 0)
timer := timing.Start("Walking filesystem")
godirwalk.Walk(dir, &godirwalk.Options{
Callback: func(path string, ent *godirwalk.Dirent) error {
if IsInIgnoreList(path) {
if IsDestDir(path) {
logrus.Tracef("Skipping paths under %s, as it is a ignored directory", path)
return filepath.SkipDir
}
return nil
}
if t, err := f(path); err != nil {
return err
} else if t {
foundPaths = append(foundPaths, path)
}
return nil
},
Unsorted: true,
},
)
timing.DefaultRun.Stop(timer)
return foundPaths
}