From 904575d0cbd476321571671b88cec2ee06c3ab60 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 26 Apr 2018 15:40:41 -0700 Subject: [PATCH] support multi stage builds --- cmd/executor/cmd/root.go | 23 ++- .../dockerfiles/Dockerfile_test_multistage | 9 + .../dockerfiles/config_test_multistage.json | 12 ++ integration_tests/integration_test_yaml.go | 18 +- pkg/commands/copy.go | 13 ++ pkg/commands/env.go | 6 + pkg/dockerfile/dockerfile.go | 96 +++++++++- pkg/dockerfile/dockerfile_test.go | 96 ++++++++++ pkg/executor/executor.go | 181 +++++++++++------- pkg/util/command_util.go | 2 +- pkg/util/fs_util.go | 35 +++- pkg/util/fs_util_test.go | 3 - 12 files changed, 402 insertions(+), 92 deletions(-) create mode 100644 integration_tests/dockerfiles/Dockerfile_test_multistage create mode 100644 integration_tests/dockerfiles/config_test_multistage.json create mode 100644 pkg/dockerfile/dockerfile_test.go diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 6a4fd546e..adeed0e58 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -32,14 +32,13 @@ import ( ) var ( - dockerfilePath string - destination string - srcContext string - snapshotMode string - bucket string - dockerInsecureSkipTLSVerify bool - logLevel string - force bool + dockerfilePath string + destination string + srcContext string + snapshotMode string + bucket string + logLevel string + force bool ) func init() { @@ -48,7 +47,6 @@ func init() { RootCmd.PersistentFlags().StringVarP(&bucket, "bucket", "b", "", "Name of the GCS bucket from which to access build context as tarball.") RootCmd.PersistentFlags().StringVarP(&destination, "destination", "d", "", "Registry the final image should be pushed to (ex: gcr.io/test/example:latest)") RootCmd.PersistentFlags().StringVarP(&snapshotMode, "snapshotMode", "", "full", "Set this flag to change the file attributes inspected during snapshotting") - RootCmd.PersistentFlags().BoolVarP(&dockerInsecureSkipTLSVerify, "insecure-skip-tls-verify", "", false, "Push to insecure registry ignoring TLS verify") RootCmd.PersistentFlags().StringVarP(&logLevel, "verbosity", "v", constants.DefaultLogLevel, "Log level (debug, info, warn, error, fatal, panic") RootCmd.PersistentFlags().BoolVarP(&force, "force", "", false, "Force building outside of a container") } @@ -76,7 +74,12 @@ var RootCmd = &cobra.Command{ logrus.Error(err) os.Exit(1) } - if err := executor.DoBuild(dockerfilePath, srcContext, destination, snapshotMode, dockerInsecureSkipTLSVerify); err != nil { + ref, image, err := executor.DoBuild(dockerfilePath, srcContext, snapshotMode) + if err != nil { + logrus.Error(err) + os.Exit(1) + } + if err := executor.DoPush(ref, image, destination); err != nil { logrus.Error(err) os.Exit(1) } diff --git a/integration_tests/dockerfiles/Dockerfile_test_multistage b/integration_tests/dockerfiles/Dockerfile_test_multistage new file mode 100644 index 000000000..18fa2d2c1 --- /dev/null +++ b/integration_tests/dockerfiles/Dockerfile_test_multistage @@ -0,0 +1,9 @@ +FROM gcr.io/distroless/base:latest +COPY . . + +FROM scratch as second +ENV foopath context/foo +COPY --from=0 $foopath context/b* /foo/ + +FROM gcr.io/distroless/base:latest +COPY --from=second /foo /foo2 diff --git a/integration_tests/dockerfiles/config_test_multistage.json b/integration_tests/dockerfiles/config_test_multistage.json new file mode 100644 index 000000000..9aa0494cb --- /dev/null +++ b/integration_tests/dockerfiles/config_test_multistage.json @@ -0,0 +1,12 @@ +[ + { + "Image1": "gcr.io/kaniko-test/docker-test-multistage:latest", + "Image2": "gcr.io/kaniko-test/kaniko-test-multistage:latest", + "DiffType": "File", + "Diff": { + "Adds": null, + "Dels": null, + "Mods": null + } + } +] \ No newline at end of file diff --git a/integration_tests/integration_test_yaml.go b/integration_tests/integration_test_yaml.go index 7c1d64607..a9f09e3a4 100644 --- a/integration_tests/integration_test_yaml.go +++ b/integration_tests/integration_test_yaml.go @@ -115,14 +115,6 @@ var fileTests = []struct { kanikoContext: buildcontextPath, repo: "test-add", }, - { - description: "test mv add", - dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_mv_add", - configPath: "/workspace/integration_tests/dockerfiles/config_test_mv_add.json", - dockerContext: buildcontextPath, - kanikoContext: buildcontextPath, - repo: "test-mv-add", - }, { description: "test registry", dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_registry", @@ -147,6 +139,14 @@ var fileTests = []struct { kanikoContext: buildcontextPath, repo: "test-scratch", }, + { + description: "test multistage", + dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_multistage", + configPath: "/workspace/integration_tests/dockerfiles/config_test_multistage.json", + dockerContext: buildcontextPath, + kanikoContext: buildcontextPath, + repo: "test-multistage", + }, } var structureTests = []struct { @@ -288,7 +288,7 @@ func main() { } compareOutputs := step{ Name: ubuntuImage, - Args: []string{"cmp", "-b", test.configPath, containerDiffOutputFile}, + Args: []string{"cmp", test.configPath, containerDiffOutputFile}, } y.Steps = append(y.Steps, dockerBuild, kaniko, pullKanikoImage, containerDiff, catContainerDiffOutput, compareOutputs) diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index 91e60d82d..2610799b9 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -17,6 +17,7 @@ limitations under the License. package commands import ( + "github.com/GoogleContainerTools/kaniko/pkg/constants" "os" "path/filepath" "strings" @@ -40,6 +41,10 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config) error { logrus.Infof("cmd: copy %s", srcs) logrus.Infof("dest: %s", dest) + // Resolve from + if c.cmd.From != "" { + c.buildcontext = filepath.Join(constants.BuildContextDir, c.cmd.From) + } // First, resolve any environment replacement resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.cmd.SourcesAndDest, config.Env, true) if err != nil { @@ -58,11 +63,19 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config) error { if err != nil { return err } + cwd := config.WorkingDir + if cwd == "" { + cwd = constants.RootDir + } destPath, err := util.DestinationFilepath(src, dest, config.WorkingDir) if err != nil { return err } if fi.IsDir() { + if !filepath.IsAbs(dest) { + // we need to add '/' to the end to indicate the destination is a directory + dest = filepath.Join(cwd, dest) + "/" + } if err := util.CopyDir(fullPath, dest); err != nil { return err } diff --git a/pkg/commands/env.go b/pkg/commands/env.go index 4e576a3f8..1cb8e974a 100644 --- a/pkg/commands/env.go +++ b/pkg/commands/env.go @@ -29,6 +29,12 @@ type EnvCommand struct { cmd *instructions.EnvCommand } +func NewEnvCommand(cmd *instructions.EnvCommand) EnvCommand { + return EnvCommand{ + cmd: cmd, + } +} + func (e *EnvCommand) ExecuteCommand(config *v1.Config) error { logrus.Info("cmd: ENV") newEnvs := e.cmd.Env diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index f755a547e..c3eb8ed95 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -18,10 +18,20 @@ package dockerfile import ( "bytes" - "strings" - + "github.com/GoogleContainerTools/kaniko/pkg/commands" + "github.com/GoogleContainerTools/kaniko/pkg/constants" + "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/docker/docker/builder/dockerfile/instructions" "github.com/docker/docker/builder/dockerfile/parser" + "github.com/google/go-containerregistry/authn" + "github.com/google/go-containerregistry/name" + "github.com/google/go-containerregistry/v1" + "github.com/google/go-containerregistry/v1/empty" + "github.com/google/go-containerregistry/v1/remote" + "net/http" + "path/filepath" + "strconv" + "strings" ) // Parse parses the contents of a Dockerfile and returns a list of commands @@ -37,6 +47,25 @@ func Parse(b []byte) ([]instructions.Stage, error) { return stages, err } +// ResolveStages resolves any calls to previous stages with names to indices +// Ex. --from=second_stage should be --from=1 for easier processing later on +func ResolveStages(stages []instructions.Stage) { + nameToIndex := make(map[string]string) + for i, stage := range stages { + index := strconv.Itoa(i) + nameToIndex[stage.Name] = index + nameToIndex[index] = index + for _, cmd := range stage.Commands { + switch c := cmd.(type) { + case *instructions.CopyCommand: + if c.From != "" { + c.From = nameToIndex[c.From] + } + } + } + } +} + // ParseCommands parses an array of commands into an array of instructions.Command; used for onbuild func ParseCommands(cmdArray []string) ([]instructions.Command, error) { var cmds []instructions.Command @@ -54,3 +83,66 @@ func ParseCommands(cmdArray []string) ([]instructions.Command, error) { } return cmds, nil } + +// Dependencies returns a list of files in this stage that will be needed in later stages +func Dependencies(index int, stages []instructions.Stage) ([]string, error) { + var dependencies []string + for stageIndex, stage := range stages { + if stageIndex <= index { + continue + } + var sourceImage v1.Image + if stage.BaseName == constants.NoBaseImage { + sourceImage = empty.Image + } else { + // Initialize source image + ref, err := name.ParseReference(stage.BaseName, name.WeakValidation) + if err != nil { + return nil, err + + } + auth, err := authn.DefaultKeychain.Resolve(ref.Context().Registry) + if err != nil { + return nil, err + } + sourceImage, err = remote.Image(ref, auth, http.DefaultTransport) + if err != nil { + return nil, err + } + } + imageConfig, err := sourceImage.ConfigFile() + if err != nil { + return nil, err + } + for _, cmd := range stage.Commands { + switch c := cmd.(type) { + case *instructions.EnvCommand: + envCommand := commands.NewEnvCommand(c) + if err := envCommand.ExecuteCommand(&imageConfig.Config); err != nil { + return nil, err + } + case *instructions.CopyCommand: + if c.From != strconv.Itoa(index) { + continue + } + // First, resolve any environment replacement + resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.SourcesAndDest, imageConfig.Config.Env, true) + if err != nil { + return nil, err + } + // Resolve wildcards and get a list of resolved sources + srcs, err := util.ResolveSources(resolvedEnvs, constants.RootDir) + if err != nil { + return nil, err + } + for index, src := range srcs { + if !filepath.IsAbs(src) { + srcs[index] = filepath.Join(constants.RootDir, src) + } + } + dependencies = append(dependencies, srcs...) + } + } + } + return dependencies, nil +} diff --git a/pkg/dockerfile/dockerfile_test.go b/pkg/dockerfile/dockerfile_test.go new file mode 100644 index 000000000..4c4c7e39c --- /dev/null +++ b/pkg/dockerfile/dockerfile_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockerfile + +import ( + "fmt" + "github.com/GoogleContainerTools/kaniko/testutil" + "github.com/docker/docker/builder/dockerfile/instructions" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "testing" +) + +var dockerfile = ` +FROM scratch +RUN echo hi > /hi + +FROM scratch AS second +COPY --from=0 /hi /hi2 + +FROM scratch +COPY --from=second /hi2 /hi3 +` + +func Test_ResolveStages(t *testing.T) { + stages, err := Parse([]byte(dockerfile)) + if err != nil { + t.Fatal(err) + } + ResolveStages(stages) + for index, stage := range stages { + if index == 0 { + continue + } + copyCmd := stage.Commands[0].(*instructions.CopyCommand) + expectedStage := strconv.Itoa(index - 1) + if copyCmd.From != expectedStage { + t.Fatalf("unexpected copy command: %s resolved to stage %s, expected %s", copyCmd.String(), copyCmd.From, expectedStage) + } + } +} + +func Test_Dependencies(t *testing.T) { + testDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + helloPath := filepath.Join(testDir, "hello") + if err := os.Mkdir(helloPath, 0755); err != nil { + t.Fatal(err) + } + + dockerfile := fmt.Sprintf(` + FROM scratch + COPY %s %s + + FROM scratch AS second + ENV hienv %s + COPY a b + COPY --from=0 /$hienv %s /hi2/ + `, helloPath, helloPath, helloPath, testDir) + + stages, err := Parse([]byte(dockerfile)) + if err != nil { + t.Fatal(err) + } + + expectedDependencies := [][]string{ + { + helloPath, + testDir, + }, + nil, + } + + for index := range stages { + actualDeps, err := Dependencies(index, stages) + testutil.CheckErrorAndDeepEqual(t, false, err, expectedDependencies[index], actualDeps) + } +} diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index d3b2039b1..8f3976da5 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -19,10 +19,13 @@ package executor import ( "bytes" "fmt" + "github.com/GoogleContainerTools/kaniko/pkg/snapshot" "io" "io/ioutil" "net/http" "os" + "path/filepath" + "strconv" "github.com/google/go-containerregistry/v1/empty" @@ -37,98 +40,88 @@ import ( "github.com/GoogleContainerTools/kaniko/pkg/commands" "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" - "github.com/GoogleContainerTools/kaniko/pkg/image" - "github.com/GoogleContainerTools/kaniko/pkg/snapshot" "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/docker/docker/builder/dockerfile/instructions" "github.com/sirupsen/logrus" ) -func DoBuild(dockerfilePath, srcContext, destination, snapshotMode string, dockerInsecureSkipTLSVerify bool) error { +func DoBuild(dockerfilePath, srcContext, snapshotMode string) (name.Reference, v1.Image, error) { // Parse dockerfile and unpack base image to root d, err := ioutil.ReadFile(dockerfilePath) if err != nil { - return err + return nil, nil, err } stages, err := dockerfile.Parse(d) if err != nil { - return err - } - baseImage := stages[0].BaseName - - // Unpack file system to root - var sourceImage v1.Image - var ref name.Reference - logrus.Infof("Unpacking filesystem of %s...", baseImage) - if baseImage == constants.NoBaseImage { - logrus.Info("No base image, nothing to extract") - sourceImage = empty.Image - } else { - // Initialize source image - ref, err = name.ParseReference(baseImage, name.WeakValidation) - if err != nil { - return err - } - auth, err := authn.DefaultKeychain.Resolve(ref.Context().Registry) - if err != nil { - return err - } - sourceImage, err = remote.Image(ref, auth, http.DefaultTransport) - if err != nil { - return err - } - } - if err := util.GetFSFromImage(sourceImage); err != nil { - return err + return nil, nil, err } + dockerfile.ResolveStages(stages) hasher, err := getHasher(snapshotMode) if err != nil { - return err + return nil, nil, err } - l := snapshot.NewLayeredMap(hasher) - snapshotter := snapshot.NewSnapshotter(l, constants.RootDir) - - // Take initial snapshot - if err := snapshotter.Init(); err != nil { - return err - } - - destRef, err := name.ParseReference(destination, name.WeakValidation) - if err != nil { - return err - } - // Set environment variables within the image - if err := image.SetEnvVariables(sourceImage); err != nil { - return err - } - - imageConfig, err := sourceImage.ConfigFile() - if err != nil { - return err - } - // Currently only supports single stage builds - for _, stage := range stages { + for index, stage := range stages { + baseImage := stage.BaseName + finalStage := index == len(stages)-1 + // Unpack file system to root + logrus.Infof("Unpacking filesystem of %s...", baseImage) + var sourceImage v1.Image + var ref name.Reference + if baseImage == constants.NoBaseImage { + logrus.Info("No base image, nothing to extract") + sourceImage = empty.Image + } else { + // Initialize source image + ref, err = name.ParseReference(baseImage, name.WeakValidation) + if err != nil { + return nil, nil, err + } + auth, err := authn.DefaultKeychain.Resolve(ref.Context().Registry) + if err != nil { + return nil, nil, err + } + sourceImage, err = remote.Image(ref, auth, http.DefaultTransport) + if err != nil { + return nil, nil, err + } + } + if err := util.GetFSFromImage(sourceImage); err != nil { + return nil, nil, err + } + l := snapshot.NewLayeredMap(hasher) + snapshotter := snapshot.NewSnapshotter(l, constants.RootDir) + // Take initial snapshot + if err := snapshotter.Init(); err != nil { + return nil, nil, err + } + imageConfig, err := sourceImage.ConfigFile() + if err != nil { + return nil, nil, err + } if err := resolveOnBuild(&stage, &imageConfig.Config); err != nil { - return err + return nil, nil, err } for _, cmd := range stage.Commands { dockerCommand, err := commands.GetCommand(cmd, srcContext) if err != nil { - return err + return nil, nil, err } if dockerCommand == nil { continue } if err := dockerCommand.ExecuteCommand(&imageConfig.Config); err != nil { - return err + return nil, nil, err + } + if !finalStage { + continue } // Now, we get the files to snapshot from this command and take the snapshot snapshotFiles := dockerCommand.FilesToSnapshot() contents, err := snapshotter.TakeSnapshot(snapshotFiles) if err != nil { - return err + return nil, nil, err } util.MoveVolumeWhitelistToWhitelist() if contents == nil { @@ -141,7 +134,7 @@ func DoBuild(dockerfilePath, srcContext, destination, snapshotMode string, docke } layer, err := tarball.LayerFromOpener(opener) if err != nil { - return err + return nil, nil, err } sourceImage, err = mutate.Append(sourceImage, mutate.Addendum{ @@ -152,15 +145,36 @@ func DoBuild(dockerfilePath, srcContext, destination, snapshotMode string, docke }, ) if err != nil { - return err + return nil, nil, err } } + if finalStage { + return ref, sourceImage, nil + } + if err := saveStageDependencies(index, stages); err != nil { + return nil, nil, err + } + // Delete the filesystem + if err := util.DeleteFilesystem(); err != nil { + return nil, nil, err + } } + return nil, nil, nil +} + +func DoPush(ref name.Reference, image v1.Image, destination string) error { // Push the image if err := setDefaultEnv(); err != nil { return err } - + imageConfig, err := image.ConfigFile() + if err != nil { + return err + } + destRef, err := name.ParseReference(destination, name.WeakValidation) + if err != nil { + return err + } wo := remote.WriteOptions{} if ref != nil { wo.MountPaths = []name.Repository{ref.Context()} @@ -169,12 +183,47 @@ func DoBuild(dockerfilePath, srcContext, destination, snapshotMode string, docke if err != nil { return err } - sourceImage, err = mutate.Config(sourceImage, imageConfig.Config) + image, err = mutate.Config(image, imageConfig.Config) if err != nil { return err } - - return remote.Write(destRef, sourceImage, pushAuth, http.DefaultTransport, wo) + return remote.Write(destRef, image, pushAuth, http.DefaultTransport, wo) +} +func saveStageDependencies(index int, stages []instructions.Stage) error { + // First, get the files in this stage later stages will need + dependencies, err := dockerfile.Dependencies(index, stages) + logrus.Infof("saving dependencies %s", dependencies) + if err != nil { + return err + } + // Then, create the directory they will exist in + i := strconv.Itoa(index) + dependencyDir := filepath.Join(constants.BuildContextDir, i) + if err := os.MkdirAll(dependencyDir, 0755); err != nil { + return err + } + // Now, copy over dependencies to this dir + for _, d := range dependencies { + fi, err := os.Lstat(d) + if err != nil { + return err + } + dest := filepath.Join(dependencyDir, d) + if fi.IsDir() { + if err := util.CopyDir(d, dest); err != nil { + return err + } + } else if fi.Mode()&os.ModeSymlink != 0 { + if err := util.CopySymlink(d, dest); err != nil { + return err + } + } else { + if err := util.CopyFile(d, dest); err != nil { + return err + } + } + } + return nil } func getHasher(snapshotMode string) (func(string) (string, error), error) { diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go index 20efd19a3..c8788a494 100644 --- a/pkg/util/command_util.go +++ b/pkg/util/command_util.go @@ -178,7 +178,7 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri } if len(resolvedSources) == 1 { - fi, err := os.Stat(filepath.Join(root, resolvedSources[0])) + fi, err := os.Lstat(filepath.Join(root, resolvedSources[0])) if err != nil { return err } diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index d639740df..a103e8bfb 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -104,6 +104,33 @@ func GetFSFromImage(img v1.Image) error { return nil } +// DeleteFilesystem deletes the extracted image file system +func DeleteFilesystem() error { + logrus.Info("Deleting filesystem...") + err := filepath.Walk(constants.RootDir, func(path string, info os.FileInfo, err error) error { + if PathInWhitelist(path, constants.RootDir) || ChildDirInWhitelist(path, constants.RootDir) { + logrus.Debugf("Not deleting %s, as it's whitelisted", path) + return nil + } + if path == constants.RootDir { + return nil + } + return os.RemoveAll(path) + }) + return err +} + +// ChildDirInWhitelist returns true if there is a child file or directory of the path in the whitelist +func ChildDirInWhitelist(path, directory string) bool { + for _, d := range whitelist { + dirPath := filepath.Join(directory, d) + if HasFilepathPrefix(dirPath, path) { + return true + } + } + return false +} + func unTar(r io.Reader, dest string) error { tr := tar.NewReader(r) for { @@ -269,6 +296,9 @@ func RelativeFiles(fp string, root string) ([]string, error) { if err != nil { return err } + if PathInWhitelist(path, root) { + return nil + } relPath, err := filepath.Rel(root, path) if err != nil { return err @@ -284,6 +314,9 @@ func Files(root string) ([]string, error) { var files []string logrus.Debugf("Getting files and contents at root %s", root) err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if PathInWhitelist(path, root) { + return nil + } files = append(files, path) return err }) @@ -368,7 +401,7 @@ func CopyDir(src, dest string) error { } for _, file := range files { fullPath := filepath.Join(src, file) - fi, err := os.Stat(fullPath) + fi, err := os.Lstat(fullPath) if err != nil { return err } diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index 0334d8c56..62e9b6091 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -103,16 +103,13 @@ var tests = []struct { files: map[string]string{ "/workspace/foo/a": "baz1", "/workspace/foo/b": "baz2", - "/kaniko/file": "file", }, directory: "", expectedFiles: []string{ "workspace/foo/a", "workspace/foo/b", - "kaniko/file", "workspace", "workspace/foo", - "kaniko", ".", }, },