Extract intermediate stages to filesystem (#266)

* WIP

* save and extract stage tarballs if there are dependencies

integration/dockerfiles/Dockerfile_test_multistage

@@ -5,6 +5,9 @@ FROM scratch as second
 ENV foopath context/foo
 COPY --from=0 $foopath context/b* /foo/
 
+FROM second
+COPY --from=base /context/foo /new/foo
+
 FROM base
 ARG file
-COPY --from=second /foo $file
+COPY --from=second /foo ${file}

pkg/dockerfile/dockerfile.go

@@ -19,16 +19,32 @@ package dockerfile
 import (
 	"bytes"
 	"fmt"
-	"path/filepath"
+	"io/ioutil"
 	"strconv"
 	"strings"
 
-	"github.com/GoogleContainerTools/kaniko/pkg/constants"
-	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
 )
 
+// Stages reads the Dockerfile, validates it's contents, and returns stages
+func Stages(dockerfilePath, target string) ([]instructions.Stage, error) {
+	d, err := ioutil.ReadFile(dockerfilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	stages, err := Parse(d)
+	if err != nil {
+		return nil, err
+	}
+	if err := ValidateTarget(stages, target); err != nil {
+		return nil, err
+	}
+	ResolveStages(stages)
+	return stages, nil
+}
+
 // Parse parses the contents of a Dockerfile and returns a list of commands
 func Parse(b []byte) ([]instructions.Stage, error) {
 	p, err := parser.Parse(bytes.NewReader(b))
@@ -43,6 +59,9 @@ func Parse(b []byte) ([]instructions.Stage, error) {
 }
 
 func ValidateTarget(stages []instructions.Stage, target string) error {
+	if target == "" {
+		return nil
+	}
 	for _, stage := range stages {
 		if stage.Name == target {
 			return nil
@@ -91,53 +110,23 @@ func ParseCommands(cmdArray []string) ([]instructions.Command, error) {
 	return cmds, nil
 }
 
-// Dependencies returns a list of files in this stage that will be needed in later stages
+// SaveStage returns true if the current stage will be needed later in the Dockerfile
-func Dependencies(index int, stages []instructions.Stage, buildArgs *BuildArgs) ([]string, error) {
+func SaveStage(index int, stages []instructions.Stage) bool {
-	dependencies := []string{}
 	for stageIndex, stage := range stages {
 		if stageIndex <= index {
 			continue
 		}
-		sourceImage, err := util.RetrieveSourceImage(stageIndex, buildArgs.ReplacementEnvs(nil), stages)
+		if stage.Name == stages[index].BaseName {
-		if err != nil {
+			return true
-			return nil, err
-		}
-		imageConfig, err := sourceImage.ConfigFile()
-		if err != nil {
-			return nil, err
 		}
 		for _, cmd := range stage.Commands {
 			switch c := cmd.(type) {
-			case *instructions.EnvCommand:
-				replacementEnvs := buildArgs.ReplacementEnvs(imageConfig.Config.Env)
-				if err := util.UpdateConfigEnv(c.Env, &imageConfig.Config, replacementEnvs); err != nil {
-					return nil, err
-				}
-			case *instructions.ArgCommand:
-				buildArgs.AddArg(c.Key, c.Value)
 			case *instructions.CopyCommand:
-				if c.From != strconv.Itoa(index) {
+				if c.From == strconv.Itoa(index) {
-					continue
+					return true
 				}
-				// First, resolve any environment replacement
-				replacementEnvs := buildArgs.ReplacementEnvs(imageConfig.Config.Env)
-				resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.SourcesAndDest, replacementEnvs, true)
-				if err != nil {
-					return nil, err
-				}
-				// Resolve wildcards and get a list of resolved sources
-				srcs, err := util.ResolveSources(resolvedEnvs, constants.RootDir)
-				if err != nil {
-					return nil, err
-				}
-				for index, src := range srcs {
-					if !filepath.IsAbs(src) {
-						srcs[index] = filepath.Join(constants.RootDir, src)
-					}
-				}
-				dependencies = append(dependencies, srcs...)
 			}
 		}
 	}
-	return dependencies, nil
+	return false
 }

pkg/dockerfile/dockerfile_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package dockerfile
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -95,82 +94,59 @@ func Test_ValidateTarget(t *testing.T) {
 	}
 }
 
-func Test_Dependencies(t *testing.T) {
+func Test_SaveStage(t *testing.T) {
-	testDir, err := ioutil.TempDir("", "")
+	tempDir, err := ioutil.TempDir("", "")
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("couldn't create temp dir: %v", err)
 	}
-	helloPath := filepath.Join(testDir, "hello")
+	defer os.RemoveAll(tempDir)
-	if err := os.Mkdir(helloPath, 0755); err != nil {
+	files := map[string]string{
-		t.Fatal(err)
+		"Dockerfile": `
-	}
+		FROM scratch
-
+		RUN echo hi > /hi
-	dockerfile := fmt.Sprintf(`
+		
-	FROM scratch
+		FROM scratch AS second
-	COPY %s %s
+		COPY --from=0 /hi /hi2
 	
-	FROM scratch AS second
+		FROM second
-	ENV hienv %s
+		RUN xxx
-	COPY a b
+		
-	COPY --from=0 /$hienv %s /hi2/
+		FROM scratch
-	`, helloPath, helloPath, helloPath, testDir)
+		COPY --from=second /hi2 /hi3
-
+		`,
-	stages, err := Parse([]byte(dockerfile))
+	}
+	if err := testutil.SetupFiles(tempDir, files); err != nil {
+		t.Fatalf("couldn't create dockerfile: %v", err)
+	}
+	stages, err := Stages(filepath.Join(tempDir, "Dockerfile"), "")
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("couldn't retrieve stages from Dockerfile: %v", err)
 	}
-
+	tests := []struct {
-	expectedDependencies := [][]string{
+		name     string
+		index    int
+		expected bool
+	}{
 		{
-			helloPath,
+			name:     "reference stage in later copy command",
-			testDir,
+			index:    0,
+			expected: true,
+		},
+		{
+			name:     "reference stage in later from command",
+			index:    1,
+			expected: true,
+		},
+		{
+			name:     "don't reference stage later",
+			index:    2,
+			expected: false,
 		},
-		{},
 	}
-
+	for _, test := range tests {
-	for index := range stages {
+		t.Run(test.name, func(t *testing.T) {
-		buildArgs := NewBuildArgs([]string{})
+			actual := SaveStage(test.index, stages)
-		actualDeps, err := Dependencies(index, stages, buildArgs)
+			testutil.CheckErrorAndDeepEqual(t, false, nil, test.expected, actual)
-		testutil.CheckErrorAndDeepEqual(t, false, err, expectedDependencies[index], actualDeps)
+		})
-	}
-}
-
-func Test_DependenciesWithArg(t *testing.T) {
-	testDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	helloPath := filepath.Join(testDir, "hello")
-	if err := os.Mkdir(helloPath, 0755); err != nil {
-		t.Fatal(err)
-	}
-
-	dockerfile := fmt.Sprintf(`
-	FROM scratch
-	COPY %s %s
-	
-	FROM scratch AS second
-	ARG hienv
-	COPY a b
-	COPY --from=0 /$hienv %s /hi2/
-	`, helloPath, helloPath, testDir)
-
-	stages, err := Parse([]byte(dockerfile))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	expectedDependencies := [][]string{
-		{
-			helloPath,
-			testDir,
-		},
-		{},
-	}
-	buildArgs := NewBuildArgs([]string{fmt.Sprintf("hienv=%s", helloPath)})
-
-	for index := range stages {
-		actualDeps, err := Dependencies(index, stages, buildArgs)
-		testutil.CheckErrorAndDeepEqual(t, false, err, expectedDependencies[index], actualDeps)
 	}
 }

pkg/executor/executor.go

@@ -58,32 +58,23 @@ type KanikoBuildArgs struct {
 
 func DoBuild(k KanikoBuildArgs) (v1.Image, error) {
 	// Parse dockerfile and unpack base image to root
-	d, err := ioutil.ReadFile(k.DockerfilePath)
+	stages, err := dockerfile.Stages(k.DockerfilePath, k.Target)
 	if err != nil {
 		return nil, err
 	}
 
-	stages, err := dockerfile.Parse(d)
-	if err != nil {
-		return nil, err
-	}
-	if err := dockerfile.ValidateTarget(stages, k.Target); err != nil {
-		return nil, err
-	}
-	dockerfile.ResolveStages(stages)
-
 	hasher, err := getHasher(k.SnapshotMode)
 	if err != nil {
 		return nil, err
 	}
 	for index, stage := range stages {
-		finalStage := (index == len(stages)-1) || (k.Target == stage.Name)
+		finalStage := finalStage(index, k.Target, stages)
 		// Unpack file system to root
 		sourceImage, err := util.RetrieveSourceImage(index, k.Args, stages)
 		if err != nil {
 			return nil, err
 		}
-		if err := util.GetFSFromImage(sourceImage); err != nil {
+		if err := util.GetFSFromImage(constants.RootDir, sourceImage); err != nil {
 			return nil, err
 		}
 		l := snapshot.NewLayeredMap(hasher)
@@ -168,11 +159,13 @@ func DoBuild(k KanikoBuildArgs) (v1.Image, error) {
 			}
 			return sourceImage, nil
 		}
-		if err := saveStageAsTarball(index, sourceImage); err != nil {
+		if dockerfile.SaveStage(index, stages) {
-			return nil, err
+			if err := saveStageAsTarball(index, sourceImage); err != nil {
-		}
+				return nil, err
-		if err := saveStageDependencies(index, stages, buildArgs.Clone()); err != nil {
+			}
-			return nil, err
+			if err := extractImageToDependecyDir(index, sourceImage); err != nil {
+				return nil, err
+			}
 		}
 		// Delete the filesystem
 		if err := util.DeleteFilesystem(); err != nil {
@@ -225,44 +218,24 @@ func DoPush(image v1.Image, destinations []string, tarPath string) error {
 	}
 	return nil
 }
-func saveStageDependencies(index int, stages []instructions.Stage, buildArgs *dockerfile.BuildArgs) error {
+
-	// First, get the files in this stage later stages will need
+func finalStage(index int, target string, stages []instructions.Stage) bool {
-	dependencies, err := dockerfile.Dependencies(index, stages, buildArgs)
+	if index == len(stages)-1 {
-	logrus.Infof("saving dependencies %s", dependencies)
+		return true
-	if err != nil {
-		return err
 	}
-	if len(dependencies) == 0 {
+	if target == "" {
-		return nil
+		return false
 	}
-	// Then, create the directory they will exist in
+	return target == stages[index].Name
-	i := strconv.Itoa(index)
+}
-	dependencyDir := filepath.Join(constants.KanikoDir, i)
+
+func extractImageToDependecyDir(index int, image v1.Image) error {
+	dependencyDir := filepath.Join(constants.KanikoDir, strconv.Itoa(index))
 	if err := os.MkdirAll(dependencyDir, 0755); err != nil {
 		return err
 	}
-	// Now, copy over dependencies to this dir
+	logrus.Infof("trying to extract to %s", dependencyDir)
-	for _, d := range dependencies {
+	return util.GetFSFromImage(dependencyDir, image)
-		fi, err := os.Lstat(d)
-		if err != nil {
-			return err
-		}
-		dest := filepath.Join(dependencyDir, d)
-		if fi.IsDir() {
-			if err := util.CopyDir(d, dest); err != nil {
-				return err
-			}
-		} else if fi.Mode()&os.ModeSymlink != 0 {
-			if err := util.CopySymlink(d, dest); err != nil {
-				return err
-			}
-		} else {
-			if err := util.CopyFile(d, dest); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
 }
 
 func saveStageAsTarball(stageIndex int, image v1.Image) error {

pkg/util/fs_util.go

@@ -42,7 +42,7 @@ var whitelist = []string{
 }
 var volumeWhitelist = []string{}
 
-func GetFSFromImage(img v1.Image) error {
+func GetFSFromImage(root string, img v1.Image) error {
 	whitelist, err := fileSystemWhitelist(constants.WhitelistPath)
 	if err != nil {
 		return err
@@ -72,7 +72,7 @@ func GetFSFromImage(img v1.Image) error {
 			if err != nil {
 				return err
 			}
-			path := filepath.Join("/", filepath.Clean(hdr.Name))
+			path := filepath.Join(root, filepath.Clean(hdr.Name))
 			base := filepath.Base(path)
 			dir := filepath.Dir(path)
 			if strings.HasPrefix(base, ".wh.") {
@@ -91,7 +91,7 @@ func GetFSFromImage(img v1.Image) error {
 				continue
 			}
 
-			if CheckWhitelist(path) {
+			if CheckWhitelist(path) && !checkWhitelistRoot(root) {
 				logrus.Infof("Not adding %s because it is whitelisted", path)
 				continue
 			}
@@ -103,7 +103,7 @@ func GetFSFromImage(img v1.Image) error {
 			}
 			fs[path] = struct{}{}
 
-			if err := extractFile("/", hdr, tr); err != nil {
+			if err := extractFile(root, hdr, tr); err != nil {
 				return err
 			}
 		}
@@ -256,6 +256,18 @@ func CheckWhitelist(path string) bool {
 	return false
 }
 
+func checkWhitelistRoot(root string) bool {
+	if root == constants.RootDir {
+		return false
+	}
+	for _, wl := range whitelist {
+		if HasFilepathPrefix(root, wl) {
+			return true
+		}
+	}
+	return false
+}
+
 // Get whitelist from roots of mounted files
 // Each line of /proc/self/mountinfo is in the form:
 // 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue

pkg/util/image_util.go

@@ -83,5 +83,4 @@ func remoteImage(image string) (v1.Image, error) {
 	}
 	kc := authn.NewMultiKeychain(authn.DefaultKeychain, k8sc)
 	return remote.Image(ref, remote.WithAuthFromKeychain(kc))
-
 }