feat: kaniko dir config option (#1997)

* remove configurables from constants

* add configurables to config init

* add kaniko dir flag

* cleanup pkg constants

* replace buildcontext constants pkg occurrences with config

* add KanikoDir to KanikoOptions

* replace executor constants pkg occurrences with config

* remove redundant KanikoDir assignment

* replace constants to config for IntermediateStagesDir

* fix imports

* add default kaniko directory into constants

* add check for kanikoDir on use

* update init to use default path constant

* update executor kanikoDir check

Co-authored-by: Jason Hall <jasonhall@redhat.com>

* alter checkKanikoDir parameter

* add TestKanikoDir func

* update error handling style

Co-authored-by: Jason Hall <jasonhall@redhat.com>

* remove shorthand flag usage from test

Co-authored-by: Jason Hall <jasonhall@redhat.com>

* add docstring to integration test

Co-authored-by: Jason Hall <jasonhall@redhat.com>

* remove shorthand flag from kaniko-dir

Co-authored-by: Jason Hall <jasonhall@redhat.com>

cmd/executor/cmd/root.go

@@ -67,6 +67,11 @@ var RootCmd = &cobra.Command{
 	Use: "executor",
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 		if cmd.Use == "executor" {
+
+			if err := checkKanikoDir(config.KanikoDir); err != nil {
+				return err
+			}
+
 			resolveEnvironmentBuildArgs(opts.BuildArgs, os.Getenv)
 
 			if err := logging.Configure(logLevel, logFormat, logTimestamp); err != nil {
@@ -182,6 +187,7 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().BoolVarP(&opts.SkipTLSVerifyPull, "skip-tls-verify-pull", "", false, "Pull from insecure registry ignoring TLS verify")
 	RootCmd.PersistentFlags().IntVar(&opts.PushRetry, "push-retry", 0, "Number of retries for the push operation")
 	RootCmd.PersistentFlags().IntVar(&opts.ImageFSExtractRetry, "image-fs-extract-retry", 0, "Number of retries for image FS extraction")
+	RootCmd.PersistentFlags().StringVarP(&opts.KanikoDir, "kaniko-dir", "", "/kaniko", "Path to the kaniko directory")
 	RootCmd.PersistentFlags().StringVarP(&opts.TarPath, "tarPath", "", "", "Path to save the image in as a tarball instead of pushing")
 	RootCmd.PersistentFlags().BoolVarP(&opts.SingleSnapshot, "single-snapshot", "", false, "Take a single snapshot at the end of the build.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.Reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible")
@@ -233,6 +239,22 @@ func addHiddenFlags(cmd *cobra.Command) {
 	cmd.PersistentFlags().MarkHidden("bucket")
 }
 
+// checkKanikoDir will check whether the executor is operating in the default '/kaniko' directory,
+// conducting the relevant operations if it is not
+func checkKanikoDir(dir string) error {
+	if dir != constants.DefaultKanikoPath {
+
+		if err := os.MkdirAll(dir, os.ModeDir); err != nil {
+			return err
+		}
+
+		if err := os.Rename(constants.DefaultKanikoPath, dir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func checkContained() bool {
 	return proc.GetContainerRuntime(0, 0) != proc.RuntimeNotFound
 }
@@ -289,16 +311,16 @@ func resolveEnvironmentBuildArgs(arguments []string, resolver func(string) strin
 // copy Dockerfile to /kaniko/Dockerfile so that if it's specified in the .dockerignore
 // it won't be copied into the image
 func copyDockerfile() error {
-	if _, err := util.CopyFile(opts.DockerfilePath, constants.DockerfilePath, util.FileContext{}, util.DoNotChangeUID, util.DoNotChangeGID); err != nil {
+	if _, err := util.CopyFile(opts.DockerfilePath, config.DockerfilePath, util.FileContext{}, util.DoNotChangeUID, util.DoNotChangeGID); err != nil {
 		return errors.Wrap(err, "copying dockerfile")
 	}
 	dockerignorePath := opts.DockerfilePath + ".dockerignore"
 	if util.FilepathExists(dockerignorePath) {
-		if _, err := util.CopyFile(dockerignorePath, constants.DockerfilePath+".dockerignore", util.FileContext{}, util.DoNotChangeUID, util.DoNotChangeGID); err != nil {
+		if _, err := util.CopyFile(dockerignorePath, config.DockerfilePath+".dockerignore", util.FileContext{}, util.DoNotChangeUID, util.DoNotChangeGID); err != nil {
 			return errors.Wrap(err, "copying Dockerfile.dockerignore")
 		}
 	}
-	opts.DockerfilePath = constants.DockerfilePath
+	opts.DockerfilePath = config.DockerfilePath
 	return nil
 }
 

integration/integration_test.go

@@ -375,6 +375,46 @@ func TestBuildViaRegistryMirrors(t *testing.T) {
 	checkContainerDiffOutput(t, diff, expected)
 }
 
+// TestKanikoDir tests that a build that sets --kaniko-dir produces the same output as the equivalent docker build.
+func TestKanikoDir(t *testing.T) {
+	repo := getGitRepo(false)
+	dockerfile := fmt.Sprintf("%s/%s/Dockerfile_registry_mirror", integrationPath, dockerfilesPath)
+
+	// Build with docker
+	dockerImage := GetDockerImage(config.imageRepo, "Dockerfile_registry_mirror")
+	dockerCmd := exec.Command("docker",
+		append([]string{"build",
+			"-t", dockerImage,
+			"-f", dockerfile,
+			repo})...)
+	out, err := RunCommandWithoutTest(dockerCmd)
+	if err != nil {
+		t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out))
+	}
+
+	// Build with kaniko
+	kanikoImage := GetKanikoImage(config.imageRepo, "Dockerfile_registry_mirror")
+	dockerRunFlags := []string{"run", "--net=host"}
+	dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount)
+	dockerRunFlags = append(dockerRunFlags, ExecutorImage,
+		"-f", dockerfile,
+		"-d", kanikoImage,
+		"--kaniko-dir", "/not-kaniko",
+		"-c", fmt.Sprintf("git://%s", repo))
+
+	kanikoCmd := exec.Command("docker", dockerRunFlags...)
+
+	out, err = RunCommandWithoutTest(kanikoCmd)
+	if err != nil {
+		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
+	}
+
+	diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache")
+
+	expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage)
+	checkContainerDiffOutput(t, diff, expected)
+}
+
 func TestBuildWithLabels(t *testing.T) {
 	repo := getGitRepo(false)
 	dockerfile := fmt.Sprintf("%s/%s/Dockerfile_test_label", integrationPath, dockerfilesPath)

pkg/buildcontext/azureblob.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
+	kConfig "github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 )
@@ -55,7 +56,7 @@ func (b *AzureBlob) UnpackTarFromBuildContext() (string, error) {
 	}
 
 	// Create directory and target file for downloading the context file
-	directory := constants.BuildContextDir
+	directory := kConfig.BuildContextDir
 	tarPath := filepath.Join(directory, constants.ContextTar)
 	file, err := util.CreateTargetTarfile(tarPath)
 	if err != nil {

pkg/buildcontext/gcs.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 
 	"cloud.google.com/go/storage"
+	kConfig "github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/sirupsen/logrus"
@@ -36,7 +37,7 @@ type GCS struct {
 
 func (g *GCS) UnpackTarFromBuildContext() (string, error) {
 	bucket, item := util.GetBucketAndItem(g.context)
-	return constants.BuildContextDir, unpackTarFromGCSBucket(bucket, item, constants.BuildContextDir)
+	return kConfig.BuildContextDir, unpackTarFromGCSBucket(bucket, item, kConfig.BuildContextDir)
 }
 
 func UploadToBucket(r io.Reader, dest string) error {

pkg/buildcontext/git.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"strings"
 
+	kConfig "github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
@@ -31,8 +32,6 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/storage/filesystem"
 	"github.com/sirupsen/logrus"
-
-	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 )
 
 const (
@@ -57,7 +56,7 @@ type Git struct {
 
 // UnpackTarFromBuildContext will provide the directory where Git Repository is Cloned
 func (g *Git) UnpackTarFromBuildContext() (string, error) {
-	directory := constants.BuildContextDir
+	directory := kConfig.BuildContextDir
 	parts := strings.Split(g.context, "#")
 	url := getGitPullMethod() + "://" + parts[0]
 	options := git.CloneOptions{

pkg/buildcontext/https.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path/filepath"
 
+	kConfig "github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/sirupsen/logrus"
@@ -39,7 +40,7 @@ func (h *HTTPSTar) UnpackTarFromBuildContext() (directory string, err error) {
 	logrus.Info("Retrieving https tar file")
 
 	// Create directory and target file for downloading the context file
-	directory = constants.BuildContextDir
+	directory = kConfig.BuildContextDir
 	tarPath := filepath.Join(directory, constants.ContextTar)
 	file, err := util.CreateTargetTarfile(tarPath)
 	if err != nil {

pkg/buildcontext/s3.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	kConfig "github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/aws/aws-sdk-go/aws"
@@ -56,7 +57,7 @@ func (s *S3) UnpackTarFromBuildContext() (string, error) {
 		return bucket, err
 	}
 	downloader := s3manager.NewDownloader(sess)
-	directory := constants.BuildContextDir
+	directory := kConfig.BuildContextDir
 	tarPath := filepath.Join(directory, constants.ContextTar)
 	if err := os.MkdirAll(directory, 0750); err != nil {
 		return directory, err

pkg/buildcontext/tar.go

@@ -21,7 +21,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	kConfig "github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -34,7 +34,7 @@ type Tar struct {
 
 // UnpackTarFromBuildContext unpack the compressed tar file
 func (t *Tar) UnpackTarFromBuildContext() (string, error) {
-	directory := constants.BuildContextDir
+	directory := kConfig.BuildContextDir
 	if err := os.MkdirAll(directory, 0750); err != nil {
 		return "", errors.Wrap(err, "unpacking tar from build context")
 	}

pkg/config/init.go

@@ -17,15 +17,36 @@ limitations under the License.
 package config
 
 import (
+	"fmt"
+	"os"
+
 	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 )
 
 var RootDir string
-var KanikoDir string
+
+// KanikoDir is the path to the Kaniko directory
+var KanikoDir = func() string {
+	if kd, ok := os.LookupEnv("KANIKO_DIR"); ok {
+		return kd
+	}
+	return constants.DefaultKanikoPath
+}()
+
+// DockerfilePath is the path the Dockerfile is copied to
+var DockerfilePath = fmt.Sprintf("%s/Dockerfile", KanikoDir)
+
+// BuildContextDir is the directory a build context will be unpacked into,
+// for example, a tarball from a GCS bucket will be unpacked here
+var BuildContextDir = fmt.Sprintf("%s/buildcontext/", KanikoDir)
+
+// KanikoIntermediateStagesDir is where we will store intermediate stages
+// as tarballs in case they are needed later on
+var KanikoIntermediateStagesDir = fmt.Sprintf("%s/stages/", KanikoDir)
+
 var IgnoreListPath string
 
 func init() {
 	RootDir = constants.RootDir
-	KanikoDir = constants.KanikoDir
 	IgnoreListPath = constants.IgnoreListPath
 }

pkg/config/options.go

@@ -58,6 +58,7 @@ type KanikoOptions struct {
 	CustomPlatform         string
 	Bucket                 string
 	TarPath                string
+	KanikoDir              string
 	Target                 string
 	CacheRepo              string
 	DigestFile             string

pkg/constants/constants.go

@@ -20,27 +20,15 @@ const (
 	// RootDir is the path to the root directory
 	RootDir = "/"
 
-	// KanikoDir is the path to the Kaniko directory
-	KanikoDir = "/kaniko"
-
 	IgnoreListPath = "/proc/self/mountinfo"
 
+	DefaultKanikoPath = "/kaniko"
+
 	Author = "kaniko"
 
-	// DockerfilePath is the path the Dockerfile is copied to
-	DockerfilePath = "/kaniko/Dockerfile"
-
 	// ContextTar is the default name of the tar uploaded to GCS buckets
 	ContextTar = "context.tar.gz"
 
-	// BuildContextDir is the directory a build context will be unpacked into,
-	// for example, a tarball from a GCS bucket will be unpacked here
-	BuildContextDir = "/kaniko/buildcontext/"
-
-	// KanikoIntermediateStagesDir is where we will store intermediate stages
-	// as tarballs in case they are needed later on
-	KanikoIntermediateStagesDir = "/kaniko/stages"
-
 	// Various snapshot modes:
 	SnapshotModeTime = "time"
 	SnapshotModeFull = "full"

pkg/executor/build.go

@@ -829,7 +829,7 @@ func saveStageAsTarball(path string, image v1.Image) error {
 	if err != nil {
 		return err
 	}
-	tarPath := filepath.Join(constants.KanikoIntermediateStagesDir, path)
+	tarPath := filepath.Join(config.KanikoIntermediateStagesDir, path)
 	logrus.Infof("Storing source image from stage %s at path %s", path, tarPath)
 	if err := os.MkdirAll(filepath.Dir(tarPath), 0750); err != nil {
 		return err

pkg/executor/copy_multistage_test.go

@@ -177,7 +177,6 @@ func setupMultistageTests(t *testing.T) (string, func()) {
 	}
 	config.IgnoreListPath = mFile
 	return testDir, func() {
-		config.KanikoDir = constants.KanikoDir
 		config.RootDir = constants.RootDir
 		config.IgnoreListPath = constants.IgnoreListPath
 	}

pkg/image/image_util.go

@@ -92,7 +92,7 @@ func RetrieveSourceImage(stage config.KanikoStage, opts *config.KanikoOptions) (
 }
 
 func tarballImage(index int) (v1.Image, error) {
-	tarPath := filepath.Join(constants.KanikoIntermediateStagesDir, strconv.Itoa(index))
+	tarPath := filepath.Join(config.KanikoIntermediateStagesDir, strconv.Itoa(index))
 	logrus.Infof("Base image from previous stage %d found, using saved tar at path %s", index, tarPath)
 	return tarball.ImageFromPath(tarPath, nil)
 }