Merge pull request #975 from cvgw/u/cgwippern/fix-issue-506

Fix #506 add option additional-whitelist

README.md

@@ -63,6 +63,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME
     - [--single-snapshot](#--single-snapshot)
     - [--skip-tls-verify](#--skip-tls-verify)
     - [--skip-tls-verify-pull](#--skip-tls-verify-pull)
+    - [--additional-whitelist](#--additional-whitelist)
     - [--snapshotMode](#--snapshotmode)
     - [--target](#--target)
     - [--tarPath](#--tarpath)
@@ -493,6 +494,9 @@ Set this flag to skip TLS certificate validation when pushing to a registry. It
 
 Set this flag to skip TLS certificate validation when pulling from a registry. It is supposed to be used for testing purposes only and should not be used in production!
 
+#### --additional-whitelist
+Set this flag with a list of filepaths and Kaniko will ignore these paths during the build. Useful for improving build performance on large filesystems.
+
 #### --snapshotMode
 
 You can set the `--snapshotMode=<full (default), time>` flag to set how kaniko will snapshot the filesystem.

cmd/executor/cmd/root.go

@@ -38,9 +38,10 @@ import (
 )
 
 var (
-	opts     = &config.KanikoOptions{}
-	logLevel string
-	force    bool
+	opts                = &config.KanikoOptions{}
+	logLevel            string
+	force               bool
+	additionalWhitelist []string
 )
 
 func init() {
@@ -73,6 +74,10 @@ var RootCmd = &cobra.Command{
 			if len(opts.Destinations) == 0 && opts.ImageNameDigestFile != "" {
 				return errors.New("You must provide --destination if setting ImageNameDigestFile")
 			}
+
+			for _, path := range additionalWhitelist {
+				util.AddToWhitelist(path)
+			}
 		}
 		return nil
 	},
@@ -144,6 +149,10 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.")
 	RootCmd.PersistentFlags().VarP(&opts.InsecureRegistries, "insecure-registry", "", "Insecure registry using plain HTTP to push and pull. Set it repeatedly for multiple registries.")
 	RootCmd.PersistentFlags().VarP(&opts.SkipTLSVerifyRegistries, "skip-tls-verify-registry", "", "Insecure registry ignoring TLS verify to push and pull. Set it repeatedly for multiple registries.")
+
+	// We use nil as the default value so we can differentiate between the flag passed
+	// with an empty list and the flag not set
+	RootCmd.PersistentFlags().StringSliceVar(&additionalWhitelist, "additional-whitelist", []string{}, "Paths to whitelist. These will be ignored by kaniko to improve performance.")
 }
 
 // addHiddenFlags marks certain flags as hidden from the executor help text

pkg/util/fs_util.go

@@ -48,13 +48,6 @@ var initialWhitelist = []WhitelistEntry{
 		Path:            "/kaniko",
 		PrefixMatchOnly: false,
 	},
-	{
-		// /var/run is a special case. It's common to mount in /var/run/docker.sock or something similar
-		// which leads to a special mount on the /var/run/docker.sock file itself, but the directory to exist
-		// in the image with no way to tell if it came from the base image or not.
-		Path:            "/var/run",
-		PrefixMatchOnly: false,
-	},
 	{
 		// similarly, we whitelist /etc/mtab, since there is no way to know if the file was mounted or came
 		// from the base image
@@ -69,6 +62,10 @@ var volumes = []string{}
 
 var excluded []string
 
+func AddToWhitelist(path string) {
+	initialWhitelist = append(initialWhitelist, WhitelistEntry{Path: path})
+}
+
 type ExtractFunction func(string, *tar.Header, io.Reader) error
 
 type FSConfig struct {

pkg/util/fs_util_test.go

@@ -38,43 +38,96 @@ import (
 )
 
 func Test_DetectFilesystemWhitelist(t *testing.T) {
-	testDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("Error creating tempdir: %s", err)
-	}
-	fileContents := `
-	228 122 0:90 / / rw,relatime - aufs none rw,si=f8e2406af90782bc,dio,dirperm1
-	229 228 0:98 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
-	230 228 0:99 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
-	231 230 0:100 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
-	232 228 0:101 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro`
-
-	path := filepath.Join(testDir, "mountinfo")
-	if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
-		t.Fatalf("Error creating tempdir: %s", err)
-	}
-	if err := ioutil.WriteFile(path, []byte(fileContents), 0644); err != nil {
-		t.Fatalf("Error writing file contents to %s: %s", path, err)
+	type testcase struct {
+		desc                string
+		additionalWhitelist []string
+		expectedWhitelist   []WhitelistEntry
 	}
 
-	err = DetectFilesystemWhitelist(path)
 	expectedWhitelist := []WhitelistEntry{
 		{"/kaniko", false},
 		{"/proc", false},
 		{"/dev", false},
 		{"/dev/pts", false},
 		{"/sys", false},
-		{"/var/run", false},
 		{"/etc/mtab", false},
 	}
-	actualWhitelist := whitelist
-	sort.Slice(actualWhitelist, func(i, j int) bool {
-		return actualWhitelist[i].Path < actualWhitelist[j].Path
-	})
-	sort.Slice(expectedWhitelist, func(i, j int) bool {
-		return expectedWhitelist[i].Path < expectedWhitelist[j].Path
-	})
-	testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist)
+
+	testCases := []testcase{
+		{
+			desc:              "no additional whitelist",
+			expectedWhitelist: expectedWhitelist,
+		},
+		{
+			desc:                "one additional whitelist - /var/run",
+			additionalWhitelist: []string{"/var/run"},
+			expectedWhitelist:   append(expectedWhitelist, WhitelistEntry{"/var/run", false}),
+		},
+		{
+			desc:                "two additional whitelist - /var/run, /usr/bin",
+			additionalWhitelist: []string{"/var/run", "/usr/bin"},
+			expectedWhitelist: append(
+				expectedWhitelist,
+				WhitelistEntry{"/var/run", false},
+				WhitelistEntry{"/usr/bin", false},
+			),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			expectedWhitelist := tc.expectedWhitelist
+			additionalWhitelist := tc.additionalWhitelist
+
+			tmpWhitelist := make([]WhitelistEntry, len(initialWhitelist))
+			copy(tmpWhitelist, initialWhitelist)
+
+			testDir, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			fileContents := `
+	228 122 0:90 / / rw,relatime - aufs none rw,si=f8e2406af90782bc,dio,dirperm1
+	229 228 0:98 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+	230 228 0:99 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755
+	231 230 0:100 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+	232 228 0:101 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro`
+
+			path := filepath.Join(testDir, "mountinfo")
+			if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
+				t.Fatalf("Error creating tempdir: %s", err)
+			}
+			if err := ioutil.WriteFile(path, []byte(fileContents), 0644); err != nil {
+				t.Fatalf("Error writing file contents to %s: %s", path, err)
+			}
+
+			for _, wl := range additionalWhitelist {
+				AddToWhitelist(wl)
+			}
+
+			err = DetectFilesystemWhitelist(path)
+			actualWhitelist := whitelist
+
+			if len(actualWhitelist) != len(expectedWhitelist) {
+				t.Errorf(
+					"expected whitelist to have %d items but was %d",
+					len(expectedWhitelist),
+					len(actualWhitelist),
+				)
+			}
+
+			sort.Slice(actualWhitelist, func(i, j int) bool {
+				return actualWhitelist[i].Path < actualWhitelist[j].Path
+			})
+			sort.Slice(expectedWhitelist, func(i, j int) bool {
+				return expectedWhitelist[i].Path < expectedWhitelist[j].Path
+			})
+
+			testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist)
+
+			initialWhitelist = tmpWhitelist
+		})
+	}
 }
 
 var tests = []struct {