Adding TTL to caching (#488)

* Adding TTL to caching

* uncomment added lines

* cache TTL works for layers now

* remove debugging

* parse booleans correctly

* parse booleans correctly everywhere

* fix boolean parsing condition

* refactor benchmarking calls

* defer file properly

cmd/executor/cmd/root.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/buildcontext"
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
@@ -109,6 +110,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) {
 	RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image")
 	RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end")
+	RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.")
 }
 
 // addHiddenFlags marks certain flags as hidden from the executor help text

integration/images.go

@@ -188,7 +188,7 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do
 	}
 
 	benchmarkEnv := "BENCHMARK_FILE=false"
-	if os.Getenv("BENCHMARK") == "true" {
+	if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b {
 		os.Mkdir("benchmarks", 0755)
 		benchmarkEnv = "BENCHMARK_FILE=/workspace/benchmarks/" + dockerfile
 	}
@@ -247,11 +247,17 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesP
 	cacheFlag := "--cache=true"
 
 	for dockerfile := range d.TestCacheDockerfiles {
+		benchmarkEnv := "BENCHMARK_FILE=false"
+		if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b {
+			os.Mkdir("benchmarks", 0755)
+			benchmarkEnv = "BENCHMARK_FILE=/workspace/benchmarks/" + dockerfile
+		}
 		kanikoImage := GetVersionedKanikoImage(imageRepo, dockerfile, version)
 		kanikoCmd := exec.Command("docker",
 			append([]string{"run",
 				"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
 				"-v", cwd + ":/workspace",
+				"-e", benchmarkEnv,
 				ExecutorImage,
 				"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
 				"-d", kanikoImage,
@@ -261,7 +267,10 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesP
 				"--cache-dir", cacheDir})...,
 		)
 
-		if _, err := RunCommandWithoutTest(kanikoCmd); err != nil {
+		timer := timing.Start(dockerfile + "_kaniko_cached_" + strconv.Itoa(version))
+		_, err := RunCommandWithoutTest(kanikoCmd)
+		timing.DefaultRun.Stop(timer)
+		if err != nil {
 			return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %s", kanikoImage, kanikoCmd.Args, err)
 		}
 	}

integration/integration_test.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -217,14 +218,9 @@ func TestRun(t *testing.T) {
 		})
 	}
 
-	if os.Getenv("BENCHMARK") == "true" {
+	err := logBenchmarks("benchmark")
-		f, err := os.Create("benchmark")
+	if err != nil {
-		if err != nil {
+		t.Logf("Failed to create benchmark file: %v", err)
-			t.Logf("Failed to create benchmark file")
-		} else {
-			f.WriteString(timing.Summary())
-		}
-		defer f.Close()
 	}
 }
 
@@ -252,6 +248,11 @@ func TestLayers(t *testing.T) {
 			checkLayers(t, dockerImage, kanikoImage, offset[dockerfile])
 		})
 	}
+
+	err := logBenchmarks("benchmark_layers")
+	if err != nil {
+		t.Logf("Failed to create benchmark file: %v", err)
+	}
 }
 
 // Build each image with kaniko twice, and then make sure they're exactly the same
@@ -284,6 +285,11 @@ func TestCache(t *testing.T) {
 			checkContainerDiffOutput(t, diff, expected)
 		})
 	}
+
+	err := logBenchmarks("benchmark_cache")
+	if err != nil {
+		t.Logf("Failed to create benchmark file: %v", err)
+	}
 }
 
 type fileDiff struct {
@@ -386,3 +392,15 @@ func getImageDetails(image string) (*imageDetails, error) {
 		digest:    digest.Hex,
 	}, nil
 }
+
+func logBenchmarks(benchmark string) error {
+	if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b {
+		f, err := os.Create(benchmark)
+		if err != nil {
+			return err
+		}
+		f.WriteString(timing.Summary())
+		defer f.Close()
+	}
+	return nil
+}

pkg/cache/cache.go

@@ -18,7 +18,9 @@ package cache
 
 import (
 	"fmt"
+	"os"
 	"path"
+	"time"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/google/go-containerregistry/pkg/authn"
@@ -31,14 +33,17 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// LayerCache is the layer cache
 type LayerCache interface {
 	RetrieveLayer(string) (v1.Image, error)
 }
 
+// RegistryCache is the registry cache
 type RegistryCache struct {
 	Opts *config.KanikoOptions
 }
 
+// RetrieveLayer retrieves a layer from the cache given the cache key ck.
 func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) {
 	cache, err := Destination(rc.Opts, ck)
 	if err != nil {
@@ -59,6 +64,19 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	cf, err := img.ConfigFile()
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("retrieving config file for %s", cache))
+	}
+
+	expiry := cf.Created.Add(rc.Opts.CacheTTL)
+	// Layer is stale, rebuild it.
+	if expiry.Before(time.Now()) {
+		logrus.Infof("Cache entry expired: %s", cache)
+		return nil, errors.New(fmt.Sprintf("Cache entry expired: %s", cache))
+	}
+
 	// Force the manifest to be populated
 	if _, err := img.RawManifest(); err != nil {
 		return nil, err
@@ -81,6 +99,7 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) {
 	return fmt.Sprintf("%s:%s", cache, cacheKey), nil
 }
 
+// LocalSource retieves a source image from a local cache given cacheKey
 func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) {
 	cache := opts.CacheDir
 	if cache == "" {
@@ -89,6 +108,18 @@ func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error)
 
 	path := path.Join(cache, cacheKey)
 
+	fi, err := os.Stat(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "geting file info")
+	}
+
+	// A stale cache is a bad cache
+	expiry := fi.ModTime().Add(opts.CacheTTL)
+	if expiry.Before(time.Now()) {
+		logrus.Debugf("Cached image is too old: %v", fi.ModTime())
+		return nil, nil
+	}
+
 	imgTar, err := tarball.ImageFromPath(path, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting image from path")

pkg/config/options.go

@@ -16,6 +16,10 @@ limitations under the License.
 
 package config
 
+import (
+	"time"
+)
+
 // KanikoOptions are options that are set by command line arguments
 type KanikoOptions struct {
 	DockerfilePath    string
@@ -37,6 +41,7 @@ type KanikoOptions struct {
 	NoPush            bool
 	Cache             bool
 	Cleanup           bool
+	CacheTTL          time.Duration
 }
 
 // WarmerOptions are options that are set by command line arguments to the cache warmer.

pkg/executor/push.go

@@ -20,6 +20,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/cache"
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
@@ -118,6 +119,11 @@ func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath strin
 	}
 	logrus.Infof("Pushing layer %s to cache now", cache)
 	empty := empty.Image
+	empty, err = mutate.CreatedAt(empty, v1.Time{Time: time.Now()})
+	if err != nil {
+		return errors.Wrap(err, "setting empty image created time")
+	}
+
 	empty, err = mutate.Append(empty,
 		mutate.Addendum{
 			Layer: layer,