Merge pull request #927 from cvgw/u/cgwippern/issue-926_fix_cache_warmer_and_method_sig

Fix #926 cache warmer and method signature

pkg/cache/cache.go

@@ -124,15 +124,17 @@ func LocalSource(opts *config.CacheOptions, cacheKey string) (v1.Image, error) {
 
 	fi, err := os.Stat(path)
 	if err != nil {
-		logrus.Debugf("No file found for cache key %v %v", cacheKey, err)
-		return nil, nil
+		msg := fmt.Sprintf("No file found for cache key %v %v", cacheKey, err)
+		logrus.Debug(msg)
+		return nil, NotFoundErr{msg: msg}
 	}
 
 	// A stale cache is a bad cache
 	expiry := fi.ModTime().Add(opts.CacheTTL)
 	if expiry.Before(time.Now()) {
-		logrus.Debugf("Cached image is too old: %v", fi.ModTime())
-		return nil, nil
+		msg := fmt.Sprintf("Cached image is too old: %v", fi.ModTime())
+		logrus.Debug(msg)
+		return nil, ExpiredErr{msg: msg}
 	}
 
 	logrus.Infof("Found %s in local cache", cacheKey)

pkg/cache/doc_test.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2019 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"bytes"
+	"log"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/config"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+)
+
+func ExampleWarmer_Warm() {
+	tarBuf := new(bytes.Buffer)
+	manifestBuf := new(bytes.Buffer)
+	w := &Warmer{
+		Remote:         remote.Image,
+		Local:          LocalSource,
+		TarWriter:      tarBuf,
+		ManifestWriter: manifestBuf,
+	}
+
+	options := &config.WarmerOptions{}
+
+	digest, err := w.Warm("ubuntu:latest", options)
+	if err != nil {
+		if !IsAlreadyCached(err) {
+			log.Fatal(err)
+		}
+	}
+
+	log.Printf("digest %v tar len %d\nmanifest:\n%s\n", digest, tarBuf.Len(), manifestBuf.String())
+}

pkg/cache/errors.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2019 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+// IsAlreadyCached returns true if the supplied error is of the type AlreadyCachedErr
+// otherwise it returns false.
+func IsAlreadyCached(err error) bool {
+	switch err.(type) {
+	case AlreadyCachedErr:
+		return true
+	}
+
+	return false
+}
+
+// AlreadyCachedErr is returned when the Docker image requested for caching is already
+// present in the cache.
+type AlreadyCachedErr struct {
+	msg string
+}
+
+func (a AlreadyCachedErr) Error() string {
+	return a.msg
+}
+
+// IsNotFound returns true if the supplied error is of the type NotFoundErr
+// otherwise it returns false.
+func IsNotFound(e error) bool {
+	switch e.(type) {
+	case NotFoundErr:
+		return true
+	}
+
+	return false
+}
+
+// NotFoundErr is returned when the requested Docker image is not present in the cache.
+type NotFoundErr struct {
+	msg string
+}
+
+func (e NotFoundErr) Error() string {
+	return e.msg
+}
+
+// IsExpired returns true if the supplied error is of the type ExpiredErr
+// otherwise it returns false.
+func IsExpired(e error) bool {
+	switch e.(type) {
+	case ExpiredErr:
+		return true
+	}
+
+	return false
+}
+
+// ExpiredErr is returned when the requested Docker image is present in the cache, but is
+// expired according to the supplied TTL.
+type ExpiredErr struct {
+	msg string
+}
+
+func (e ExpiredErr) Error() string {
+	return e.msg
+}

pkg/cache/warm.go

@@ -17,12 +17,15 @@ limitations under the License.
 package cache
 
 import (
-	"fmt"
+	"bytes"
+	"io"
 	"io/ioutil"
+	"os"
 	"path"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/pkg/errors"
@@ -37,42 +40,111 @@ func WarmCache(opts *config.WarmerOptions) error {
 	logrus.Debugf("%s\n", images)
 
 	for _, image := range images {
-		cacheRef, err := name.NewTag(image, name.WeakValidation)
-		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image))
-		}
-		img, err := remote.Image(cacheRef)
-		if err != nil || img == nil {
-			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image))
+		tarBuf := new(bytes.Buffer)
+		manifestBuf := new(bytes.Buffer)
+
+		cw := &Warmer{
+			Remote:         remote.Image,
+			Local:          LocalSource,
+			TarWriter:      tarBuf,
+			ManifestWriter: manifestBuf,
 		}
 
-		digest, err := img.Digest()
+		digest, err := cw.Warm(image, opts)
 		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image))
+			if !IsAlreadyCached(err) {
+				return err
+			}
+
+			continue
 		}
+
 		cachePath := path.Join(cacheDir, digest.String())
 
-		if !opts.Force {
-			_, err := LocalSource(&opts.CacheOptions, digest.String())
-			if err == nil {
-				continue
-			}
+		if err := writeBufsToFile(cachePath, tarBuf, manifestBuf); err != nil {
+			return err
 		}
 
-		err = tarball.WriteToFile(cachePath, cacheRef, img)
-		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image))
-		}
-
-		mfst, err := img.RawManifest()
-		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve manifest for %s", image))
-		}
-		mfstPath := cachePath + ".json"
-		if err := ioutil.WriteFile(mfstPath, mfst, 0666); err != nil {
-			return errors.Wrap(err, fmt.Sprintf("Failed to save manifest for %s", image))
-		}
 		logrus.Debugf("Wrote %s to cache", image)
 	}
 	return nil
 }
+
+func writeBufsToFile(cachePath string, tarBuf, manifestBuf *bytes.Buffer) error {
+	f, err := os.Create(cachePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if _, err := f.Write(tarBuf.Bytes()); err != nil {
+		return errors.Wrap(err, "Failed to save tar to file")
+	}
+
+	mfstPath := cachePath + ".json"
+	if err := ioutil.WriteFile(mfstPath, manifestBuf.Bytes(), 0666); err != nil {
+		return errors.Wrap(err, "Failed to save manifest to file")
+	}
+
+	return nil
+}
+
+// FetchRemoteImage retrieves a Docker image manifest from a remote source.
+// github.com/google/go-containerregistry/pkg/v1/remote.Image can be used as
+// this type.
+type FetchRemoteImage func(name.Reference, ...remote.Option) (v1.Image, error)
+
+// FetchLocalSource retrieves a Docker image manifest from a local source.
+// github.com/GoogleContainerTools/kaniko/cache.LocalSource can be used as
+// this type.
+type FetchLocalSource func(*config.CacheOptions, string) (v1.Image, error)
+
+// Warmer is used to prepopulate the cache with a Docker image
+type Warmer struct {
+	Remote         FetchRemoteImage
+	Local          FetchLocalSource
+	TarWriter      io.Writer
+	ManifestWriter io.Writer
+}
+
+// Warm retrieves a Docker image and populates the supplied buffer with the image content and manifest
+// or returns an AlreadyCachedErr if the image is present in the cache.
+func (w *Warmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error) {
+	cacheRef, err := name.NewTag(image, name.WeakValidation)
+	if err != nil {
+		return v1.Hash{}, errors.Wrapf(err, "Failed to verify image name: %s", image)
+	}
+
+	img, err := w.Remote(cacheRef)
+	if err != nil || img == nil {
+		return v1.Hash{}, errors.Wrapf(err, "Failed to retrieve image: %s", image)
+	}
+
+	digest, err := img.Digest()
+	if err != nil {
+		return v1.Hash{}, errors.Wrapf(err, "Failed to retrieve digest: %s", image)
+	}
+
+	if !opts.Force {
+		_, err := w.Local(&opts.CacheOptions, digest.String())
+		if err == nil || IsExpired(err) {
+			return v1.Hash{}, AlreadyCachedErr{}
+		}
+	}
+
+	err = tarball.Write(cacheRef, img, w.TarWriter)
+	if err != nil {
+		return v1.Hash{}, errors.Wrapf(err, "Failed to write %s to tar buffer", image)
+	}
+
+	mfst, err := img.RawManifest()
+	if err != nil {
+		return v1.Hash{}, errors.Wrapf(err, "Failed to retrieve manifest for %s", image)
+	}
+
+	if _, err := w.ManifestWriter.Write(mfst); err != nil {
+		return v1.Hash{}, errors.Wrapf(err, "Failed to save manifest to buffer for %s", image)
+	}
+
+	return digest, nil
+}

pkg/cache/warm_test.go

@@ -0,0 +1,116 @@
+/*
+Copyright 2019 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/config"
+	"github.com/GoogleContainerTools/kaniko/pkg/fakes"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+)
+
+const (
+	image = "foo:latest"
+)
+
+func Test_Warmer_Warm_not_in_cache(t *testing.T) {
+	tarBuf := new(bytes.Buffer)
+	manifestBuf := new(bytes.Buffer)
+
+	cw := &Warmer{
+		Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) {
+			return fakes.FakeImage{}, nil
+		},
+		Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) {
+			return nil, NotFoundErr{}
+		},
+		TarWriter:      tarBuf,
+		ManifestWriter: manifestBuf,
+	}
+
+	opts := &config.WarmerOptions{}
+
+	_, err := cw.Warm(image, opts)
+	if err != nil {
+		t.Errorf("expected error to be nil but was %v", err)
+		t.FailNow()
+	}
+
+	if len(tarBuf.Bytes()) == 0 {
+		t.Error("expected image to be written but buffer was empty")
+	}
+}
+
+func Test_Warmer_Warm_in_cache_not_expired(t *testing.T) {
+	tarBuf := new(bytes.Buffer)
+	manifestBuf := new(bytes.Buffer)
+
+	cw := &Warmer{
+		Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) {
+			return fakes.FakeImage{}, nil
+		},
+		Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) {
+			return fakes.FakeImage{}, nil
+		},
+		TarWriter:      tarBuf,
+		ManifestWriter: manifestBuf,
+	}
+
+	opts := &config.WarmerOptions{}
+
+	_, err := cw.Warm(image, opts)
+	if !IsAlreadyCached(err) {
+		t.Errorf("expected error to be already cached err but was %v", err)
+		t.FailNow()
+	}
+
+	if len(tarBuf.Bytes()) != 0 {
+		t.Errorf("expected nothing to be written")
+	}
+}
+
+func Test_Warmer_Warm_in_cache_expired(t *testing.T) {
+	tarBuf := new(bytes.Buffer)
+	manifestBuf := new(bytes.Buffer)
+
+	cw := &Warmer{
+		Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) {
+			return fakes.FakeImage{}, nil
+		},
+		Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) {
+			return fakes.FakeImage{}, ExpiredErr{}
+		},
+		TarWriter:      tarBuf,
+		ManifestWriter: manifestBuf,
+	}
+
+	opts := &config.WarmerOptions{}
+
+	_, err := cw.Warm(image, opts)
+	if !IsAlreadyCached(err) {
+		t.Errorf("expected error to be already cached err but was %v", err)
+		t.FailNow()
+	}
+
+	if len(tarBuf.Bytes()) != 0 {
+		t.Errorf("expected nothing to be written")
+	}
+}

pkg/fakes/image.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2019 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fakes
+
+import (
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/types"
+)
+
+type FakeImage struct {
+	Hash v1.Hash
+}
+
+func (f FakeImage) Layers() ([]v1.Layer, error) {
+	return nil, nil
+}
+func (f FakeImage) MediaType() (types.MediaType, error) {
+	return "", nil
+}
+func (f FakeImage) Size() (int64, error) {
+	return 0, nil
+}
+func (f FakeImage) ConfigName() (v1.Hash, error) {
+	return v1.Hash{}, nil
+}
+func (f FakeImage) ConfigFile() (*v1.ConfigFile, error) {
+	return &v1.ConfigFile{}, nil
+}
+func (f FakeImage) RawConfigFile() ([]byte, error) {
+	return []byte{}, nil
+}
+func (f FakeImage) Digest() (v1.Hash, error) {
+	return f.Hash, nil
+}
+func (f FakeImage) Manifest() (*v1.Manifest, error) {
+	return &v1.Manifest{}, nil
+}
+func (f FakeImage) RawManifest() ([]byte, error) {
+	return []byte{}, nil
+}
+func (f FakeImage) LayerByDigest(v1.Hash) (v1.Layer, error) {
+	return nil, nil
+}
+func (f FakeImage) LayerByDiffID(v1.Hash) (v1.Layer, error) {
+	return nil, nil
+}

pkg/util/image_util.go

@@ -75,12 +75,19 @@ func RetrieveSourceImage(stage config.KanikoStage, opts *config.KanikoOptions) (
 	if opts.CacheDir != "" {
 		cachedImage, err := cachedImage(opts, currentBaseName)
 		if err != nil {
-			logrus.Errorf("Error while retrieving image from cache: %v %v", currentBaseName, err)
+			switch {
+			case cache.IsNotFound(err):
+				logrus.Debugf("Image %v not found in cache", currentBaseName)
+			case cache.IsExpired(err):
+				logrus.Debugf("Image %v found in cache but was expired", currentBaseName)
+			default:
+				logrus.Errorf("Error while retrieving image from cache: %v %v", currentBaseName, err)
+			}
 		} else if cachedImage != nil {
 			return cachedImage, nil
 		}
 	}
-	logrus.Infof("Image %v not found in cache", currentBaseName)
+
 	// Otherwise, initialize image as usual
 	return RetrieveRemoteImage(currentBaseName, opts)
 }