feat: cache dockerfile images through warmer (#2499)

* feat: cache dockerfile images through warmer

* Fix logical error in conditional statement

* Addressed review feedback

1. Updated help text for the --build-arg flag to indicate it should be used with the dockerfile flag.
2. Updated the documentation to include the optional --build-arg flag.
3. Added unit tests for `ParseDockerfile`, covering scenarios for missing Dockerfile, invalid Dockerfile, single stage Dockerfile, multi-stage Dockerfile and Args Dockerfile

---------

Co-authored-by: 连奔驰 <benchi.lian@thoughtworks.com>

README.md

@@ -505,9 +505,12 @@ provide a kaniko cache warming image at `gcr.io/kaniko-project/warmer`:
 
 ```shell
 docker run -v $(pwd):/workspace gcr.io/kaniko-project/warmer:latest --cache-dir=/workspace/cache --image=<image to cache> --image=<another image to cache>
+docker run -v $(pwd):/workspace gcr.io/kaniko-project/warmer:latest --cache-dir=/workspace/cache --dockerfile=<path to dockerfile>
+docker run -v $(pwd):/workspace gcr.io/kaniko-project/warmer:latest --cache-dir=/workspace/cache --dockerfile=<path to dockerfile> --build-args version=1.19
 ```
 
-`--image` can be specified for any number of desired images. This command will
+`--image` can be specified for any number of desired images. `--dockerfile` can 
+be specified for the path of dockerfile for cache.These command will combined to 
 cache those images by digest in a local directory named `cache`. Once the cache
 is populated, caching is opted into with the same `--cache=true` flag as above.
 The location of the local cache is provided via the `--cache-dir` flag,

cmd/warmer/cmd/root.go

@@ -19,11 +19,14 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"regexp"
 	"time"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/cache"
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/logging"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/containerd/containerd/platforms"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/pkg/errors"
@@ -54,9 +57,14 @@ var RootCmd = &cobra.Command{
 			return err
 		}
 
-		if len(opts.Images) == 0 {
+		if len(opts.Images) == 0 && opts.DockerfilePath == "" {
-			return errors.New("You must select at least one image to cache")
+			return errors.New("You must select at least one image to cache or a dockerfilepath to parse")
 		}
+
+		if err := validateDockerfilePath(); err != nil {
+			return errors.Wrap(err, "error validating dockerfile path")
+		}
+
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
@@ -89,6 +97,8 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().VarP(&opts.RegistriesClientCertificates, "registry-client-cert", "", "Use the provided client certificate for mutual TLS (mTLS) communication with the given registry. Expected format is 'my.registry.url=/path/to/client/cert,/path/to/client/key'.")
 	RootCmd.PersistentFlags().VarP(&opts.RegistryMirrors, "registry-mirror", "", "Registry mirror to use as pull-through cache instead of docker.io. Set it repeatedly for multiple mirrors.")
 	RootCmd.PersistentFlags().StringVarP(&opts.CustomPlatform, "customPlatform", "", "", "Specify the build platform if different from the current host")
+	RootCmd.PersistentFlags().StringVarP(&opts.DockerfilePath, "dockerfile", "d", "Dockerfile", "Path to the dockerfile to be cached. The kaniko warmer will parse and write out each stage's base image layers to the cache-dir. Using the same dockerfile path as what you plan to build in the kaniko executor is the expected usage.")
+	RootCmd.PersistentFlags().VarP(&opts.BuildArgs, "build-arg", "", "This flag should be used in conjunction with the dockerfile flag for scenarios where dynamic replacement of the base image is required.")
 
 	// Default the custom platform flag to our current platform, and validate it.
 	if opts.CustomPlatform == "" {
@@ -104,6 +114,29 @@ func addHiddenFlags() {
 	RootCmd.PersistentFlags().MarkHidden("azure-container-registry-config")
 }
 
+func validateDockerfilePath() error {
+	if isURL(opts.DockerfilePath) {
+		return nil
+	}
+	if util.FilepathExists(opts.DockerfilePath) {
+		abs, err := filepath.Abs(opts.DockerfilePath)
+		if err != nil {
+			return errors.Wrap(err, "getting absolute path for dockerfile")
+		}
+		opts.DockerfilePath = abs
+		return nil
+	}
+
+	return errors.New("please provide a valid path to a Dockerfile within the build context with --dockerfile")
+}
+
+func isURL(path string) bool {
+	if match, _ := regexp.MatchString("^https?://", path); match {
+		return true
+	}
+	return false
+}
+
 func exit(err error) {
 	fmt.Println(err)
 	os.Exit(1)

pkg/cache/warm.go

@@ -18,13 +18,18 @@ package cache
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
+	"regexp"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
+	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
 	"github.com/GoogleContainerTools/kaniko/pkg/image/remote"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
@@ -34,8 +39,21 @@ import (
 
 // WarmCache populates the cache
 func WarmCache(opts *config.WarmerOptions) error {
+	var dockerfileImages []string
 	cacheDir := opts.CacheDir
 	images := opts.Images
+
+	// if opts.image is empty,we need to parse dockerfilepath to get images list
+	if opts.DockerfilePath != "" {
+		var err error
+		if dockerfileImages, err = ParseDockerfile(opts); err != nil {
+			return errors.Wrap(err, "failed to parse Dockerfile")
+		}
+	}
+
+	// TODO: Implement deduplication logic later.
+	images = append(images, dockerfileImages...)
+
 	logrus.Debugf("%s\n", cacheDir)
 	logrus.Debugf("%s\n", images)
 
@@ -157,3 +175,41 @@ func (w *Warmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error)
 
 	return digest, nil
 }
+
+func ParseDockerfile(opts *config.WarmerOptions) ([]string, error) {
+	var err error
+	var d []uint8
+	var baseNames []string
+	match, _ := regexp.MatchString("^https?://", opts.DockerfilePath)
+	if match {
+		response, e := http.Get(opts.DockerfilePath) //nolint:noctx
+		if e != nil {
+			return nil, e
+		}
+		d, err = ioutil.ReadAll(response.Body)
+	} else {
+		d, err = ioutil.ReadFile(opts.DockerfilePath)
+	}
+
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("reading dockerfile at path %s", opts.DockerfilePath))
+	}
+
+	stages, _, err := dockerfile.Parse(d)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing dockerfile")
+	}
+
+	for i, s := range stages {
+		resolvedBaseName, err := util.ResolveEnvironmentReplacement(s.BaseName, opts.BuildArgs, false)
+		if err != nil {
+			return nil, errors.Wrap(err, fmt.Sprintf("resolving base name %s", s.BaseName))
+		}
+		if s.BaseName != resolvedBaseName {
+			stages[i].BaseName = resolvedBaseName
+		}
+		baseNames = append(baseNames, resolvedBaseName)
+	}
+	return baseNames, nil
+
+}

pkg/cache/warm_test.go

@@ -18,6 +18,8 @@ package cache
 
 import (
 	"bytes"
+	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
@@ -112,3 +114,136 @@ func Test_Warmer_Warm_in_cache_expired(t *testing.T) {
 		t.Errorf("expected nothing to be written")
 	}
 }
+
+func TestParseDockerfile_SingleStageDockerfile(t *testing.T) {
+	dockerfile := `FROM alpine:latest
+LABEL maintainer="alexezio"
+`
+	tmpfile, err := ioutil.TempFile("", "example")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	if _, err := tmpfile.Write([]byte(dockerfile)); err != nil {
+		t.Fatal(err)
+	}
+	if err := tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	opts := &config.WarmerOptions{DockerfilePath: tmpfile.Name()}
+	baseNames, err := ParseDockerfile(opts)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if len(baseNames) != 1 {
+		t.Fatalf("expected 1 base name, got %d", len(baseNames))
+	}
+	if baseNames[0] != "alpine:latest" {
+		t.Fatalf("expected 'alpine:latest', got '%s'", baseNames[0])
+	}
+}
+
+func TestParseDockerfile_MultiStageDockerfile(t *testing.T) {
+	dockerfile := `FROM golang:1.20 as BUILDER
+LABEL maintainer="alexezio"
+
+FROM alpine:latest as RUNNER
+LABEL maintainer="alexezio"
+`
+	tmpfile, err := ioutil.TempFile("", "example")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	if _, err := tmpfile.Write([]byte(dockerfile)); err != nil {
+		t.Fatal(err)
+	}
+	if err := tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	opts := &config.WarmerOptions{DockerfilePath: tmpfile.Name()}
+	baseNames, err := ParseDockerfile(opts)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if len(baseNames) != 2 {
+		t.Fatalf("expected 2 base name, got %d", len(baseNames))
+	}
+	if baseNames[0] != "golang:1.20" {
+		t.Fatalf("expected 'golang:1.20', got '%s'", baseNames[0])
+	}
+
+	if baseNames[1] != "alpine:latest" {
+		t.Fatalf("expected 'alpine:latest', got '%s'", baseNames[0])
+	}
+}
+
+func TestParseDockerfile_ArgsDockerfile(t *testing.T) {
+	dockerfile := `ARG version=latest
+FROM golang:${version}
+`
+	tmpfile, err := ioutil.TempFile("", "example")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	if _, err := tmpfile.Write([]byte(dockerfile)); err != nil {
+		t.Fatal(err)
+	}
+	if err := tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	opts := &config.WarmerOptions{DockerfilePath: tmpfile.Name(), BuildArgs: []string{"version=1.20"}}
+	baseNames, err := ParseDockerfile(opts)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if len(baseNames) != 1 {
+		t.Fatalf("expected 1 base name, got %d", len(baseNames))
+	}
+	if baseNames[0] != "golang:1.20" {
+		t.Fatalf("expected 'golang:1.20', got '%s'", baseNames[0])
+	}
+}
+
+func TestParseDockerfile_MissingsDockerfile(t *testing.T) {
+	opts := &config.WarmerOptions{DockerfilePath: "dummy-nowhere"}
+	baseNames, err := ParseDockerfile(opts)
+	if err == nil {
+		t.Fatal("expected an error, got nil")
+	}
+	if len(baseNames) != 0 {
+		t.Fatalf("expected no base names, got %d", len(baseNames))
+	}
+}
+
+func TestParseDockerfile_InvalidsDockerfile(t *testing.T) {
+	dockerfile := "This is a invalid dockerfile"
+	tmpfile, err := ioutil.TempFile("", "example")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	if _, err := tmpfile.Write([]byte(dockerfile)); err != nil {
+		t.Fatal(err)
+	}
+	if err := tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+	opts := &config.WarmerOptions{DockerfilePath: tmpfile.Name()}
+	baseNames, err := ParseDockerfile(opts)
+	if err == nil {
+		t.Fatal("expected an error, got nil")
+	}
+
+	if len(baseNames) != 0 {
+		t.Fatalf("expected no base names, got %d", len(baseNames))
+	}
+}

pkg/config/options.go

@@ -162,4 +162,6 @@ type WarmerOptions struct {
 	CustomPlatform string
 	Images         multiArg
 	Force          bool
+	DockerfilePath string
+	BuildArgs      multiArg
 }