public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

feat: cache dockerfile images through warmer (#2499) 

* feat: cache dockerfile images through warmer

* Fix logical error in conditional statement

* Addressed review feedback

1. Updated help text for the --build-arg flag to indicate it should be used with the dockerfile flag.
2. Updated the documentation to include the optional --build-arg flag.
3. Added unit tests for `ParseDockerfile`, covering scenarios for missing Dockerfile, invalid Dockerfile, single stage Dockerfile, multi-stage Dockerfile and Args Dockerfile

---------

Co-authored-by: 连奔驰 <benchi.lian@thoughtworks.com>
This commit is contained in:
alexezio 2023-06-22 03:00:22 +08:00 committed by GitHub
parent 7cd39d14e3
commit 0743c19176
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 232 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -505,9 +505,12 @@ provide a kaniko cache warming image at `gcr.io/kaniko-project/warmer`:
```shell
docker run -v $(pwd):/workspace gcr.io/kaniko-project/warmer:latest --cache-dir=/workspace/cache --image=<image to cache> --image=<another image to cache>
docker run -v $(pwd):/workspace gcr.io/kaniko-project/warmer:latest --cache-dir=/workspace/cache --dockerfile=<path to dockerfile>
docker run -v $(pwd):/workspace gcr.io/kaniko-project/warmer:latest --cache-dir=/workspace/cache --dockerfile=<path to dockerfile> --build-args version=1.19
```
`--image` can be specified for any number of desired images. This command will
`--image` can be specified for any number of desired images. `--dockerfile` can
be specified for the path of dockerfile for cache.These command will combined to
cache those images by digest in a local directory named `cache`. Once the cache
is populated, caching is opted into with the same `--cache=true` flag as above.
The location of the local cache is provided via the `--cache-dir` flag,

37
cmd/warmer/cmd/root.go
View File

@ -19,11 +19,14 @@ package cmd
import (
"fmt"
"os"
"path/filepath"
"regexp"
"time"
"github.com/GoogleContainerTools/kaniko/pkg/cache"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/logging"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containerd/containerd/platforms"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/pkg/errors"
@ -54,9 +57,14 @@ var RootCmd = &cobra.Command{
return err
}
if len(opts.Images) == 0 {
return errors.New("You must select at least one image to cache")
if len(opts.Images) == 0 && opts.DockerfilePath == "" {
return errors.New("You must select at least one image to cache or a dockerfilepath to parse")
}
if err := validateDockerfilePath(); err != nil {
return errors.Wrap(err, "error validating dockerfile path")
}
return nil
},
Run: func(cmd *cobra.Command, args []string) {
@ -89,6 +97,8 @@ func addKanikoOptionsFlags() {
RootCmd.PersistentFlags().VarP(&opts.RegistriesClientCertificates, "registry-client-cert", "", "Use the provided client certificate for mutual TLS (mTLS) communication with the given registry. Expected format is 'my.registry.url=/path/to/client/cert,/path/to/client/key'.")
RootCmd.PersistentFlags().VarP(&opts.RegistryMirrors, "registry-mirror", "", "Registry mirror to use as pull-through cache instead of docker.io. Set it repeatedly for multiple mirrors.")
RootCmd.PersistentFlags().StringVarP(&opts.CustomPlatform, "customPlatform", "", "", "Specify the build platform if different from the current host")
RootCmd.PersistentFlags().StringVarP(&opts.DockerfilePath, "dockerfile", "d", "Dockerfile", "Path to the dockerfile to be cached. The kaniko warmer will parse and write out each stage's base image layers to the cache-dir. Using the same dockerfile path as what you plan to build in the kaniko executor is the expected usage.")
RootCmd.PersistentFlags().VarP(&opts.BuildArgs, "build-arg", "", "This flag should be used in conjunction with the dockerfile flag for scenarios where dynamic replacement of the base image is required.")
// Default the custom platform flag to our current platform, and validate it.
if opts.CustomPlatform == "" {
@ -104,6 +114,29 @@ func addHiddenFlags() {
RootCmd.PersistentFlags().MarkHidden("azure-container-registry-config")
}
func validateDockerfilePath() error {
if isURL(opts.DockerfilePath) {
return nil
}
if util.FilepathExists(opts.DockerfilePath) {
abs, err := filepath.Abs(opts.DockerfilePath)
if err != nil {
return errors.Wrap(err, "getting absolute path for dockerfile")
}
opts.DockerfilePath = abs
return nil
}
return errors.New("please provide a valid path to a Dockerfile within the build context with --dockerfile")
}
func isURL(path string) bool {
if match, _ := regexp.MatchString("^https?://", path); match {
return true
}
return false
}
func exit(err error) {
fmt.Println(err)
os.Exit(1)

56
pkg/cache/warm.go vendored
View File

@ -18,13 +18,18 @@ package cache
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"regexp"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/GoogleContainerTools/kaniko/pkg/image/remote"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/tarball"
@ -34,8 +39,21 @@ import (
// WarmCache populates the cache
func WarmCache(opts *config.WarmerOptions) error {
var dockerfileImages []string
cacheDir := opts.CacheDir
images := opts.Images
// if opts.image is empty,we need to parse dockerfilepath to get images list
if opts.DockerfilePath != "" {
var err error
if dockerfileImages, err = ParseDockerfile(opts); err != nil {
return errors.Wrap(err, "failed to parse Dockerfile")
}
}
// TODO: Implement deduplication logic later.
images = append(images, dockerfileImages...)
logrus.Debugf("%s\n", cacheDir)
logrus.Debugf("%s\n", images)
@ -157,3 +175,41 @@ func (w *Warmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error)
return digest, nil
}
func ParseDockerfile(opts *config.WarmerOptions) ([]string, error) {
var err error
var d []uint8
var baseNames []string
match, _ := regexp.MatchString("^https?://", opts.DockerfilePath)
if match {
response, e := http.Get(opts.DockerfilePath) //nolint:noctx
if e != nil {
return nil, e
}
d, err = ioutil.ReadAll(response.Body)
} else {
d, err = ioutil.ReadFile(opts.DockerfilePath)
}
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("reading dockerfile at path %s", opts.DockerfilePath))
}
stages, _, err := dockerfile.Parse(d)
if err != nil {
return nil, errors.Wrap(err, "parsing dockerfile")
}
for i, s := range stages {
resolvedBaseName, err := util.ResolveEnvironmentReplacement(s.BaseName, opts.BuildArgs, false)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("resolving base name %s", s.BaseName))
}
if s.BaseName != resolvedBaseName {
stages[i].BaseName = resolvedBaseName
}
baseNames = append(baseNames, resolvedBaseName)
}
return baseNames, nil
}

135
pkg/cache/warm_test.go vendored
View File

@ -18,6 +18,8 @@ package cache
import (
"bytes"
"io/ioutil"
"os"
"testing"
"github.com/GoogleContainerTools/kaniko/pkg/config"
@ -112,3 +114,136 @@ func Test_Warmer_Warm_in_cache_expired(t *testing.T) {
t.Errorf("expected nothing to be written")
}
}
func TestParseDockerfile_SingleStageDockerfile(t *testing.T) {
dockerfile := `FROM alpine:latest
LABEL maintainer="alexezio"
`
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
if _, err := tmpfile.Write([]byte(dockerfile)); err != nil {
t.Fatal(err)
}
if err := tmpfile.Close(); err != nil {
t.Fatal(err)
}
opts := &config.WarmerOptions{DockerfilePath: tmpfile.Name()}
baseNames, err := ParseDockerfile(opts)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(baseNames) != 1 {
t.Fatalf("expected 1 base name, got %d", len(baseNames))
}
if baseNames[0] != "alpine:latest" {
t.Fatalf("expected 'alpine:latest', got '%s'", baseNames[0])
}
}
func TestParseDockerfile_MultiStageDockerfile(t *testing.T) {
dockerfile := `FROM golang:1.20 as BUILDER
LABEL maintainer="alexezio"
FROM alpine:latest as RUNNER
LABEL maintainer="alexezio"
`
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
if _, err := tmpfile.Write([]byte(dockerfile)); err != nil {
t.Fatal(err)
}
if err := tmpfile.Close(); err != nil {
t.Fatal(err)
}
opts := &config.WarmerOptions{DockerfilePath: tmpfile.Name()}
baseNames, err := ParseDockerfile(opts)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(baseNames) != 2 {
t.Fatalf("expected 2 base name, got %d", len(baseNames))
}
if baseNames[0] != "golang:1.20" {
t.Fatalf("expected 'golang:1.20', got '%s'", baseNames[0])
}
if baseNames[1] != "alpine:latest" {
t.Fatalf("expected 'alpine:latest', got '%s'", baseNames[0])
}
}
func TestParseDockerfile_ArgsDockerfile(t *testing.T) {
dockerfile := `ARG version=latest
FROM golang:${version}
`
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
if _, err := tmpfile.Write([]byte(dockerfile)); err != nil {
t.Fatal(err)
}
if err := tmpfile.Close(); err != nil {
t.Fatal(err)
}
opts := &config.WarmerOptions{DockerfilePath: tmpfile.Name(), BuildArgs: []string{"version=1.20"}}
baseNames, err := ParseDockerfile(opts)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(baseNames) != 1 {
t.Fatalf("expected 1 base name, got %d", len(baseNames))
}
if baseNames[0] != "golang:1.20" {
t.Fatalf("expected 'golang:1.20', got '%s'", baseNames[0])
}
}
func TestParseDockerfile_MissingsDockerfile(t *testing.T) {
opts := &config.WarmerOptions{DockerfilePath: "dummy-nowhere"}
baseNames, err := ParseDockerfile(opts)
if err == nil {
t.Fatal("expected an error, got nil")
}
if len(baseNames) != 0 {
t.Fatalf("expected no base names, got %d", len(baseNames))
}
}
func TestParseDockerfile_InvalidsDockerfile(t *testing.T) {
dockerfile := "This is a invalid dockerfile"
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
if _, err := tmpfile.Write([]byte(dockerfile)); err != nil {
t.Fatal(err)
}
if err := tmpfile.Close(); err != nil {
t.Fatal(err)
}
opts := &config.WarmerOptions{DockerfilePath: tmpfile.Name()}
baseNames, err := ParseDockerfile(opts)
if err == nil {
t.Fatal("expected an error, got nil")
}
if len(baseNames) != 0 {
t.Fatalf("expected no base names, got %d", len(baseNames))
}
}

2
pkg/config/options.go
View File

@ -162,4 +162,6 @@ type WarmerOptions struct {
CustomPlatform string
Images multiArg
Force bool
DockerfilePath string
BuildArgs multiArg
}