290 lines
9.6 KiB
Go
290 lines
9.6 KiB
Go
/*
|
|
Copyright 2018 Google LLC
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package integration
|
|
|
|
import (
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"os/exec"
|
|
"path"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/GoogleContainerTools/kaniko/pkg/timing"
|
|
)
|
|
|
|
const (
|
|
// ExecutorImage is the name of the kaniko executor image
|
|
ExecutorImage = "executor-image"
|
|
//WarmerImage is the name of the kaniko cache warmer image
|
|
WarmerImage = "warmer-image"
|
|
|
|
dockerPrefix = "docker-"
|
|
kanikoPrefix = "kaniko-"
|
|
buildContextPath = "/workspace"
|
|
cacheDir = "/workspace/cache"
|
|
baseImageToCache = "gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0"
|
|
)
|
|
|
|
// Arguments to build Dockerfiles with, used for both docker and kaniko builds
|
|
var argsMap = map[string][]string{
|
|
"Dockerfile_test_run": {"file=/file"},
|
|
"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
|
|
"Dockerfile_test_add": {"file=context/foo"},
|
|
"Dockerfile_test_onbuild": {"file=/tmp/onbuild"},
|
|
"Dockerfile_test_scratch": {
|
|
"image=scratch",
|
|
"hello=hello-value",
|
|
"file=context/foo",
|
|
"file3=context/b*",
|
|
},
|
|
"Dockerfile_test_multistage": {"file=/foo2"},
|
|
}
|
|
|
|
// Arguments to build Dockerfiles with when building with docker
|
|
var additionalDockerFlagsMap = map[string][]string{
|
|
"Dockerfile_test_target": {"--target=second"},
|
|
}
|
|
|
|
// Arguments to build Dockerfiles with when building with kaniko
|
|
var additionalKanikoFlagsMap = map[string][]string{
|
|
"Dockerfile_test_add": {"--single-snapshot"},
|
|
"Dockerfile_test_scratch": {"--single-snapshot"},
|
|
"Dockerfile_test_target": {"--target=second"},
|
|
}
|
|
|
|
var bucketContextTests = []string{"Dockerfile_test_copy_bucket"}
|
|
var reproducibleTests = []string{"Dockerfile_test_reproducible"}
|
|
|
|
// GetDockerImage constructs the name of the docker image that would be built with
|
|
// dockerfile if it was tagged with imageRepo.
|
|
func GetDockerImage(imageRepo, dockerfile string) string {
|
|
return strings.ToLower(imageRepo + dockerPrefix + dockerfile)
|
|
}
|
|
|
|
// GetKanikoImage constructs the name of the kaniko image that would be built with
|
|
// dockerfile if it was tagged with imageRepo.
|
|
func GetKanikoImage(imageRepo, dockerfile string) string {
|
|
return strings.ToLower(imageRepo + kanikoPrefix + dockerfile)
|
|
}
|
|
|
|
// GetVersionedKanikoImage versions constructs the name of the kaniko image that would be built
|
|
// with the dockerfile and versions it for cache testing
|
|
func GetVersionedKanikoImage(imageRepo, dockerfile string, version int) string {
|
|
return strings.ToLower(imageRepo + kanikoPrefix + dockerfile + strconv.Itoa(version))
|
|
}
|
|
|
|
// FindDockerFiles will look for test docker files in the directory dockerfilesPath.
|
|
// These files must start with `Dockerfile_test`. If the file is one we are intentionally
|
|
// skipping, it will not be included in the returned list.
|
|
func FindDockerFiles(dockerfilesPath string) ([]string, error) {
|
|
allDockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*"))
|
|
if err != nil {
|
|
return []string{}, fmt.Errorf("Failed to find docker files at %s: %s", dockerfilesPath, err)
|
|
}
|
|
|
|
var dockerfiles []string
|
|
for _, dockerfile := range allDockerfiles {
|
|
// Remove the leading directory from the path
|
|
dockerfile = dockerfile[len("dockerfiles/"):]
|
|
dockerfiles = append(dockerfiles, dockerfile)
|
|
|
|
}
|
|
return dockerfiles, err
|
|
}
|
|
|
|
// DockerFileBuilder knows how to build docker files using both Kaniko and Docker and
|
|
// keeps track of which files have been built.
|
|
type DockerFileBuilder struct {
|
|
// Holds all available docker files and whether or not they've been built
|
|
FilesBuilt map[string]bool
|
|
DockerfilesToIgnore map[string]struct{}
|
|
TestCacheDockerfiles map[string]struct{}
|
|
}
|
|
|
|
// NewDockerFileBuilder will create a DockerFileBuilder initialized with dockerfiles, which
|
|
// it will assume are all as yet unbuilt.
|
|
func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder {
|
|
d := DockerFileBuilder{FilesBuilt: map[string]bool{}}
|
|
for _, f := range dockerfiles {
|
|
d.FilesBuilt[f] = false
|
|
}
|
|
d.DockerfilesToIgnore = map[string]struct{}{
|
|
// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
|
|
"Dockerfile_test_user_run": {},
|
|
}
|
|
d.TestCacheDockerfiles = map[string]struct{}{
|
|
"Dockerfile_test_cache": {},
|
|
"Dockerfile_test_cache_install": {},
|
|
"Dockerfile_test_cache_perm": {},
|
|
"Dockerfile_test_cache_copy": {},
|
|
}
|
|
return &d
|
|
}
|
|
|
|
// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
|
|
// The resulting image will be tagged with imageRepo. If the dockerfile will be built with
|
|
// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
|
|
func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, dockerfile string) error {
|
|
_, ex, _, _ := runtime.Caller(0)
|
|
cwd := filepath.Dir(ex)
|
|
|
|
fmt.Printf("Building images for Dockerfile %s\n", dockerfile)
|
|
|
|
var buildArgs []string
|
|
buildArgFlag := "--build-arg"
|
|
for _, arg := range argsMap[dockerfile] {
|
|
buildArgs = append(buildArgs, buildArgFlag)
|
|
buildArgs = append(buildArgs, arg)
|
|
}
|
|
// build docker image
|
|
additionalFlags := append(buildArgs, additionalDockerFlagsMap[dockerfile]...)
|
|
dockerImage := strings.ToLower(imageRepo + dockerPrefix + dockerfile)
|
|
dockerCmd := exec.Command("docker",
|
|
append([]string{"build",
|
|
"-t", dockerImage,
|
|
"-f", path.Join(dockerfilesPath, dockerfile),
|
|
"."},
|
|
additionalFlags...)...,
|
|
)
|
|
|
|
timer := timing.Start(dockerfile + "_docker")
|
|
out, err := RunCommandWithoutTest(dockerCmd)
|
|
timing.DefaultRun.Stop(timer)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out))
|
|
}
|
|
|
|
contextFlag := "-c"
|
|
contextPath := buildContextPath
|
|
for _, d := range bucketContextTests {
|
|
if d == dockerfile {
|
|
contextFlag = "-b"
|
|
contextPath = gcsBucket
|
|
}
|
|
}
|
|
|
|
reproducibleFlag := ""
|
|
for _, d := range reproducibleTests {
|
|
if d == dockerfile {
|
|
reproducibleFlag = "--reproducible"
|
|
break
|
|
}
|
|
}
|
|
|
|
benchmarkEnv := "BENCHMARK_FILE=false"
|
|
benchmarkDir, err := ioutil.TempDir("", "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b {
|
|
benchmarkEnv = "BENCHMARK_FILE=/kaniko/benchmarks/" + dockerfile
|
|
benchmarkFile := path.Join(benchmarkDir, dockerfile)
|
|
fileName := fmt.Sprintf("run_%s_%s", time.Now().Format("2006-01-02-15:04"), dockerfile)
|
|
dst := path.Join("benchmarks", fileName)
|
|
defer UploadFileToBucket(gcsBucket, benchmarkFile, dst)
|
|
}
|
|
|
|
// build kaniko image
|
|
additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
|
|
kanikoImage := GetKanikoImage(imageRepo, dockerfile)
|
|
kanikoCmd := exec.Command("docker",
|
|
append([]string{"run",
|
|
"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
|
|
"-v", benchmarkDir + ":/kaniko/benchmarks",
|
|
"-v", cwd + ":/workspace",
|
|
"-e", benchmarkEnv,
|
|
ExecutorImage,
|
|
"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
|
|
"-d", kanikoImage, reproducibleFlag,
|
|
contextFlag, contextPath},
|
|
additionalFlags...)...,
|
|
)
|
|
|
|
timer = timing.Start(dockerfile + "_kaniko")
|
|
out, err = RunCommandWithoutTest(kanikoCmd)
|
|
timing.DefaultRun.Stop(timer)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out))
|
|
}
|
|
|
|
d.FilesBuilt[dockerfile] = true
|
|
return nil
|
|
}
|
|
|
|
func populateVolumeCache() error {
|
|
_, ex, _, _ := runtime.Caller(0)
|
|
cwd := filepath.Dir(ex)
|
|
warmerCmd := exec.Command("docker",
|
|
append([]string{"run",
|
|
"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
|
|
"-v", cwd + ":/workspace",
|
|
WarmerImage,
|
|
"-c", cacheDir,
|
|
"-i", baseImageToCache},
|
|
)...,
|
|
)
|
|
|
|
if _, err := RunCommandWithoutTest(warmerCmd); err != nil {
|
|
return fmt.Errorf("Failed to warm kaniko cache: %s", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
|
|
func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath string, version int) error {
|
|
_, ex, _, _ := runtime.Caller(0)
|
|
cwd := filepath.Dir(ex)
|
|
|
|
cacheFlag := "--cache=true"
|
|
|
|
for dockerfile := range d.TestCacheDockerfiles {
|
|
benchmarkEnv := "BENCHMARK_FILE=false"
|
|
if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b {
|
|
os.Mkdir("benchmarks", 0755)
|
|
benchmarkEnv = "BENCHMARK_FILE=/workspace/benchmarks/" + dockerfile
|
|
}
|
|
kanikoImage := GetVersionedKanikoImage(imageRepo, dockerfile, version)
|
|
kanikoCmd := exec.Command("docker",
|
|
append([]string{"run",
|
|
"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
|
|
"-v", cwd + ":/workspace",
|
|
"-e", benchmarkEnv,
|
|
ExecutorImage,
|
|
"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
|
|
"-d", kanikoImage,
|
|
"-c", buildContextPath,
|
|
cacheFlag,
|
|
"--cache-repo", cacheRepo,
|
|
"--cache-dir", cacheDir})...,
|
|
)
|
|
|
|
timer := timing.Start(dockerfile + "_kaniko_cached_" + strconv.Itoa(version))
|
|
_, err := RunCommandWithoutTest(kanikoCmd)
|
|
timing.DefaultRun.Stop(timer)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %s", kanikoImage, kanikoCmd.Args, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|