public_git
/
kaniko
mirror of https://github.com/GoogleContainerTools/kaniko
1
0
Fork 0
Code Issues Releases Wiki Activity

Fix the e2e K8s test (#1842) 

* Debugging the integration testing.

It seems to be consistently failing (also fix a typo in the file/leg name).

I'm disabling `-v` for `go test` because it interleaves the test output in ways that make it painful to read a single failing test's output when `t.Parallel` is involved.

* Try swapping registry:2 for the registry add-on

* Drop logf and make Logger private
This commit is contained in:
Matt Moore 2021-12-17 11:43:38 -08:00 committed by GitHub
parent 78808c9070
commit 49f0154b3f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 97 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/intergration-layers-tests.yaml → .github/workflows/integration-layers-tests.yaml vendored
View File

@ -28,4 +28,5 @@ jobs:
- name: Run integration-test-layers
run: |
make travis-setup
make minikube-setup
make integration-test-layers

1
.github/workflows/integration-mis.yaml → .github/workflows/integration-misc.yaml vendored
View File

@ -37,4 +37,5 @@ jobs:
- name: Run integration-test-misc
run : |
make travis-setup
make minikube-setup
make integration-test-misc

1
.github/workflows/integration-run-tests.yaml vendored
View File

@ -27,4 +27,5 @@ jobs:
- name: Run integration-test-run
run: |
make travis-setup
make minikube-setup
make integration-test-run

4
.github/workflows/integration_k8_tests.yaml vendored
View File

@ -1,6 +1,6 @@
name: Intergration tests (K8s)
name: Integration tests (K8s)
# Triggers the workflow on push or pull request events
# Triggers the workflow on pull request events
on: [pull_request]
concurrency:

12
integration/benchmark_test.go
View File

@ -59,7 +59,7 @@ func TestSnapshotBenchmark(t *testing.T) {
kanikoImage := fmt.Sprintf("%s_%d", GetKanikoImage(config.imageRepo, dockerfile), num)
buildArgs := []string{"--build-arg", fmt.Sprintf("NUM=%d", num)}
var benchmarkDir string
benchmarkDir, *err = buildKanikoImage("", dockerfile,
benchmarkDir, *err = buildKanikoImage(t.Logf, "", dockerfile,
buildArgs, []string{}, kanikoImage, contextDir, config.gcsBucket,
config.serviceAccount, false)
if *err != nil {
@ -77,11 +77,11 @@ func TestSnapshotBenchmark(t *testing.T) {
}
wg.Wait()
fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
t.Log("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
timeMap.Range(func(key interface{}, value interface{}) bool {
d, _ := key.(int)
v, _ := value.(result)
fmt.Println(fmt.Sprintf("%d,%f,%f,%f", d, v.totalBuildTime, v.walkingFiles, v.resolvingFiles))
t.Logf("%d,%f,%f,%f", d, v.totalBuildTime, v.walkingFiles, v.resolvingFiles)
return true
})
@ -111,7 +111,7 @@ func newResult(t *testing.T, f string) result {
if c, ok := current["Hashing files"]; ok {
r.hashingFiles = c.Seconds()
}
fmt.Println(r)
t.Log(r)
return r
}
@ -128,7 +128,7 @@ func TestSnapshotBenchmarkGcloud(t *testing.T) {
nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000}
var wg sync.WaitGroup
fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
t.Log("Number of Files,Total Build Time,Walking Filesystem, Resolving Files")
for _, num := range nums {
t.Run(fmt.Sprintf("test_benchmark_%d", num), func(t *testing.T) {
wg.Add(1)
@ -139,7 +139,7 @@ func TestSnapshotBenchmarkGcloud(t *testing.T) {
return
}
r := newResult(t, filepath.Join(dir, "results"))
fmt.Println(fmt.Sprintf("%d,%f,%f,%f, %f", num, r.totalBuildTime, r.walkingFiles, r.resolvingFiles, r.hashingFiles))
t.Log(fmt.Sprintf("%d,%f,%f,%f, %f", num, r.totalBuildTime, r.walkingFiles, r.resolvingFiles, r.hashingFiles))
wg.Done()
defer os.Remove(dir)
defer os.Chdir(cwd)

26
integration/images.go
View File

@ -27,6 +27,7 @@ import (
"runtime"
"strconv"
"strings"
"testing"
"time"
"github.com/GoogleContainerTools/kaniko/pkg/timing"
@ -183,10 +184,14 @@ type DockerFileBuilder struct {
TestCacheDockerfiles map[string]struct{}
}
type logger func(string, ...interface{})
// NewDockerFileBuilder will create a DockerFileBuilder initialized with dockerfiles, which
// it will assume are all as yet unbuilt.
func NewDockerFileBuilder() *DockerFileBuilder {
d := DockerFileBuilder{filesBuilt: map[string]struct{}{}}
d := DockerFileBuilder{
filesBuilt: map[string]struct{}{},
}
d.DockerfilesToIgnore = map[string]struct{}{
"Dockerfile_test_add_404": {},
// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
@ -218,8 +223,8 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string {
return flags
}
func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir string) error {
fmt.Printf("Building image for Dockerfile %s\n", dockerfile)
func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfilesPath, dockerfile, contextDir string) error {
t.Logf("Building image for Dockerfile %s\n", dockerfile)
var buildArgs []string
buildArgFlag := "--build-arg"
@ -252,21 +257,21 @@ func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerf
if err != nil {
return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out))
}
fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out)
t.Logf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out)
return nil
}
// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
// The resulting image will be tagged with imageRepo. If the dockerfile will be built with
// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
func (d *DockerFileBuilder) BuildImage(t *testing.T, config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
_, ex, _, _ := runtime.Caller(0)
cwd := filepath.Dir(ex)
return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd)
return d.BuildImageWithContext(t, config, dockerfilesPath, dockerfile, cwd)
}
func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error {
func (d *DockerFileBuilder) BuildImageWithContext(t *testing.T, config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error {
if _, present := d.filesBuilt[dockerfile]; present {
return nil
}
@ -279,7 +284,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
}
timer := timing.Start(dockerfile + "_docker")
d.BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir)
d.BuildDockerImage(t, imageRepo, dockerfilesPath, dockerfile, contextDir)
timing.DefaultRun.Stop(timer)
contextFlag := "-c"
@ -302,7 +307,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
kanikoImage := GetKanikoImage(imageRepo, dockerfile)
timer = timing.Start(dockerfile + "_kaniko")
if _, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
if _, err := buildKanikoImage(t.Logf, dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage,
contextDir, gcsBucket, serviceAccount, true); err != nil {
return err
}
@ -420,6 +425,7 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi
}
func buildKanikoImage(
logf logger,
dockerfilesPath string,
dockerfile string,
buildArgs []string,
@ -447,7 +453,7 @@ func buildKanikoImage(
// build kaniko image
additionalFlags := append(buildArgs, kanikoArgs...)
fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags)
logf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags)
dockerRunFlags := []string{"run", "--net=host",
"-e", benchmarkEnv,

49
integration/integration_test.go
View File

@ -87,8 +87,8 @@ func getDockerMajorVersion() int {
}
return ver
}
func launchTests(m *testing.M) (int, error) {
func launchTests(m *testing.M) (int, error) {
if config.isGcrRepository() {
contextFile, err := CreateIntegrationTarball()
if err != nil {
@ -141,32 +141,25 @@ func buildRequiredImages() error {
setupCommands := []struct {
name string
command []string
}{
{
name: "Building kaniko image",
command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."},
},
{
name: "Building cache warmer image",
command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."},
},
{
name: "Building onbuild base image",
command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."},
},
{
name: "Pushing onbuild base image",
command: []string{"docker", "push", config.onbuildBaseImage},
},
{
name: "Building hardlink base image",
command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."},
},
{
name: "Pushing hardlink base image",
command: []string{"docker", "push", config.hardlinkBaseImage},
},
}
}{{
name: "Building kaniko image",
command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."},
}, {
name: "Building cache warmer image",
command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."},
}, {
name: "Building onbuild base image",
command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."},
}, {
name: "Pushing onbuild base image",
command: []string{"docker", "push", config.onbuildBaseImage},
}, {
name: "Building hardlink base image",
command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."},
}, {
name: "Pushing hardlink base image",
command: []string{"docker", "push", config.hardlinkBaseImage},
}}
for _, setupCmd := range setupCommands {
fmt.Println(setupCmd.name)
@ -522,7 +515,7 @@ func TestLayers(t *testing.T) {
}
func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder) {
if err := imageBuilder.BuildImage(config, dockerfilesPath, dockerfile); err != nil {
if err := imageBuilder.BuildImage(t, config, dockerfilesPath, dockerfile); err != nil {
t.Errorf("Error building image: %s", err)
t.FailNow()
}

2
integration/integration_with_context_test.go
View File

@ -47,7 +47,7 @@ func TestWithContext(t *testing.T) {
t.Parallel()
if err := builder.BuildImageWithContext(
config, "", name, testDir,
t, config, "", name, testDir,
); err != nil {
t.Fatal(err)
}

5
integration/k8s-job.yaml
View File

@ -9,11 +9,12 @@ spec:
containers:
- name: kaniko
image: localhost:5000/executor:latest
workingDir: /workspace
args: [ "--context=dir:///workspace",
"--destination={{.KanikoImage}}"]
volumeMounts:
- name: context
mountPath: /workspace
- name: context
mountPath: /workspace
restartPolicy: Never
volumes:
- name: context

30
integration/k8s_test.go
View File

@ -56,7 +56,7 @@ func TestK8s(t *testing.T) {
t.Parallel()
if err := builder.BuildDockerImage(
config.imageRepo, "", name, testDir,
t, config.imageRepo, "", name, testDir,
); err != nil {
t.Fatal(err)
}
@ -75,21 +75,21 @@ func TestK8s(t *testing.T) {
t.Fatal(err)
}
fmt.Printf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n",
t.Logf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n",
testDir, kanikoImage)
content, err := ioutil.ReadFile(tmpfile.Name())
if err != nil {
log.Fatal(err)
}
fmt.Printf("K8s template %s:\n%s\n", tmpfile.Name(), content)
t.Logf("K8s template %s:\n%s\n", tmpfile.Name(), content)
kubeCmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name())
RunCommand(kubeCmd, t)
fmt.Printf("Waiting for K8s kaniko build job to finish: %s\n",
t.Logf("Waiting for K8s kaniko build job to finish: %s\n",
"job/kaniko-test-"+job.Name)
kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=120s",
kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=1m",
"job/kaniko-test-"+job.Name)
if out, errR := RunCommandWithoutTest(kubeWaitCmd); errR != nil {
t.Log(kubeWaitCmd.Args)
@ -100,9 +100,25 @@ func TestK8s(t *testing.T) {
t.Error(errD)
} else {
t.Log(string(outD))
t.Error(errR)
}
t.FailNow()
descCmd = exec.Command("kubectl", "describe", "pods", "--selector", "job-name=kaniko-test-"+job.Name)
outD, errD = RunCommandWithoutTest(descCmd)
if errD != nil {
t.Error(errD)
} else {
t.Log(string(outD))
}
logsCmd := exec.Command("kubectl", "logs", "--all-containers", "job/kaniko-test-"+job.Name)
outL, errL := RunCommandWithoutTest(logsCmd)
if errL != nil {
t.Error(errL)
} else {
t.Log(string(outL))
}
t.Fatal(errR)
}
diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache")

2
scripts/integration-test.sh
View File

@ -24,4 +24,4 @@ docker version
echo "Running integration tests..."
make out/executor
make out/warmer
go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@"
go test ./integration/... --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@"

24
scripts/minikube-setup.sh
View File

@ -15,6 +15,24 @@
set -ex
curl -Lo minikube https://storage.googleapis.com/minikube-builds/master/minikube-linux-amd64
sudo install minikube /usr/local/bin/minikube
minikube start --profile=minikube --driver=docker
# conntrack is required for minikube 1.19 and higher for none driver
if ! conntrack --version &>/dev/null; then
echo "WARNING: No contrack is not installed"
sudo apt-get update -qq
sudo apt-get -qq -y install conntrack
fi
curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
chmod +x minikube
sudo mv minikube /usr/local/bin/
sudo apt-get update
sudo apt-get install -y liblz4-tool
cat /proc/cpuinfo
minikube start --vm-driver=none --force
minikube status
minikube addons enable registry
kubectl cluster-info
kubectl port-forward --namespace kube-system service/registry 5000:80 &

1
scripts/travis-setup.sh
View File

@ -20,7 +20,6 @@ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubun
sudo apt-get update
sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && sudo mv container-diff-linux-amd64 /usr/local/bin/container-diff
docker run -d -p 5000:5000 --restart always --name registry registry:2
mkdir -p $HOME/.docker/
echo '{}' > $HOME/.docker/config.json