Add multi-arch image via Bazel (#1452)

* Add multi-arch image via Bazel

* Drop the commented  tags

* Move tests to //integration

.gitignore

@@ -1,4 +1,7 @@
 out/
+bazel-*
+*~
+BUILD.bazel
 .idea
 *.iml
 .vagrant

.travis.yml

@@ -1,6 +1,7 @@
 language: go
 os: linux
 dist: bionic
+
 env:
   global:
     - IMAGE_REPO=localhost:5000 REGISTRY=localhost:5000
@@ -35,3 +36,28 @@ jobs:
         - make travis-setup
       script:
         - make integration-test-misc
+
+    - name: bazel amd64
+      arch: amd64
+      env: CPU=k8
+      before_install: &before_install_multiarch
+        - export PATH=$PATH:$HOME/bin && mkdir -p $HOME/bin
+        - eval $(go env)
+        # install bazelisk as bazel to install the appropriate bazel version
+        - wget https://github.com/bazelbuild/bazelisk/releases/download/v1.6.1/bazelisk-linux-${GOARCH} && chmod +x bazelisk-linux-${GOARCH} && mv bazelisk-linux-${GOARCH} $HOME/bin/bazel
+      script: &script_multiarch
+        # Generate BUILD.bazel files (we do not check them in)
+        - bazel run //:gazelle
+        - bazel build --cpu=${CPU} --curses=no //integration:all
+        # Build all targets tagged with our architecture:
+        - bazel build --cpu=${CPU} --curses=no $(bazel query 'attr("tags", "'${GOARCH}'", "//...")')
+        # Run all tests not tagged as "manual":
+        - bazel test  --cpu=${CPU} --curses=no --test_output=errors --test_timeout=900 //integration:all
+        # Run all tests tagged with our architecture:
+        - bazel test  --cpu=${CPU} --curses=no --test_output=errors --test_timeout=900 $(bazel query 'attr("tags", "'${GOARCH}'", "//...")')
+
+    - name: bazel arm64
+      arch: arm64
+      env: CPU=aarch64
+      before_install: *before_install_multiarch
+      script: *script_multiarch

BUILD

@@ -0,0 +1,4 @@
+load("@bazel_gazelle//:def.bzl", "gazelle")
+
+# gazelle:prefix github.com/GoogleContainerTools/kaniko
+gazelle(name = "gazelle")

WORKSPACE

@@ -0,0 +1,53 @@
+workspace(name = "kaniko")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "io_bazel_rules_go",
+    sha256 = "b725e6497741d7fc2d55fcc29a276627d10e43fa5d0bb692692890ae30d98d00",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.24.3/rules_go-v0.24.3.tar.gz",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.24.3/rules_go-v0.24.3.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "bazel_gazelle",
+    sha256 = "b85f48fa105c4403326e9525ad2b2cc437babaa6e15a3fc0b1dbab0ab064bc7c",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.22.2/bazel-gazelle-v0.22.2.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.22.2/bazel-gazelle-v0.22.2.tar.gz",
+    ],
+)
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains()
+
+gazelle_dependencies()
+
+# Docker rules.
+http_archive(
+    name = "io_bazel_rules_docker",
+    sha256 = "cf53839c398e464b10ec2fbeb11aedb446f078c28e3b4ce372461bb105ef435c",
+    strip_prefix = "rules_docker-f8478e57ab7457e403fda474f06ac0bb120d92a7",
+    urls = ["https://github.com/bazelbuild/rules_docker/archive/f8478e57ab7457e403fda474f06ac0bb120d92a7.tar.gz"],
+)
+
+load(
+    "@io_bazel_rules_docker//repositories:repositories.bzl",
+    container_repositories = "repositories",
+)
+
+container_repositories()
+
+load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")
+
+container_deps()
+
+load("@io_bazel_rules_docker//repositories:pip_repositories.bzl", "pip_deps")
+
+pip_deps()

cmd/executor/BUILD

@@ -0,0 +1,60 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@io_bazel_rules_docker//container:container.bzl", "container_image")
+
+go_library(
+    name = "executor_lib",
+    srcs = ["main.go"],
+    importpath = "github.com/GoogleContainerTools/kaniko/cmd/executor",
+    visibility = ["//visibility:private"],
+    deps = ["//cmd/executor/cmd"],
+)
+
+go_binary(
+    name = "executor",
+    embed = [":executor_lib"],
+    pure = "on",
+    visibility = ["//visibility:public"],
+)
+
+ARCHITECTURES = [
+    "amd64",
+    "arm64",
+]
+
+[
+    go_binary(
+        name = "executor_" + arch,
+        embed = [":executor_lib"],
+        goarch = arch,
+        goos = "linux",
+        pure = "on",
+        visibility = ["//visibility:public"],
+    )
+    for arch in ARCHITECTURES
+]
+
+[
+    container_image(
+        name = "image_" + arch,
+        architecture = arch,
+        base = "//files:image",
+        directory = "/kaniko",
+        entrypoint = ["/kaniko/executor_" + arch],
+        env = {
+            "HOME": "/root",
+            "USER": "root",
+            "PATH": "/usr/local/bin:/kaniko",
+            "SSL_CERT_DIR": "/kaniko/ssl/certs",
+            "DOCKER_CONFIG": "/kaniko/.docker/",
+        },
+        files = [
+            ":executor_" + arch,
+        ],
+        symlinks = {
+            "/kaniko/executor": "/kaniko/executor_" + arch,
+        },
+        visibility = ["//visibility:public"],
+        workdir = "/workspace",
+    )
+    for arch in ARCHITECTURES
+]

cmd/executor/cmd/root.go

@@ -95,8 +95,10 @@ var RootCmd = &cobra.Command{
 			}
 			logrus.Warn("kaniko is being run outside of a container. This can have dangerous effects on your system")
 		}
-		if err := executor.CheckPushPermissions(opts); err != nil {
+		if !opts.NoPush {
-			exit(errors.Wrap(err, "error checking push permissions -- make sure you entered the correct tag name, and that you are authenticated correctly, and try again"))
+			if err := executor.CheckPushPermissions(opts); err != nil {
+				exit(errors.Wrap(err, "error checking push permissions -- make sure you entered the correct tag name, and that you are authenticated correctly, and try again"))
+			}
 		}
 		if err := resolveRelativePaths(); err != nil {
 			exit(errors.Wrap(err, "error resolving relative paths to absolute paths"))

deploy/cloudbuild-release.yaml

@@ -1,5 +1,8 @@
 # This cloudbuild is run on the creation of new tags, which should signify releases.
+timeout: 1800s
+
 steps:
+
   # First, build kaniko
   - name: "gcr.io/cloud-builders/docker"
     args: ["build", "-f", "deploy/Dockerfile",
@@ -21,6 +24,73 @@ steps:
   - name: "gcr.io/cloud-builders/docker"
     args: ["tag", "gcr.io/kaniko-project/warmer:$TAG_NAME", 
            "gcr.io/kaniko-project/warmer:latest"]
+
+
+  # Build each of the multi-arch images with Bazel and load them into the Docker daemon.
+  - name: gcr.io/cloud-marketplace-containers/google/bazel:3.4.1
+    entrypoint: sh
+    args:
+    - -c
+    - |
+      #!/bin/sh
+      set -o errexit
+      set -o xtrace
+
+      bazel run //:gazelle
+      bazel run --host_force_python=PY2 //cmd/executor:image_amd64
+      bazel run --host_force_python=PY2 //cmd/executor:image_arm64
+
+  # Publish the individual container images
+  - name: docker
+    entrypoint: sh
+    args:
+    - -c
+    - |
+      #!/bin/sh
+      set -o errexit
+      set -o xtrace
+
+      docker tag bazel/cmd/executor:image_amd64     gcr.io/kaniko-project/executor:amd64
+      docker tag bazel/cmd/executor:image_amd64     gcr.io/kaniko-project/executor:amd64-$TAG_NAME
+      docker tag bazel/cmd/executor:image_arm64     gcr.io/kaniko-project/executor:arm64
+      docker tag bazel/cmd/executor:image_arm64     gcr.io/kaniko-project/executor:arm64-$TAG_NAME
+
+      docker push gcr.io/kaniko-project/executor:amd64
+      docker push gcr.io/kaniko-project/executor:amd64-$TAG_NAME
+      docker push gcr.io/kaniko-project/executor:arm64
+      docker push gcr.io/kaniko-project/executor:arm64-$TAG_NAME
+
+  # Enable "manifest list" support in docker, and publish one covering the per-architecture
+  # images published above.
+  - name: docker
+    entrypoint: sh
+    args:
+    - -c
+    - |
+      #!/bin/sh
+      set -o errexit
+      set -o xtrace
+
+      # Publish manifest lists second, after all of the binary material
+      # has been uploaded, so that it is fast.  We want fast because enabling
+      # the experimental features in docker changes ~/.docker/config.json, which
+      # GCB periodically tramples.
+      #
+      # Enable support for 'docker manifest create'
+      # https://docs.docker.com/engine/reference/commandline/manifest_create/
+      sed -i 's/^{/{"experimental": "enabled",/g' ~/.docker/config.json
+
+      docker manifest create gcr.io/kaniko-project/executor:multi-arch \
+         gcr.io/kaniko-project/executor:amd64 \
+         gcr.io/kaniko-project/executor:arm64
+      docker manifest push gcr.io/kaniko-project/executor:multi-arch
+
+      docker manifest create gcr.io/kaniko-project/executor:multi-arch-$TAG_NAME \
+         gcr.io/kaniko-project/executor:amd64-$TAG_NAME \
+         gcr.io/kaniko-project/executor:arm64-$TAG_NAME
+      docker manifest push gcr.io/kaniko-project/executor:multi-arch-$TAG_NAME
+
+
 images: ["gcr.io/kaniko-project/executor:$TAG_NAME",
          "gcr.io/kaniko-project/executor:latest",
          "gcr.io/kaniko-project/executor:debug-$TAG_NAME",

deploy/cloudbuild.yaml

@@ -1,4 +1,7 @@
+timeout: 1800s
+
 steps:
+
   # First, build kaniko
   - name: "gcr.io/cloud-builders/docker"
     args: ["build", "-f", "deploy/Dockerfile",
@@ -14,9 +17,68 @@ steps:
   - name: "gcr.io/cloud-builders/docker"
     args: ["build", "-f", "deploy/Dockerfile_warmer",
            "-t", "gcr.io/$PROJECT_ID/${_WARMER_IMAGE_NAME}:${COMMIT_SHA}", "."]
+
+
+  # Build each of the multi-arch images with Bazel and load them into the Docker daemon.
+  - name: gcr.io/cloud-marketplace-containers/google/bazel:3.4.1
+    entrypoint: sh
+    args:
+    - -c
+    - |
+      #!/bin/sh
+      set -o errexit
+      set -o xtrace
+
+      bazel run //:gazelle
+      bazel run --host_force_python=PY2 //cmd/executor:image_amd64
+      bazel run --host_force_python=PY2 //cmd/executor:image_arm64
+
+  # Publish the individual container images
+  - name: docker
+    entrypoint: sh
+    args:
+    - -c
+    - |
+      #!/bin/sh
+      set -o errexit
+      set -o xtrace
+
+      docker tag bazel/cmd/executor:image_amd64     gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:amd64-${COMMIT_SHA}
+      docker tag bazel/cmd/executor:image_arm64     gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:arm64-${COMMIT_SHA}
+
+      docker push gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:amd64-${COMMIT_SHA}
+      docker push gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:arm64-${COMMIT_SHA}
+
+  # Enable "manifest list" support in docker, and publish one covering the per-architecture
+  # images published above.
+  - name: docker
+    entrypoint: sh
+    args:
+    - -c
+    - |
+      #!/bin/sh
+      set -o errexit
+      set -o xtrace
+
+      # Publish manifest lists second, after all of the binary material
+      # has been uploaded, so that it is fast.  We want fast because enabling
+      # the experimental features in docker changes ~/.docker/config.json, which
+      # GCB periodically tramples.
+      #
+      # Enable support for 'docker manifest create'
+      # https://docs.docker.com/engine/reference/commandline/manifest_create/
+      sed -i 's/^{/{"experimental": "enabled",/g' ~/.docker/config.json
+
+      docker manifest create gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:multi-arch-${COMMIT_SHA} \
+         gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:amd64-${COMMIT_SHA} \
+         gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:arm64-${COMMIT_SHA}
+      docker manifest push gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:multi-arch-${COMMIT_SHA}
+
+
 images: ["gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:${COMMIT_SHA}",
          "gcr.io/$PROJECT_ID/${_EXECUTOR_IMAGE_NAME}:debug-${COMMIT_SHA}",
          "gcr.io/$PROJECT_ID/${_WARMER_IMAGE_NAME}:${COMMIT_SHA}"]
+
 substitutions:
   _EXECUTOR_IMAGE_NAME: executor
   _WARMER_IMAGE_NAME: warmer

files/BUILD

@@ -0,0 +1,16 @@
+load("@io_bazel_rules_docker//container:container.bzl", "container_image")
+
+container_image(
+    name = "nsswitch",
+    directory = "etc",
+    files = [":nsswitch.conf"],
+    visibility = ["//visibility:private"],
+)
+
+container_image(
+    name = "image",
+    base = ":nsswitch",
+    directory = "kaniko/ssl/certs",
+    files = [":ca-certificates.crt"],
+    visibility = ["//visibility:public"],
+)

integration/BUILD

@@ -0,0 +1,108 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "integration",
+    srcs = [
+        "cleanup.go",
+        "cmd.go",
+        "config.go",
+        "gcs.go",
+        "images.go",
+    ],
+    importpath = "github.com/GoogleContainerTools/kaniko/integration",
+    tags = ["manual"],
+    visibility = ["//visibility:public"],
+    deps = ["//pkg/timing"],
+)
+
+go_test(
+    name = "integration_test",
+    srcs = [
+        "benchmark_test.go",
+        "integration_test.go",
+        "integration_with_context_test.go",
+        "integration_with_stdin_test.go",
+        "k8s_test.go",
+    ],
+    embed = [":integration"],
+    tags = ["manual"],
+    deps = [
+        "//pkg/timing",
+        "//pkg/util",
+        "//testutil",
+        "//vendor/github.com/google/go-containerregistry/pkg/name",
+        "//vendor/github.com/google/go-containerregistry/pkg/v1/daemon",
+        "//vendor/github.com/pkg/errors",
+    ],
+)
+
+load("@io_bazel_rules_docker//container:container.bzl", "container_image")
+load("@io_bazel_rules_docker//contrib:test.bzl", "container_test")
+
+ARCHITECTURES = [
+    "amd64",
+    "arm64",
+]
+
+# Image with testdata
+[
+    container_image(
+        name = "buildtest_image_" + arch,
+        architecture = arch,
+        base = "//cmd/executor:image_" + arch,
+        directory = "/workspace",
+        files = [
+            ":testdata/Dockerfile.trivial",
+        ],
+    )
+    for arch in ARCHITECTURES
+]
+
+# Non-executable tests can run on any architecture,
+# so do not tag them.
+[
+    container_test(
+        name = "image_files_" + arch + "_test",
+        configs = ["testdata/files.yaml"],
+        image = "//cmd/executor:image_" + arch,
+    )
+    for arch in ARCHITECTURES
+]
+
+[
+    container_test(
+        name = "buildtest_image_" + arch + "_test",
+        configs = [
+            "testdata/files.yaml",
+            "testdata/testfiles.yaml",
+        ],
+        image = ":buildtest_image_" + arch,
+    )
+    for arch in ARCHITECTURES
+]
+
+[
+    container_test(
+        name = "image_exec_" + arch + "_test",
+        configs = ["testdata/exec.yaml"],
+        image = "//cmd/executor:image_" + arch,
+        tags = [
+            "manual",
+            arch,
+        ],
+    )
+    for arch in ARCHITECTURES
+]
+
+[
+    container_test(
+        name = "image_build_" + arch + "_test",
+        configs = ["testdata/build.yaml"],
+        image = ":buildtest_image_" + arch,
+        tags = [
+            "manual",
+            arch,
+        ],
+    )
+    for arch in ARCHITECTURES
+]

integration/testdata/Dockerfile.trivial

@@ -0,0 +1,2 @@
+FROM ubuntu
+RUN echo Hello

integration/testdata/build.yaml

@@ -0,0 +1,5 @@
+schemaVersion: "1.0.0"
+commandTests:
+- name: Trivial build
+  command: ["/kaniko/executor", "--no-push", "--dockerfile", "./Dockerfile.trivial"]
+  exitCode: 0

integration/testdata/exec.yaml

@@ -0,0 +1,10 @@
+schemaVersion: "1.0.0"
+commandTests:
+- name: check fixed name
+  command: ["/kaniko/executor"]
+  expectedError: ['Usage:']
+  exitCode: 1
+- name: check PATH
+  command: ["executor"]
+  expectedError: ['Usage:']
+  exitCode: 1

integration/testdata/files.yaml

@@ -0,0 +1,12 @@
+schemaVersion: "1.0.0"
+fileExistenceTests:
+# Basic FS sanity checks.
+- name: root
+  path: '/'
+  shouldExist: true
+- name: certs
+  path: '/kaniko/ssl/certs/ca-certificates.crt'
+  shouldExist: true
+- name: certs
+  path: '/etc/nsswitch.conf'
+  shouldExist: true

integration/testdata/testfiles.yaml

@@ -0,0 +1,6 @@
+schemaVersion: "1.0.0"
+fileExistenceTests:
+# Basic FS sanity checks.
+- name: trivial testdata Dockerfile
+  path: '/workspace/Dockerfile.trivial'
+  shouldExist: true