diff --git a/Makefile b/Makefile index 0d4ace0da..ff4ef67eb 100644 --- a/Makefile +++ b/Makefile @@ -92,16 +92,10 @@ k8s-executor-build-push: DOCKER_BUILDKIT=1 docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:latest -f deploy/Dockerfile . docker push $(REGISTRY)/executor:latest -files/kaniko: - mkdir files/kaniko - -files/kaniko.tar: files/kaniko - chmod 777 files/kaniko - cd files && tar cf kaniko.tar kaniko .PHONY: images images: DOCKER_BUILDKIT=1 -images: files/kaniko.tar +images: docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:latest -f deploy/Dockerfile . docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug . docker build ${BUILD_ARG} --build-arg=GOARCH=$(GOARCH) -t $(REGISTRY)/executor:slim -f deploy/Dockerfile_slim . diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 1603755ee..2e760585c 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -36,9 +36,9 @@ RUN mkdir -p /kaniko/.docker COPY . . RUN \ - --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg \ - make GOARCH=$TARGETARCH + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + make GOARCH=$TARGETARCH # Generate latest ca-certificates FROM debian:bullseye-slim AS certs @@ -46,7 +46,9 @@ RUN apt update && apt install -y ca-certificates FROM scratch # Create kaniko directory with world write permission to allow non root run -ADD files/kaniko.tar / +# use musl busybox since it's staticly compiled +RUN --mount=from=busybox:musl,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] + COPY --from=0 /src/out/executor /kaniko/executor COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /usr/local/bin/docker-credential-ecr-login /kaniko/docker-credential-ecr-login diff --git a/deploy/Dockerfile_debug b/deploy/Dockerfile_debug index 130a5b48e..12c614d9f 100644 --- a/deploy/Dockerfile_debug +++ b/deploy/Dockerfile_debug @@ -36,27 +36,27 @@ RUN mkdir -p /kaniko/.docker COPY . . RUN \ - --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg \ - make GOARCH=$TARGETARCH && \ - make GOARCH=$TARGETARCH out/warmer + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + make GOARCH=$TARGETARCH && \ + make GOARCH=$TARGETARCH out/warmer # Generate latest ca-certificates FROM debian:bullseye-slim AS certs RUN apt update && apt install -y ca-certificates +# use musl busybox since it's staticly compiled on all platforms +FROM busybox:musl as busybox FROM scratch +# Create kaniko directory with world write permission to allow non root run +RUN --mount=from=busybox,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] + COPY --from=0 /src/out/executor /kaniko/executor COPY --from=0 /src/out/warmer /kaniko/warmer COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /usr/local/bin/docker-credential-ecr-login /kaniko/docker-credential-ecr-login COPY --from=0 /usr/local/bin/docker-credential-acr-env /kaniko/docker-credential-acr-env -COPY --from=busybox:1.32.0 /bin /busybox -# Since busybox needs some lib files which lie in /lib directory to run the executables on s390x, -# the below COPY command is added to address "ld64.so.1 not found" issue. This extra copy action will not -# happen on amd64 or arm64 platforms since /lib does not exist in amd64 or arm64 version of busybox container. -# Similar issues could be found in https://github.com/multiarch/qemu-user-static/issues/110#issuecomment-652951564. -COPY --from=busybox:1.32.0 /*lib /lib +COPY --from=busybox /bin /busybox # Declare /busybox as a volume to get it automatically in the path to ignore VOLUME /busybox diff --git a/deploy/Dockerfile_slim b/deploy/Dockerfile_slim index fc3a71a7f..5b176de39 100644 --- a/deploy/Dockerfile_slim +++ b/deploy/Dockerfile_slim @@ -31,6 +31,10 @@ FROM debian:bullseye-slim AS certs RUN apt update && apt install -y ca-certificates FROM scratch + +# Create kaniko directory with world write permission to allow non root run +# use musl busybox since it's staticly compiled +RUN --mount=from=busybox:musl,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] COPY --from=0 /src/out/executor /kaniko/executor COPY files/nsswitch.conf /etc/nsswitch.conf COPY --from=certs /etc/ssl/certs/ca-certificates.crt /kaniko/ssl/certs/ diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer index 9faaedb6f..062343c8a 100644 --- a/deploy/Dockerfile_warmer +++ b/deploy/Dockerfile_warmer @@ -36,15 +36,19 @@ RUN mkdir -p /kaniko/.docker COPY . . RUN \ - --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg \ - make GOARCH=$TARGETARCH out/warmer + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + make GOARCH=$TARGETARCH out/warmer # Generate latest ca-certificates FROM debian:bullseye-slim AS certs RUN apt update && apt install -y ca-certificates FROM scratch +# Create kaniko directory with world write permission to allow non root run +# use musl busybox since it's staticly compiled +RUN --mount=from=busybox:musl,dst=/usr/ ["busybox", "sh", "-c", "mkdir -p /kaniko && chmod 777 /kaniko"] + COPY --from=0 /src/out/warmer /kaniko/warmer COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr COPY --from=0 /usr/local/bin/docker-credential-ecr-login /kaniko/docker-credential-ecr-login diff --git a/scripts/minikube-setup.sh b/scripts/minikube-setup.sh index 885865e82..59cc389b8 100755 --- a/scripts/minikube-setup.sh +++ b/scripts/minikube-setup.sh @@ -22,13 +22,22 @@ if ! conntrack --version &>/dev/null; then sudo apt-get -qq -y install conntrack fi -if ! command -v minikube; then - curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 - chmod +x minikube - sudo mv minikube /usr/local/bin/ +# taken from https://github.com/kubernetes/minikube/blob/b45b29c5df6f88c6ac0afd60079a6190dc1e32c9/hack/jenkins/linux_integration_tests_none.sh#L38 +if ! kubeadm &>/dev/null; then + echo "WARNING: kubeadm is not installed. will try to install." + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubeadm" + sudo install kubeadm /usr/local/bin/kubeadm fi +# "none" driver specific cleanup from previous runs. +sudo kubeadm reset -f --cri-socket unix:///var/run/cri-dockerd.sock || true +# kubeadm reset may not stop pods immediately +docker rm -f $(docker ps -aq) >/dev/null 2>&1 || true +# always install minikube, because version inconsistency is possible and could lead to weird errors +curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +chmod +x minikube +sudo mv minikube /usr/local/bin/ # Minikube needs cri-dockerd to run clusters 1.24+ CRI_DOCKERD_VERSION="${CRI_DOCKERD_VERSION:-0.2.3}" @@ -36,7 +45,8 @@ CRI_DOCKERD_BINARY_URL="https://github.com/Mirantis/cri-dockerd/releases/downloa curl -Lo cri-dockerd.tgz $CRI_DOCKERD_BINARY_URL tar xfz cri-dockerd.tgz -sudo mv cri-dockerd/cri-dockerd /usr/bin/cri-docker +chmod +x cri-dockerd/cri-dockerd +sudo mv cri-dockerd/cri-dockerd /usr/bin/cri-dockerd git clone https://github.com/Mirantis/cri-dockerd.git /tmp/cri-dockerd sudo cp /tmp/cri-dockerd/packaging/systemd/* /etc/systemd/system @@ -44,19 +54,15 @@ sudo systemctl daemon-reload sudo systemctl enable cri-docker.service sudo systemctl enable --now cri-docker.socket -if ! command -v crictl; then - CRICTL_VERSION="v1.24.1" - curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz --output crictl-${CRICTL_VERSION}-linux-amd64.tar.gz - sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin - rm -f crictl-$CRICTL_VERSION-linux-amd64.tar.gz -fi +CRICTL_VERSION="v1.17.0" +curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz --output crictl-${CRICTL_VERSION}-linux-amd64.tar.gz +sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin +rm -f crictl-$CRICTL_VERSION-linux-amd64.tar.gz sudo apt-get update sudo apt-get install -y liblz4-tool cat /proc/cpuinfo -sudo systemctl unmask docker -minikube start --vm-driver=none --force +minikube start --vm-driver=none --force --addons="registry,default-storageclass,storage-provisioner" || minikube logs; minikube status -minikube addons enable registry kubectl cluster-info