feat: Add container to propagate host network MTU (#1201)

* feat: Add container to propagate host network MTU

Some network environments use non-standard MTU values. In these
situations, the `DockerMTU` setting might be used to specify the MTU
setting for the `bridge` network created by Docker. However, when the
Github Actions workflow creates networks, it doesn't propagate the
`bridge` network MTU which can lead to `connection reset by peer`
messages.

To overcome this, I've created a new docker image called
`summerwind/actions-runner-mtu` that shims the docker binary in order to
propagate the MTU setting to networks created by Github workflows.

This is a follow-up on the discussion in
(#1046)[https://github.com/actions-runner-controller/actions-runner-controller/issues/1046]
and uses a separate image since there might be some unintended
side-effects with this approach.

* fixup! feat: Add container to propagate host network MTU

Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>

TROUBLESHOOTING.md

@@ -256,8 +256,28 @@ spec:
       env: []
 ```
 
-There may be more places you need to tweak for MTU.
-Please consult issues like #651 for more information.
+If the issue still persists, you can set the `ARC_DOCKER_MTU_PROPAGATION` to propagate the host MTU to networks created
+by the GitHub Runner. For instance:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: github-runner
+  namespace: github-system
+spec:
+  replicas: 6
+  template:
+    spec:
+      dockerMTU: 1400
+      repository: $username/$repo
+      env:
+        - name: ARC_DOCKER_MTU_PROPAGATION
+          value: "true"
+```
+
+You can read the discussion regarding this issue in
+(#1406)[https://github.com/actions-runner-controller/actions-runner-controller/issues/1046].
 
 ## Unable to scale to zero with TotalNumberOfQueuedAndInProgressWorkflowRuns
 

runner/Makefile

@@ -1,4 +1,5 @@
 DOCKER_USER ?= summerwind
+DOCKER ?= docker
 NAME ?= ${DOCKER_USER}/actions-runner
 DIND_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind
 TAG ?= latest
@@ -26,14 +27,14 @@ else
 endif
 
 docker-build-ubuntu:
-	docker build \
+	${DOCKER} build \
 	  --build-arg TARGETPLATFORM=${TARGETPLATFORM} \
 	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
 	  --build-arg RUNNER_CONTAINER_HOOKS_VERSION=${RUNNER_CONTAINER_HOOKS_VERSION} \
 	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
 	  -f actions-runner.dockerfile \
 	  -t ${NAME}:${TAG} .
-	docker build \
+	${DOCKER} build \
 	  --build-arg TARGETPLATFORM=${TARGETPLATFORM} \
 	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
 	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
@@ -41,8 +42,8 @@ docker-build-ubuntu:
 	  -t ${DIND_RUNNER_NAME}:${TAG} .
 
 docker-push-ubuntu:
-	docker push ${NAME}:${TAG}
-	docker push ${DIND_RUNNER_NAME}:${TAG}
+	${DOCKER} push ${NAME}:${TAG}
+	${DOCKER} push ${DIND_RUNNER_NAME}:${TAG}
 
 docker-buildx-ubuntu:
 	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
@@ -50,14 +51,14 @@ docker-buildx-ubuntu:
 	@if ! docker buildx ls | grep -q container-builder; then\
 	  docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
 	fi
-	docker buildx build --platform ${PLATFORMS} \
+	${DOCKER} buildx build --platform ${PLATFORMS} \
 	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
 	  --build-arg RUNNER_CONTAINER_HOOKS_VERSION=${RUNNER_CONTAINER_HOOKS_VERSION} \
 	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
 	  -f actions-runner.dockerfile \
 	  -t "${NAME}:${TAG}" \
 	  . ${PUSH_ARG}
-	docker buildx build --platform ${PLATFORMS} \
+	${DOCKER} buildx build --platform ${PLATFORMS} \
 	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
 	  --build-arg RUNNER_CONTAINER_HOOKS_VERSION=${RUNNER_CONTAINER_HOOKS_VERSION} \
 	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \

runner/actions-runner-dind.dockerfile

@@ -69,7 +69,7 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
 	tar --extract \
 		--file docker.tgz \
 		--strip-components 1 \
-		--directory /usr/local/bin/ \
+		--directory /usr/bin/ \
 	; \
 	rm docker.tgz; \
 	dockerd --version; \

runner/actions-runner.dockerfile

@@ -60,7 +60,7 @@ RUN set -vx; \
     && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x86_64 ; fi \
     && curl -f -L -o docker.tgz https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${ARCH}/docker-${DOCKER_VERSION}.tgz \
     && tar zxvf docker.tgz \
-    && install -o root -g root -m 755 docker/docker /usr/local/bin/docker \
+    && install -o root -g root -m 755 docker/docker /usr/bin/docker \
     && rm -rf docker docker.tgz \
     && adduser --disabled-password --gecos "" --uid 1000 runner \
     && groupadd docker \
@@ -119,6 +119,10 @@ RUN mkdir /opt/hostedtoolcache \
 # override them with scripts of the same name placed in `/usr/local/bin`.
 COPY entrypoint.sh logger.bash update-status /usr/bin/
 
+# Copy the docker shim which propagates the docker MTU to underlying networks
+# to replace the docker binary in the PATH.
+COPY docker-shim.sh /usr/local/bin/docker
+
 # Configure hooks folder structure.
 COPY hooks /etc/arc/hooks/
 

runner/docker-shim.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+if [[ ${ARC_DOCKER_MTU_PROPAGATION:-false} == true ]] &&
+  (($# >= 2)) && [[ $1 == network && $2 == create ]] &&
+  mtu=$(/usr/bin/docker network inspect bridge --format '{{index .Options "com.docker.network.driver.mtu"}}' 2>/dev/null); then
+  shift 2
+  set -- network create --opt com.docker.network.driver.mtu="$mtu" "$@"
+fi
+
+exec /usr/bin/docker "$@"

runner/supervisor/dockerd.conf

@@ -1,5 +1,5 @@
 [program:dockerd]
-command=/usr/local/bin/dockerd
+command=/usr/bin/dockerd
 autostart=true
 autorestart=true
 stderr_logfile=/var/log/dockerd.err.log