From e7a21cfc5341609055679569d3788ef9a061865a Mon Sep 17 00:00:00 2001 From: Tiago Melo Date: Fri, 23 Sep 2022 09:08:28 +0100 Subject: [PATCH] feat: Add container to propagate host network MTU (#1201) * feat: Add container to propagate host network MTU Some network environments use non-standard MTU values. In these situations, the `DockerMTU` setting might be used to specify the MTU setting for the `bridge` network created by Docker. However, when the Github Actions workflow creates networks, it doesn't propagate the `bridge` network MTU which can lead to `connection reset by peer` messages. To overcome this, I've created a new docker image called `summerwind/actions-runner-mtu` that shims the docker binary in order to propagate the MTU setting to networks created by Github workflows. This is a follow-up on the discussion in (#1046)[https://github.com/actions-runner-controller/actions-runner-controller/issues/1046] and uses a separate image since there might be some unintended side-effects with this approach. * fixup! feat: Add container to propagate host network MTU Co-authored-by: Yusuke Kuoka --- TROUBLESHOOTING.md | 24 ++++++++++++++++++++++-- runner/Makefile | 13 +++++++------ runner/actions-runner-dind.dockerfile | 2 +- runner/actions-runner.dockerfile | 6 +++++- runner/docker-shim.sh | 12 ++++++++++++ runner/supervisor/dockerd.conf | 2 +- 6 files changed, 48 insertions(+), 11 deletions(-) create mode 100755 runner/docker-shim.sh diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 4c5b842a..2e2bb1e1 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -256,8 +256,28 @@ spec: env: [] ``` -There may be more places you need to tweak for MTU. -Please consult issues like #651 for more information. +If the issue still persists, you can set the `ARC_DOCKER_MTU_PROPAGATION` to propagate the host MTU to networks created +by the GitHub Runner. For instance: + +```yaml +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: github-runner + namespace: github-system +spec: + replicas: 6 + template: + spec: + dockerMTU: 1400 + repository: $username/$repo + env: + - name: ARC_DOCKER_MTU_PROPAGATION + value: "true" +``` + +You can read the discussion regarding this issue in +(#1406)[https://github.com/actions-runner-controller/actions-runner-controller/issues/1046]. ## Unable to scale to zero with TotalNumberOfQueuedAndInProgressWorkflowRuns diff --git a/runner/Makefile b/runner/Makefile index 848d3c62..0b950a1a 100644 --- a/runner/Makefile +++ b/runner/Makefile @@ -1,4 +1,5 @@ DOCKER_USER ?= summerwind +DOCKER ?= docker NAME ?= ${DOCKER_USER}/actions-runner DIND_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind TAG ?= latest @@ -26,14 +27,14 @@ else endif docker-build-ubuntu: - docker build \ + ${DOCKER} build \ --build-arg TARGETPLATFORM=${TARGETPLATFORM} \ --build-arg RUNNER_VERSION=${RUNNER_VERSION} \ --build-arg RUNNER_CONTAINER_HOOKS_VERSION=${RUNNER_CONTAINER_HOOKS_VERSION} \ --build-arg DOCKER_VERSION=${DOCKER_VERSION} \ -f actions-runner.dockerfile \ -t ${NAME}:${TAG} . - docker build \ + ${DOCKER} build \ --build-arg TARGETPLATFORM=${TARGETPLATFORM} \ --build-arg RUNNER_VERSION=${RUNNER_VERSION} \ --build-arg DOCKER_VERSION=${DOCKER_VERSION} \ @@ -41,8 +42,8 @@ docker-build-ubuntu: -t ${DIND_RUNNER_NAME}:${TAG} . docker-push-ubuntu: - docker push ${NAME}:${TAG} - docker push ${DIND_RUNNER_NAME}:${TAG} + ${DOCKER} push ${NAME}:${TAG} + ${DOCKER} push ${DIND_RUNNER_NAME}:${TAG} docker-buildx-ubuntu: export DOCKER_CLI_EXPERIMENTAL=enabled ;\ @@ -50,14 +51,14 @@ docker-buildx-ubuntu: @if ! docker buildx ls | grep -q container-builder; then\ docker buildx create --platform ${PLATFORMS} --name container-builder --use;\ fi - docker buildx build --platform ${PLATFORMS} \ + ${DOCKER} buildx build --platform ${PLATFORMS} \ --build-arg RUNNER_VERSION=${RUNNER_VERSION} \ --build-arg RUNNER_CONTAINER_HOOKS_VERSION=${RUNNER_CONTAINER_HOOKS_VERSION} \ --build-arg DOCKER_VERSION=${DOCKER_VERSION} \ -f actions-runner.dockerfile \ -t "${NAME}:${TAG}" \ . ${PUSH_ARG} - docker buildx build --platform ${PLATFORMS} \ + ${DOCKER} buildx build --platform ${PLATFORMS} \ --build-arg RUNNER_VERSION=${RUNNER_VERSION} \ --build-arg RUNNER_CONTAINER_HOOKS_VERSION=${RUNNER_CONTAINER_HOOKS_VERSION} \ --build-arg DOCKER_VERSION=${DOCKER_VERSION} \ diff --git a/runner/actions-runner-dind.dockerfile b/runner/actions-runner-dind.dockerfile index a32af931..43ab6cd4 100644 --- a/runner/actions-runner-dind.dockerfile +++ b/runner/actions-runner-dind.dockerfile @@ -69,7 +69,7 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \ tar --extract \ --file docker.tgz \ --strip-components 1 \ - --directory /usr/local/bin/ \ + --directory /usr/bin/ \ ; \ rm docker.tgz; \ dockerd --version; \ diff --git a/runner/actions-runner.dockerfile b/runner/actions-runner.dockerfile index 79c37e7a..26e684ef 100644 --- a/runner/actions-runner.dockerfile +++ b/runner/actions-runner.dockerfile @@ -60,7 +60,7 @@ RUN set -vx; \ && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x86_64 ; fi \ && curl -f -L -o docker.tgz https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${ARCH}/docker-${DOCKER_VERSION}.tgz \ && tar zxvf docker.tgz \ - && install -o root -g root -m 755 docker/docker /usr/local/bin/docker \ + && install -o root -g root -m 755 docker/docker /usr/bin/docker \ && rm -rf docker docker.tgz \ && adduser --disabled-password --gecos "" --uid 1000 runner \ && groupadd docker \ @@ -119,6 +119,10 @@ RUN mkdir /opt/hostedtoolcache \ # override them with scripts of the same name placed in `/usr/local/bin`. COPY entrypoint.sh logger.bash update-status /usr/bin/ +# Copy the docker shim which propagates the docker MTU to underlying networks +# to replace the docker binary in the PATH. +COPY docker-shim.sh /usr/local/bin/docker + # Configure hooks folder structure. COPY hooks /etc/arc/hooks/ diff --git a/runner/docker-shim.sh b/runner/docker-shim.sh new file mode 100755 index 00000000..56d8f5b5 --- /dev/null +++ b/runner/docker-shim.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +if [[ ${ARC_DOCKER_MTU_PROPAGATION:-false} == true ]] && + (($# >= 2)) && [[ $1 == network && $2 == create ]] && + mtu=$(/usr/bin/docker network inspect bridge --format '{{index .Options "com.docker.network.driver.mtu"}}' 2>/dev/null); then + shift 2 + set -- network create --opt com.docker.network.driver.mtu="$mtu" "$@" +fi + +exec /usr/bin/docker "$@" diff --git a/runner/supervisor/dockerd.conf b/runner/supervisor/dockerd.conf index 08767893..27aa44f7 100644 --- a/runner/supervisor/dockerd.conf +++ b/runner/supervisor/dockerd.conf @@ -1,5 +1,5 @@ [program:dockerd] -command=/usr/local/bin/dockerd +command=/usr/bin/dockerd autostart=true autorestart=true stderr_logfile=/var/log/dockerd.err.log