diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 2e5c92c2..38070051 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -304,3 +304,27 @@ If you noticed that it takes several minutes for sidecar dind container to be cr **Solution** The solution is to switch to using faster storage, if you are experiencing this issue you are probably using HDD storage. Switching to SSD storage fixed the problem in my case. Most cloud providers have a list of storage options to use just pick something faster that your current disk, for on prem clusters you will need to invest in some SSDs. + +### Dockerd no space left on device + +**Problem** + +If you are running many containers on your runner you might encounter an issue where docker daemon is unable to start new containers and you see error `no space left on device`. + +**Solution** + +Add a `dockerVarRunVolumeSizeLimit` key in your runner's spec with a higher size limit (the default is 1M) For instance: + +```yaml +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: github-runner + namespace: github-system +spec: + replicas: 6 + template: + spec: + dockerVarRunVolumeSizeLimit: 50M + env: [] +``` \ No newline at end of file diff --git a/apis/actions.summerwind.net/v1alpha1/runner_types.go b/apis/actions.summerwind.net/v1alpha1/runner_types.go index 7986efad..ca62238e 100644 --- a/apis/actions.summerwind.net/v1alpha1/runner_types.go +++ b/apis/actions.summerwind.net/v1alpha1/runner_types.go @@ -70,6 +70,8 @@ type RunnerConfig struct { // +optional DockerRegistryMirror *string `json:"dockerRegistryMirror,omitempty"` // +optional + DockerVarRunVolumeSizeLimit *resource.Quantity `json:"dockerVarRunVolumeSizeLimit,omitempty"` + // +optional VolumeSizeLimit *resource.Quantity `json:"volumeSizeLimit,omitempty"` // +optional VolumeStorageMedium *string `json:"volumeStorageMedium,omitempty"` diff --git a/apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go b/apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go index bc450b7c..ac21ce49 100644 --- a/apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go +++ b/apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go @@ -436,6 +436,11 @@ func (in *RunnerConfig) DeepCopyInto(out *RunnerConfig) { *out = new(string) **out = **in } + if in.DockerVarRunVolumeSizeLimit != nil { + in, out := &in.DockerVarRunVolumeSizeLimit, &out.DockerVarRunVolumeSizeLimit + x := (*in).DeepCopy() + *out = &x + } if in.VolumeSizeLimit != nil { in, out := &in.VolumeSizeLimit, &out.VolumeSizeLimit x := (*in).DeepCopy() diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml index cb5f54ae..4291b6b6 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml @@ -1497,6 +1497,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml index a2e6aba7..877bd518 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml @@ -1479,6 +1479,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml index beaea51b..c688cc6d 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml @@ -1432,6 +1432,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml index 7cdb8156..11409281 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml @@ -55,6 +55,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerdWithinRunnerContainer: type: boolean effectiveTime: diff --git a/config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml b/config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml index cb5f54ae..4291b6b6 100644 --- a/config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml +++ b/config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml @@ -1497,6 +1497,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml b/config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml index a2e6aba7..877bd518 100644 --- a/config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml +++ b/config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml @@ -1479,6 +1479,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/config/crd/bases/actions.summerwind.dev_runners.yaml b/config/crd/bases/actions.summerwind.dev_runners.yaml index beaea51b..c688cc6d 100644 --- a/config/crd/bases/actions.summerwind.dev_runners.yaml +++ b/config/crd/bases/actions.summerwind.dev_runners.yaml @@ -1432,6 +1432,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerVolumeMounts: items: description: VolumeMount describes a mounting of a Volume within a container. diff --git a/config/crd/bases/actions.summerwind.dev_runnersets.yaml b/config/crd/bases/actions.summerwind.dev_runnersets.yaml index 7cdb8156..11409281 100644 --- a/config/crd/bases/actions.summerwind.dev_runnersets.yaml +++ b/config/crd/bases/actions.summerwind.dev_runnersets.yaml @@ -55,6 +55,12 @@ spec: type: integer dockerRegistryMirror: type: string + dockerVarRunVolumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true dockerdWithinRunnerContainer: type: boolean effectiveTime: diff --git a/controllers/actions.summerwind.net/runner_controller.go b/controllers/actions.summerwind.net/runner_controller.go index 574d08aa..a711fd8c 100644 --- a/controllers/actions.summerwind.net/runner_controller.go +++ b/controllers/actions.summerwind.net/runner_controller.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "k8s.io/apimachinery/pkg/api/resource" "reflect" "strconv" "strings" @@ -30,7 +31,6 @@ import ( "github.com/go-logr/logr" kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" @@ -810,6 +810,11 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru dockerRegistryMirror = *runnerSpec.DockerRegistryMirror } + if runnerSpec.DockerVarRunVolumeSizeLimit == nil { + runnerSpec.DockerVarRunVolumeSizeLimit = resource.NewScaledQuantity(1, resource.Mega) + + } + // Be aware some of the environment variables are used // in the runner entrypoint script env := []corev1.EnvVar{ @@ -1080,7 +1085,7 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{ Medium: corev1.StorageMediumMemory, - SizeLimit: resource.NewScaledQuantity(1, resource.Mega), + SizeLimit: runnerSpec.DockerVarRunVolumeSizeLimit, }, }, },