From 3c4d403168ae9d0ddb02e5e75393153e697d19df Mon Sep 17 00:00:00 2001 From: prosper-sre <37417368+prosper-sre@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:20:29 -0700 Subject: [PATCH 1/5] Update _helpers.tpl --- charts/gha-runner-scale-set/templates/_helpers.tpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/gha-runner-scale-set/templates/_helpers.tpl b/charts/gha-runner-scale-set/templates/_helpers.tpl index bd71ed64..0b44166f 100644 --- a/charts/gha-runner-scale-set/templates/_helpers.tpl +++ b/charts/gha-runner-scale-set/templates/_helpers.tpl @@ -104,6 +104,8 @@ args: env: - name: DOCKER_GROUP_GID value: "123" + - name: DOCKER_IPTABLES_LEGACY + value: "1" securityContext: privileged: true volumeMounts: @@ -227,7 +229,7 @@ env: {{- end }} {{- if $setRunnerWaitDocker }} - name: RUNNER_WAIT_FOR_DOCKER_IN_SECONDS - value: "120" + value: "240" {{- end }} {{- if $setNodeExtraCaCerts }} - name: NODE_EXTRA_CA_CERTS From ed1b322a8873c705df6009ad03309d69d731cdc7 Mon Sep 17 00:00:00 2001 From: prosper-sre <37417368+prosper-sre@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:20:49 -0700 Subject: [PATCH 2/5] Update values.yaml --- charts/gha-runner-scale-set/values.yaml | 131 +++++++++++++----------- 1 file changed, 69 insertions(+), 62 deletions(-) diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml index 6018b7d0..c6214b9c 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml @@ -1,10 +1,10 @@ ## githubConfigUrl is the GitHub url for where you want to configure runners ## ex: https://github.com/myorg/myrepo or https://github.com/myorg -githubConfigUrl: "" +githubConfigUrl: "https://github.com/prosperllc" ## githubConfigSecret is the k8s secrets to use when auth with GitHub API. ## You can choose to use GitHub App or a PAT token -githubConfigSecret: +githubConfigSecret: pre-defined-secret ### GitHub Apps Configuration ## NOTE: IDs MUST be strings, use quotes #github_app_id: "" @@ -12,7 +12,7 @@ githubConfigSecret: #github_app_private_key: | ### GitHub PAT Configuration - github_token: "" +##github_token: "" ## If you have a pre-define Kubernetes secret in the same namespace the gha-runner-scale-set is going to deploy, ## you can also reference it via `githubConfigSecret: pre-defined-secret`. ## You need to make sure your predefined secret has all the required secret data set properly. @@ -37,16 +37,16 @@ githubConfigSecret: # - example.org ## maxRunners is the max number of runners the autoscaling runner set will scale up to. -# maxRunners: 5 +maxRunners: 20 ## minRunners is the min number of idle runners. The target number of runners created will be ## calculated as a sum of minRunners and the number of jobs assigned to the scale set. -# minRunners: 0 +minRunners: 5 -# runnerGroup: "default" +#runner-scale-set1" ## name of the runner scale set to create. Defaults to the helm release name -# runnerScaleSetName: "" +#runnerScaleSetName: "runner-scale-set1" ## A self-signed CA certificate for communication with the GitHub server can be ## provided using a config map key selector. If `runnerMountPath` is set, for @@ -75,8 +75,8 @@ githubConfigSecret: ## ## If any customization is required for dind or kubernetes mode, containerMode should remain ## empty, and configuration should be applied to the template. -# containerMode: -# type: "dind" ## type can be set to dind or kubernetes +containerMode: + type: "dind" ## type can be set to dind or kubernetes # ## the following is required when containerMode.type=kubernetes # kubernetesModeWorkVolumeClaim: # accessModes: ["ReadWriteOnce"] @@ -88,7 +88,7 @@ githubConfigSecret: # kubernetesModeServiceAccount: # annotations: -## listenerTemplate is the PodSpec for each listener Pod +## template is the PodSpec for each listener Pod ## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec # listenerTemplate: # spec: @@ -107,55 +107,55 @@ githubConfigSecret: ## template is the PodSpec for each runner Pod ## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec + template: - ## template.spec will be modified if you change the container mode - ## with containerMode.type=dind, we will populate the template.spec with following pod spec - ## template: - ## spec: - ## initContainers: - ## - name: init-dind-externals - ## image: ghcr.io/actions/actions-runner:latest - ## command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"] - ## volumeMounts: - ## - name: dind-externals - ## mountPath: /home/runner/tmpDir - ## containers: - ## - name: runner - ## image: ghcr.io/actions/actions-runner:latest - ## command: ["/home/runner/run.sh"] - ## env: - ## - name: DOCKER_HOST - ## value: unix:///var/run/docker.sock - ## volumeMounts: - ## - name: work - ## mountPath: /home/runner/_work - ## - name: dind-sock - ## mountPath: /var/run - ## - name: dind - ## image: docker:dind - ## args: - ## - dockerd - ## - --host=unix:///var/run/docker.sock - ## - --group=$(DOCKER_GROUP_GID) - ## env: - ## - name: DOCKER_GROUP_GID - ## value: "123" - ## securityContext: - ## privileged: true - ## volumeMounts: - ## - name: work - ## mountPath: /home/runner/_work - ## - name: dind-sock - ## mountPath: /var/run - ## - name: dind-externals - ## mountPath: /home/runner/externals - ## volumes: - ## - name: work - ## emptyDir: {} - ## - name: dind-sock - ## emptyDir: {} - ## - name: dind-externals - ## emptyDir: {} + spec: + initContainers: + - name: init-dind-externals + image: us-docker.pkg.dev/prosper-prod-registry/images/actions-runner:latest + command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"] + volumeMounts: + - name: dind-externals + mountPath: /home/runner/tmpDir + containers: + - name: runner + image: ghcr.io/actions/actions-runner:latest + command: ["/home/runner/run.sh"] + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + volumeMounts: + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + - name: dind + image: docker:dind + args: + - dockerd + - --host=unix:///var/run/docker.sock + - --group=$(DOCKER_GROUP_GID) + env: + - name: DOCKER_GROUP_GID + value: "123" + - name: DOCKER_IPTABLES_LEGACY + value: '1' + securityContext: + privileged: true + volumeMounts: + - name: work + mountPath: /home/runner/_work + - name: dind-sock + mountPath: /var/run + - name: dind-externals + mountPath: /home/runner/externals + volumes: + - name: work + emptyDir: {} + - name: dind-sock + emptyDir: {} + - name: dind-externals + emptyDir: {} ###################################################################################################### ## with containerMode.type=kubernetes, we will populate the template.spec with following pod spec ## template: @@ -189,8 +189,15 @@ template: spec: containers: - name: runner - image: ghcr.io/actions/actions-runner:latest + image: us-docker.pkg.dev/prosper-prod-registry/images/actions-runner:latest command: ["/home/runner/run.sh"] + resources: + limits: + cpu: "4" + memory: 8Gi + request: + cpu: "2" + memory: 4Gi ## Optional controller service account that needs to have required Role and RoleBinding ## to operate this gha-runner-scale-set installation. @@ -198,6 +205,6 @@ template: ## In case the helm chart can't find the right service account, you can explicitly pass in the following value ## to help it finish RoleBinding with the right service account. ## Note: if your controller is installed to only watch a single namespace, you have to pass these values explicitly. -# controllerServiceAccount: -# namespace: arc-system -# name: test-arc-gha-runner-scale-set-controller +controllerServiceAccount: + namespace: prosper-runners + name: prosper-runner-controller-gha-rs-controller From 3418ddb0c5997296fa03db1059712cb94e705ac7 Mon Sep 17 00:00:00 2001 From: prosper-sre Date: Wed, 17 Apr 2024 14:40:24 -0700 Subject: [PATCH 3/5] added readme file --- charts/gha-runner-scale-set/README.md | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 charts/gha-runner-scale-set/README.md diff --git a/charts/gha-runner-scale-set/README.md b/charts/gha-runner-scale-set/README.md new file mode 100644 index 00000000..9abf90c0 --- /dev/null +++ b/charts/gha-runner-scale-set/README.md @@ -0,0 +1,29 @@ +Runner Scale set installation steps + + +GCP Porject : prosper-nonprod-platform +cluster : gke_prosper-nonprod-platform_us-west1_nonprod-us-west1-util02 +namespace : prosper-runners + + +helm ls + +❯ helm ls +NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION +prosper-linux-np prosper-runners 5 2024-04-17 14:10:37.697746 -0700 PDT deployed gha-runner-scale-set-0.9.0 0.9.0 +prosper-runner-controller prosper-runners 1 2024-04-15 13:09:45.65406 -0700 PDT deployed gha-runner-scale-set-controller-0.9.0 0.9.0 + + + +Clone the actions runner controller repo + + +Goto --> Charts --- gha-runner-scale-set-controller + + helm upgrade prosper-runner-controller . + +Goto --> Charts --- gha-runner-scale-set-controller + + helm upgrade prosper-linux-np . + + From 1b8ac765445dcd4f39eaee47bb402abdf71d5294 Mon Sep 17 00:00:00 2001 From: prosper-sre <37417368+prosper-sre@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:42:21 -0700 Subject: [PATCH 4/5] Update README.md --- charts/gha-runner-scale-set/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/gha-runner-scale-set/README.md b/charts/gha-runner-scale-set/README.md index 9abf90c0..1937d1fc 100644 --- a/charts/gha-runner-scale-set/README.md +++ b/charts/gha-runner-scale-set/README.md @@ -9,6 +9,7 @@ namespace : prosper-runners helm ls ❯ helm ls + NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION prosper-linux-np prosper-runners 5 2024-04-17 14:10:37.697746 -0700 PDT deployed gha-runner-scale-set-0.9.0 0.9.0 prosper-runner-controller prosper-runners 1 2024-04-15 13:09:45.65406 -0700 PDT deployed gha-runner-scale-set-controller-0.9.0 0.9.0 From d725bc41d232a65cf7544768ab952cec22e3ce39 Mon Sep 17 00:00:00 2001 From: prosper-sre <37417368+prosper-sre@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:51:48 -0700 Subject: [PATCH 5/5] Update values.yaml --- .../values.yaml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/charts/gha-runner-scale-set-controller/values.yaml b/charts/gha-runner-scale-set-controller/values.yaml index efd39958..dd434c7c 100644 --- a/charts/gha-runner-scale-set-controller/values.yaml +++ b/charts/gha-runner-scale-set-controller/values.yaml @@ -72,8 +72,6 @@ tolerations: [] affinity: {} -topologySpreadConstraints: [] - # Mount volumes in the container. volumes: [] volumeMounts: [] @@ -83,16 +81,16 @@ volumeMounts: [] # PriorityClass: system-cluster-critical priorityClassName: "" -## If `metrics:` object is not provided, or commented out, the following flags -## will be applied the controller-manager and listener pods with empty values: -## `--metrics-addr`, `--listener-metrics-addr`, `--listener-metrics-endpoint`. +## If `metrics:` object is not provided, or commented out, the following flags +## will be applied the controller-manager and listener pods with empty values: +## `--metrics-addr`, `--listener-metrics-addr`, `--listener-metrics-endpoint`. ## This will disable metrics. ## ## To enable metrics, uncomment the following lines. -# metrics: -# controllerManagerAddr: ":8080" -# listenerAddr: ":8080" -# listenerEndpoint: "/metrics" +metrics: + controllerManagerAddr: ":8080" + listenerAddr: ":8080" + listenerEndpoint: "/metrics" flags: ## Log level can be set here with one of the following values: "debug", "info", "warn", "error". @@ -104,7 +102,7 @@ flags: ## Restricts the controller to only watch resources in the desired namespace. ## Defaults to watch all namespaces when unset. - # watchSingleNamespace: "" + watchSingleNamespace: "prosper-runners" ## Defines how the controller should handle upgrades while having running jobs. ##