diff --git a/apis/actions.github.com/v1alpha1/autoscalinglistener_types.go b/apis/actions.github.com/v1alpha1/autoscalinglistener_types.go index 68eb7664..e4e5c383 100644 --- a/apis/actions.github.com/v1alpha1/autoscalinglistener_types.go +++ b/apis/actions.github.com/v1alpha1/autoscalinglistener_types.go @@ -54,11 +54,13 @@ type AutoscalingListenerSpec struct { // Required ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + + // +optional + Proxy *ProxyConfig `json:"proxy,omitempty"` } // AutoscalingListenerStatus defines the observed state of AutoscalingListener -type AutoscalingListenerStatus struct { -} +type AutoscalingListenerStatus struct{} //+kubebuilder:object:root=true //+kubebuilder:subresource:status diff --git a/apis/actions.github.com/v1alpha1/autoscalingrunnerset_types.go b/apis/actions.github.com/v1alpha1/autoscalingrunnerset_types.go index a842ff83..ba8af2fc 100644 --- a/apis/actions.github.com/v1alpha1/autoscalingrunnerset_types.go +++ b/apis/actions.github.com/v1alpha1/autoscalingrunnerset_types.go @@ -17,7 +17,13 @@ limitations under the License. package v1alpha1 import ( + "fmt" + "net/http" + "net/url" + "strings" + "github.com/actions/actions-runner-controller/hash" + "golang.org/x/net/http/httpproxy" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -80,6 +86,94 @@ type ProxyConfig struct { // +optional HTTPS *ProxyServerConfig `json:"https,omitempty"` + + // +optional + NoProxy []string `json:"noProxy,omitempty"` +} + +func (c *ProxyConfig) toHTTPProxyConfig(secretFetcher func(string) (*corev1.Secret, error)) (*httpproxy.Config, error) { + config := &httpproxy.Config{ + NoProxy: strings.Join(c.NoProxy, ","), + } + + if c.HTTP != nil { + u, err := url.Parse(c.HTTP.Url) + if err != nil { + return nil, fmt.Errorf("failed to parse proxy http url %q: %w", c.HTTP.Url, err) + } + + if c.HTTP.CredentialSecretRef != "" { + secret, err := secretFetcher(c.HTTP.CredentialSecretRef) + if err != nil { + return nil, fmt.Errorf( + "failed to get secret %s for http proxy: %w", + c.HTTP.CredentialSecretRef, + err, + ) + } + + u.User = url.UserPassword( + string(secret.Data["username"]), + string(secret.Data["password"]), + ) + } + + config.HTTPProxy = u.String() + } + + if c.HTTPS != nil { + u, err := url.Parse(c.HTTPS.Url) + if err != nil { + return nil, fmt.Errorf("failed to parse proxy https url %q: %w", c.HTTPS.Url, err) + } + + if c.HTTPS.CredentialSecretRef != "" { + secret, err := secretFetcher(c.HTTPS.CredentialSecretRef) + if err != nil { + return nil, fmt.Errorf( + "failed to get secret %s for https proxy: %w", + c.HTTPS.CredentialSecretRef, + err, + ) + } + + u.User = url.UserPassword( + string(secret.Data["username"]), + string(secret.Data["password"]), + ) + } + + config.HTTPSProxy = u.String() + } + + return config, nil +} + +func (c *ProxyConfig) ToSecretData(secretFetcher func(string) (*corev1.Secret, error)) (map[string][]byte, error) { + config, err := c.toHTTPProxyConfig(secretFetcher) + if err != nil { + return nil, err + } + + data := map[string][]byte{} + data["http_proxy"] = []byte(config.HTTPProxy) + data["https_proxy"] = []byte(config.HTTPSProxy) + data["no_proxy"] = []byte(config.NoProxy) + + return data, nil +} + +func (c *ProxyConfig) ProxyFunc(secretFetcher func(string) (*corev1.Secret, error)) (func(*http.Request) (*url.URL, error), error) { + config, err := c.toHTTPProxyConfig(secretFetcher) + if err != nil { + return nil, err + } + + proxyFunc := func(req *http.Request) (*url.URL, error) { + return config.ProxyFunc()(req.URL) + } + + return proxyFunc, nil } type ProxyServerConfig struct { @@ -88,9 +182,6 @@ type ProxyServerConfig struct { // +optional CredentialSecretRef string `json:"credentialSecretRef,omitempty"` - - // +optional - NoProxy []string `json:"noProxy,omitempty"` } // AutoscalingRunnerSetStatus defines the observed state of AutoscalingRunnerSet diff --git a/apis/actions.github.com/v1alpha1/ephemeralrunner_types.go b/apis/actions.github.com/v1alpha1/ephemeralrunner_types.go index dbfe040e..631abde3 100644 --- a/apis/actions.github.com/v1alpha1/ephemeralrunner_types.go +++ b/apis/actions.github.com/v1alpha1/ephemeralrunner_types.go @@ -59,6 +59,9 @@ type EphemeralRunnerSpec struct { // +optional Proxy *ProxyConfig `json:"proxy,omitempty"` + // +optional + ProxySecretRef string `json:"proxySecretRef,omitempty"` + // +optional GitHubServerTLS *GitHubServerTLSConfig `json:"githubServerTLS,omitempty"` diff --git a/apis/actions.github.com/v1alpha1/proxy_config_test.go b/apis/actions.github.com/v1alpha1/proxy_config_test.go new file mode 100644 index 00000000..9291cde4 --- /dev/null +++ b/apis/actions.github.com/v1alpha1/proxy_config_test.go @@ -0,0 +1,118 @@ +package v1alpha1_test + +import ( + "net/http" + "testing" + + corev1 "k8s.io/api/core/v1" + + "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProxyConfig_ToSecret(t *testing.T) { + config := &v1alpha1.ProxyConfig{ + HTTP: &v1alpha1.ProxyServerConfig{ + Url: "http://proxy.example.com:8080", + CredentialSecretRef: "my-secret", + }, + HTTPS: &v1alpha1.ProxyServerConfig{ + Url: "https://proxy.example.com:8080", + CredentialSecretRef: "my-secret", + }, + NoProxy: []string{ + "noproxy.example.com", + "noproxy2.example.com", + }, + } + + secretFetcher := func(string) (*corev1.Secret, error) { + return &corev1.Secret{ + Data: map[string][]byte{ + "username": []byte("username"), + "password": []byte("password"), + }, + }, nil + } + + result, err := config.ToSecretData(secretFetcher) + require.NoError(t, err) + require.NotNil(t, result) + + assert.Equal(t, "http://username:password@proxy.example.com:8080", string(result["http_proxy"])) + assert.Equal(t, "https://username:password@proxy.example.com:8080", string(result["https_proxy"])) + assert.Equal(t, "noproxy.example.com,noproxy2.example.com", string(result["no_proxy"])) +} + +func TestProxyConfig_ProxyFunc(t *testing.T) { + config := &v1alpha1.ProxyConfig{ + HTTP: &v1alpha1.ProxyServerConfig{ + Url: "http://proxy.example.com:8080", + CredentialSecretRef: "my-secret", + }, + HTTPS: &v1alpha1.ProxyServerConfig{ + Url: "https://proxy.example.com:8080", + CredentialSecretRef: "my-secret", + }, + NoProxy: []string{ + "noproxy.example.com", + "noproxy2.example.com", + }, + } + + secretFetcher := func(string) (*corev1.Secret, error) { + return &corev1.Secret{ + Data: map[string][]byte{ + "username": []byte("username"), + "password": []byte("password"), + }, + }, nil + } + + result, err := config.ProxyFunc(secretFetcher) + require.NoError(t, err) + + tests := []struct { + name string + in string + out string + }{ + { + name: "http target", + in: "http://target.com", + out: "http://username:password@proxy.example.com:8080", + }, + { + name: "https target", + in: "https://target.com", + out: "https://username:password@proxy.example.com:8080", + }, + { + name: "no proxy", + in: "https://noproxy.example.com", + out: "", + }, + { + name: "no proxy 2", + in: "https://noproxy2.example.com", + out: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, err := http.NewRequest("GET", test.in, nil) + require.NoError(t, err) + u, err := result(req) + require.NoError(t, err) + + if test.out == "" { + assert.Nil(t, u) + return + } + + assert.Equal(t, test.out, u.String()) + }) + } +} diff --git a/apis/actions.github.com/v1alpha1/zz_generated.deepcopy.go b/apis/actions.github.com/v1alpha1/zz_generated.deepcopy.go index 753dd7fb..324707b2 100644 --- a/apis/actions.github.com/v1alpha1/zz_generated.deepcopy.go +++ b/apis/actions.github.com/v1alpha1/zz_generated.deepcopy.go @@ -93,6 +93,11 @@ func (in *AutoscalingListenerSpec) DeepCopyInto(out *AutoscalingListenerSpec) { *out = make([]v1.LocalObjectReference, len(*in)) copy(*out, *in) } + if in.Proxy != nil { + in, out := &in.Proxy, &out.Proxy + *out = new(ProxyConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingListenerSpec. @@ -448,12 +453,17 @@ func (in *ProxyConfig) DeepCopyInto(out *ProxyConfig) { if in.HTTP != nil { in, out := &in.HTTP, &out.HTTP *out = new(ProxyServerConfig) - (*in).DeepCopyInto(*out) + **out = **in } if in.HTTPS != nil { in, out := &in.HTTPS, &out.HTTPS *out = new(ProxyServerConfig) - (*in).DeepCopyInto(*out) + **out = **in + } + if in.NoProxy != nil { + in, out := &in.NoProxy, &out.NoProxy + *out = make([]string, len(*in)) + copy(*out, *in) } } @@ -470,11 +480,6 @@ func (in *ProxyConfig) DeepCopy() *ProxyConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyServerConfig) DeepCopyInto(out *ProxyServerConfig) { *out = *in - if in.NoProxy != nil { - in, out := &in.NoProxy, &out.NoProxy - *out = make([]string, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyServerConfig. diff --git a/charts/actions-runner-controller-2/crds/actions.github.com_autoscalinglisteners.yaml b/charts/actions-runner-controller-2/crds/actions.github.com_autoscalinglisteners.yaml index 18946cb3..f0f3f8fb 100644 --- a/charts/actions-runner-controller-2/crds/actions.github.com_autoscalinglisteners.yaml +++ b/charts/actions-runner-controller-2/crds/actions.github.com_autoscalinglisteners.yaml @@ -76,6 +76,29 @@ spec: description: Required minimum: 0 type: integer + proxy: + properties: + http: + properties: + credentialSecretRef: + type: string + url: + description: Required + type: string + type: object + https: + properties: + credentialSecretRef: + type: string + url: + description: Required + type: string + type: object + noProxy: + items: + type: string + type: array + type: object runnerScaleSetId: description: Required type: integer diff --git a/charts/actions-runner-controller-2/crds/actions.github.com_autoscalingrunnersets.yaml b/charts/actions-runner-controller-2/crds/actions.github.com_autoscalingrunnersets.yaml index 9542f522..00775406 100644 --- a/charts/actions-runner-controller-2/crds/actions.github.com_autoscalingrunnersets.yaml +++ b/charts/actions-runner-controller-2/crds/actions.github.com_autoscalingrunnersets.yaml @@ -67,10 +67,6 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string @@ -79,14 +75,14 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string type: object + noProxy: + items: + type: string + type: array type: object runnerGroup: type: string diff --git a/charts/actions-runner-controller-2/crds/actions.github.com_ephemeralrunners.yaml b/charts/actions-runner-controller-2/crds/actions.github.com_ephemeralrunners.yaml index f321a85a..41cdc81b 100644 --- a/charts/actions-runner-controller-2/crds/actions.github.com_ephemeralrunners.yaml +++ b/charts/actions-runner-controller-2/crds/actions.github.com_ephemeralrunners.yaml @@ -94,10 +94,6 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string @@ -106,15 +102,17 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string type: object + noProxy: + items: + type: string + type: array type: object + proxySecretRef: + type: string runnerScaleSetId: type: integer spec: diff --git a/charts/actions-runner-controller-2/crds/actions.github.com_ephemeralrunnersets.yaml b/charts/actions-runner-controller-2/crds/actions.github.com_ephemeralrunnersets.yaml index d4b2d351..072cd265 100644 --- a/charts/actions-runner-controller-2/crds/actions.github.com_ephemeralrunnersets.yaml +++ b/charts/actions-runner-controller-2/crds/actions.github.com_ephemeralrunnersets.yaml @@ -76,10 +76,6 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string @@ -88,15 +84,17 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string type: object + noProxy: + items: + type: string + type: array type: object + proxySecretRef: + type: string runnerScaleSetId: type: integer spec: diff --git a/charts/auto-scaling-runner-set/templates/autoscalingrunnerset.yaml b/charts/auto-scaling-runner-set/templates/autoscalingrunnerset.yaml index e29ec157..d12c886a 100644 --- a/charts/auto-scaling-runner-set/templates/autoscalingrunnerset.yaml +++ b/charts/auto-scaling-runner-set/templates/autoscalingrunnerset.yaml @@ -12,6 +12,23 @@ spec: runnerGroup: {{ . }} {{- end }} + {{- if .Values.proxy }} + proxy: + {{- if .Values.proxy.http }} + http: + url: {{ .Values.proxy.http.url }} + credentialSecretRef: {{ .Values.proxy.http.credentialSecretRef }} + {{ end }} + {{- if .Values.proxy.https }} + https: + url: {{ .Values.proxy.https.url }} + credentialSecretRef: {{ .Values.proxy.https.credentialSecretRef }} + {{ end }} + {{- if and .Values.proxy.noProxy (kindIs "slice" .Values.proxy.noProxy) }} + noProxy: {{ .Values.proxy.noProxy | toYaml | nindent 6}} + {{ end }} + {{ end }} + {{- if and (or (kindIs "int64" .Values.minRunners) (kindIs "float64" .Values.minRunners)) (or (kindIs "int64" .Values.maxRunners) (kindIs "float64" .Values.maxRunners)) }} {{- if gt .Values.minRunners .Values.maxRunners }} {{- fail "maxRunners has to be greater or equal to minRunners" }} diff --git a/charts/auto-scaling-runner-set/tests/template_test.go b/charts/auto-scaling-runner-set/tests/template_test.go index 89439b0f..96a8e894 100644 --- a/charts/auto-scaling-runner-set/tests/template_test.go +++ b/charts/auto-scaling-runner-set/tests/template_test.go @@ -737,3 +737,46 @@ func TestTemplateRenderedAutoScalingRunnerSet_ErrorOnEmptyPredefinedSecret(t *te assert.ErrorContains(t, err, "Values.githubConfigSecret is required for setting auth with GitHub server") } + +func TestTemplateRenderedWithProxy(t *testing.T) { + t.Parallel() + + // Path to the helm chart we will test + helmChartPath, err := filepath.Abs("../../auto-scaling-runner-set") + require.NoError(t, err) + + releaseName := "test-runners" + namespaceName := "test-" + strings.ToLower(random.UniqueId()) + + options := &helm.Options{ + SetValues: map[string]string{ + "githubConfigUrl": "https://github.com/actions", + "githubConfigSecret": "pre-defined-secrets", + "proxy.http.url": "http://proxy.example.com", + "proxy.http.credentialSecretRef": "http-secret", + "proxy.https.url": "https://proxy.example.com", + "proxy.https.credentialSecretRef": "https-secret", + "proxy.noProxy": "{example.com,example.org}", + }, + KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName), + } + + output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/autoscalingrunnerset.yaml"}) + + var ars v1alpha1.AutoscalingRunnerSet + helm.UnmarshalK8SYaml(t, output, &ars) + + require.NotNil(t, ars.Spec.Proxy) + require.NotNil(t, ars.Spec.Proxy.HTTP) + assert.Equal(t, "http://proxy.example.com", ars.Spec.Proxy.HTTP.Url) + assert.Equal(t, "http-secret", ars.Spec.Proxy.HTTP.CredentialSecretRef) + + require.NotNil(t, ars.Spec.Proxy.HTTPS) + assert.Equal(t, "https://proxy.example.com", ars.Spec.Proxy.HTTPS.Url) + assert.Equal(t, "https-secret", ars.Spec.Proxy.HTTPS.CredentialSecretRef) + + require.NotNil(t, ars.Spec.Proxy.NoProxy) + require.Len(t, ars.Spec.Proxy.NoProxy, 2) + assert.Contains(t, ars.Spec.Proxy.NoProxy, "example.com") + assert.Contains(t, ars.Spec.Proxy.NoProxy, "example.org") +} diff --git a/charts/auto-scaling-runner-set/values.yaml b/charts/auto-scaling-runner-set/values.yaml index 6494ecda..0e3b10be 100644 --- a/charts/auto-scaling-runner-set/values.yaml +++ b/charts/auto-scaling-runner-set/values.yaml @@ -22,6 +22,20 @@ githubConfigSecret: ## > kubectl create secret generic pre-defined-secret --namespace=my_namespace --from-literal=github_app_id=123456 --from-literal=github_app_installation_id=654321 --from-literal=github_app_private_key='-----BEGIN CERTIFICATE-----*******' # githubConfigSecret: pre-defined-secret +## proxy can be used to define proxy settings that will be used by the +## controller, the listener and the runner of this scale set. +# +# proxy: +# http: +# url: http://proxy.com:1234 +# credentialSecretRef: proxy-auth # a secret with `username` and `password` keys +# https: +# url: http://proxy.com:1234 +# credentialSecretRef: proxy-auth # a secret with `username` and `password` keys +# noProxy: +# - example.com +# - example.org + ## maxRunners is the max number of runners the auto scaling runner set will scale up to. # maxRunners: 5 diff --git a/config/crd/bases/actions.github.com_autoscalinglisteners.yaml b/config/crd/bases/actions.github.com_autoscalinglisteners.yaml index 18946cb3..f0f3f8fb 100644 --- a/config/crd/bases/actions.github.com_autoscalinglisteners.yaml +++ b/config/crd/bases/actions.github.com_autoscalinglisteners.yaml @@ -76,6 +76,29 @@ spec: description: Required minimum: 0 type: integer + proxy: + properties: + http: + properties: + credentialSecretRef: + type: string + url: + description: Required + type: string + type: object + https: + properties: + credentialSecretRef: + type: string + url: + description: Required + type: string + type: object + noProxy: + items: + type: string + type: array + type: object runnerScaleSetId: description: Required type: integer diff --git a/config/crd/bases/actions.github.com_autoscalingrunnersets.yaml b/config/crd/bases/actions.github.com_autoscalingrunnersets.yaml index 9542f522..00775406 100644 --- a/config/crd/bases/actions.github.com_autoscalingrunnersets.yaml +++ b/config/crd/bases/actions.github.com_autoscalingrunnersets.yaml @@ -67,10 +67,6 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string @@ -79,14 +75,14 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string type: object + noProxy: + items: + type: string + type: array type: object runnerGroup: type: string diff --git a/config/crd/bases/actions.github.com_ephemeralrunners.yaml b/config/crd/bases/actions.github.com_ephemeralrunners.yaml index f321a85a..41cdc81b 100644 --- a/config/crd/bases/actions.github.com_ephemeralrunners.yaml +++ b/config/crd/bases/actions.github.com_ephemeralrunners.yaml @@ -94,10 +94,6 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string @@ -106,15 +102,17 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string type: object + noProxy: + items: + type: string + type: array type: object + proxySecretRef: + type: string runnerScaleSetId: type: integer spec: diff --git a/config/crd/bases/actions.github.com_ephemeralrunnersets.yaml b/config/crd/bases/actions.github.com_ephemeralrunnersets.yaml index d4b2d351..072cd265 100644 --- a/config/crd/bases/actions.github.com_ephemeralrunnersets.yaml +++ b/config/crd/bases/actions.github.com_ephemeralrunnersets.yaml @@ -76,10 +76,6 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string @@ -88,15 +84,17 @@ spec: properties: credentialSecretRef: type: string - noProxy: - items: - type: string - type: array url: description: Required type: string type: object + noProxy: + items: + type: string + type: array type: object + proxySecretRef: + type: string runnerScaleSetId: type: integer spec: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index a07ac823..e7063a8d 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: jokicnikola07/actions-runner-controller + newName: summerwind/actions-runner-controller newTag: dev diff --git a/controllers/actions.github.com/autoscalinglistener_controller.go b/controllers/actions.github.com/autoscalinglistener_controller.go index faf2e4e6..3110a748 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller.go +++ b/controllers/actions.github.com/autoscalinglistener_controller.go @@ -40,6 +40,7 @@ import ( ) const ( + autoscalingListenerContainerName = "autoscaler" autoscalingListenerOwnerKey = ".metadata.controller" autoscalingListenerFinalizerName = "autoscalinglistener.actions.github.com/finalizer" ) @@ -202,6 +203,21 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl. return r.createRoleBindingForListener(ctx, autoscalingListener, listenerRole, serviceAccount, log) } + // Create a secret containing proxy config if specifiec + if autoscalingListener.Spec.Proxy != nil { + proxySecret := new(corev1.Secret) + if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Namespace, Name: proxyListenerSecretName(autoscalingListener)}, proxySecret); err != nil { + if !kerrors.IsNotFound(err) { + log.Error(err, "Unable to get listener proxy secret", "namespace", autoscalingListener.Namespace, "name", proxyListenerSecretName(autoscalingListener)) + return ctrl.Result{}, err + } + + // Create a mirror secret for the listener pod in the Controller namespace for listener pod to use + log.Info("Creating a listener proxy secret for the listener pod") + return r.createProxySecret(ctx, autoscalingListener, log) + } + } + // TODO: make sure the role binding has the up-to-date role and service account listenerPod := new(corev1.Pod) @@ -307,6 +323,25 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au } logger.Info("Listener pod is deleted") + if autoscalingListener.Spec.Proxy != nil { + logger.Info("Cleaning up the listener proxy secret") + proxySecret := new(corev1.Secret) + err = r.Get(ctx, types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingListener.Namespace}, proxySecret) + switch { + case err == nil: + if proxySecret.ObjectMeta.DeletionTimestamp.IsZero() { + logger.Info("Deleting the listener proxy secret") + if err := r.Delete(ctx, proxySecret); err != nil { + return false, fmt.Errorf("failed to delete listener proxy secret: %v", err) + } + } + return false, nil + case err != nil && !kerrors.IsNotFound(err): + return false, fmt.Errorf("failed to get listener proxy secret: %v", err) + } + logger.Info("Listener proxy secret is deleted") + } + logger.Info("Cleaning up the listener service account") listenerSa := new(corev1.ServiceAccount) err = r.Get(ctx, types.NamespacedName{Name: scaleSetListenerServiceAccountName(autoscalingListener), Namespace: autoscalingListener.Namespace}, listenerSa) @@ -345,7 +380,49 @@ func (r *AutoscalingListenerReconciler) createServiceAccountForListener(ctx cont } func (r *AutoscalingListenerReconciler) createListenerPod(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, autoscalingListener *v1alpha1.AutoscalingListener, serviceAccount *corev1.ServiceAccount, secret *corev1.Secret, logger logr.Logger) (ctrl.Result, error) { - newPod := r.resourceBuilder.newScaleSetListenerPod(autoscalingListener, serviceAccount, secret) + var envs []corev1.EnvVar + if autoscalingListener.Spec.Proxy != nil { + httpURL := corev1.EnvVar{ + Name: "http_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, + Key: "http_proxy", + }, + }, + } + if autoscalingListener.Spec.Proxy.HTTP != nil { + envs = append(envs, httpURL) + } + + httpsURL := corev1.EnvVar{ + Name: "https_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, + Key: "https_proxy", + }, + }, + } + if autoscalingListener.Spec.Proxy.HTTPS != nil { + envs = append(envs, httpsURL) + } + + noProxy := corev1.EnvVar{ + Name: "no_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, + Key: "no_proxy", + }, + }, + } + if len(autoscalingListener.Spec.Proxy.NoProxy) > 0 { + envs = append(envs, noProxy) + } + } + + newPod := r.resourceBuilder.newScaleSetListenerPod(autoscalingListener, serviceAccount, secret, envs...) if err := ctrl.SetControllerReference(autoscalingListener, newPod, r.Scheme); err != nil { return ctrl.Result{}, err @@ -378,6 +455,45 @@ func (r *AutoscalingListenerReconciler) createSecretsForListener(ctx context.Con return ctrl.Result{}, nil } +func (r *AutoscalingListenerReconciler) createProxySecret(ctx context.Context, autoscalingListener *v1alpha1.AutoscalingListener, logger logr.Logger) (ctrl.Result, error) { + data, err := autoscalingListener.Spec.Proxy.ToSecretData(func(s string) (*corev1.Secret, error) { + var secret corev1.Secret + err := r.Get(ctx, types.NamespacedName{Name: s, Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, &secret) + if err != nil { + return nil, fmt.Errorf("failed to get secret %s: %w", s, err) + } + return &secret, nil + }) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to convert proxy config to secret data: %w", err) + } + + newProxySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: proxyListenerSecretName(autoscalingListener), + Namespace: autoscalingListener.Namespace, + Labels: map[string]string{ + "auto-scaling-runner-set-namespace": autoscalingListener.Spec.AutoscalingRunnerSetNamespace, + "auto-scaling-runner-set-name": autoscalingListener.Spec.AutoscalingRunnerSetName, + }, + }, + Data: data, + } + if err := ctrl.SetControllerReference(autoscalingListener, newProxySecret, r.Scheme); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create listener proxy secret: %w", err) + } + + logger.Info("Creating listener proxy secret", "namespace", newProxySecret.Namespace, "name", newProxySecret.Name) + if err := r.Create(ctx, newProxySecret); err != nil { + logger.Error(err, "Unable to create listener secret", "namespace", newProxySecret.Namespace, "name", newProxySecret.Name) + return ctrl.Result{}, err + } + + logger.Info("Created listener proxy secret", "namespace", newProxySecret.Namespace, "name", newProxySecret.Name) + + return ctrl.Result{}, nil +} + func (r *AutoscalingListenerReconciler) updateSecretsForListener(ctx context.Context, secret *corev1.Secret, mirrorSecret *corev1.Secret, logger logr.Logger) (ctrl.Result, error) { dataHash := hash.ComputeTemplateHash(secret.Data) updatedMirrorSecret := mirrorSecret.DeepCopy() diff --git a/controllers/actions.github.com/autoscalinglistener_controller_test.go b/controllers/actions.github.com/autoscalinglistener_controller_test.go index 09961efd..d5cf3280 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller_test.go +++ b/controllers/actions.github.com/autoscalinglistener_controller_test.go @@ -13,7 +13,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" ) @@ -222,7 +224,7 @@ var _ = Describe("Test AutoScalingListener controller", func() { Context("When deleting a new AutoScalingListener", func() { It("It should cleanup all resources for a deleting AutoScalingListener before removing it", func() { - // Waiting for the pod is created + // Waiting for the pod to be created pod := new(corev1.Pod) Eventually( func() (string, error) { @@ -391,3 +393,234 @@ var _ = Describe("Test AutoScalingListener controller", func() { }) }) }) + +var _ = Describe("Test AutoScalingListener controller with proxy", func() { + var ctx context.Context + var cancel context.CancelFunc + autoscalingNS := new(corev1.Namespace) + autoscalingRunnerSet := new(actionsv1alpha1.AutoscalingRunnerSet) + configSecret := new(corev1.Secret) + autoscalingListener := new(actionsv1alpha1.AutoscalingListener) + + createRunnerSetAndListener := func(proxy *actionsv1alpha1.ProxyConfig) { + min := 1 + max := 10 + autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-asrs", + Namespace: autoscalingNS.Name, + }, + Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{ + GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigSecret: configSecret.Name, + MaxRunners: &max, + MinRunners: &min, + Proxy: proxy, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "runner", + Image: "ghcr.io/actions/runner", + }, + }, + }, + }, + }, + } + + err := k8sClient.Create(ctx, autoscalingRunnerSet) + Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") + + autoscalingListener = &actionsv1alpha1.AutoscalingListener{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-asl", + Namespace: autoscalingNS.Name, + }, + Spec: actionsv1alpha1.AutoscalingListenerSpec{ + GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigSecret: configSecret.Name, + RunnerScaleSetId: 1, + AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, + AutoscalingRunnerSetName: autoscalingRunnerSet.Name, + EphemeralRunnerSetName: "test-ers", + MaxRunners: 10, + MinRunners: 1, + Image: "ghcr.io/owner/repo", + Proxy: proxy, + }, + } + + err = k8sClient.Create(ctx, autoscalingListener) + Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingListener") + } + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.TODO()) + autoscalingNS = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "testns-autoscaling-listener" + RandStringRunes(5)}, + } + + err := k8sClient.Create(ctx, autoscalingNS) + Expect(err).NotTo(HaveOccurred(), "failed to create test namespace for AutoScalingRunnerSet") + + configSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "github-config-secret", + Namespace: autoscalingNS.Name, + }, + Data: map[string][]byte{ + "github_token": []byte(autoscalingListenerTestGitHubToken), + }, + } + + err = k8sClient.Create(ctx, configSecret) + Expect(err).NotTo(HaveOccurred(), "failed to create config secret") + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Namespace: autoscalingNS.Name, + MetricsBindAddress: "0", + }) + Expect(err).NotTo(HaveOccurred(), "failed to create manager") + + controller := &AutoscalingListenerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: logf.Log, + } + err = controller.SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred(), "failed to setup controller") + + go func() { + defer GinkgoRecover() + + err := mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to start manager") + }() + }) + + AfterEach(func() { + defer cancel() + + err := k8sClient.Delete(ctx, autoscalingNS) + Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace for AutoScalingRunnerSet") + }) + + It("should create a secret in the listener namespace containing proxy details, use it to populate env vars on the pod and should delete it as part of cleanup", func() { + proxyCredentials := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxy-credentials", + Namespace: autoscalingNS.Name, + }, + Data: map[string][]byte{ + "username": []byte("test"), + "password": []byte("password"), + }, + } + + err := k8sClient.Create(ctx, proxyCredentials) + Expect(err).NotTo(HaveOccurred(), "failed to create proxy credentials secret") + + proxy := &actionsv1alpha1.ProxyConfig{ + HTTP: &actionsv1alpha1.ProxyServerConfig{ + Url: "http://localhost:8080", + CredentialSecretRef: "proxy-credentials", + }, + HTTPS: &actionsv1alpha1.ProxyServerConfig{ + Url: "https://localhost:8443", + CredentialSecretRef: "proxy-credentials", + }, + NoProxy: []string{ + "example.com", + "example.org", + }, + } + + createRunnerSetAndListener(proxy) + + var proxySecret corev1.Secret + Eventually( + func(g Gomega) { + err := k8sClient.Get( + ctx, + types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingNS.Name}, + &proxySecret, + ) + g.Expect(err).NotTo(HaveOccurred(), "failed to get secret") + expected, err := autoscalingListener.Spec.Proxy.ToSecretData(func(s string) (*corev1.Secret, error) { + var secret corev1.Secret + err := k8sClient.Get(ctx, types.NamespacedName{Name: s, Namespace: autoscalingNS.Name}, &secret) + if err != nil { + return nil, err + } + return &secret, nil + }) + g.Expect(err).NotTo(HaveOccurred(), "failed to convert proxy config to secret data") + g.Expect(proxySecret.Data).To(Equal(expected)) + }, + autoscalingRunnerSetTestTimeout, + autoscalingRunnerSetTestInterval, + ).Should(Succeed(), "failed to create secret with proxy details") + + // wait for listener pod to be created + Eventually( + func(g Gomega) { + pod := new(corev1.Pod) + err := k8sClient.Get( + ctx, + client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, + pod, + ) + g.Expect(err).NotTo(HaveOccurred(), "failed to get pod") + + g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ + Name: "http_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, + Key: "http_proxy", + }, + }, + }), "http_proxy environment variable not found") + + g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ + Name: "https_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, + Key: "https_proxy", + }, + }, + }), "https_proxy environment variable not found") + + g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ + Name: "no_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, + Key: "no_proxy", + }, + }, + }), "no_proxy environment variable not found") + }, + autoscalingListenerTestTimeout, + autoscalingListenerTestInterval).Should(Succeed(), "failed to create listener pod with proxy details") + + // Delete the AutoScalingListener + err = k8sClient.Delete(ctx, autoscalingListener) + Expect(err).NotTo(HaveOccurred(), "failed to delete test AutoScalingListener") + + Eventually( + func(g Gomega) { + var proxySecret corev1.Secret + err := k8sClient.Get( + ctx, + types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingNS.Name}, + &proxySecret, + ) + g.Expect(kerrors.IsNotFound(err)).To(BeTrue()) + }, + autoscalingListenerTestTimeout, + autoscalingListenerTestInterval).Should(Succeed(), "failed to delete secret with proxy details") + }) +}) diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller.go b/controllers/actions.github.com/autoscalingrunnerset_controller.go index b956d281..99fa0cb5 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller.go @@ -42,7 +42,6 @@ import ( const ( // TODO: Replace with shared image. - name = "autoscaler" autoscalingRunnerSetOwnerKey = ".metadata.controller" LabelKeyRunnerSpecHash = "runner-spec-hash" LabelKeyAutoScaleRunnerSetName = "auto-scale-runner-set-name" @@ -495,7 +494,31 @@ func (r *AutoscalingRunnerSetReconciler) actionsClientFor(ctx context.Context, a return nil, fmt.Errorf("failed to find GitHub config secret: %w", err) } - return r.ActionsClient.GetClientFromSecret(ctx, autoscalingRunnerSet.Spec.GitHubConfigUrl, autoscalingRunnerSet.Namespace, configSecret.Data) + var opts []actions.ClientOption + if autoscalingRunnerSet.Spec.Proxy != nil { + proxyFunc, err := autoscalingRunnerSet.Spec.Proxy.ProxyFunc(func(s string) (*corev1.Secret, error) { + var secret corev1.Secret + err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingRunnerSet.Namespace, Name: s}, &secret) + if err != nil { + return nil, fmt.Errorf("failed to get proxy secret %s: %w", s, err) + } + + return &secret, nil + }) + if err != nil { + return nil, fmt.Errorf("failed to get proxy func: %w", err) + } + + opts = append(opts, actions.WithProxy(proxyFunc)) + } + + return r.ActionsClient.GetClientFromSecret( + ctx, + autoscalingRunnerSet.Spec.GitHubConfigUrl, + autoscalingRunnerSet.Namespace, + configSecret.Data, + opts..., + ) } // SetupWithManager sets up the controller with the Manager. diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller_test.go b/controllers/actions.github.com/autoscalingrunnerset_controller_test.go index 65bebe8c..4b1ac8b9 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller_test.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller_test.go @@ -2,7 +2,11 @@ package actionsgithubcom import ( "context" + "encoding/base64" "fmt" + "net/http" + "net/http/httptest" + "strings" "time" corev1 "k8s.io/api/core/v1" @@ -11,13 +15,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" + "github.com/actions/actions-runner-controller/github/actions" "github.com/actions/actions-runner-controller/github/actions/fake" + "github.com/actions/actions-runner-controller/github/actions/testserver" ) const ( @@ -570,3 +577,206 @@ var _ = Describe("Test AutoscalingController creation failures", func() { }) }) }) + +var _ = Describe("Test Client optional configuration", func() { + Context("When specifying a proxy", func() { + var ctx context.Context + var cancel context.CancelFunc + + autoscalingNS := new(corev1.Namespace) + configSecret := new(corev1.Secret) + var mgr ctrl.Manager + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.TODO()) + autoscalingNS = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "testns-autoscaling" + RandStringRunes(5)}, + } + + err := k8sClient.Create(ctx, autoscalingNS) + Expect(err).NotTo(HaveOccurred(), "failed to create test namespace for AutoScalingRunnerSet") + + configSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "github-config-secret", + Namespace: autoscalingNS.Name, + }, + Data: map[string][]byte{ + "github_token": []byte(autoscalingRunnerSetTestGitHubToken), + }, + } + + err = k8sClient.Create(ctx, configSecret) + Expect(err).NotTo(HaveOccurred(), "failed to create config secret") + + mgr, err = ctrl.NewManager(cfg, ctrl.Options{ + Namespace: autoscalingNS.Name, + }) + Expect(err).NotTo(HaveOccurred(), "failed to create manager") + + go func() { + defer GinkgoRecover() + + err := mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to start manager") + }() + }) + + AfterEach(func() { + defer cancel() + + err := k8sClient.Delete(ctx, autoscalingNS) + Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace for AutoScalingRunnerSet") + }) + + It("should be able to make requests to a server using a proxy", func() { + controller := &AutoscalingRunnerSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: logf.Log, + ControllerNamespace: autoscalingNS.Name, + DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", + ActionsClient: actions.NewMultiClient("test", logr.Discard()), + } + err := controller.SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred(), "failed to setup controller") + + serverSuccessfullyCalled := false + proxy := testserver.New(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + serverSuccessfullyCalled = true + w.WriteHeader(http.StatusOK) + })) + + min := 1 + max := 10 + autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-asrs", + Namespace: autoscalingNS.Name, + }, + Spec: v1alpha1.AutoscalingRunnerSetSpec{ + GitHubConfigUrl: "http://example.com/org/repo", + GitHubConfigSecret: configSecret.Name, + MaxRunners: &max, + MinRunners: &min, + RunnerGroup: "testgroup", + Proxy: &v1alpha1.ProxyConfig{ + HTTP: &v1alpha1.ProxyServerConfig{ + Url: proxy.URL, + }, + }, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "runner", + Image: "ghcr.io/actions/runner", + }, + }, + }, + }, + }, + } + + err = k8sClient.Create(ctx, autoscalingRunnerSet) + Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") + + // wait for server to be called + Eventually( + func() (bool, error) { + return serverSuccessfullyCalled, nil + }, + autoscalingRunnerSetTestTimeout, + 1*time.Nanosecond, + ).Should(BeTrue(), "server was not called") + }) + + It("should be able to make requests to a server using a proxy with user info", func() { + controller := &AutoscalingRunnerSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: logf.Log, + ControllerNamespace: autoscalingNS.Name, + DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", + ActionsClient: actions.NewMultiClient("test", logr.Discard()), + } + err := controller.SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred(), "failed to setup controller") + + serverSuccessfullyCalled := false + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + header := r.Header.Get("Proxy-Authorization") + Expect(header).NotTo(BeEmpty()) + + header = strings.TrimPrefix(header, "Basic ") + decoded, err := base64.StdEncoding.DecodeString(header) + Expect(err).NotTo(HaveOccurred()) + Expect(string(decoded)).To(Equal("test:password")) + + serverSuccessfullyCalled = true + w.WriteHeader(http.StatusOK) + })) + GinkgoT().Cleanup(func() { + proxy.Close() + }) + + secretCredentials := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxy-credentials", + Namespace: autoscalingNS.Name, + }, + Data: map[string][]byte{ + "username": []byte("test"), + "password": []byte("password"), + }, + } + + err = k8sClient.Create(ctx, secretCredentials) + Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") + + min := 1 + max := 10 + autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-asrs", + Namespace: autoscalingNS.Name, + }, + Spec: v1alpha1.AutoscalingRunnerSetSpec{ + GitHubConfigUrl: "http://example.com/org/repo", + GitHubConfigSecret: configSecret.Name, + MaxRunners: &max, + MinRunners: &min, + RunnerGroup: "testgroup", + Proxy: &v1alpha1.ProxyConfig{ + HTTP: &v1alpha1.ProxyServerConfig{ + Url: proxy.URL, + CredentialSecretRef: "proxy-credentials", + }, + }, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "runner", + Image: "ghcr.io/actions/runner", + }, + }, + }, + }, + }, + } + + err = k8sClient.Create(ctx, autoscalingRunnerSet) + Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") + + // wait for server to be called + Eventually( + func() (bool, error) { + return serverSuccessfullyCalled, nil + }, + autoscalingRunnerSetTestTimeout, + 1*time.Nanosecond, + ).Should(BeTrue(), "server was not called") + }) + }) +}) diff --git a/controllers/actions.github.com/constants.go b/controllers/actions.github.com/constants.go index 70f39628..613db79c 100644 --- a/controllers/actions.github.com/constants.go +++ b/controllers/actions.github.com/constants.go @@ -9,3 +9,10 @@ const ( EnvVarRunnerJITConfig = "ACTIONS_RUNNER_INPUT_JITCONFIG" EnvVarRunnerExtraUserAgent = "GITHUB_ACTIONS_RUNNER_EXTRA_USER_AGENT" ) + +// Environment variable names used to set proxy variables for containers +const ( + EnvVarHTTPProxy = "http_proxy" + EnvVarHTTPSProxy = "https_proxy" + EnvVarNoProxy = "no_proxy" +) diff --git a/controllers/actions.github.com/ephemeralrunner_controller.go b/controllers/actions.github.com/ephemeralrunner_controller.go index e6bfc9cb..516526a3 100644 --- a/controllers/actions.github.com/ephemeralrunner_controller.go +++ b/controllers/actions.github.com/ephemeralrunner_controller.go @@ -557,8 +557,56 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con } func (r *EphemeralRunnerReconciler) createPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, secret *corev1.Secret, log logr.Logger) (ctrl.Result, error) { + var envs []corev1.EnvVar + if runner.Spec.ProxySecretRef != "" { + http := corev1.EnvVar{ + Name: "http_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: runner.Spec.ProxySecretRef, + }, + Key: "http_proxy", + }, + }, + } + if runner.Spec.Proxy.HTTP != nil { + envs = append(envs, http) + } + + https := corev1.EnvVar{ + Name: "https_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: runner.Spec.ProxySecretRef, + }, + Key: "https_proxy", + }, + }, + } + if runner.Spec.Proxy.HTTPS != nil { + envs = append(envs, https) + } + + noProxy := corev1.EnvVar{ + Name: "no_proxy", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: runner.Spec.ProxySecretRef, + }, + Key: "no_proxy", + }, + }, + } + if len(runner.Spec.Proxy.NoProxy) > 0 { + envs = append(envs, noProxy) + } + } + log.Info("Creating new pod for ephemeral runner") - newPod := r.resourceBuilder.newEphemeralRunnerPod(ctx, runner, secret) + newPod := r.resourceBuilder.newEphemeralRunnerPod(ctx, runner, secret, envs...) if err := ctrl.SetControllerReference(runner, newPod, r.Scheme); err != nil { log.Error(err, "Failed to set controller reference to a new pod") @@ -632,7 +680,31 @@ func (r *EphemeralRunnerReconciler) actionsClientFor(ctx context.Context, runner return nil, fmt.Errorf("failed to get secret: %w", err) } - return r.ActionsClient.GetClientFromSecret(ctx, runner.Spec.GitHubConfigUrl, runner.Namespace, secret.Data) + var opts []actions.ClientOption + if runner.Spec.Proxy != nil { + proxyFunc, err := runner.Spec.Proxy.ProxyFunc(func(s string) (*corev1.Secret, error) { + var secret corev1.Secret + err := r.Get(ctx, types.NamespacedName{Namespace: runner.Namespace, Name: s}, &secret) + if err != nil { + return nil, fmt.Errorf("failed to get proxy secret %s: %w", s, err) + } + + return &secret, nil + }) + if err != nil { + return nil, fmt.Errorf("failed to get proxy func: %w", err) + } + + opts = append(opts, actions.WithProxy(proxyFunc)) + } + + return r.ActionsClient.GetClientFromSecret( + ctx, + runner.Spec.GitHubConfigUrl, + runner.Namespace, + secret.Data, + opts..., + ) } // runnerRegisteredWithService checks if the runner is still registered with the service diff --git a/controllers/actions.github.com/ephemeralrunner_controller_test.go b/controllers/actions.github.com/ephemeralrunner_controller_test.go index ba5d9fb2..3f1747ab 100644 --- a/controllers/actions.github.com/ephemeralrunner_controller_test.go +++ b/controllers/actions.github.com/ephemeralrunner_controller_test.go @@ -2,12 +2,16 @@ package actionsgithubcom import ( "context" + "encoding/base64" "fmt" "net/http" + "net/http/httptest" + "strings" "time" "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" "github.com/actions/actions-runner-controller/github/actions" + "github.com/go-logr/logr" "github.com/actions/actions-runner-controller/github/actions/fake" . "github.com/onsi/ginkgo/v2" @@ -773,4 +777,185 @@ var _ = Describe("EphemeralRunner", func() { }, timeout, interval).Should(BeEquivalentTo(corev1.PodSucceeded)) }) }) + + Describe("Pod proxy config", func() { + var ctx context.Context + var cancel context.CancelFunc + + autoScalingNS := new(corev1.Namespace) + configSecret := new(corev1.Secret) + controller := new(EphemeralRunnerReconciler) + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) + autoScalingNS = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testns-autoscaling-runner" + RandStringRunes(5), + }, + } + err := k8sClient.Create(ctx, autoScalingNS) + Expect(err).To(BeNil(), "failed to create test namespace for EphemeralRunner") + + configSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "github-config-secret", + Namespace: autoScalingNS.Name, + }, + Data: map[string][]byte{ + "github_token": []byte(gh_token), + }, + } + + err = k8sClient.Create(ctx, configSecret) + Expect(err).To(BeNil(), "failed to create config secret") + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Namespace: autoScalingNS.Name, + MetricsBindAddress: "0", + }) + Expect(err).To(BeNil(), "failed to create manager") + + controller = &EphemeralRunnerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: logf.Log, + ActionsClient: fake.NewMultiClient(), + } + + err = controller.SetupWithManager(mgr) + Expect(err).To(BeNil(), "failed to setup controller") + + go func() { + defer GinkgoRecover() + + err := mgr.Start(ctx) + Expect(err).To(BeNil(), "failed to start manager") + }() + }) + + AfterEach(func() { + defer cancel() + + err := k8sClient.Delete(ctx, autoScalingNS) + Expect(err).To(BeNil(), "failed to delete test namespace for EphemeralRunner") + }) + + It("uses an actions client with proxy transport", func() { + // Use an actual client + controller.ActionsClient = actions.NewMultiClient("test", logr.Discard()) + + proxySuccessfulllyCalled := false + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + header := r.Header.Get("Proxy-Authorization") + Expect(header).NotTo(BeEmpty()) + + header = strings.TrimPrefix(header, "Basic ") + decoded, err := base64.StdEncoding.DecodeString(header) + Expect(err).NotTo(HaveOccurred()) + Expect(string(decoded)).To(Equal("test:password")) + + proxySuccessfulllyCalled = true + w.WriteHeader(http.StatusOK) + })) + GinkgoT().Cleanup(func() { + proxy.Close() + }) + + secretCredentials := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxy-credentials", + Namespace: autoScalingNS.Name, + }, + Data: map[string][]byte{ + "username": []byte("test"), + "password": []byte("password"), + }, + } + + err := k8sClient.Create(ctx, secretCredentials) + Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") + + ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) + ephemeralRunner.Spec.GitHubConfigUrl = "http://example.com/org/repo" + ephemeralRunner.Spec.Proxy = &v1alpha1.ProxyConfig{ + HTTP: &v1alpha1.ProxyServerConfig{ + Url: proxy.URL, + CredentialSecretRef: "proxy-credentials", + }, + } + + err = k8sClient.Create(ctx, ephemeralRunner) + Expect(err).To(BeNil(), "failed to create ephemeral runner") + + Eventually( + func() bool { + return proxySuccessfulllyCalled + }, + 2*time.Second, + interval, + ).Should(BeEquivalentTo(true)) + }) + + It("It should create EphemeralRunner with proxy environment variables using ProxySecretRef", func() { + ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) + ephemeralRunner.Spec.Proxy = &v1alpha1.ProxyConfig{ + HTTP: &v1alpha1.ProxyServerConfig{ + Url: "http://proxy.example.com:8080", + }, + HTTPS: &v1alpha1.ProxyServerConfig{ + Url: "http://proxy.example.com:8080", + }, + NoProxy: []string{"example.com"}, + } + ephemeralRunner.Spec.ProxySecretRef = "proxy-secret" + err := k8sClient.Create(ctx, ephemeralRunner) + Expect(err).To(BeNil(), "failed to create ephemeral runner") + + pod := new(corev1.Pod) + Eventually( + func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) + g.Expect(err).To(BeNil(), "failed to get ephemeral runner pod") + }, + timeout, + interval, + ).Should(Succeed(), "failed to get ephemeral runner pod") + + Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ + Name: EnvVarHTTPProxy, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ephemeralRunner.Spec.ProxySecretRef, + }, + Key: "http_proxy", + }, + }, + })) + + Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ + Name: EnvVarHTTPSProxy, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ephemeralRunner.Spec.ProxySecretRef, + }, + Key: "https_proxy", + }, + }, + })) + + Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ + Name: EnvVarNoProxy, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ephemeralRunner.Spec.ProxySecretRef, + }, + Key: "no_proxy", + }, + }, + })) + }) + }) }) diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller.go b/controllers/actions.github.com/ephemeralrunnerset_controller.go index e1840a4e..29f40e0d 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller.go @@ -122,6 +122,24 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil } + // Create proxy secret if not present + if ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy != nil { + proxySecret := new(corev1.Secret) + if err := r.Get(ctx, types.NamespacedName{Namespace: ephemeralRunnerSet.Namespace, Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet)}, proxySecret); err != nil { + if !kerrors.IsNotFound(err) { + log.Error(err, "Unable to get ephemeralRunnerSet proxy secret", "namespace", ephemeralRunnerSet.Namespace, "name", proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet)) + return ctrl.Result{}, err + } + + // Create a compiled secret for the runner pods in the runnerset namespace + log.Info("Creating a ephemeralRunnerSet proxy secret for the runner pods") + if err := r.createProxySecret(ctx, ephemeralRunnerSet, log); err != nil { + log.Error(err, "Unable to create ephemeralRunnerSet proxy secret", "namespace", ephemeralRunnerSet.Namespace, "set-name", ephemeralRunnerSet.Name) + return ctrl.Result{}, err + } + } + } + // Find all EphemeralRunner with matching namespace and own by this EphemeralRunnerSet. ephemeralRunnerList := new(v1alpha1.EphemeralRunnerList) err := r.List( @@ -196,15 +214,39 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil } -func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) (done bool, err error) { +func (r *EphemeralRunnerSetReconciler) cleanUpProxySecret(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) error { + if ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy == nil { + return nil + } + log.Info("Deleting proxy secret") + + proxySecret := new(corev1.Secret) + proxySecret.Namespace = ephemeralRunnerSet.Namespace + proxySecret.Name = proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet) + + if err := r.Delete(ctx, proxySecret); err != nil && !kerrors.IsNotFound(err) { + return fmt.Errorf("failed to delete proxy secret: %v", err) + } + + log.Info("Deleted proxy secret") + + return nil +} + +func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) (bool, error) { ephemeralRunnerList := new(v1alpha1.EphemeralRunnerList) - err = r.List(ctx, ephemeralRunnerList, client.InNamespace(ephemeralRunnerSet.Namespace), client.MatchingFields{ephemeralRunnerSetReconcilerOwnerKey: ephemeralRunnerSet.Name}) + err := r.List(ctx, ephemeralRunnerList, client.InNamespace(ephemeralRunnerSet.Namespace), client.MatchingFields{ephemeralRunnerSetReconcilerOwnerKey: ephemeralRunnerSet.Name}) if err != nil { return false, fmt.Errorf("failed to list child ephemeral runners: %v", err) } + log.Info("Actual Ephemeral runner counts", "count", len(ephemeralRunnerList.Items)) // only if there are no ephemeral runners left, return true if len(ephemeralRunnerList.Items) == 0 { + err := r.cleanUpProxySecret(ctx, ephemeralRunnerSet, log) + if err != nil { + return false, err + } log.Info("All ephemeral runners are deleted") return true, nil } @@ -269,6 +311,9 @@ func (r *EphemeralRunnerSetReconciler) createEphemeralRunners(ctx context.Contex errs := make([]error, 0) for i := 0; i < count; i++ { ephemeralRunner := r.resourceBuilder.newEphemeralRunner(runnerSet) + if runnerSet.Spec.EphemeralRunnerSpec.Proxy != nil { + ephemeralRunner.Spec.ProxySecretRef = proxyEphemeralRunnerSetSecretName(runnerSet) + } // Make sure that we own the resource we create. if err := ctrl.SetControllerReference(runnerSet, ephemeralRunner, r.Scheme); err != nil { @@ -290,6 +335,45 @@ func (r *EphemeralRunnerSetReconciler) createEphemeralRunners(ctx context.Contex return multierr.Combine(errs...) } +func (r *EphemeralRunnerSetReconciler) createProxySecret(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) error { + proxySecretData, err := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy.ToSecretData(func(s string) (*corev1.Secret, error) { + secret := new(corev1.Secret) + err := r.Get(ctx, types.NamespacedName{Namespace: ephemeralRunnerSet.Namespace, Name: s}, secret) + return secret, err + }) + if err != nil { + return fmt.Errorf("failed to convert proxy config to secret data: %w", err) + } + + runnerPodProxySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), + Namespace: ephemeralRunnerSet.Namespace, + Labels: map[string]string{ + // TODO: figure out autoScalingRunnerSet name and set it as a label for this secret + // "auto-scaling-runner-set-namespace": ephemeralRunnerSet.Namespace, + // "auto-scaling-runner-set-name": ephemeralRunnerSet.Name, + }, + }, + Data: proxySecretData, + } + + // Make sure that we own the resource we create. + if err := ctrl.SetControllerReference(ephemeralRunnerSet, runnerPodProxySecret, r.Scheme); err != nil { + log.Error(err, "failed to set controller reference on proxy secret") + return err + } + + log.Info("Creating new proxy secret") + if err := r.Create(ctx, runnerPodProxySecret); err != nil { + log.Error(err, "failed to create proxy secret") + return err + } + + log.Info("Created new proxy secret") + return nil +} + // deleteIdleEphemeralRunners try to deletes `count` number of v1alpha1.EphemeralRunner resources in the cluster. // It will only delete `v1alpha1.EphemeralRunner` that has registered with Actions service // which has a `v1alpha1.EphemeralRunner.Status.RunnerId` set. @@ -366,8 +450,31 @@ func (r *EphemeralRunnerSetReconciler) actionsClientFor(ctx context.Context, rs if err := r.Get(ctx, types.NamespacedName{Namespace: rs.Namespace, Name: rs.Spec.EphemeralRunnerSpec.GitHubConfigSecret}, secret); err != nil { return nil, fmt.Errorf("failed to get secret: %w", err) } + var opts []actions.ClientOption + if rs.Spec.EphemeralRunnerSpec.Proxy != nil { + proxyFunc, err := rs.Spec.EphemeralRunnerSpec.Proxy.ProxyFunc(func(s string) (*corev1.Secret, error) { + var secret corev1.Secret + err := r.Get(ctx, types.NamespacedName{Namespace: rs.Namespace, Name: s}, &secret) + if err != nil { + return nil, fmt.Errorf("failed to get secret %s: %w", s, err) + } - return r.ActionsClient.GetClientFromSecret(ctx, rs.Spec.EphemeralRunnerSpec.GitHubConfigUrl, rs.Namespace, secret.Data) + return &secret, nil + }) + if err != nil { + return nil, fmt.Errorf("failed to get proxy func: %w", err) + } + + opts = append(opts, actions.WithProxy(proxyFunc)) + } + + return r.ActionsClient.GetClientFromSecret( + ctx, + rs.Spec.EphemeralRunnerSpec.GitHubConfigUrl, + rs.Namespace, + secret.Data, + opts..., + ) } // SetupWithManager sets up the controller with the Manager. diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller_test.go b/controllers/actions.github.com/ephemeralrunnerset_controller_test.go index e51d91dd..cba4eb2d 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller_test.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller_test.go @@ -2,7 +2,11 @@ package actionsgithubcom import ( "context" + "encoding/base64" "fmt" + "net/http" + "net/http/httptest" + "strings" "time" corev1 "k8s.io/api/core/v1" @@ -11,11 +15,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" + v1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" + "github.com/actions/actions-runner-controller/github/actions" "github.com/actions/actions-runner-controller/github/actions/fake" ) @@ -585,3 +592,315 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() { }) }) }) + +var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func() { + var ctx context.Context + var cancel context.CancelFunc + autoscalingNS := new(corev1.Namespace) + ephemeralRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet) + configSecret := new(corev1.Secret) + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.TODO()) + autoscalingNS = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "testns-autoscaling-runnerset" + RandStringRunes(5)}, + } + + err := k8sClient.Create(ctx, autoscalingNS) + Expect(err).NotTo(HaveOccurred(), "failed to create test namespace for EphemeralRunnerSet") + + configSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "github-config-secret", + Namespace: autoscalingNS.Name, + }, + Data: map[string][]byte{ + "github_token": []byte(ephemeralRunnerSetTestGitHubToken), + }, + } + + err = k8sClient.Create(ctx, configSecret) + Expect(err).NotTo(HaveOccurred(), "failed to create config secret") + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Namespace: autoscalingNS.Name, + MetricsBindAddress: "0", + }) + Expect(err).NotTo(HaveOccurred(), "failed to create manager") + + controller := &EphemeralRunnerSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: logf.Log, + ActionsClient: actions.NewMultiClient("test", logr.Discard()), + } + err = controller.SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred(), "failed to setup controller") + + go func() { + defer GinkgoRecover() + + err := mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to start manager") + }() + }) + + AfterEach(func() { + defer cancel() + + err := k8sClient.Delete(ctx, autoscalingNS) + Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace for EphemeralRunnerSet") + }) + + It("should create a proxy secret and delete the proxy secreat after the runner-set is deleted", func() { + secretCredentials := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxy-credentials", + Namespace: autoscalingNS.Name, + }, + Data: map[string][]byte{ + "username": []byte("username"), + "password": []byte("password"), + }, + } + + err := k8sClient.Create(ctx, secretCredentials) + Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") + + ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-asrs", + Namespace: autoscalingNS.Name, + }, + Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ + Replicas: 1, + EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ + GitHubConfigUrl: "http://example.com/owner/repo", + GitHubConfigSecret: configSecret.Name, + RunnerScaleSetId: 100, + Proxy: &v1alpha1.ProxyConfig{ + HTTP: &v1alpha1.ProxyServerConfig{ + Url: "http://proxy.example.com", + CredentialSecretRef: secretCredentials.Name, + }, + HTTPS: &v1alpha1.ProxyServerConfig{ + Url: "https://proxy.example.com", + CredentialSecretRef: secretCredentials.Name, + }, + NoProxy: []string{"example.com", "example.org"}, + }, + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "runner", + Image: "ghcr.io/actions/runner", + }, + }, + }, + }, + }, + }, + } + + err = k8sClient.Create(ctx, ephemeralRunnerSet) + Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") + + Eventually(func(g Gomega) { + // Compiled / flattened proxy secret should exist at this point + actualProxySecret := &corev1.Secret{} + err = k8sClient.Get(ctx, client.ObjectKey{ + Namespace: autoscalingNS.Name, + Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), + }, actualProxySecret) + g.Expect(err).NotTo(HaveOccurred(), "failed to get compiled / flattened proxy secret") + + secretFetcher := func(name string) (*corev1.Secret, error) { + secret := &corev1.Secret{} + err = k8sClient.Get(ctx, client.ObjectKey{ + Namespace: autoscalingNS.Name, + Name: name, + }, secret) + return secret, err + } + + // Assert that the proxy secret is created with the correct values + expectedData, err := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy.ToSecretData(secretFetcher) + g.Expect(err).NotTo(HaveOccurred(), "failed to get proxy secret data") + g.Expect(actualProxySecret.Data).To(Equal(expectedData)) + }, + ephemeralRunnerSetTestTimeout, + ephemeralRunnerSetTestInterval, + ).Should(Succeed(), "compiled / flattened proxy secret should exist") + + Eventually(func(g Gomega) { + runnerList := new(actionsv1alpha1.EphemeralRunnerList) + err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) + g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunners") + + for _, runner := range runnerList.Items { + g.Expect(runner.Spec.ProxySecretRef).To(Equal(proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet))) + } + }, ephemeralRunnerSetTestTimeout, ephemeralRunnerSetTestInterval).Should(Succeed(), "EphemeralRunners should have a reference to the proxy secret") + + // patch ephemeral runner set to have 0 replicas + patch := client.MergeFrom(ephemeralRunnerSet.DeepCopy()) + ephemeralRunnerSet.Spec.Replicas = 0 + err = k8sClient.Patch(ctx, ephemeralRunnerSet, patch) + Expect(err).NotTo(HaveOccurred(), "failed to patch EphemeralRunnerSet") + + // Set pods to PodSucceeded to simulate an actual EphemeralRunner stopping + Eventually( + func(g Gomega) (int, error) { + runnerList := new(actionsv1alpha1.EphemeralRunnerList) + err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) + if err != nil { + return -1, err + } + + // Set status to simulate a configured EphemeralRunner + refetch := false + for i, runner := range runnerList.Items { + if runner.Status.RunnerId == 0 { + updatedRunner := runner.DeepCopy() + updatedRunner.Status.Phase = corev1.PodSucceeded + updatedRunner.Status.RunnerId = i + 100 + err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) + Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") + refetch = true + } + } + + if refetch { + err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) + if err != nil { + return -1, err + } + } + + return len(runnerList.Items), nil + }, + ephemeralRunnerSetTestTimeout, + ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(1), "1 EphemeralRunner should exist") + + // Delete the EphemeralRunnerSet + err = k8sClient.Delete(ctx, ephemeralRunnerSet) + Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunnerSet") + + // Assert that the proxy secret is deleted + Eventually(func(g Gomega) { + proxySecret := &corev1.Secret{} + err = k8sClient.Get(ctx, client.ObjectKey{ + Namespace: autoscalingNS.Name, + Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), + }, proxySecret) + g.Expect(err).To(HaveOccurred(), "proxy secret should be deleted") + g.Expect(kerrors.IsNotFound(err)).To(BeTrue(), "proxy secret should be deleted") + }, + ephemeralRunnerSetTestTimeout, + ephemeralRunnerSetTestInterval, + ).Should(Succeed(), "proxy secret should be deleted") + }) + + It("should configure the actions client to use proxy details", func() { + secretCredentials := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxy-credentials", + Namespace: autoscalingNS.Name, + }, + Data: map[string][]byte{ + "username": []byte("test"), + "password": []byte("password"), + }, + } + + err := k8sClient.Create(ctx, secretCredentials) + Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") + + proxySuccessfulllyCalled := false + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + header := r.Header.Get("Proxy-Authorization") + Expect(header).NotTo(BeEmpty()) + + header = strings.TrimPrefix(header, "Basic ") + decoded, err := base64.StdEncoding.DecodeString(header) + Expect(err).NotTo(HaveOccurred()) + Expect(string(decoded)).To(Equal("test:password")) + + proxySuccessfulllyCalled = true + w.WriteHeader(http.StatusOK) + })) + GinkgoT().Cleanup(func() { + proxy.Close() + }) + + ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-asrs", + Namespace: autoscalingNS.Name, + }, + Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ + Replicas: 1, + EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ + GitHubConfigUrl: "http://example.com/owner/repo", + GitHubConfigSecret: configSecret.Name, + RunnerScaleSetId: 100, + Proxy: &v1alpha1.ProxyConfig{ + HTTP: &v1alpha1.ProxyServerConfig{ + Url: proxy.URL, + CredentialSecretRef: "proxy-credentials", + }, + }, + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "runner", + Image: "ghcr.io/actions/runner", + }, + }, + }, + }, + }, + }, + } + + err = k8sClient.Create(ctx, ephemeralRunnerSet) + Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") + + runnerList := new(actionsv1alpha1.EphemeralRunnerList) + Eventually(func() (int, error) { + err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) + if err != nil { + return -1, err + } + + return len(runnerList.Items), nil + }, + ephemeralRunnerSetTestTimeout, + ephemeralRunnerSetTestInterval, + ).Should(BeEquivalentTo(1), "failed to create ephemeral runner") + + runner := runnerList.Items[0].DeepCopy() + runner.Status.Phase = corev1.PodRunning + runner.Status.RunnerId = 100 + err = k8sClient.Status().Patch(ctx, runner, client.MergeFrom(&runnerList.Items[0])) + Expect(err).NotTo(HaveOccurred(), "failed to update ephemeral runner status") + + updatedRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet) + err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, updatedRunnerSet) + Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") + + updatedRunnerSet.Spec.Replicas = 0 + err = k8sClient.Update(ctx, updatedRunnerSet) + Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") + + Eventually( + func() bool { + return proxySuccessfulllyCalled + }, + 2*time.Second, + interval, + ).Should(BeEquivalentTo(true)) + }) +}) diff --git a/controllers/actions.github.com/resourcebuilder.go b/controllers/actions.github.com/resourcebuilder.go index f6aa1a47..a7c58cae 100644 --- a/controllers/actions.github.com/resourcebuilder.go +++ b/controllers/actions.github.com/resourcebuilder.go @@ -18,10 +18,9 @@ const ( jitTokenKey = "jitToken" ) -type resourceBuilder struct { -} +type resourceBuilder struct{} -func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.AutoscalingListener, serviceAccount *corev1.ServiceAccount, secret *corev1.Secret) *corev1.Pod { +func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.AutoscalingListener, serviceAccount *corev1.ServiceAccount, secret *corev1.Secret, envs ...corev1.EnvVar) *corev1.Pod { newLabels := map[string]string{} newLabels[scaleSetListenerLabel] = fmt.Sprintf("%v-%v", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, autoscalingListener.Spec.AutoscalingRunnerSetName) @@ -51,6 +50,7 @@ func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.A Value: strconv.Itoa(autoscalingListener.Spec.RunnerScaleSetId), }, } + listenerEnv = append(listenerEnv, envs...) if _, ok := secret.Data["github_token"]; ok { listenerEnv = append(listenerEnv, corev1.EnvVar{ @@ -112,7 +112,7 @@ func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.A ServiceAccountName: serviceAccount.Name, Containers: []corev1.Container{ { - Name: name, + Name: autoscalingListenerContainerName, Image: autoscalingListener.Spec.Image, Env: listenerEnv, ImagePullPolicy: corev1.PullIfNotPresent, @@ -299,6 +299,7 @@ func (b *resourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1. MaxRunners: effectiveMaxRunners, Image: image, ImagePullSecrets: imagePullSecrets, + Proxy: autoscalingRunnerSet.Spec.Proxy, }, } @@ -316,7 +317,7 @@ func (b *resourceBuilder) newEphemeralRunner(ephemeralRunnerSet *v1alpha1.Epheme } } -func (b *resourceBuilder) newEphemeralRunnerPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, secret *corev1.Secret) *corev1.Pod { +func (b *resourceBuilder) newEphemeralRunnerPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, secret *corev1.Secret, envs ...corev1.EnvVar) *corev1.Pod { var newPod corev1.Pod labels := map[string]string{} @@ -374,7 +375,9 @@ func (b *resourceBuilder) newEphemeralRunnerPod(ctx context.Context, runner *v1a corev1.EnvVar{ Name: EnvVarRunnerExtraUserAgent, Value: fmt.Sprintf("actions-runner-controller/%s", build.Version), - }) + }, + ) + c.Env = append(c.Env, envs...) } newPod.Spec.Containers = append(newPod.Spec.Containers, c) @@ -427,6 +430,22 @@ func scaleSetListenerSecretMirrorName(autoscalingListener *v1alpha1.AutoscalingL return fmt.Sprintf("%v-%v-listener", autoscalingListener.Spec.AutoscalingRunnerSetName, namespaceHash) } +func proxyListenerSecretName(autoscalingListener *v1alpha1.AutoscalingListener) string { + namespaceHash := hash.FNVHashString(autoscalingListener.Spec.AutoscalingRunnerSetNamespace) + if len(namespaceHash) > 8 { + namespaceHash = namespaceHash[:8] + } + return fmt.Sprintf("%v-%v-listener-proxy", autoscalingListener.Spec.AutoscalingRunnerSetName, namespaceHash) +} + +func proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet) string { + namespaceHash := hash.FNVHashString(ephemeralRunnerSet.Namespace) + if len(namespaceHash) > 8 { + namespaceHash = namespaceHash[:8] + } + return fmt.Sprintf("%v-%v-runner-proxy", ephemeralRunnerSet.Name, namespaceHash) +} + func rulesForListenerRole(resourceNames []string) []rbacv1.PolicyRule { return []rbacv1.PolicyRule{ { diff --git a/github/actions/client.go b/github/actions/client.go index 2d3c7906..4574b354 100644 --- a/github/actions/client.go +++ b/github/actions/client.go @@ -76,8 +76,12 @@ type Client struct { rootCAs *x509.CertPool tlsInsecureSkipVerify bool + + proxyFunc ProxyFunc } +type ProxyFunc func(req *http.Request) (*url.URL, error) + type ClientOption func(*Client) func WithUserAgent(userAgent string) ClientOption { @@ -116,6 +120,12 @@ func WithoutTLSVerify() ClientOption { } } +func WithProxy(proxyFunc ProxyFunc) ClientOption { + return func(c *Client) { + c.proxyFunc = proxyFunc + } +} + func NewClient(githubConfigURL string, creds *ActionsAuth, options ...ClientOption) (*Client, error) { config, err := ParseGitHubConfigFromURL(githubConfigURL) if err != nil { @@ -160,6 +170,8 @@ func NewClient(githubConfigURL string, creds *ActionsAuth, options ...ClientOpti transport.TLSClientConfig.InsecureSkipVerify = true } + transport.Proxy = ac.proxyFunc + retryClient.HTTPClient.Transport = transport ac.Client = retryClient.StandardClient() diff --git a/github/actions/client_proxy_test.go b/github/actions/client_proxy_test.go new file mode 100644 index 00000000..c63d41a2 --- /dev/null +++ b/github/actions/client_proxy_test.go @@ -0,0 +1,39 @@ +package actions_test + +import ( + "net/http" + "net/url" + "testing" + + "github.com/actions/actions-runner-controller/github/actions" + "github.com/actions/actions-runner-controller/github/actions/testserver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/http/httpproxy" +) + +func TestClientProxy(t *testing.T) { + serverCalled := false + + proxy := testserver.New(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + serverCalled = true + })) + + proxyConfig := &httpproxy.Config{ + HTTPProxy: proxy.URL, + } + proxyFunc := func(req *http.Request) (*url.URL, error) { + return proxyConfig.ProxyFunc()(req.URL) + } + + c, err := actions.NewClient("http://github.com/org/repo", nil, actions.WithProxy(proxyFunc)) + require.NoError(t, err) + + req, err := http.NewRequest(http.MethodGet, "http://example.com", nil) + require.NoError(t, err) + + _, err = c.Do(req) + require.NoError(t, err) + + assert.True(t, serverCalled) +} diff --git a/github/actions/multi_client.go b/github/actions/multi_client.go index bfef8893..37316870 100644 --- a/github/actions/multi_client.go +++ b/github/actions/multi_client.go @@ -125,7 +125,7 @@ func (m *multiClient) GetClientFromSecret(ctx context.Context, githubConfigURL, if hasToken { auth.Token = token - return m.GetClientFor(ctx, githubConfigURL, auth, namespace) + return m.GetClientFor(ctx, githubConfigURL, auth, namespace, options...) } parsedAppID, err := strconv.ParseInt(appID, 10, 64) @@ -139,7 +139,7 @@ func (m *multiClient) GetClientFromSecret(ctx context.Context, githubConfigURL, } auth.AppCreds = &GitHubAppAuth{AppID: parsedAppID, AppInstallationID: parsedAppInstallationID, AppPrivateKey: appPrivateKey} - return m.GetClientFor(ctx, githubConfigURL, auth, namespace) + return m.GetClientFor(ctx, githubConfigURL, auth, namespace, options...) } func RootCAsFromConfigMap(configMapData map[string][]byte) (*x509.CertPool, error) { diff --git a/github/actions/multi_client_test.go b/github/actions/multi_client_test.go index 80d54a3f..8606353e 100644 --- a/github/actions/multi_client_test.go +++ b/github/actions/multi_client_test.go @@ -55,24 +55,48 @@ func TestMultiClientOptions(t *testing.T) { defaultNamespace := "default" defaultConfigURL := "https://github.com/org/repo" - defaultCreds := &ActionsAuth{ - Token: "token", - } - multiClient := NewMultiClient("test-user-agent", logger) - service, err := multiClient.GetClientFor( - ctx, - defaultConfigURL, - *defaultCreds, - defaultNamespace, - WithUserAgent("test-option"), - ) - require.NoError(t, err) + t.Run("GetClientFor", func(t *testing.T) { + defaultCreds := &ActionsAuth{ + Token: "token", + } - client := service.(*Client) - req, err := client.NewGitHubAPIRequest(ctx, "GET", "/test", nil) - require.NoError(t, err) - assert.Equal(t, "test-option", req.Header.Get("User-Agent")) + multiClient := NewMultiClient("test-user-agent", logger) + service, err := multiClient.GetClientFor( + ctx, + defaultConfigURL, + *defaultCreds, + defaultNamespace, + WithUserAgent("test-option"), + ) + require.NoError(t, err) + + client := service.(*Client) + req, err := client.NewGitHubAPIRequest(ctx, "GET", "/test", nil) + require.NoError(t, err) + assert.Equal(t, "test-option", req.Header.Get("User-Agent")) + }) + + t.Run("GetClientFromSecret", func(t *testing.T) { + secret := map[string][]byte{ + "github_token": []byte("token"), + } + + multiClient := NewMultiClient("test-user-agent", logger) + service, err := multiClient.GetClientFromSecret( + ctx, + defaultConfigURL, + defaultNamespace, + secret, + WithUserAgent("test-option"), + ) + require.NoError(t, err) + + client := service.(*Client) + req, err := client.NewGitHubAPIRequest(ctx, "GET", "/test", nil) + require.NoError(t, err) + assert.Equal(t, "test-option", req.Header.Get("User-Agent")) + }) } func TestCreateJWT(t *testing.T) { diff --git a/go.mod b/go.mod index 237baa21..179672d3 100644 --- a/go.mod +++ b/go.mod @@ -84,10 +84,10 @@ require ( github.com/urfave/cli v1.22.2 // indirect go.uber.org/atomic v1.7.0 // indirect golang.org/x/crypto v0.1.0 // indirect - golang.org/x/net v0.5.0 // indirect - golang.org/x/sys v0.4.0 // indirect - golang.org/x/term v0.4.0 // indirect - golang.org/x/text v0.6.0 // indirect + golang.org/x/net v0.6.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index 64d3957d..7542f60e 100644 --- a/go.sum +++ b/go.sum @@ -450,6 +450,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -518,10 +520,14 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -531,6 +537,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=