diff --git a/Makefile b/Makefile index acff8dfc..236dd2c1 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ KUBE_RBAC_PROXY_VERSION ?= v0.11.0 SHELLCHECK_VERSION ?= 0.8.0 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) -CRD_OPTIONS ?= "crd:generateEmbeddedObjectMeta=true" +CRD_OPTIONS ?= "crd:generateEmbeddedObjectMeta=true,allowDangerousTypes=true" # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) diff --git a/apis/actions.github.com/v1alpha1/autoscalingrunnerset_types.go b/apis/actions.github.com/v1alpha1/autoscalingrunnerset_types.go index cfc0d8b3..921c27c4 100644 --- a/apis/actions.github.com/v1alpha1/autoscalingrunnerset_types.go +++ b/apis/actions.github.com/v1alpha1/autoscalingrunnerset_types.go @@ -18,7 +18,6 @@ package v1alpha1 import ( "crypto/x509" - "encoding/json" "fmt" "net/http" "net/url" @@ -76,7 +75,7 @@ type AutoscalingRunnerSetSpec struct { Template corev1.PodTemplateSpec `json:"template,omitempty"` // +optional - ListenerMetrics *MetricsConfig `json:"metrics,omitempty"` + ListenerMetrics *MetricsConfig `json:"listenerMetrics,omitempty"` // +optional ListenerTemplate *corev1.PodTemplateSpec `json:"listenerTemplate,omitempty"` @@ -255,8 +254,8 @@ type GaugeMetric struct { // HistogramMetric holds configuration of a single metric of type Histogram type HistogramMetric struct { - Labels []string `json:"labels"` - Buckets []json.Number `json:"buckets,omitempty"` + Labels []string `json:"labels"` + Buckets []float64 `json:"buckets,omitempty"` } // AutoscalingRunnerSetStatus defines the observed state of AutoscalingRunnerSet diff --git a/apis/actions.github.com/v1alpha1/zz_generated.deepcopy.go b/apis/actions.github.com/v1alpha1/zz_generated.deepcopy.go index 1cee1d5f..dd7553f0 100644 --- a/apis/actions.github.com/v1alpha1/zz_generated.deepcopy.go +++ b/apis/actions.github.com/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,6 @@ limitations under the License. package v1alpha1 import ( - "encoding/json" "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -527,7 +526,7 @@ func (in *HistogramMetric) DeepCopyInto(out *HistogramMetric) { } if in.Buckets != nil { in, out := &in.Buckets, &out.Buckets - *out = make([]json.Number, len(*in)) + *out = make([]float64, len(*in)) copy(*out, *in) } } diff --git a/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalinglisteners.yaml b/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalinglisteners.yaml index 143c4c27..e5920395 100644 --- a/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalinglisteners.yaml +++ b/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalinglisteners.yaml @@ -152,8 +152,7 @@ spec: properties: buckets: items: - description: A Number represents a JSON number literal. - type: string + type: number type: array labels: items: diff --git a/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalingrunnersets.yaml b/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalingrunnersets.yaml index bb4a5863..c7a16da3 100644 --- a/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalingrunnersets.yaml +++ b/charts/gha-runner-scale-set-controller/crds/actions.github.com_autoscalingrunnersets.yaml @@ -99,6 +99,54 @@ spec: x-kubernetes-map-type: atomic type: object type: object + listenerMetrics: + description: MetricsConfig holds configuration parameters for each metric type + properties: + counters: + additionalProperties: + description: CounterMetric holds configuration of a single metric of type Counter + properties: + labels: + items: + type: string + type: array + required: + - labels + type: object + type: object + gauges: + additionalProperties: + description: GaugeMetric holds configuration of a single metric of type Gauge + properties: + labels: + items: + type: string + type: array + required: + - labels + type: object + type: object + histograms: + additionalProperties: + description: HistogramMetric holds configuration of a single metric of type Histogram + properties: + buckets: + items: + type: number + type: array + labels: + items: + type: string + type: array + required: + - labels + type: object + type: object + required: + - counters + - gauges + - histograms + type: object listenerTemplate: description: PodTemplateSpec describes the data a pod should have when created from a template properties: @@ -7772,55 +7820,6 @@ spec: maxRunners: minimum: 0 type: integer - metrics: - description: MetricsConfig holds configuration parameters for each metric type - properties: - counters: - additionalProperties: - description: CounterMetric holds configuration of a single metric of type Counter - properties: - labels: - items: - type: string - type: array - required: - - labels - type: object - type: object - gauges: - additionalProperties: - description: GaugeMetric holds configuration of a single metric of type Gauge - properties: - labels: - items: - type: string - type: array - required: - - labels - type: object - type: object - histograms: - additionalProperties: - description: HistogramMetric holds configuration of a single metric of type Histogram - properties: - buckets: - items: - description: A Number represents a JSON number literal. - type: string - type: array - labels: - items: - type: string - type: array - required: - - labels - type: object - type: object - required: - - counters - - gauges - - histograms - type: object minRunners: minimum: 0 type: integer diff --git a/charts/gha-runner-scale-set/templates/autoscalingrunnerset.yaml b/charts/gha-runner-scale-set/templates/autoscalingrunnerset.yaml index c5ad2e38..9abcf513 100644 --- a/charts/gha-runner-scale-set/templates/autoscalingrunnerset.yaml +++ b/charts/gha-runner-scale-set/templates/autoscalingrunnerset.yaml @@ -106,11 +106,16 @@ spec: minRunners: {{ .Values.minRunners | int }} {{- end }} - {{- with .Values.listenerTemplate}} + {{- with .Values.listenerTemplate }} listenerTemplate: {{- toYaml . | nindent 4}} {{- end }} + {{- with .Values.listenerMetrics }} + listenerMetrics: + {{- toYaml . | nindent 4 }} + {{- end }} + template: {{- with .Values.template.metadata }} metadata: diff --git a/charts/gha-runner-scale-set/values.yaml b/charts/gha-runner-scale-set/values.yaml index 831e26d0..d02ef3a6 100644 --- a/charts/gha-runner-scale-set/values.yaml +++ b/charts/gha-runner-scale-set/values.yaml @@ -122,9 +122,18 @@ githubConfigSecret: listenerMetrics: counters: gha_started_jobs_total: - labels: ["repository", "organization", "enterprise", "job_name", "event_name"] + labels: + ["repository", "organization", "enterprise", "job_name", "event_name"] gha_completed_jobs_total: - labels: ["repository", "organization", "enterprise", "job_name", "event_name", "job_result"] + labels: + [ + "repository", + "organization", + "enterprise", + "job_name", + "event_name", + "job_result", + ] gauges: gha_assigned_jobs: labels: ["name", "namespace", "repository", "organization", "enterprise"] @@ -144,11 +153,114 @@ listenerMetrics: labels: ["name", "namespace", "repository", "organization", "enterprise"] histograms: gha_job_startup_duration_seconds: - labels: ["repository", "organization", "enterprise", "job_name", "event_name"] - buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 15, 18, 20, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 150, 180, 210, 240, 300, 360, 420, 480, 540, 600, 900, 1200, 1800, 2400, 3000, 3600] + labels: + ["repository", "organization", "enterprise", "job_name", "event_name"] + buckets: + [ + 0.01, + 0.05, + 0.1, + 0.5, + 1.0, + 2.0, + 3.0, + 4.0, + 5.0, + 6.0, + 7.0, + 8.0, + 9.0, + 10.0, + 12.0, + 15.0, + 18.0, + 20.0, + 25.0, + 30.0, + 40.0, + 50.0, + 60.0, + 70.0, + 80.0, + 90.0, + 100.0, + 110.0, + 120.0, + 150.0, + 180.0, + 210.0, + 240.0, + 300.0, + 360.0, + 420.0, + 480.0, + 540.0, + 600.0, + 900.0, + 1200.0, + 1800.0, + 2400.0, + 3000.0, + 3600.0, + ] gha_job_execution_duration_seconds: - labels: ["repository", "organization", "enterprise", "job_name", "event_name", "job_result"] - buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 15, 18, 20, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 150, 180, 210, 240, 300, 360, 420, 480, 540, 600, 900, 1200, 1800, 2400, 3000, 3600] + labels: + [ + "repository", + "organization", + "enterprise", + "job_name", + "event_name", + "job_result", + ] + buckets: + [ + 0.01, + 0.05, + 0.1, + 0.5, + 1.0, + 2.0, + 3.0, + 4.0, + 5.0, + 6.0, + 7.0, + 8.0, + 9.0, + 10.0, + 12.0, + 15.0, + 18.0, + 20.0, + 25.0, + 30.0, + 40.0, + 50.0, + 60.0, + 70.0, + 80.0, + 90.0, + 100.0, + 110.0, + 120.0, + 150.0, + 180.0, + 210.0, + 240.0, + 300.0, + 360.0, + 420.0, + 480.0, + 540.0, + 600.0, + 900.0, + 1200.0, + 1800.0, + 2400.0, + 3000.0, + 3600.0, + ] ## template is the PodSpec for each runner Pod ## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec @@ -236,7 +348,6 @@ template: - name: runner image: ghcr.io/actions/actions-runner:latest command: ["/home/runner/run.sh"] - ## Optional controller service account that needs to have required Role and RoleBinding ## to operate this gha-runner-scale-set installation. ## The helm chart will try to find the controller deployment and its service account at installation time. diff --git a/cmd/ghalistener/app/app.go b/cmd/ghalistener/app/app.go index badcaea4..a55363f9 100644 --- a/cmd/ghalistener/app/app.go +++ b/cmd/ghalistener/app/app.go @@ -69,6 +69,7 @@ func New(config config.Config) (*App, error) { Repository: ghConfig.Repository, ServerAddr: config.MetricsAddr, ServerEndpoint: config.MetricsEndpoint, + Metrics: *config.Metrics, }) } diff --git a/cmd/ghalistener/metrics/metrics.go b/cmd/ghalistener/metrics/metrics.go index f3ab58e8..a614cfa4 100644 --- a/cmd/ghalistener/metrics/metrics.go +++ b/cmd/ghalistener/metrics/metrics.go @@ -142,76 +142,7 @@ type ExporterConfig struct { func NewExporter(config ExporterConfig) ServerExporter { reg := prometheus.NewRegistry() - metrics := &metrics{ - counters: make(map[string]*counterMetric, len(config.Metrics.Gauges)), - gauges: make(map[string]*gaugeMetric, len(config.Metrics.Counters)), - histograms: make(map[string]*histogramMetric, len(config.Metrics.Histograms)), - } - for name, cfg := range config.Metrics.Gauges { - g := prometheus.V2.NewGaugeVec(prometheus.GaugeVecOpts{ - GaugeOpts: prometheus.GaugeOpts{ - Subsystem: githubScaleSetSubsystem, - Name: strings.TrimPrefix(name, githubScaleSetSubsystem), - Help: metricsHelp[name], - }, - VariableLabels: prometheus.UnconstrainedLabels(cfg.Labels), - }) - reg.MustRegister(g) - metrics.gauges[name] = &gaugeMetric{ - gauge: g, - config: cfg, - } - } - - for name, cfg := range config.Metrics.Counters { - c := prometheus.V2.NewCounterVec(prometheus.CounterVecOpts{ - CounterOpts: prometheus.CounterOpts{ - Subsystem: githubScaleSetSubsystem, - Name: strings.TrimPrefix(name, githubScaleSetSubsystem), - Help: metricsHelp[name], - }, - VariableLabels: prometheus.UnconstrainedLabels(cfg.Labels), - }) - reg.MustRegister(c) - metrics.counters[name] = &counterMetric{ - counter: c, - config: cfg, - } - } - - for name, cfg := range config.Metrics.Histograms { - buckets := defaultRuntimeBuckets - if len(cfg.Buckets) > 0 { - b := make([]float64, 0, len(cfg.Buckets)) - ok := true - for _, v := range cfg.Buckets { - f, err := v.Float64() - if err != nil { - ok = false - config.Logger.Error(err, "Failed to parse number in %q bucket: %w; continuing with the default buckets", name, err) - break - } - b = append(b, f) - } - if ok { - buckets = b - } - } - h := prometheus.V2.NewHistogramVec(prometheus.HistogramVecOpts{ - HistogramOpts: prometheus.HistogramOpts{ - Subsystem: githubScaleSetSubsystem, - Name: strings.TrimPrefix(name, githubScaleSetSubsystem), - Help: metricsHelp[name], - Buckets: buckets, - }, - VariableLabels: prometheus.UnconstrainedLabels(cfg.Labels), - }) - reg.MustRegister(h) - metrics.histograms[name] = &histogramMetric{ - histogram: h, - config: cfg, - } - } + metrics := installMetrics(config.Metrics, reg) mux := http.NewServeMux() mux.Handle( @@ -236,6 +167,68 @@ func NewExporter(config ExporterConfig) ServerExporter { } } +func installMetrics(config v1alpha1.MetricsConfig, reg *prometheus.Registry) *metrics { + metrics := &metrics{ + counters: make(map[string]*counterMetric, len(config.Gauges)), + gauges: make(map[string]*gaugeMetric, len(config.Counters)), + histograms: make(map[string]*histogramMetric, len(config.Histograms)), + } + for name, cfg := range config.Gauges { + g := prometheus.V2.NewGaugeVec(prometheus.GaugeVecOpts{ + GaugeOpts: prometheus.GaugeOpts{ + Subsystem: githubScaleSetSubsystem, + Name: strings.TrimPrefix(name, githubScaleSetSubsystem), + Help: metricsHelp[name], + }, + VariableLabels: prometheus.UnconstrainedLabels(cfg.Labels), + }) + reg.MustRegister(g) + metrics.gauges[name] = &gaugeMetric{ + gauge: g, + config: cfg, + } + } + + for name, cfg := range config.Counters { + c := prometheus.V2.NewCounterVec(prometheus.CounterVecOpts{ + CounterOpts: prometheus.CounterOpts{ + Subsystem: githubScaleSetSubsystem, + Name: strings.TrimPrefix(name, githubScaleSetSubsystem), + Help: metricsHelp[name], + }, + VariableLabels: prometheus.UnconstrainedLabels(cfg.Labels), + }) + reg.MustRegister(c) + metrics.counters[name] = &counterMetric{ + counter: c, + config: cfg, + } + } + + for name, cfg := range config.Histograms { + buckets := defaultRuntimeBuckets + if len(cfg.Buckets) > 0 { + buckets = cfg.Buckets + } + h := prometheus.V2.NewHistogramVec(prometheus.HistogramVecOpts{ + HistogramOpts: prometheus.HistogramOpts{ + Subsystem: githubScaleSetSubsystem, + Name: strings.TrimPrefix(name, githubScaleSetSubsystem), + Help: metricsHelp[name], + Buckets: buckets, + }, + VariableLabels: prometheus.UnconstrainedLabels(cfg.Labels), + }) + reg.MustRegister(h) + metrics.histograms[name] = &histogramMetric{ + histogram: h, + config: cfg, + } + } + + return metrics +} + func (e *exporter) ListenAndServe(ctx context.Context) error { e.logger.Info("starting metrics server", "addr", e.srv.Addr) go func() { diff --git a/config/crd/bases/actions.github.com_autoscalinglisteners.yaml b/config/crd/bases/actions.github.com_autoscalinglisteners.yaml index 143c4c27..e5920395 100644 --- a/config/crd/bases/actions.github.com_autoscalinglisteners.yaml +++ b/config/crd/bases/actions.github.com_autoscalinglisteners.yaml @@ -152,8 +152,7 @@ spec: properties: buckets: items: - description: A Number represents a JSON number literal. - type: string + type: number type: array labels: items: diff --git a/config/crd/bases/actions.github.com_autoscalingrunnersets.yaml b/config/crd/bases/actions.github.com_autoscalingrunnersets.yaml index bb4a5863..c7a16da3 100644 --- a/config/crd/bases/actions.github.com_autoscalingrunnersets.yaml +++ b/config/crd/bases/actions.github.com_autoscalingrunnersets.yaml @@ -99,6 +99,54 @@ spec: x-kubernetes-map-type: atomic type: object type: object + listenerMetrics: + description: MetricsConfig holds configuration parameters for each metric type + properties: + counters: + additionalProperties: + description: CounterMetric holds configuration of a single metric of type Counter + properties: + labels: + items: + type: string + type: array + required: + - labels + type: object + type: object + gauges: + additionalProperties: + description: GaugeMetric holds configuration of a single metric of type Gauge + properties: + labels: + items: + type: string + type: array + required: + - labels + type: object + type: object + histograms: + additionalProperties: + description: HistogramMetric holds configuration of a single metric of type Histogram + properties: + buckets: + items: + type: number + type: array + labels: + items: + type: string + type: array + required: + - labels + type: object + type: object + required: + - counters + - gauges + - histograms + type: object listenerTemplate: description: PodTemplateSpec describes the data a pod should have when created from a template properties: @@ -7772,55 +7820,6 @@ spec: maxRunners: minimum: 0 type: integer - metrics: - description: MetricsConfig holds configuration parameters for each metric type - properties: - counters: - additionalProperties: - description: CounterMetric holds configuration of a single metric of type Counter - properties: - labels: - items: - type: string - type: array - required: - - labels - type: object - type: object - gauges: - additionalProperties: - description: GaugeMetric holds configuration of a single metric of type Gauge - properties: - labels: - items: - type: string - type: array - required: - - labels - type: object - type: object - histograms: - additionalProperties: - description: HistogramMetric holds configuration of a single metric of type Histogram - properties: - buckets: - items: - description: A Number represents a JSON number literal. - type: string - type: array - labels: - items: - type: string - type: array - required: - - labels - type: object - type: object - required: - - counters - - gauges - - histograms - type: object minRunners: minimum: 0 type: integer