Add basic Prometheus metrics (#82)

Fixes #71

internal/controller/api.go

@@ -12,6 +12,7 @@ import (
 	"github.com/deckarep/golang-set/v2"
 	"github.com/gin-gonic/gin"
 	"github.com/go-openapi/runtime/middleware"
+	"github.com/penglongli/gin-metrics/ginmetrics"
 	"google.golang.org/grpc/metadata"
 	"net/http"
 	"strings"
@@ -24,6 +25,11 @@ var ErrUnauthorized = errors.New("unauthorized")
 func (controller *Controller) initAPI() *gin.Engine {
 	ginEngine := gin.Default()
 
+	// expose metrics
+	monitor := ginmetrics.GetMonitor()
+	monitor.SetMetricPath("/metrics")
+	monitor.Use(ginEngine)
+
 	// v1 API
 	v1 := ginEngine.Group("/v1")
 

internal/controller/scheduler/scheduler.go

@@ -6,6 +6,8 @@ import (
 	storepkg "github.com/cirruslabs/orchard/internal/controller/store"
 	"github.com/cirruslabs/orchard/pkg/resource/v1"
 	"github.com/cirruslabs/orchard/rpc"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"go.uber.org/zap"
 	"sort"
 	"time"
@@ -13,6 +15,18 @@ import (
 
 const schedulerInterval = 5 * time.Second
 
+var (
+	schedulerLoopIterationStat = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "orchard_scheduler_loop_iteration_total",
+	})
+	workersStat = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "orchard_workers",
+	}, []string{"status"})
+	vmsStat = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "orchard_vms",
+	}, []string{"status"})
+)
+
 type Scheduler struct {
 	store                storepkg.Store
 	notifier             *notifier.Notifier
@@ -49,10 +63,25 @@ func (scheduler *Scheduler) Run() {
 		}
 		if err := scheduler.schedulingLoopIteration(); err != nil {
 			scheduler.logger.Errorf("Failed to schedule VMs: %v", err)
+		} else {
+			schedulerLoopIterationStat.Inc()
 		}
 	}
 }
 
+func (scheduler *Scheduler) reportStats(workers []v1.Worker, vms []v1.VM) {
+	for _, worker := range workers {
+		if worker.Offline(scheduler.workerOfflineTimeout) {
+			workersStat.With(map[string]string{"status": "offline"}).Inc()
+		} else {
+			workersStat.With(map[string]string{"status": "online"}).Inc()
+		}
+	}
+	for _, vm := range vms {
+		vmsStat.With(map[string]string{"status": string(vm.Status)}).Inc()
+	}
+}
+
 func (scheduler *Scheduler) RequestScheduling() {
 	select {
 	case scheduler.schedulingRequested <- true:
@@ -162,6 +191,8 @@ func (scheduler *Scheduler) healthCheckingLoopIteration() error {
 			nameToWorker[worker.Name] = worker
 		}
 
+		scheduler.reportStats(workers, vms)
+
 		// Process scheduled VMs
 		for _, scheduledVM := range scheduledVMs {
 			if err := scheduler.healthCheckVM(txn, nameToWorker, scheduledVM); err != nil {