Fail VMs if the worker had crashed/is unhealthy (#70)

* Fail VMs if the worker had crashed/is unhealthy

* OnDiskName: properly handle cases when VM's name contains hyphens

* Worker: introduce Offline() method and check it before scheduling

* tart.List(): use Tart's JSON output

* OnDiskName: remove empty parts check

* Scheduler: move health-checking logic to a separate function

* Only fail "running" VMs

* Only fail orphaned VMs if they're in terminal state

* Integration tests

* Run healthCheckingLoopIteration() before schedulingLoopIteration()

* Worker: sync on-disk VMs only once at start

internal/controller/controller.go

@@ -43,14 +43,16 @@ type Controller struct {
 	workerNotifier       *notifier.Notifier
 	proxy                *proxy.Proxy
 	enableSwaggerDocs    bool
+	workerOfflineTimeout time.Duration
 
 	rpc.UnimplementedControllerServer
 }
 
 func New(opts ...Option) (*Controller, error) {
 	controller := &Controller{
-		workerNotifier: notifier.NewNotifier(),
-		proxy:          proxy.NewProxy(),
+		workerNotifier:       notifier.NewNotifier(),
+		proxy:                proxy.NewProxy(),
+		workerOfflineTimeout: 3 * time.Minute,
 	}
 
 	// Apply options
@@ -76,7 +78,8 @@ func New(opts ...Option) (*Controller, error) {
 		return nil, err
 	}
 	controller.store = store
-	controller.scheduler = scheduler.NewScheduler(store, controller.workerNotifier, controller.logger)
+	controller.scheduler = scheduler.NewScheduler(store, controller.workerNotifier,
+		controller.workerOfflineTimeout, controller.logger)
 
 	listener, err := net.Listen("tcp", controller.listenAddr)
 	if err != nil {

internal/controller/option.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"crypto/tls"
 	"go.uber.org/zap"
+	"time"
 )
 
 type Option func(*Controller)
@@ -37,6 +38,12 @@ func WithSwaggerDocs() Option {
 	}
 }
 
+func WithWorkerOfflineTimeout(workerOfflineTimeout time.Duration) Option {
+	return func(controller *Controller) {
+		controller.workerOfflineTimeout = workerOfflineTimeout
+	}
+}
+
 func WithLogger(logger *zap.Logger) Option {
 	return func(controller *Controller) {
 		controller.logger = logger.Sugar()

internal/controller/scheduler/scheduler.go

@@ -14,18 +14,25 @@ import (
 const schedulerInterval = 5 * time.Second
 
 type Scheduler struct {
-	store               storepkg.Store
-	notifier            *notifier.Notifier
-	logger              *zap.SugaredLogger
-	schedulingRequested chan bool
+	store                storepkg.Store
+	notifier             *notifier.Notifier
+	workerOfflineTimeout time.Duration
+	logger               *zap.SugaredLogger
+	schedulingRequested  chan bool
 }
 
-func NewScheduler(store storepkg.Store, notifier *notifier.Notifier, logger *zap.SugaredLogger) *Scheduler {
+func NewScheduler(
+	store storepkg.Store,
+	notifier *notifier.Notifier,
+	workerOfflineTimeout time.Duration,
+	logger *zap.SugaredLogger,
+) *Scheduler {
 	return &Scheduler{
-		store:               store,
-		notifier:            notifier,
-		logger:              logger,
-		schedulingRequested: make(chan bool, 1),
+		store:                store,
+		notifier:             notifier,
+		workerOfflineTimeout: workerOfflineTimeout,
+		logger:               logger,
+		schedulingRequested:  make(chan bool, 1),
 	}
 }
 
@@ -36,6 +43,10 @@ func (scheduler *Scheduler) Run() {
 		case <-scheduler.schedulingRequested:
 		case <-time.After(schedulerInterval):
 		}
+
+		if err := scheduler.healthCheckingLoopIteration(); err != nil {
+			scheduler.logger.Errorf("Failed to health-check VMs: %v", err)
+		}
 		if err := scheduler.schedulingLoopIteration(); err != nil {
 			scheduler.logger.Errorf("Failed to schedule VMs: %v", err)
 		}
@@ -53,6 +64,7 @@ func (scheduler *Scheduler) RequestScheduling() {
 
 func (scheduler *Scheduler) schedulingLoopIteration() error {
 	affectedWorkers := map[string]bool{}
+
 	err := scheduler.store.Update(func(txn storepkg.Transaction) error {
 		vms, err := txn.ListVMs()
 		if err != nil {
@@ -71,7 +83,8 @@ func (scheduler *Scheduler) schedulingLoopIteration() error {
 				resourcesUsed := workerToResources.Get(worker.Name)
 				resourcesRemaining := worker.Resources.Subtracted(resourcesUsed)
 
-				if resourcesRemaining.CanFit(unscheduledVM.Resources) {
+				if resourcesRemaining.CanFit(unscheduledVM.Resources) &&
+					!worker.Offline(scheduler.workerOfflineTimeout) {
 					unscheduledVM.Worker = worker.Name
 
 					if err := txn.SetVM(unscheduledVM); err != nil {
@@ -86,6 +99,7 @@ func (scheduler *Scheduler) schedulingLoopIteration() error {
 
 		return nil
 	})
+
 	syncVMsInstruction := rpc.WatchInstruction{
 		Action: &rpc.WatchInstruction_SyncVmsAction{},
 	}
@@ -96,6 +110,7 @@ func (scheduler *Scheduler) schedulingLoopIteration() error {
 			scheduler.logger.Errorf("Failed to reactively sync VMs on worker %s: %v", workerToPoke, notifyErr)
 		}
 	}
+
 	return err
 }
 
@@ -118,3 +133,62 @@ func processVMs(vms []v1.VM) ([]v1.VM, WorkerToResources) {
 
 	return unscheduledVMs, workerToResources
 }
+
+func (scheduler *Scheduler) healthCheckingLoopIteration() error {
+	return scheduler.store.Update(func(txn storepkg.Transaction) error {
+		// Retrieve scheduled VMs
+		vms, err := txn.ListVMs()
+		if err != nil {
+			return err
+		}
+
+		var scheduledVMs []v1.VM
+
+		for _, vm := range vms {
+			if vm.Worker != "" {
+				scheduledVMs = append(scheduledVMs, vm)
+			}
+		}
+
+		// Retrieve and index workers by name
+		workers, err := txn.ListWorkers()
+		if err != nil {
+			return err
+		}
+
+		nameToWorker := map[string]v1.Worker{}
+		for _, worker := range workers {
+			nameToWorker[worker.Name] = worker
+		}
+
+		// Process scheduled VMs
+		for _, scheduledVM := range scheduledVMs {
+			if err := scheduler.healthCheckVM(txn, nameToWorker, scheduledVM); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func (scheduler *Scheduler) healthCheckVM(txn storepkg.Transaction, nameToWorker map[string]v1.Worker, vm v1.VM) error {
+	worker, ok := nameToWorker[vm.Worker]
+	if !ok {
+		vm.Status = v1.VMStatusFailed
+		vm.StatusMessage = "VM is assigned to a worker that " +
+			"doesn't exist anymore"
+
+		return txn.SetVM(vm)
+	}
+
+	if worker.Offline(scheduler.workerOfflineTimeout) && !vm.TerminalState() {
+		vm.Status = v1.VMStatusFailed
+		vm.StatusMessage = "VM is assigned to a worker that " +
+			"lost connection with the controller"
+
+		return txn.SetVM(vm)
+	}
+
+	return nil
+}