Implement restart policy for VMs (#83)

* Implement restart policy for VMs

* Do not update VM.Resource, we only use it as a read-only specification

* Err()/setErr(): use atomic.Pointer instead of sync.Mutex

internal/controller/api_vms.go

@@ -37,6 +37,8 @@ func (controller *Controller) createVM(ctx *gin.Context) responder.Responder {
 
 	vm.Status = v1.VMStatusPending
 	vm.CreatedAt = time.Now()
+	vm.RestartedAt = time.Time{}
+	vm.RestartCount = 0
 	vm.UID = uuid.New().String()
 
 	// Provide resource defaults
@@ -47,6 +49,16 @@ func (controller *Controller) createVM(ctx *gin.Context) responder.Responder {
 		vm.Resources[v1.ResourceTartVMs] = 1
 	}
 
+	// Validate restart policy and provide a default value if it's missing
+	if vm.RestartPolicy != "" {
+		if _, err := v1.NewRestartPolicyFromString(string(vm.RestartPolicy)); err != nil {
+			return responder.JSON(http.StatusPreconditionFailed,
+				NewErrorResponse("unsupported restart policy: %q", vm.RestartPolicy))
+		}
+	} else {
+		vm.RestartPolicy = v1.RestartPolicyNever
+	}
+
 	response := controller.storeUpdate(func(txn storepkg.Transaction) responder.Responder {
 		// Does the VM resource with this name already exists?
 		_, err := txn.GetVM(vm.Name)

internal/controller/scheduler/scheduler.go

@@ -13,7 +13,11 @@ import (
 	"time"
 )
 
-const schedulerInterval = 5 * time.Second
+const (
+	schedulerInterval = 5 * time.Second
+
+	schedulerVMRestartDelay = 15 * time.Second
+)
 
 var (
 	schedulerLoopIterationStat = promauto.NewCounter(prometheus.CounterOpts{
@@ -205,6 +209,25 @@ func (scheduler *Scheduler) healthCheckingLoopIteration() error {
 }
 
 func (scheduler *Scheduler) healthCheckVM(txn storepkg.Transaction, nameToWorker map[string]v1.Worker, vm v1.VM) error {
+	logger := scheduler.logger.With("vm_name", vm.Name, "vm_uid", vm.UID, "vm_restart_count", vm.RestartCount)
+
+	// Schedule a VM restart if the restart policy mandates it
+	needsRestart := vm.RestartPolicy == v1.RestartPolicyOnFailure &&
+		vm.Status == v1.VMStatusFailed &&
+		time.Since(vm.RestartedAt) > schedulerVMRestartDelay
+
+	if needsRestart {
+		logger.Debugf("restarting VM")
+
+		vm.Status = v1.VMStatusPending
+		vm.StatusMessage = ""
+		vm.Worker = ""
+		vm.RestartedAt = time.Now()
+		vm.RestartCount++
+
+		return txn.SetVM(vm)
+	}
+
 	worker, ok := nameToWorker[vm.Worker]
 	if !ok {
 		vm.Status = v1.VMStatusFailed