Ability to set VM's power state and retrieve backing Tart VM's name (#373)

* Ability to set VM's power state and retrieve backing Tart VM's name

* Validate user-provided "powerState" field

* Introduce TestSpecUpdatePowerStateSuspend

* Introduce TestSpecUpdatePowerStateStopped

* OpenAPI specification: add note about suspended VMs to "tartName" desc.

* Sometimes we need to wait more than 30 seconds

internal/controller/api_vms.go

@@ -10,6 +10,7 @@ import (
 	storepkg "github.com/cirruslabs/orchard/internal/controller/store"
 	"github.com/cirruslabs/orchard/internal/responder"
 	"github.com/cirruslabs/orchard/internal/simplename"
+	"github.com/cirruslabs/orchard/internal/worker/ondiskname"
 	"github.com/cirruslabs/orchard/pkg/resource/v1"
 	"github.com/gin-gonic/gin"
 	"github.com/google/go-cmp/cmp"
@@ -43,6 +44,16 @@ func (controller *Controller) createVM(ctx *gin.Context) responder.Responder {
 	vm.RestartedAt = time.Time{}
 	vm.RestartCount = 0
 	vm.UID = uuid.New().String()
+	vm.PowerState = v1.PowerStateRunning
+	vm.TartName = ondiskname.New(vm.Name, vm.UID, vm.RestartCount).String()
+	vm.Generation = 0
+	vm.ObservedGeneration = 0
+	vm.Conditions = []v1.Condition{
+		{
+			Type:  v1.ConditionTypeScheduled,
+			State: v1.ConditionStateFalse,
+		},
+	}
 
 	// Softnet-specific logic: automatically enable Softnet when NetSoftnetAllow or NetSoftnetBlock are set
 	// and propagate deprecated and non-deprecated boolean fields into each other
@@ -150,6 +161,20 @@ func (controller *Controller) updateVMSpec(ctx *gin.Context) responder.Responder
 				"toggled for suspendable VMs"))
 		}
 
+		// Power state-specific sanity checks
+		if !userVM.PowerState.Valid() {
+			return responder.JSON(http.StatusPreconditionFailed, NewErrorResponse("invalid \"powerState\" "+
+				"value: %s", userVM.PowerState))
+		}
+		if dbVM.PowerState.TerminalState() {
+			return responder.JSON(http.StatusPreconditionFailed, NewErrorResponse("invalid \"powerState\" "+
+				"transition: cannot transition from a terminal power state"))
+		}
+		if !dbVM.Suspendable && userVM.PowerState == v1.PowerStateSuspended {
+			return responder.JSON(http.StatusPreconditionFailed, NewErrorResponse("invalid \"powerState\" "+
+				"transition: only suspendable VMs can be suspended"))
+		}
+
 		if cmp.Equal(dbVM.VMSpec, userVM.VMSpec) {
 			// Nothing was changed
 			return responder.JSON(http.StatusOK, dbVM)

internal/controller/scheduler/scheduler.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cirruslabs/orchard/internal/controller/notifier"
 	storepkg "github.com/cirruslabs/orchard/internal/controller/store"
 	"github.com/cirruslabs/orchard/internal/opentelemetry"
+	"github.com/cirruslabs/orchard/internal/worker/ondiskname"
 	"github.com/cirruslabs/orchard/pkg/resource/v1"
 	"github.com/cirruslabs/orchard/rpc"
 	mapset "github.com/deckarep/golang-set/v2"
@@ -267,13 +268,22 @@ NextVM:
 					return ErrVMSchedulingSkipped
 				}
 
-				if currentUnscheduledVM.Status != v1.VMStatusPending ||
-					currentUnscheduledVM.Worker != "" {
+				if unscheduledVM.IsScheduled() {
 					// Unscheduled VM is not unscheduled anymore,
 					// so there's nothing to do
 					return ErrVMSchedulingSkipped
 				}
 
+				if unscheduledVM.TerminalState() {
+					// We don't support re-scheduling of VMs in terminal state at the moment
+					return ErrVMSchedulingSkipped
+				}
+
+				if unscheduledVM.PowerState.TerminalState() {
+					// We don't support re-scheduling of stopped/suspended VMs at the moment
+					return ErrVMSchedulingSkipped
+				}
+
 				currentWorker, err := txn.GetWorker(worker.Name)
 				if err != nil {
 					if errors.Is(err, storepkg.ErrNotFound) {
@@ -298,6 +308,10 @@ NextVM:
 
 				unscheduledVM.Worker = worker.Name
 				unscheduledVM.ScheduledAt = time.Now()
+				v1.ConditionsSet(&unscheduledVM.Conditions, v1.Condition{
+					Type:  v1.ConditionTypeScheduled,
+					State: v1.ConditionStateTrue,
+				})
 
 				// Fill out the actual CPU allocation
 				if unscheduledVM.CPU == 0 {
@@ -376,10 +390,10 @@ func ProcessVMs(vms []v1.VM) ([]v1.VM, WorkerInfos) {
 	workerToResources := make(WorkerInfos)
 
 	for _, vm := range vms {
-		if vm.Worker == "" {
-			unscheduledVMs = append(unscheduledVMs, vm)
-		} else if !vm.TerminalState() {
+		if vm.IsScheduled() {
 			workerToResources.AddVM(vm.Worker, vm.Resources)
+		} else {
+			unscheduledVMs = append(unscheduledVMs, vm)
 		}
 	}
 
@@ -427,7 +441,7 @@ func (scheduler *Scheduler) healthCheckingLoopIteration() (int, int, error) {
 	// transaction, re-checking that the VM still exists
 	// and it is still scheduled
 	for _, vm := range vms {
-		if vm.Worker == "" {
+		if !vm.IsScheduled() {
 			// Not a scheduled VM
 			//
 			// We'll re-check this below, but this allows us
@@ -447,7 +461,7 @@ func (scheduler *Scheduler) healthCheckingLoopIteration() (int, int, error) {
 				return err
 			}
 
-			if currentVM.Worker == "" {
+			if !vm.IsScheduled() {
 				// Not a scheduled VM, nothing to do
 				return nil
 			}
@@ -483,6 +497,14 @@ func (scheduler *Scheduler) healthCheckVM(txn storepkg.Transaction, vm v1.VM) er
 		vm.RestartCount++
 		vm.ScheduledAt = time.Time{}
 		vm.StartedAt = time.Time{}
+		vm.PowerState = v1.PowerStateRunning
+		vm.TartName = ondiskname.New(vm.Name, vm.UID, vm.RestartCount).String()
+		vm.Conditions = []v1.Condition{
+			{
+				Type:  v1.ConditionTypeScheduled,
+				State: v1.ConditionStateFalse,
+			},
+		}
 
 		return txn.SetVM(vm)
 	}
@@ -508,5 +530,33 @@ func (scheduler *Scheduler) healthCheckVM(txn storepkg.Transaction, vm v1.VM) er
 		return txn.SetVM(vm)
 	}
 
+	if vm.PowerState.TerminalState() && v1.ConditionIsFalse(vm.Conditions, v1.ConditionTypeRunning) {
+		// VM has entered a terminal power state and stopped running,
+		// de-schedule it to free up resources
+		v1.ConditionsSet(&vm.Conditions, v1.Condition{
+			Type:  v1.ConditionTypeScheduled,
+			State: v1.ConditionStateFalse,
+		})
+
+		return txn.SetVM(vm)
+	}
+
+	if vm.TerminalState() {
+		// VM has entered a terminal state,
+		// de-schedule it to free up resources
+		v1.ConditionsSet(&vm.Conditions, v1.Condition{
+			Type:  v1.ConditionTypeScheduled,
+			State: v1.ConditionStateFalse,
+		})
+
+		// Also correct the conditions for the worker
+		v1.ConditionsSet(&vm.Conditions, v1.Condition{
+			Type:  v1.ConditionTypeRunning,
+			State: v1.ConditionStateFalse,
+		})
+
+		return txn.SetVM(vm)
+	}
+
 	return nil
 }