Fail VMs if the worker had crashed/is unhealthy (#70)

* Fail VMs if the worker had crashed/is unhealthy

* OnDiskName: properly handle cases when VM's name contains hyphens

* Worker: introduce Offline() method and check it before scheduling

* tart.List(): use Tart's JSON output

* OnDiskName: remove empty parts check

* Scheduler: move health-checking logic to a separate function

* Only fail "running" VMs

* Only fail orphaned VMs if they're in terminal state

* Integration tests

* Run healthCheckingLoopIteration() before schedulingLoopIteration()

* Worker: sync on-disk VMs only once at start
This commit is contained in:
Nikolay Edigaryev
2023-04-03 16:47:49 +04:00
committed by GitHub
parent ea1e5c8578
commit 4eafec99a5
14 changed files with 523 additions and 48 deletions
+19 -4
View File
@@ -52,7 +52,8 @@ func runDev(cmd *cobra.Command, args []string) error {
}
devController, devWorker, err := CreateDevControllerAndWorker(devDataDirPath,
fmt.Sprintf(":%d", netconstants.DefaultControllerPort), resources)
fmt.Sprintf(":%d", netconstants.DefaultControllerPort), resources,
nil, nil)
if err != nil {
return err
@@ -79,6 +80,8 @@ func CreateDevControllerAndWorker(
devDataDirPath string,
controllerListenAddr string,
resources v1.Resources,
additionalControllerOpts []controller.Option,
additionalWorkerOpts []worker.Option,
) (*controller.Controller, *worker.Worker, error) {
// Initialize the logger
logger, err := zap.NewDevelopment()
@@ -96,13 +99,17 @@ func CreateDevControllerAndWorker(
return nil, nil, err
}
devController, err := controller.New(
controllerOpts := []controller.Option{
controller.WithDataDir(dataDir),
controller.WithListenAddr(controllerListenAddr),
controller.WithInsecureAuthDisabled(),
controller.WithSwaggerDocs(),
controller.WithLogger(logger),
)
}
controllerOpts = append(controllerOpts, additionalControllerOpts...)
devController, err := controller.New(controllerOpts...)
if err != nil {
return nil, nil, err
}
@@ -111,7 +118,15 @@ func CreateDevControllerAndWorker(
if err != nil {
return nil, nil, err
}
devWorker, err := worker.New(defaultClient, worker.WithResources(resources), worker.WithLogger(logger))
workerOpts := []worker.Option{
worker.WithResources(resources),
worker.WithLogger(logger),
}
workerOpts = append(workerOpts, additionalWorkerOpts...)
devWorker, err := worker.New(defaultClient, workerOpts...)
if err != nil {
return nil, nil, err
}