301 lines
8.3 KiB
Go
301 lines
8.3 KiB
Go
package tests_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"os/signal"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/cirruslabs/orchard/internal/controller"
|
|
"github.com/cirruslabs/orchard/internal/tests/platformdependent"
|
|
"github.com/cirruslabs/orchard/internal/tests/wait"
|
|
"github.com/cirruslabs/orchard/internal/worker"
|
|
"github.com/cirruslabs/orchard/pkg/client"
|
|
v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
const (
|
|
controllerRestartHelperEnv = "ORCHARD_CONTROLLER_RESTART_TEST_HELPER"
|
|
controllerRestartHelperDataDirEnv = "ORCHARD_CONTROLLER_RESTART_TEST_DATA_DIR"
|
|
controllerRestartHelperListenAddrEnv = "ORCHARD_CONTROLLER_RESTART_TEST_LISTEN_ADDR"
|
|
controllerRestartHelperOfflineTimeoutEnv = "ORCHARD_CONTROLLER_RESTART_TEST_OFFLINE_TIMEOUT"
|
|
controllerRestartWorkerOfflineTimeout = 20 * time.Second
|
|
controllerRestartWorkerDisconnectDuration = 21 * time.Second
|
|
)
|
|
|
|
func TestControllerRestartDoesNotFailRunningVMs(t *testing.T) {
|
|
ctx := t.Context()
|
|
listenAddr := unusedTCPAddress(t)
|
|
dataDir := t.TempDir()
|
|
|
|
devClient, err := client.New(client.WithAddress("http://" + listenAddr))
|
|
require.NoError(t, err)
|
|
|
|
controllerProcess := startControllerProcess(t, devClient, dataDir, listenAddr)
|
|
devWorker := startSyntheticWorker(t, devClient)
|
|
|
|
var workerName string
|
|
require.True(t, wait.Wait(30*time.Second, func() bool {
|
|
workers, err := devClient.Workers().List(ctx)
|
|
if err != nil || len(workers) != 1 {
|
|
return false
|
|
}
|
|
|
|
workerName = workers[0].Name
|
|
|
|
return true
|
|
}), "failed to wait for the worker to register")
|
|
|
|
const vmName = "controller-restart-vm"
|
|
|
|
require.NoError(t, devClient.VMs().Create(ctx, platformdependent.VM(vmName)))
|
|
require.True(t, wait.Wait(30*time.Second, func() bool {
|
|
vm, err := devClient.VMs().Get(ctx, vmName)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
t.Logf("Waiting for the synthetic VM to start. Current status: %s", vm.Status)
|
|
|
|
return vm.Status == v1.VMStatusRunning
|
|
}), "failed to wait for the synthetic VM to start")
|
|
|
|
vmBeforeRestart, err := devClient.VMs().Get(ctx, vmName)
|
|
require.NoError(t, err)
|
|
require.Equal(t, workerName, vmBeforeRestart.Worker)
|
|
|
|
// Stop the worker's controller session without stopping its running VMs, then
|
|
// keep the controller down until the persisted worker heartbeat is stale.
|
|
devWorker.stop(t)
|
|
require.NoError(t, controllerProcess.stop())
|
|
time.Sleep(controllerRestartWorkerDisconnectDuration)
|
|
|
|
restartedAt := time.Now()
|
|
startControllerProcess(t, devClient, dataDir, listenAddr)
|
|
|
|
// Keep the worker disconnected beyond the first post-restart scheduler
|
|
// health check. The VM should remain running during the startup grace period.
|
|
observationDeadline := time.Now().Add(7 * time.Second)
|
|
for time.Now().Before(observationDeadline) {
|
|
vm, err := devClient.VMs().Get(ctx, vmName)
|
|
require.NoError(t, err)
|
|
require.Equal(t, v1.VMStatusRunning, vm.Status)
|
|
require.Equal(t, vmBeforeRestart.UID, vm.UID)
|
|
require.Equal(t, workerName, vm.Worker)
|
|
|
|
time.Sleep(250 * time.Millisecond)
|
|
}
|
|
|
|
devWorker.start(t)
|
|
require.True(t, wait.Wait(30*time.Second, func() bool {
|
|
workerResource, err := devClient.Workers().Get(ctx, workerName)
|
|
if err != nil || !workerResource.LastSeen.After(restartedAt) {
|
|
return false
|
|
}
|
|
|
|
vm, err := devClient.VMs().Get(ctx, vmName)
|
|
|
|
return err == nil && vm.Status == v1.VMStatusRunning
|
|
}), "worker did not reconnect with its VM still running after the controller restart")
|
|
}
|
|
|
|
// TestControllerRestartHelperProcess runs an Orchard Controller in a separate
|
|
// process so that its Badger database is released and can be reopened during a
|
|
// realistic controller restart.
|
|
func TestControllerRestartHelperProcess(t *testing.T) {
|
|
if os.Getenv(controllerRestartHelperEnv) != "1" {
|
|
return
|
|
}
|
|
|
|
dataDirPath := requireEnv(t, controllerRestartHelperDataDirEnv)
|
|
listenAddr := requireEnv(t, controllerRestartHelperListenAddrEnv)
|
|
offlineTimeout, err := time.ParseDuration(requireEnv(t, controllerRestartHelperOfflineTimeoutEnv))
|
|
require.NoError(t, err)
|
|
|
|
dataDir, err := controller.NewDataDir(dataDirPath)
|
|
require.NoError(t, err)
|
|
|
|
devController, err := controller.New(
|
|
controller.WithDataDir(dataDir),
|
|
controller.WithListenAddr(listenAddr),
|
|
controller.WithInsecureAuthDisabled(),
|
|
controller.WithExperimentalRPCV2(),
|
|
controller.WithWorkerOfflineTimeout(offlineTimeout),
|
|
controller.WithLogger(zap.NewNop()),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
|
defer cancel()
|
|
|
|
err = devController.Run(ctx)
|
|
require.True(t, err == nil || errors.Is(err, context.Canceled) || errors.Is(err, http.ErrServerClosed),
|
|
"controller failed: %v", err)
|
|
}
|
|
|
|
type controllerTestProcess struct {
|
|
cmd *exec.Cmd
|
|
output bytes.Buffer
|
|
stopped bool
|
|
}
|
|
|
|
func startControllerProcess(
|
|
t *testing.T,
|
|
devClient *client.Client,
|
|
dataDir string,
|
|
listenAddr string,
|
|
) *controllerTestProcess {
|
|
t.Helper()
|
|
|
|
process := &controllerTestProcess{}
|
|
process.cmd = exec.Command(os.Args[0], "-test.run=^TestControllerRestartHelperProcess$", "-test.v")
|
|
process.cmd.Env = append(os.Environ(),
|
|
controllerRestartHelperEnv+"=1",
|
|
controllerRestartHelperDataDirEnv+"="+dataDir,
|
|
controllerRestartHelperListenAddrEnv+"="+listenAddr,
|
|
controllerRestartHelperOfflineTimeoutEnv+"="+controllerRestartWorkerOfflineTimeout.String(),
|
|
)
|
|
process.cmd.Stdout = &process.output
|
|
process.cmd.Stderr = &process.output
|
|
|
|
require.NoError(t, process.cmd.Start())
|
|
t.Cleanup(func() {
|
|
require.NoError(t, process.stop())
|
|
})
|
|
|
|
if wait.Wait(10*time.Second, func() bool {
|
|
requestCtx, requestCancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
|
|
defer requestCancel()
|
|
|
|
_, err := devClient.Controller().Info(requestCtx)
|
|
|
|
return err == nil
|
|
}) {
|
|
return process
|
|
}
|
|
|
|
stopErr := process.stop()
|
|
t.Fatalf("controller failed to become ready: %v\n%s", stopErr, process.output.String())
|
|
|
|
return nil
|
|
}
|
|
|
|
func (process *controllerTestProcess) stop() error {
|
|
if process.stopped {
|
|
return nil
|
|
}
|
|
process.stopped = true
|
|
|
|
signalErr := process.cmd.Process.Signal(os.Interrupt)
|
|
waitCh := make(chan error, 1)
|
|
go func() {
|
|
waitCh <- process.cmd.Wait()
|
|
}()
|
|
|
|
select {
|
|
case waitErr := <-waitCh:
|
|
if signalErr != nil && !errors.Is(signalErr, os.ErrProcessDone) {
|
|
return fmt.Errorf("failed to interrupt controller process: %w", signalErr)
|
|
}
|
|
if waitErr != nil {
|
|
return fmt.Errorf("controller process failed: %w\n%s", waitErr, process.output.String())
|
|
}
|
|
|
|
return nil
|
|
case <-time.After(10 * time.Second):
|
|
_ = process.cmd.Process.Kill()
|
|
<-waitCh
|
|
|
|
return fmt.Errorf("timed out waiting for controller process to stop\n%s", process.output.String())
|
|
}
|
|
}
|
|
|
|
type syntheticTestWorker struct {
|
|
worker *worker.Worker
|
|
cancel context.CancelFunc
|
|
done chan error
|
|
}
|
|
|
|
func startSyntheticWorker(t *testing.T, devClient *client.Client) *syntheticTestWorker {
|
|
t.Helper()
|
|
|
|
devWorker, err := worker.New(devClient, worker.WithSynthetic(), worker.WithLogger(zap.NewNop()))
|
|
require.NoError(t, err)
|
|
|
|
testWorker := &syntheticTestWorker{
|
|
worker: devWorker,
|
|
}
|
|
testWorker.start(t)
|
|
t.Cleanup(func() {
|
|
testWorker.stop(t)
|
|
require.NoError(t, testWorker.worker.Close())
|
|
})
|
|
|
|
return testWorker
|
|
}
|
|
|
|
func (worker *syntheticTestWorker) start(t *testing.T) {
|
|
t.Helper()
|
|
require.Nil(t, worker.cancel, "synthetic test worker is already running")
|
|
|
|
workerCtx, cancelWorker := context.WithCancel(context.Background())
|
|
workerDone := make(chan error, 1)
|
|
worker.cancel = cancelWorker
|
|
worker.done = workerDone
|
|
|
|
go func() {
|
|
workerDone <- worker.worker.Run(workerCtx)
|
|
}()
|
|
}
|
|
|
|
func (worker *syntheticTestWorker) stop(t *testing.T) {
|
|
t.Helper()
|
|
if worker.cancel == nil {
|
|
return
|
|
}
|
|
|
|
worker.cancel()
|
|
|
|
select {
|
|
case err := <-worker.done:
|
|
if err != nil && !errors.Is(err, context.Canceled) {
|
|
t.Errorf("dev worker failed: %v", err)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("timed out waiting for dev worker to stop")
|
|
}
|
|
|
|
worker.cancel = nil
|
|
worker.done = nil
|
|
}
|
|
|
|
func unusedTCPAddress(t *testing.T) string {
|
|
t.Helper()
|
|
|
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
require.NoError(t, err)
|
|
|
|
listenAddr := listener.Addr().String()
|
|
require.NoError(t, listener.Close())
|
|
|
|
return listenAddr
|
|
}
|
|
|
|
func requireEnv(t *testing.T, name string) string {
|
|
t.Helper()
|
|
|
|
value := os.Getenv(name)
|
|
require.NotEmpty(t, value, "%s must be set", name)
|
|
|
|
return value
|
|
}
|