mirror of
https://github.com/cirruslabs/orchard.git
synced 2026-09-29 19:41:15 +02:00
280 lines
9.1 KiB
Go
280 lines
9.1 KiB
Go
package worker //nolint:testpackage // The regression test exercises unexported worker reconciliation.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/cirruslabs/orchard/internal/worker/ondiskname"
|
|
"github.com/cirruslabs/orchard/internal/worker/runtime"
|
|
"github.com/cirruslabs/orchard/internal/worker/vmmanager"
|
|
"github.com/cirruslabs/orchard/internal/worker/vmmanager/base"
|
|
"github.com/cirruslabs/orchard/internal/worker/vmmanager/synthetic"
|
|
"github.com/cirruslabs/orchard/pkg/client"
|
|
v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
var errLocalVMFailed = errors.New("local VM failed")
|
|
|
|
type delayedStopVM struct {
|
|
vmmanager.VM
|
|
|
|
resource v1.VM
|
|
status v1.VMStatus
|
|
stopStarted chan struct{}
|
|
stopResult chan error
|
|
conditions []v1.Condition
|
|
starts int
|
|
}
|
|
|
|
func (vm *delayedStopVM) OnDiskName() ondiskname.OnDiskName {
|
|
return ondiskname.NewFromResource(vm.resource)
|
|
}
|
|
|
|
func (vm *delayedStopVM) Status() v1.VMStatus { return vm.status }
|
|
|
|
func (vm *delayedStopVM) Conditions() []v1.Condition { return vm.conditions }
|
|
|
|
func (vm *delayedStopVM) Running() bool {
|
|
return v1.ConditionIsTrue(vm.conditions, v1.ConditionTypeRunning) &&
|
|
!v1.ConditionIsTrue(vm.conditions, v1.ConditionTypeStopping) &&
|
|
!v1.ConditionIsTrue(vm.conditions, v1.ConditionTypeSuspending)
|
|
}
|
|
|
|
func (vm *delayedStopVM) Resource() v1.VM { return vm.resource }
|
|
|
|
func (vm *delayedStopVM) SetResource(resource v1.VM) {
|
|
vm.resource = resource
|
|
vm.resource.ObservedGeneration = resource.Generation
|
|
}
|
|
|
|
func (vm *delayedStopVM) StatusMessage() string { return "" }
|
|
|
|
func (vm *delayedStopVM) Start(streamer *client.EventStreamer) {
|
|
vm.starts++
|
|
v1.ConditionsSet(&vm.conditions, v1.Condition{
|
|
Type: v1.ConditionTypeRunning, State: v1.ConditionStateTrue,
|
|
})
|
|
_ = streamer.Close()
|
|
}
|
|
|
|
func (vm *delayedStopVM) Err() error { return errLocalVMFailed }
|
|
|
|
func (vm *delayedStopVM) Stop() <-chan error {
|
|
close(vm.stopStarted)
|
|
if vm.conditions != nil {
|
|
v1.ConditionsSet(&vm.conditions, v1.Condition{
|
|
Type: v1.ConditionTypeRunning, State: v1.ConditionStateFalse,
|
|
})
|
|
v1.ConditionsSet(&vm.conditions, v1.Condition{
|
|
Type: v1.ConditionTypeStopping, State: v1.ConditionStateTrue,
|
|
})
|
|
}
|
|
return vm.stopResult
|
|
}
|
|
|
|
//nolint:exhaustruct_v5 // Fixture fields unrelated to lifecycle transitions use their zero values.
|
|
func TestMonitorWaitsForStopBeforeApplyingGeneration(t *testing.T) {
|
|
for _, powerState := range []v1.PowerState{v1.PowerStateStopped, v1.PowerStateRunning} {
|
|
t.Run(string(powerState), func(t *testing.T) {
|
|
events := make(chan struct{}, 1)
|
|
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
|
|
response.WriteHeader(http.StatusOK)
|
|
events <- struct{}{}
|
|
}))
|
|
defer server.Close()
|
|
apiClient, err := client.New(client.WithAddress(server.URL))
|
|
require.NoError(t, err)
|
|
worker := &Worker{client: apiClient, runtime: runtime.NewSynthetic()}
|
|
vm := &delayedStopVM{
|
|
VM: &synthetic.VM{VM: base.NewVM(v1.VM{}, ondiskname.OnDiskName{}, zap.NewNop().Sugar())},
|
|
resource: v1.VM{Meta: v1.Meta{Name: "test-vm"}},
|
|
stopStarted: make(chan struct{}),
|
|
stopResult: make(chan error),
|
|
conditions: []v1.Condition{{
|
|
Type: v1.ConditionTypeRunning, State: v1.ConditionStateTrue,
|
|
}},
|
|
}
|
|
desired := vm.resource
|
|
desired.Generation = 1
|
|
desired.PowerState = powerState
|
|
update := func(context.Context, v1.VM) error { return nil }
|
|
|
|
// Stop clears Running immediately, before the command has finished.
|
|
// Neither this reconciliation nor later ones may acknowledge or restart it.
|
|
for range 2 {
|
|
require.NoError(t, worker.monitorRunningVM(t.Context(), &desired, vm, update))
|
|
require.Zero(t, vm.resource.Generation, "the specification changed before Stop completed")
|
|
require.Zero(t, desired.ObservedGeneration, "shutdown was acknowledged before it completed")
|
|
require.Zero(t, vm.starts, "the VM restarted before Stop completed")
|
|
}
|
|
|
|
close(vm.stopResult)
|
|
v1.ConditionsSet(&vm.conditions, v1.Condition{
|
|
Type: v1.ConditionTypeStopping, State: v1.ConditionStateFalse,
|
|
})
|
|
require.NoError(t, worker.monitorRunningVM(t.Context(), &desired, vm, update))
|
|
require.Equal(t, desired.Generation, vm.resource.Generation)
|
|
require.Equal(t, desired.Generation, desired.ObservedGeneration)
|
|
if powerState == v1.PowerStateRunning {
|
|
require.Equal(t, 1, vm.starts)
|
|
select {
|
|
case <-events:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("restart event stream did not close")
|
|
}
|
|
} else {
|
|
require.Zero(t, vm.starts)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMonitorPreservesRestartAfterHostProcessUpdate(t *testing.T) {
|
|
events := make(chan struct{}, 1)
|
|
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
|
|
response.WriteHeader(http.StatusOK)
|
|
events <- struct{}{}
|
|
}))
|
|
defer server.Close()
|
|
apiClient, err := client.New(client.WithAddress(server.URL))
|
|
require.NoError(t, err)
|
|
worker := &Worker{client: apiClient, runtime: runtime.NewSynthetic()}
|
|
update := func(context.Context, v1.VM) error { return nil }
|
|
|
|
// An earlier specification change has already initiated a restart
|
|
vm := &delayedStopVM{
|
|
VM: &synthetic.VM{VM: base.NewVM(v1.VM{}, ondiskname.OnDiskName{}, zap.NewNop().Sugar())},
|
|
resource: v1.VM{
|
|
Name: "test-vm",
|
|
PowerState: v1.PowerStateRunning,
|
|
HostProcesses: []v1.HostProcess{{Name: "removed", Program: "unused"}},
|
|
},
|
|
conditions: []v1.Condition{
|
|
{Type: v1.ConditionTypeRunning, State: v1.ConditionStateFalse},
|
|
{Type: v1.ConditionTypeStopping, State: v1.ConditionStateTrue},
|
|
},
|
|
}
|
|
desired := vm.resource
|
|
desired.Generation = 2
|
|
desired.HostProcesses = nil
|
|
|
|
// Removing the host process must leave the generation pending while stopping
|
|
require.NoError(t, worker.monitorRunningVM(t.Context(), &desired, vm, update))
|
|
require.Zero(t, vm.resource.Generation)
|
|
require.Zero(t, desired.ObservedGeneration)
|
|
require.Zero(t, vm.starts)
|
|
|
|
// After stopping, the pending generation must still cause the VM to start
|
|
v1.ConditionsSet(&vm.conditions, v1.Condition{
|
|
Type: v1.ConditionTypeStopping, State: v1.ConditionStateFalse,
|
|
})
|
|
require.NoError(t, worker.monitorRunningVM(t.Context(), &desired, vm, update))
|
|
require.Equal(t, desired.Generation, vm.resource.Generation)
|
|
require.Equal(t, desired.Generation, desired.ObservedGeneration)
|
|
require.Equal(t, 1, vm.starts)
|
|
require.Empty(t, vm.resource.HostProcesses)
|
|
select {
|
|
case <-events:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("restart event stream did not close")
|
|
}
|
|
}
|
|
|
|
func TestSyncVMsWaitsForVMShutdown(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
remoteStatus v1.VMStatus
|
|
localStatus v1.VMStatus
|
|
update bool
|
|
}{
|
|
{
|
|
name: "remote failed VM stops before reconciliation continues",
|
|
remoteStatus: v1.VMStatusFailed,
|
|
localStatus: v1.VMStatusRunning,
|
|
},
|
|
{
|
|
name: "local failed VM stops before reporting failure",
|
|
remoteStatus: v1.VMStatusRunning,
|
|
localStatus: v1.VMStatusFailed,
|
|
update: true,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
resource := v1.VM{
|
|
Meta: v1.Meta{Name: "test-vm"},
|
|
UID: "00112233-4455-6677-8899-aabbccddeeff",
|
|
Worker: "test-worker",
|
|
Status: test.remoteStatus,
|
|
}
|
|
|
|
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
|
if request.Method != http.MethodGet || request.URL.Path != "/v1/vms" {
|
|
t.Errorf("unexpected request: %s %s", request.Method, request.URL.Path)
|
|
return
|
|
}
|
|
|
|
response.Header().Set("Content-Type", "application/json")
|
|
if err := json.NewEncoder(response).Encode([]map[string]any{{
|
|
"name": resource.Name,
|
|
"uid": resource.UID,
|
|
"worker": resource.Worker,
|
|
"status": resource.Status,
|
|
}}); err != nil {
|
|
t.Errorf("encode VM response: %v", err)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
apiClient, err := client.New(client.WithAddress(server.URL))
|
|
require.NoError(t, err)
|
|
vm := &delayedStopVM{
|
|
resource: resource,
|
|
status: test.localStatus,
|
|
stopStarted: make(chan struct{}),
|
|
stopResult: make(chan error, 1),
|
|
}
|
|
manager := vmmanager.New()
|
|
manager.Put(vm.OnDiskName(), vm)
|
|
worker := &Worker{name: "test-worker", client: apiClient, vmm: manager, logger: zap.NewNop().Sugar()}
|
|
updated := make(chan v1.VM, 1)
|
|
finished := make(chan error, 1)
|
|
|
|
go func() {
|
|
finished <- worker.syncVMs(context.Background(), func(_ context.Context, updatedVM v1.VM) error {
|
|
updated <- updatedVM
|
|
return nil
|
|
}, nil)
|
|
}()
|
|
|
|
select {
|
|
case <-vm.stopStarted:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("VM shutdown was not requested")
|
|
}
|
|
|
|
select {
|
|
case updatedVM := <-updated:
|
|
t.Fatalf("VM was reported %q before shutdown completed", updatedVM.Status)
|
|
case err := <-finished:
|
|
t.Fatalf("reconciliation continued before VM shutdown completed: %v", err)
|
|
case <-time.After(30 * time.Millisecond):
|
|
}
|
|
|
|
vm.stopResult <- nil
|
|
require.NoError(t, <-finished)
|
|
if test.update {
|
|
require.Equal(t, v1.VMStatusFailed, (<-updated).Status)
|
|
}
|
|
})
|
|
}
|
|
}
|