mirror of
https://github.com/cirruslabs/orchard.git
synced 2026-10-02 13:03:05 +02:00
edi-oai
a3e08ffcbf
* Regenerate .pb with protoc-gen-go v1.36.11 * Support running host processes alongside VMs * Replace existing host processes when starting a set
533 lines
15 KiB
Go
533 lines
15 KiB
Go
package tests
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"runtime"
|
|
"testing"
|
|
"time"
|
|
|
|
commandssh "github.com/cirruslabs/orchard/internal/command/ssh"
|
|
"github.com/cirruslabs/orchard/internal/imageconstant"
|
|
"github.com/cirruslabs/orchard/internal/tests/devcontroller"
|
|
"github.com/cirruslabs/orchard/internal/tests/wait"
|
|
"github.com/cirruslabs/orchard/internal/worker"
|
|
"github.com/cirruslabs/orchard/internal/worker/ondiskname"
|
|
"github.com/cirruslabs/orchard/internal/worker/vmmanager"
|
|
"github.com/cirruslabs/orchard/internal/worker/vmmanager/tart"
|
|
v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
|
|
"github.com/samber/lo"
|
|
"github.com/shirou/gopsutil/v4/process"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/zap"
|
|
"golang.org/x/crypto/ssh"
|
|
)
|
|
|
|
func TestSpecUpdateSoftnet(t *testing.T) {
|
|
if runtime.GOOS != "darwin" {
|
|
t.Skip("Softnet is only supported on macOS with Tart")
|
|
}
|
|
|
|
if os.Getenv("ORCHARD_SKIP_SOFTNET_TESTS") != "" {
|
|
t.Skip("softnet tests require root")
|
|
}
|
|
|
|
devClient, _, _ := devcontroller.StartIntegrationTestEnvironmentWithAdditionalOpts(
|
|
t,
|
|
false,
|
|
nil,
|
|
false,
|
|
[]worker.Option{worker.WithSoftnetPolicyUpdates(false)},
|
|
)
|
|
|
|
// Create a VM
|
|
vmName := "test"
|
|
|
|
err := devClient.VMs().Create(t.Context(), &v1.VM{
|
|
Meta: v1.Meta{
|
|
Name: vmName,
|
|
},
|
|
Image: imageconstant.DefaultMacosImage,
|
|
CPU: 4,
|
|
Memory: 8 * 1024,
|
|
Headless: true,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Wait for the VM to start
|
|
var vm *v1.VM
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM to start. Current status: %s", vm.Status)
|
|
|
|
return vm.Status == v1.VMStatusRunning
|
|
}), "failed to start a VM")
|
|
|
|
// Ensure that Softnet is not enabled for a VM
|
|
tartVMName := ondiskname.New(vmName, vm.UID, vm.RestartCount).String()
|
|
|
|
tartRunCmdline, err := tartRunProcessCmdline(tartVMName)
|
|
require.NoError(t, err)
|
|
require.NotContains(t, tartRunCmdline, "--net-softnet")
|
|
require.NotContains(t, tartRunCmdline, "--net-softnet-allow")
|
|
require.NotContains(t, tartRunCmdline, "--net-softnet-block")
|
|
|
|
// Update the VM's specification and enable Softnet
|
|
vm.NetSoftnetAllow = []string{"10.0.0.0/16"}
|
|
vm.NetSoftnetBlock = []string{"0.0.0.0/0"}
|
|
|
|
vm, err = devClient.VMs().Update(t.Context(), *vm)
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, 1, vm.Generation)
|
|
require.EqualValues(t, 0, vm.ObservedGeneration)
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM's observed generation to be updated...")
|
|
|
|
return vm.ObservedGeneration == 1
|
|
}), "failed to wait for the VM's observed generation to be updated")
|
|
|
|
tartRunCmdline, err = tartRunProcessCmdline(tartVMName)
|
|
require.NoError(t, err)
|
|
require.Contains(t, tartRunCmdline, "--net-softnet")
|
|
require.True(t, sliceContainsAnotherSlice(tartRunCmdline, []string{"--net-softnet-allow", "10.0.0.0/16"}))
|
|
require.True(t, sliceContainsAnotherSlice(tartRunCmdline, []string{"--net-softnet-block", "0.0.0.0/0"}))
|
|
}
|
|
|
|
func TestSpecUpdateSoftnetSuspendable(t *testing.T) {
|
|
if runtime.GOOS != "darwin" {
|
|
t.Skip("Softnet is only supported on macOS with Tart")
|
|
}
|
|
|
|
if os.Getenv("ORCHARD_SKIP_SOFTNET_TESTS") != "" {
|
|
t.Skip("softnet tests require root")
|
|
}
|
|
|
|
devClient, _, _ := devcontroller.StartIntegrationTestEnvironmentWithAdditionalOpts(
|
|
t,
|
|
false,
|
|
nil,
|
|
false,
|
|
[]worker.Option{worker.WithSoftnetPolicyUpdates(false)},
|
|
)
|
|
|
|
// Create a suspendable VM with Softnet enabled
|
|
vmName := "test"
|
|
|
|
err := devClient.VMs().Create(t.Context(), &v1.VM{
|
|
Meta: v1.Meta{
|
|
Name: vmName,
|
|
},
|
|
Image: imageconstant.DefaultMacosImage,
|
|
CPU: 4,
|
|
Memory: 8 * 1024,
|
|
Headless: true,
|
|
VMSpec: v1.VMSpec{
|
|
Suspendable: true,
|
|
NetSoftnet: true,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Wait for the VM to start
|
|
var vm *v1.VM
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM to start. Current status: %s", vm.Status)
|
|
|
|
return vm.Status == v1.VMStatusRunning
|
|
}), "failed to start a VM")
|
|
|
|
// Ensure that the VM is using "--suspendable" and "--net-softnet"
|
|
tartVMName := ondiskname.New(vmName, vm.UID, vm.RestartCount).String()
|
|
|
|
tartRunCmdline, err := tartRunProcessCmdline(tartVMName)
|
|
require.NoError(t, err)
|
|
require.Contains(t, tartRunCmdline, "--suspendable")
|
|
require.Contains(t, tartRunCmdline, "--net-softnet")
|
|
|
|
// Update the VM's specification and tighten the Softnet restrictions
|
|
vm.NetSoftnetAllow = []string{"10.0.0.0/16"}
|
|
vm.NetSoftnetBlock = []string{"0.0.0.0/0"}
|
|
|
|
vm, err = devClient.VMs().Update(t.Context(), *vm)
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, 1, vm.Generation)
|
|
require.EqualValues(t, 0, vm.ObservedGeneration)
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM's observed generation to be updated...")
|
|
|
|
return vm.ObservedGeneration == 1
|
|
}), "failed to wait for the VM's observed generation to be updated")
|
|
|
|
// Ensure that the VM is using "--suspendable", "--net-softnet" and "--net-softnet-{allow,block}"
|
|
tartRunCmdline, err = tartRunProcessCmdline(tartVMName)
|
|
require.NoError(t, err)
|
|
require.Contains(t, tartRunCmdline, "--suspendable")
|
|
require.Contains(t, tartRunCmdline, "--net-softnet")
|
|
require.True(t, sliceContainsAnotherSlice(tartRunCmdline, []string{"--net-softnet-allow", "10.0.0.0/16"}))
|
|
require.True(t, sliceContainsAnotherSlice(tartRunCmdline, []string{"--net-softnet-block", "0.0.0.0/0"}))
|
|
}
|
|
|
|
//nolint:gosec,modernize,perfsprint,staticcheck // preserve the original integration test
|
|
func TestSpecUpdateSoftnetPolicy(t *testing.T) {
|
|
if runtime.GOOS != "darwin" {
|
|
t.Skip("Softnet is only supported on macOS with Tart")
|
|
}
|
|
|
|
if os.Getenv("ORCHARD_SKIP_SOFTNET_TESTS") != "" {
|
|
t.Skip("softnet tests require root")
|
|
}
|
|
|
|
devClient, _, _ := devcontroller.StartIntegrationTestEnvironmentWithAdditionalOpts(
|
|
t,
|
|
false,
|
|
nil,
|
|
false,
|
|
[]worker.Option{worker.WithSoftnetPolicyUpdates(true)},
|
|
)
|
|
|
|
// Create a VM with Softnet enabled
|
|
vmName := "test"
|
|
|
|
err := devClient.VMs().Create(t.Context(), &v1.VM{
|
|
Meta: v1.Meta{
|
|
Name: vmName,
|
|
},
|
|
Image: imageconstant.DefaultMacosImage,
|
|
CPU: 4,
|
|
Memory: 8 * 1024,
|
|
Headless: true,
|
|
VMSpec: v1.VMSpec{
|
|
NetSoftnet: true,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Wait for the VM to start
|
|
var vm *v1.VM
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM to start. Current status: %s", vm.Status)
|
|
|
|
return vm.Status == v1.VMStatusRunning
|
|
}), "failed to start a VM")
|
|
|
|
// Ensure that the VM is using "--net-softnet"
|
|
tartVMName := ondiskname.New(vmName, vm.UID, vm.RestartCount).String()
|
|
|
|
tartRunCmdline, err := tartRunProcessCmdline(tartVMName)
|
|
require.NoError(t, err)
|
|
require.Contains(t, tartRunCmdline, "--net-softnet")
|
|
|
|
// Connect to the VM over SSH
|
|
var netConn net.Conn
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
netConn, err = devClient.VMs().PortForward(t.Context(), vmName, 22, 120)
|
|
if err != nil {
|
|
t.Logf("Waiting for SSH to become available: %v", err)
|
|
}
|
|
|
|
return err == nil
|
|
}), "failed to connect to the VM over SSH")
|
|
defer netConn.Close()
|
|
|
|
username, password := commandssh.ChooseUsernameAndPassword(t.Context(), devClient, vmName, "", "")
|
|
|
|
sshConn, chans, reqs, err := ssh.NewClientConn(netConn, "", &ssh.ClientConfig{
|
|
User: username,
|
|
Auth: []ssh.AuthMethod{ssh.Password(password)},
|
|
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
sshClient := ssh.NewClient(sshConn, chans, reqs)
|
|
defer sshClient.Close()
|
|
|
|
curl := func(address string) error {
|
|
session, err := sshClient.NewSession()
|
|
require.NoError(t, err)
|
|
defer session.Close()
|
|
|
|
return session.Run(fmt.Sprintf(
|
|
"/usr/bin/curl -4 -k -sS -o /dev/null --connect-timeout 5 --max-time 10 https://%s",
|
|
address,
|
|
))
|
|
}
|
|
|
|
// Ensure that the address is reachable before blocking it
|
|
require.NoError(t, curl("1.1.1.1"))
|
|
|
|
// Update the Softnet policy
|
|
restartCountBeforePolicyUpdate := vm.RestartCount
|
|
vm.NetSoftnetBlock = []string{"1.1.1.1/32"}
|
|
|
|
vm, err = devClient.VMs().Update(t.Context(), *vm)
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, 1, vm.Generation)
|
|
require.EqualValues(t, 0, vm.ObservedGeneration)
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM's observed generation to be updated...")
|
|
|
|
return vm.ObservedGeneration == 1
|
|
}), "failed to wait for the VM's observed generation to be updated")
|
|
|
|
// Ensure that the policy was updated without restarting the VM
|
|
require.Equal(t, restartCountBeforePolicyUpdate, vm.RestartCount)
|
|
require.Equal(t, tartVMName, vm.TartName)
|
|
|
|
updatedTartRunCmdline, err := tartRunProcessCmdline(tartVMName)
|
|
require.NoError(t, err)
|
|
require.Equal(t, tartRunCmdline, updatedTartRunCmdline)
|
|
|
|
// Ensure that the new policy is applied without disrupting other traffic
|
|
require.Error(t, curl("1.1.1.1"))
|
|
require.NoError(t, curl("1.0.0.1"))
|
|
}
|
|
|
|
func TestSpecUpdatePowerStateSuspend(t *testing.T) {
|
|
if runtime.GOOS != "darwin" {
|
|
t.Skip("VM suspension is only supported on macOS with Tart")
|
|
}
|
|
|
|
devClient, _, _ := devcontroller.StartIntegrationTestEnvironment(t)
|
|
|
|
// Create a suspendable VM with Softnet enabled
|
|
vmName := "test"
|
|
|
|
err := devClient.VMs().Create(t.Context(), &v1.VM{
|
|
Meta: v1.Meta{
|
|
Name: vmName,
|
|
},
|
|
Image: imageconstant.DefaultMacosImage,
|
|
CPU: 4,
|
|
Memory: 8 * 1024,
|
|
Headless: true,
|
|
VMSpec: v1.VMSpec{
|
|
Suspendable: true,
|
|
NetSoftnet: true,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Wait for the VM to start
|
|
var vm *v1.VM
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM to start. Current status: %s", vm.Status)
|
|
|
|
return vm.Status == v1.VMStatusRunning
|
|
}), "failed to start a VM")
|
|
|
|
// Ensure that the VM is running
|
|
tartVMName := ondiskname.New(vmName, vm.UID, vm.RestartCount).String()
|
|
|
|
_, err = tartRunProcessCmdline(tartVMName)
|
|
require.NoError(t, err)
|
|
|
|
// Include a host process that would fail to start to ensure that host-process
|
|
// reconciliation cannot block the terminal power-state transition.
|
|
vm.HostProcesses = []v1.HostProcess{{
|
|
Name: "unavailable",
|
|
Program: "/does/not/exist",
|
|
}}
|
|
|
|
// Update the VM's specification and change its power state
|
|
vm.PowerState = v1.PowerStateSuspended
|
|
|
|
vm, err = devClient.VMs().Update(t.Context(), *vm)
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, 1, vm.Generation)
|
|
require.EqualValues(t, 0, vm.ObservedGeneration)
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM's observed generation to be updated...")
|
|
|
|
return vm.ObservedGeneration == 1
|
|
}), "failed to wait for the VM's observed generation to be updated")
|
|
|
|
// Ensure that the VM is not running
|
|
_, err = tartRunProcessCmdline(tartVMName)
|
|
require.Error(t, err)
|
|
|
|
// Ensure that the VM is present and is suspended
|
|
tartVMs, err := tart.List(t.Context(), zap.NewNop().Sugar())
|
|
require.NoError(t, err)
|
|
require.Contains(t, tartVMs, vmmanager.VMInfo{
|
|
Name: vm.LocalName,
|
|
Source: "local",
|
|
State: "suspended",
|
|
Running: false,
|
|
})
|
|
}
|
|
|
|
func TestSpecUpdatePowerStateStopped(t *testing.T) {
|
|
if runtime.GOOS != "darwin" {
|
|
t.Skip("VM suspension and Softnet is only supported on macOS with Tart")
|
|
}
|
|
|
|
devClient, _, _ := devcontroller.StartIntegrationTestEnvironment(t)
|
|
|
|
// Create a suspendable VM with Softnet enabled
|
|
vmName := "test"
|
|
|
|
err := devClient.VMs().Create(t.Context(), &v1.VM{
|
|
Meta: v1.Meta{
|
|
Name: vmName,
|
|
},
|
|
Image: imageconstant.DefaultMacosImage,
|
|
CPU: 4,
|
|
Memory: 8 * 1024,
|
|
Headless: true,
|
|
VMSpec: v1.VMSpec{
|
|
Suspendable: true,
|
|
NetSoftnet: true,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Wait for the VM to start
|
|
var vm *v1.VM
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM to start. Current status: %s", vm.Status)
|
|
|
|
return vm.Status == v1.VMStatusRunning
|
|
}), "failed to start a VM")
|
|
|
|
// Ensure that the VM is running
|
|
tartVMName := ondiskname.New(vmName, vm.UID, vm.RestartCount).String()
|
|
|
|
_, err = tartRunProcessCmdline(tartVMName)
|
|
require.NoError(t, err)
|
|
|
|
// Include a host process that would fail to start to ensure that host-process
|
|
// reconciliation cannot block the terminal power-state transition.
|
|
vm.HostProcesses = []v1.HostProcess{{
|
|
Name: "unavailable",
|
|
Program: "/does/not/exist",
|
|
}}
|
|
|
|
// Update the VM's specification and change its power state
|
|
vm.PowerState = v1.PowerStateStopped
|
|
|
|
vm, err = devClient.VMs().Update(t.Context(), *vm)
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, 1, vm.Generation)
|
|
require.EqualValues(t, 0, vm.ObservedGeneration)
|
|
|
|
require.True(t, wait.Wait(2*time.Minute, func() bool {
|
|
vm, err = devClient.VMs().Get(context.Background(), vmName)
|
|
require.NoError(t, err)
|
|
|
|
t.Logf("Waiting for the VM's observed generation to be updated...")
|
|
|
|
return vm.ObservedGeneration == 1
|
|
}), "failed to wait for the VM's observed generation to be updated")
|
|
|
|
// Ensure that the VM is not running
|
|
_, err = tartRunProcessCmdline(tartVMName)
|
|
require.Error(t, err)
|
|
|
|
// Ensure that the VM is present and is suspended
|
|
tartVMs, err := tart.List(t.Context(), zap.NewNop().Sugar())
|
|
require.NoError(t, err)
|
|
require.Contains(t, tartVMs, vmmanager.VMInfo{
|
|
Name: vm.LocalName,
|
|
Source: "local",
|
|
State: "stopped",
|
|
Running: false,
|
|
})
|
|
}
|
|
|
|
func tartRunProcessCmdline(vmName string) ([]string, error) {
|
|
processes, err := process.Processes()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for _, process := range processes {
|
|
name, err := process.Name()
|
|
if err != nil {
|
|
// On macOS, process.Name() returns "invalid argument" for most
|
|
// of the processes likely due to permissions, so just ignore it
|
|
continue
|
|
}
|
|
|
|
if name != "tart" {
|
|
continue
|
|
}
|
|
|
|
cmdline, err := process.CmdlineSlice()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(cmdline) < 3 {
|
|
continue
|
|
}
|
|
|
|
if cmdline[1] != "run" {
|
|
continue
|
|
}
|
|
|
|
if lo.Contains(cmdline[2:], vmName) {
|
|
return cmdline, nil
|
|
}
|
|
}
|
|
|
|
return nil, fmt.Errorf("failed to find a \"tart run\" process for VM %q", vmName)
|
|
}
|
|
|
|
func sliceContainsAnotherSlice(haystack []string, needle []string) bool {
|
|
if len(needle) == 0 {
|
|
return true
|
|
}
|
|
|
|
var needleIdx int
|
|
|
|
for _, haystackItem := range haystack {
|
|
if haystackItem == needle[needleIdx] {
|
|
needleIdx++
|
|
|
|
if needleIdx == len(needle) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|