mirror of
https://github.com/cirruslabs/orchard.git
synced 2026-10-18 13:21:00 +02:00
Support running host processes alongside VMs (#482)
* Regenerate .pb with protoc-gen-go v1.36.11 * Support running host processes alongside VMs * Replace existing host processes when starting a set
This commit is contained in:
edi-oai
@@ -0,0 +1,170 @@
|
||||
//nolint:err113,mnd // Preserve the original host-process errors and retry policy.
|
||||
package hostprocess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/avast/retry-go/v4"
|
||||
v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
|
||||
)
|
||||
|
||||
type Process struct {
|
||||
cancel context.CancelFunc
|
||||
socketPath string
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func NewProcess(
|
||||
spec v1.HostProcess,
|
||||
workerName string,
|
||||
vmName string,
|
||||
controlSocket string,
|
||||
) (*Process, error) {
|
||||
// Create a directory where the host process will create its Unix socket
|
||||
//
|
||||
// To keep the Unix socket path short enough and fit the platform limit,
|
||||
// we're specifically requesting "/var/tmp" instead of "/var/folders/.../T/".
|
||||
//
|
||||
// And unlike "/tmp", "/var/tmp" is not subject to macOS's nightly cleanup.
|
||||
runtimeDir, err := os.MkdirTemp("/var/tmp", "orchard-hp-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Use a fixed socket name within the process-specific runtime directory
|
||||
socketPath := filepath.Join(runtimeDir, "process.sock")
|
||||
|
||||
// Route the Tart control socket through the runtime directory as well
|
||||
// to work around macOS's 104-byte Unix-domain socket limit
|
||||
controlSocket, err = shortenControlSocketPath(runtimeDir, controlSocket)
|
||||
if err != nil {
|
||||
return nil, errors.Join(err, os.RemoveAll(runtimeDir))
|
||||
}
|
||||
|
||||
// Give the process an independently cancellable lifetime
|
||||
cmdCtx, cmdCtxCancel := context.WithCancel(context.Background())
|
||||
|
||||
//nolint:gosec // Executing the configured host process is the purpose of this API
|
||||
command := exec.CommandContext(
|
||||
cmdCtx,
|
||||
spec.Program,
|
||||
expandArgs(spec.Args, workerName, vmName, controlSocket, socketPath)...,
|
||||
)
|
||||
command.Dir = runtimeDir
|
||||
|
||||
// Ensure that we terminate the host process and any children it spawns
|
||||
command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
command.Cancel = func() error {
|
||||
return syscall.Kill(-command.Process.Pid, syscall.SIGKILL)
|
||||
}
|
||||
|
||||
// Craft the environment starting from the host process specification
|
||||
env := make(map[string]string)
|
||||
|
||||
maps.Copy(env, spec.Env)
|
||||
|
||||
// Preserve only PATH from the worker and make Orchard-managed variables authoritative
|
||||
env["PATH"] = os.Getenv("PATH")
|
||||
env["ORCHARD_WORKER_NAME"] = workerName
|
||||
env["ORCHARD_VM_NAME"] = vmName
|
||||
env["ORCHARD_VM_CONTROL_SOCKET"] = controlSocket
|
||||
env["ORCHARD_PROCESS_SOCKET"] = socketPath
|
||||
|
||||
// Convert the environment to the format expected by exec.Cmd
|
||||
for name, value := range env {
|
||||
command.Env = append(command.Env, name+"="+value)
|
||||
}
|
||||
|
||||
if err := command.Start(); err != nil {
|
||||
cmdCtxCancel()
|
||||
|
||||
return nil, errors.Join(err, os.RemoveAll(runtimeDir))
|
||||
}
|
||||
|
||||
process := &Process{
|
||||
cancel: cmdCtxCancel,
|
||||
socketPath: socketPath,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(process.done)
|
||||
|
||||
// Reap the process; callers observe termination through done regardless of exit status
|
||||
_ = command.Wait()
|
||||
_ = command.Cancel()
|
||||
_ = os.RemoveAll(runtimeDir)
|
||||
}()
|
||||
|
||||
return process, nil
|
||||
}
|
||||
|
||||
func shortenControlSocketPath(runtimeDir string, controlSocketPath string) (string, error) {
|
||||
// Make the symlink target absolute because a relative TART_HOME would
|
||||
// otherwise be resolved from runtimeDir, breaking the symlink
|
||||
absoluteControlSocketPath, err := filepath.Abs(controlSocketPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Place the VM's control socket alias in the runtime directory
|
||||
aliasControlSocketPath := filepath.Join(runtimeDir, "vm.sock")
|
||||
|
||||
if err := os.Symlink(absoluteControlSocketPath, aliasControlSocketPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return aliasControlSocketPath, nil
|
||||
}
|
||||
|
||||
func (process *Process) Dial(ctx context.Context) (net.Conn, error) {
|
||||
var dialer net.Dialer
|
||||
|
||||
return retry.DoWithData(func() (net.Conn, error) {
|
||||
select {
|
||||
case <-process.done:
|
||||
return nil, retry.Unrecoverable(errors.New("process exited before accepting connections"))
|
||||
default:
|
||||
// Continue
|
||||
}
|
||||
|
||||
return dialer.DialContext(ctx, "unix", process.socketPath)
|
||||
},
|
||||
retry.Context(ctx),
|
||||
retry.Attempts(200),
|
||||
retry.Delay(50*time.Millisecond),
|
||||
retry.DelayType(retry.FixedDelay),
|
||||
retry.LastErrorOnly(true),
|
||||
)
|
||||
}
|
||||
|
||||
func (process *Process) Close() {
|
||||
process.cancel()
|
||||
<-process.done
|
||||
}
|
||||
|
||||
func expandArgs(args []string, workerName string, vmName string, controlSocket string, socketPath string) []string {
|
||||
replacer := strings.NewReplacer(
|
||||
"${ORCHARD_WORKER_NAME}", workerName,
|
||||
"${ORCHARD_VM_NAME}", vmName,
|
||||
"${ORCHARD_VM_CONTROL_SOCKET}", controlSocket,
|
||||
"${ORCHARD_PROCESS_SOCKET}", socketPath,
|
||||
)
|
||||
|
||||
var expanded []string
|
||||
|
||||
for _, arg := range args {
|
||||
expanded = append(expanded, replacer.Replace(arg))
|
||||
}
|
||||
|
||||
return expanded
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
//nolint:noctx,testpackage,usetesting // Preserve the socket-path tests and their required short /var/tmp paths.
|
||||
package hostprocess
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestShortenControlSocketPath(t *testing.T) {
|
||||
// Create a control socket path that exceeds macOS's 104-byte limit
|
||||
const controlSocketName = "control.sock"
|
||||
|
||||
vmDir := filepath.Join(t.TempDir(), strings.Repeat("v", 104))
|
||||
require.NoError(t, os.MkdirAll(vmDir, 0o700))
|
||||
|
||||
controlSocketPath := filepath.Join(vmDir, controlSocketName)
|
||||
require.Greater(t, len(controlSocketPath), 104)
|
||||
|
||||
// Create Orchard's short per-process runtime directory
|
||||
runtimeDir, err := os.MkdirTemp("/var/tmp", "orchard-hp-test-")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, os.RemoveAll(runtimeDir))
|
||||
})
|
||||
|
||||
// Shorten the control socket path and verify the resulting alias
|
||||
shortControlSocketPath, err := shortenControlSocketPath(runtimeDir, controlSocketPath)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, filepath.Join(runtimeDir, "vm.sock"), shortControlSocketPath)
|
||||
require.Less(t, len(shortControlSocketPath), 104)
|
||||
|
||||
// Model Tart binding the socket relative to the long VM directory
|
||||
t.Chdir(vmDir)
|
||||
|
||||
listener, err := net.Listen("unix", controlSocketName)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, listener.Close())
|
||||
})
|
||||
|
||||
// Model the host process running from Orchard's runtime directory
|
||||
t.Chdir(runtimeDir)
|
||||
|
||||
// Verify connecting through the long absolute path fails
|
||||
connection, err := net.Dial("unix", controlSocketPath)
|
||||
require.Error(t, err)
|
||||
require.Nil(t, connection)
|
||||
|
||||
// Connect to Tart's control socket through Orchard's short alias
|
||||
connection, err = net.Dial("unix", shortControlSocketPath)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, connection.Close())
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
//nolint:contextcheck,err113 // Host processes have independent lifetimes; preserve the original errors.
|
||||
package hostprocess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
|
||||
"github.com/cirruslabs/orchard/internal/worker/ondiskname"
|
||||
v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
|
||||
)
|
||||
|
||||
// Set owns the host processes associated with one VM.
|
||||
type Set struct {
|
||||
workerName string
|
||||
vmName string
|
||||
onDiskName ondiskname.OnDiskName
|
||||
processes map[string]*Process
|
||||
started bool
|
||||
mtx sync.Mutex
|
||||
}
|
||||
|
||||
func NewSet(
|
||||
workerName string,
|
||||
vmName string,
|
||||
onDiskName ondiskname.OnDiskName,
|
||||
) *Set {
|
||||
return &Set{
|
||||
workerName: workerName,
|
||||
vmName: vmName,
|
||||
onDiskName: onDiskName,
|
||||
processes: make(map[string]*Process),
|
||||
}
|
||||
}
|
||||
|
||||
func (set *Set) Start(ctx context.Context, specs []v1.HostProcess) error {
|
||||
set.mtx.Lock()
|
||||
defer set.mtx.Unlock()
|
||||
|
||||
set.stopLocked()
|
||||
|
||||
return set.startLocked(ctx, specs)
|
||||
}
|
||||
|
||||
func (set *Set) Replace(ctx context.Context, specs []v1.HostProcess) error {
|
||||
set.mtx.Lock()
|
||||
defer set.mtx.Unlock()
|
||||
|
||||
set.stopLocked()
|
||||
|
||||
return set.startLocked(ctx, specs)
|
||||
}
|
||||
|
||||
func (set *Set) Dial(ctx context.Context, name string) (net.Conn, error) {
|
||||
process := set.Lookup(name)
|
||||
|
||||
if process == nil {
|
||||
return nil, fmt.Errorf("host process %q is not running", name)
|
||||
}
|
||||
|
||||
connection, err := process.Dial(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to host process %q: %w", name, err)
|
||||
}
|
||||
|
||||
return connection, nil
|
||||
}
|
||||
|
||||
func (set *Set) Lookup(name string) *Process {
|
||||
set.mtx.Lock()
|
||||
defer set.mtx.Unlock()
|
||||
|
||||
return set.processes[name]
|
||||
}
|
||||
|
||||
func (set *Set) Ready() bool {
|
||||
set.mtx.Lock()
|
||||
defer set.mtx.Unlock()
|
||||
|
||||
if !set.started {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, process := range set.processes {
|
||||
select {
|
||||
case <-process.done:
|
||||
return false
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (set *Set) Stop() {
|
||||
set.mtx.Lock()
|
||||
defer set.mtx.Unlock()
|
||||
|
||||
set.stopLocked()
|
||||
}
|
||||
|
||||
func (set *Set) startLocked(ctx context.Context, specs []v1.HostProcess) error {
|
||||
set.started = false
|
||||
|
||||
if len(specs) == 0 {
|
||||
set.started = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
controlSocket, err := set.onDiskName.ControlSocketPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Start and register every process
|
||||
//
|
||||
// If any process fails to start, stop the entire set
|
||||
// rather than leaving it partially running.
|
||||
for _, spec := range specs {
|
||||
process, err := NewProcess(
|
||||
spec,
|
||||
set.workerName,
|
||||
set.vmName,
|
||||
controlSocket,
|
||||
)
|
||||
if err != nil {
|
||||
set.stopLocked()
|
||||
|
||||
return fmt.Errorf("failed to start host process %q: %w", spec.Name, err)
|
||||
}
|
||||
|
||||
set.processes[spec.Name] = process
|
||||
}
|
||||
|
||||
// A host process is ready once it accepts connections on its socket
|
||||
for _, spec := range specs {
|
||||
process := set.processes[spec.Name]
|
||||
|
||||
connection, err := process.Dial(ctx)
|
||||
if err != nil {
|
||||
set.stopLocked()
|
||||
|
||||
return fmt.Errorf("failed to connect to host process %q: %w", spec.Name, err)
|
||||
}
|
||||
|
||||
_ = connection.Close()
|
||||
}
|
||||
|
||||
set.started = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (set *Set) stopLocked() {
|
||||
set.started = false
|
||||
|
||||
for name, process := range set.processes {
|
||||
delete(set.processes, name)
|
||||
|
||||
process.Close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
//nolint:goconst,noctx,testpackage // Preserve the original helper-process fixtures.
|
||||
package hostprocess
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/cirruslabs/orchard/internal/worker/ondiskname"
|
||||
v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
testHelperArg = "--orchard-host-process-test-helper"
|
||||
testHelperEnv = "ORCHARD_HOST_PROCESS_TEST_ENV"
|
||||
testHelperInheritedEnv = "ORCHARD_HOST_PROCESS_TEST_INHERITED_ENV"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if len(os.Args) == 2 && os.Args[1] == testHelperArg {
|
||||
os.Exit(runTestHelper())
|
||||
}
|
||||
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestSetStartAndStop(t *testing.T) {
|
||||
t.Setenv(testHelperInheritedEnv, "must-not-be-inherited")
|
||||
|
||||
// Use this test binary as a host process
|
||||
executable, err := os.Executable()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create an empty host process set
|
||||
set := NewSet("", "", ondiskname.OnDiskName{})
|
||||
t.Cleanup(set.Stop)
|
||||
|
||||
// Ensure that a new set is not ready until started
|
||||
require.False(t, set.Ready())
|
||||
|
||||
// Start the first host process
|
||||
require.NoError(t, set.Start(t.Context(), []v1.HostProcess{{
|
||||
Name: "first",
|
||||
Program: executable,
|
||||
Args: []string{testHelperArg},
|
||||
Env: map[string]string{
|
||||
testHelperEnv: "set",
|
||||
"PATH": "must-not-win",
|
||||
"ORCHARD_PROCESS_SOCKET": "/must-not-win",
|
||||
},
|
||||
}}))
|
||||
require.True(t, set.Ready())
|
||||
|
||||
// Ensure that the first host process is reachable
|
||||
connection, err := set.Dial(t.Context(), "first")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, connection.Close())
|
||||
|
||||
// A failed replacement leaves the set not ready
|
||||
require.Error(t, set.Replace(t.Context(), []v1.HostProcess{{
|
||||
Name: "invalid",
|
||||
Program: "/does/not/exist",
|
||||
}}))
|
||||
require.False(t, set.Ready())
|
||||
_, err = set.Dial(t.Context(), "first")
|
||||
require.Error(t, err)
|
||||
|
||||
// Replace the first host process
|
||||
require.NoError(t, set.Replace(t.Context(), []v1.HostProcess{{
|
||||
Name: "replacement",
|
||||
Program: executable,
|
||||
Args: []string{testHelperArg},
|
||||
Env: map[string]string{
|
||||
testHelperEnv: "set",
|
||||
"PATH": "must-not-win",
|
||||
"ORCHARD_PROCESS_SOCKET": "/must-not-win",
|
||||
},
|
||||
}}))
|
||||
require.True(t, set.Ready())
|
||||
|
||||
// Ensure that only the replacement host process is reachable
|
||||
_, err = set.Dial(t.Context(), "first")
|
||||
require.Error(t, err)
|
||||
|
||||
connection, err = set.Dial(t.Context(), "replacement")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, connection.Close())
|
||||
|
||||
// Stop all host processes
|
||||
set.Stop()
|
||||
require.False(t, set.Ready())
|
||||
|
||||
// Ensure that the stopped host process is no longer reachable
|
||||
_, err = set.Dial(t.Context(), "replacement")
|
||||
require.Error(t, err)
|
||||
|
||||
// Ensure that an explicitly started empty set is ready
|
||||
require.NoError(t, set.Start(t.Context(), nil))
|
||||
require.True(t, set.Ready())
|
||||
}
|
||||
|
||||
func TestSetStartReplacesExistingProcesses(t *testing.T) {
|
||||
// Use this test binary as a host process
|
||||
executable, err := os.Executable()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a host process set and start a new host process
|
||||
set := NewSet("", "", ondiskname.OnDiskName{})
|
||||
defer set.Stop()
|
||||
|
||||
hostProcesses := []v1.HostProcess{{
|
||||
Name: "process",
|
||||
Program: executable,
|
||||
Args: []string{testHelperArg},
|
||||
Env: map[string]string{
|
||||
testHelperEnv: "set",
|
||||
},
|
||||
}}
|
||||
|
||||
require.NoError(t, set.Start(t.Context(), hostProcesses))
|
||||
|
||||
// Keep track of the original process to verify it is stopped
|
||||
original := set.Lookup("process")
|
||||
require.NotNil(t, original)
|
||||
defer original.Close()
|
||||
|
||||
// Start a new host process
|
||||
require.NoError(t, set.Start(t.Context(), hostProcesses))
|
||||
|
||||
// Ensure that the original process was stopped
|
||||
select {
|
||||
case <-original.done:
|
||||
// It was stopped, nice
|
||||
default:
|
||||
require.FailNow(t, "the original process is still running")
|
||||
}
|
||||
|
||||
// Ensure that a new ready process replaced the original
|
||||
replacement := set.Lookup("process")
|
||||
require.NotNil(t, replacement)
|
||||
require.NotSame(t, original, replacement)
|
||||
require.True(t, set.Ready())
|
||||
}
|
||||
|
||||
func runTestHelper() int {
|
||||
if os.Getenv(testHelperEnv) != "set" {
|
||||
return 3
|
||||
}
|
||||
if os.Getenv(testHelperInheritedEnv) != "" {
|
||||
return 4
|
||||
}
|
||||
if os.Getenv("PATH") == "must-not-win" {
|
||||
return 5
|
||||
}
|
||||
|
||||
listener, err := net.Listen("unix", os.Getenv("ORCHARD_PROCESS_SOCKET"))
|
||||
if err != nil {
|
||||
return 2
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
for {
|
||||
connection, err := listener.Accept()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
_ = connection.Close()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user