Support Tart Guest Agent transport for VM startup scripts (#490)

* Support Tart Guest Agent transport for VM startup scripts

* TestTartGuestAgentStartupScript: make ScriptContent more readable

* Align Tart Guest Agent's startup script execution with SSH

* Use mvdan.cc/sh/v3/syntax to validate "export" shell commands

* Use Bash instead of Zsh on Linux

* Invert logic when validating vm.StartupScript.Transport for simplicity

* Document startup_script's transport field in OpenAPI specification

* Prevent unbounded buffering of startup script output

* grpc.WaitForReady(true) → retry.DoWithData()
This commit is contained in:
edi-oai
2026-09-09 08:04:25 +01:00
committed by GitHub
parent 01bf2d72a9
commit 9ac6ca6326
9 changed files with 407 additions and 19 deletions
+9 -1
View File
@@ -800,8 +800,16 @@ components:
default: admin
startup_script:
type: object
description: Startup script to run after the VM boots and becomes accessible via SSH
description: Startup script to run after the VM boots
properties:
transport:
type: string
description: |
Transport used to execute the startup script.
Omitted or empty values use SSH.
`tart-guest-agent` is supported only with the Tart runtime.
enum: [ "", ssh, tart-guest-agent ]
default: ssh
script_content:
type: string
env:
+3 -1
View File
@@ -24,6 +24,7 @@ require (
github.com/gosuri/uitable v0.0.4
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/go-version v1.9.0
github.com/lithammer/dedent v1.1.0
github.com/manifoldco/promptui v0.9.0
github.com/mitchellh/go-grpc-net-conn v0.0.0-20200427190222-eb030e4876f0
github.com/pkg/errors v0.9.1
@@ -49,13 +50,14 @@ require (
golang.org/x/net v0.55.0
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0
golang.org/x/term v0.43.0
golang.org/x/term v0.45.0
golang.org/x/text v0.37.0
google.golang.org/grpc v1.83.0
google.golang.org/protobuf v1.36.12
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/yaml.v3 v3.0.1
howett.net/plist v1.0.1
mvdan.cc/sh/v3 v3.14.1
)
require (
+7
View File
@@ -232,6 +232,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
@@ -286,6 +288,7 @@ github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRC
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM=
@@ -465,6 +468,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -523,3 +528,5 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
mvdan.cc/sh/v3 v3.14.1 h1:bXkhQWNHCs0KZEChF8hYS6FC+T2N9mUZLbQv9blditI=
mvdan.cc/sh/v3 v3.14.1/go.mod h1:syYCoFET8w9tvevxiXUtY8/ICrU+l26jHmhJDra3Vwo=
+10 -4
View File
@@ -38,6 +38,7 @@ var labels map[string]string
var randomSerial bool
var restartPolicy string
var startupScript string
var startupScriptTransport string
var hostDirsRaw []string
var imagePullPolicy string
@@ -93,6 +94,9 @@ func newCreateVMCommand() *cobra.Command {
command.Flags().StringVar(&startupScript, "startup-script", "",
"startup script (e.g. --startup-script=\"sync\") or a path to a script file prefixed with \"@\" "+
"(e.g. \"--startup-script=@script.sh\")")
command.Flags().StringVar(&startupScriptTransport, "startup-script-transport", string(v1.VMScriptTransportSSH),
fmt.Sprintf("transport for the startup script: %q or %q (Tart only)",
v1.VMScriptTransportSSH, v1.VMScriptTransportTartGuestAgent))
command.Flags().StringSliceVar(&hostDirsRaw, "host-dirs", []string{},
"directories on the Orchard Worker host to mount to a VM, can be specified multiple times "+
"and/or be comma-separated (see \"tart run\"'s --dir argument for syntax)")
@@ -169,10 +173,6 @@ func runCreateVM(cmd *cobra.Command, args []string) error {
HostDirs: hostDirs,
}
if err := vm.Validate(); err != nil {
return fmt.Errorf("%w: %v", ErrVMFailed, err)
}
// Convert resources
vm.Resources, err = v1.NewResourcesFromStringToString(resources)
if err != nil {
@@ -202,13 +202,19 @@ func runCreateVM(cmd *cobra.Command, args []string) error {
vm.StartupScript = &v1.VMScript{
ScriptContent: string(startupScriptBytes),
Transport: v1.VMScriptTransport(startupScriptTransport),
}
} else if startupScript != "" {
vm.StartupScript = &v1.VMScript{
ScriptContent: startupScript,
Transport: v1.VMScriptTransport(startupScriptTransport),
}
}
if err := vm.Validate(); err != nil {
return fmt.Errorf("%w: %v", ErrVMFailed, err)
}
client, err := client.New()
if err != nil {
return err
+51
View File
@@ -0,0 +1,51 @@
//go:build darwin
package tests_test
import (
"testing"
"time"
"github.com/cirruslabs/orchard/internal/tests/devcontroller"
"github.com/cirruslabs/orchard/internal/tests/platformdependent"
v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
"github.com/google/uuid"
"github.com/lithammer/dedent"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTartGuestAgentStartupScript(t *testing.T) {
devClient, _, _ := devcontroller.StartIntegrationTestEnvironment(t)
// Create a VM whose startup script runs through Tart Guest Agent
vm := platformdependent.VM("test-agent-startup-" + uuid.NewString())
// Agent execution must work without valid SSH credentials.
vm.Username = "invalid-ssh-user"
vm.Password = "invalid-ssh-password"
vm.StartupScript = &v1.VMScript{
Transport: v1.VMScriptTransportTartGuestAgent,
ScriptContent: dedent.Dedent(`
printf 'Hello, %s!\n' "$FOO"
printf 'startup stderr\n' >&2
exit 123
`),
Env: map[string]string{"FOO": "Tart Guest Agent"},
}
require.NoError(t, devClient.VMs().Create(t.Context(), vm))
// Ensure that the script's non-zero exit status is reported as a VM failure
require.EventuallyWithT(t, func(collect *assert.CollectT) {
current, err := devClient.VMs().Get(t.Context(), vm.Name)
require.NoError(collect, err)
require.Equal(collect, v1.VMStatusFailed, current.Status)
require.Contains(collect, current.StatusMessage, "failed to run startup script: process exited with status 123")
// Ensure that the script received FOO and both stdout and stderr were captured
logs, err := devClient.VMs().Logs(t.Context(), vm.Name)
require.NoError(collect, err)
require.Contains(collect, logs, "Hello, Tart Guest Agent!")
require.Contains(collect, logs, "startup stderr")
}, 2*time.Minute, time.Second)
}
+35 -13
View File
@@ -23,11 +23,15 @@ import (
"github.com/samber/lo"
"go.uber.org/zap"
"golang.org/x/crypto/ssh"
"mvdan.cc/sh/v3/syntax"
)
var ErrVMFailed = errors.New("VM failed")
type VM struct {
onDiskName ondiskname.OnDiskName
os v1.OS
// Backward compatibility with v1.VM specification's "Status" field
//
// "started" is always true after the first "tart run",
@@ -54,6 +58,8 @@ type VM struct {
func NewVM(vmResource v1.VM, onDiskName ondiskname.OnDiskName, logger *zap.SugaredLogger) *VM {
return &VM{
onDiskName: onDiskName,
os: vmResource.OS,
conditions: mapset.NewSet(v1.ConditionTypeCloning),
hostProcesses: hostprocess.NewSet(vmResource.Worker, vmResource.Name, onDiskName),
endpoints: endpoint.NewSet(logger),
@@ -165,7 +171,6 @@ func (vm *VM) Shell(
sshUser string,
sshPassword string,
script string,
env map[string]string,
consumeLine func(line string),
dialer dialer.Dialer,
getIP func(ctx context.Context) (string, error),
@@ -265,17 +270,7 @@ func (vm *VM) Shell(
return fmt.Errorf("%w: failed to start a shell: %v", ErrVMFailed, err)
}
var scriptBuilder strings.Builder
scriptBuilder.WriteString("set -e\n")
// don't use sess.Setenv since it requires non-default SSH server configuration
for key, value := range env {
scriptBuilder.WriteString("export " + key + "=\"" + value + "\"\n")
}
scriptBuilder.WriteString(script)
scriptBuilder.WriteString("\nexit\n")
_, err = stdinBuf.Write([]byte(scriptBuilder.String()))
_, err = stdinBuf.Write([]byte(script))
if err != nil {
return fmt.Errorf("%w: failed to start script: %v", ErrVMFailed, err)
}
@@ -312,7 +307,34 @@ func (vm *VM) RunScript(
})
}
err := vm.Shell(ctx, sshUser, sshPassword, script.ScriptContent, script.Env, consumeLine, dialer, getIP)
var scriptBuilder strings.Builder
scriptBuilder.WriteString("set -e\n")
for key, value := range script.Env {
if !syntax.ValidName(key) {
vm.SetErr(fmt.Errorf("%w: invalid environment variable name %q", ErrVMFailed, key))
return
}
quotedValue, err := syntax.Quote(value, lo.Ternary(vm.os == v1.OSDarwin, syntax.LangZsh, syntax.LangBash))
if err != nil {
vm.SetErr(fmt.Errorf("%w: failed to quote environment variable %q: %v", ErrVMFailed, key, err))
return
}
scriptBuilder.WriteString("export " + key + "=" + quotedValue + "\n")
}
scriptBuilder.WriteString(script.ScriptContent)
scriptBuilder.WriteString("\nexit\n")
var err error
switch script.Transport {
case v1.VMScriptTransportTartGuestAgent:
err = vm.shellTartGuestAgent(ctx, scriptBuilder.String(), consumeLine)
default:
err = vm.Shell(ctx, sshUser, sshPassword, scriptBuilder.String(), consumeLine, dialer, getIP)
}
if err != nil {
vm.SetErr(fmt.Errorf("%w: failed to run startup script: %v", ErrVMFailed, err))
}
@@ -0,0 +1,159 @@
package base
import (
"bufio"
"bytes"
"context"
"fmt"
"net"
"github.com/avast/retry-go/v4"
"github.com/cirruslabs/orchard/internal/worker/socketalias"
v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
guestagent "github.com/cirruslabs/tart-guest-agent/pkg/v1"
"github.com/dustin/go-humanize"
"github.com/samber/lo"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
func (vm *VM) shellTartGuestAgent(ctx context.Context, script string, consumeLine func(string)) error {
path, err := vm.onDiskName.ControlSocketPath()
if err != nil {
return err
}
conn, err := grpc.NewClient("passthrough:///tart-guest-agent",
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
return socketalias.DialContext(ctx, path)
}),
)
if err != nil {
return fmt.Errorf("failed to create Tart Guest Agent client: %w", err)
}
defer conn.Close()
stream, err := retry.DoWithData(func() (guestagent.Agent_ExecClient, error) {
return guestagent.NewAgentClient(conn).Exec(ctx)
}, retry.Context(ctx), retry.OnRetry(func(n uint, err error) {
consumeLine(fmt.Sprintf("attempt %d to open Tart Guest Agent execution stream failed: %v", n, err))
}))
if err != nil {
return fmt.Errorf("failed to open Tart Guest Agent execution stream: %w", err)
}
// Start the shell process
if err := stream.Send(&guestagent.ExecRequest{
Type: &guestagent.ExecRequest_Command_{
Command: &guestagent.ExecRequest_Command{
Name: lo.Ternary(vm.os == v1.OSDarwin, "/bin/zsh", "/bin/bash"),
Args: []string{"-l"},
Interactive: true,
},
},
}); err != nil {
return fmt.Errorf("failed to send Tart Guest Agent command: %w", err)
}
// Feed it our startup script
if err := stream.Send(&guestagent.ExecRequest{
Type: &guestagent.ExecRequest_StandardInput{
StandardInput: &guestagent.IOChunk{
Data: []byte(script),
},
},
}); err != nil {
return fmt.Errorf("failed to send Tart Guest Agent script: %w", err)
}
if err := stream.CloseSend(); err != nil {
return err
}
// Wait for the shell process to finish,
// retrieving its outputs and exit code
stdout := newScriptOutput(consumeLine)
stderr := newScriptOutput(consumeLine)
defer func() {
stdout.flush()
stderr.flush()
}()
for {
response, err := stream.Recv()
if err != nil {
return fmt.Errorf("failed to receive Tart Guest Agent output: %w", err)
}
switch result := response.GetType().(type) {
case *guestagent.ExecResponse_StandardOutput:
stdout.write(result.StandardOutput.GetData())
case *guestagent.ExecResponse_StandardError:
stderr.write(result.StandardError.GetData())
case *guestagent.ExecResponse_Exit_:
if result.Exit.GetCode() != 0 {
return fmt.Errorf("process exited with status %d", result.Exit.GetCode())
}
return nil
}
}
}
const (
scriptOutputBufferSize = 128 * humanize.KiByte
)
type scriptOutput struct {
consumeLine func(string)
pending *bytes.Buffer
stopped bool
}
func newScriptOutput(consumeLine func(string)) *scriptOutput {
return &scriptOutput{
consumeLine: consumeLine,
pending: bytes.NewBuffer(make([]byte, 0, scriptOutputBufferSize)),
}
}
func (output *scriptOutput) write(data []byte) {
if output.stopped {
return
}
for len(data) > 0 {
// Determine if we have a full line available in the incoming data
advance, _, _ := bufio.ScanLines(data, false)
if advance == 0 {
// No newline yet; buffer the remainder
output.pending.Write(data)
break
}
// Complete the pending line and emit it
output.pending.Write(data[:advance])
output.flush()
// Update slice to point to the next data chunk
data = data[advance:]
}
output.stopped = output.pending.Len() >= scriptOutputBufferSize
}
func (output *scriptOutput) flush() {
defer output.pending.Reset()
if output.stopped {
return
}
_, line, _ := bufio.ScanLines(output.pending.Bytes(), true)
if line != nil {
output.consumeLine(string(line))
}
}
@@ -0,0 +1,102 @@
//nolint:testpackage // exercise the internal output helper
package base
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestScriptOutput(t *testing.T) {
atLimit := strings.Repeat("x", scriptOutputBufferSize)
belowLimit := atLimit[:len(atLimit)-1]
oversized := atLimit + atLimit
half := atLimit[:len(atLimit)/2]
testCases := []struct {
name string
chunks []string
want []string
beforeFlush int // Number of lines emitted before flushing the final tail
stopped bool
}{
{name: "empty"},
{
name: "split lines and blanks", chunks: []string{"hel", "", "lo\n\nwor", "ld\nlast"},
want: []string{"hello", "", "world", "last"}, beforeFlush: 3,
},
{
name: "split CRLF and trailing CR", chunks: []string{"first\r", "\none\rtwo\r", "\nlast\r"},
want: []string{"first", "one\rtwo", "last"}, beforeFlush: 2,
},
{
name: "strip only one CR", chunks: []string{"line\r\r\nlast\r\r"},
want: []string{"line\r", "last\r"}, beforeFlush: 1,
},
{name: "only CR", chunks: []string{"\r"}, want: []string{""}},
{
name: "binary and split UTF8", chunks: []string{"\x00\xff\x80\nhello \xf0", "\x9f\x8c", "\x8d\n"},
want: []string{"\x00\xff\x80", "hello 🌍"}, beforeFlush: 2,
},
{name: "unfinished below limit", chunks: []string{belowLimit}, want: []string{belowLimit}},
{
name: "complete at limit", chunks: []string{atLimit + "\n"},
want: []string{atLimit}, beforeFlush: 1,
},
{
name: "unfinished reaches limit", chunks: []string{"ok\n", belowLimit, "x", "\nignored\n"},
want: []string{"ok"}, beforeFlush: 1, stopped: true,
},
{name: "unfinished overshoots limit", chunks: []string{oversized}, stopped: true},
{
name: "large complete line with tail", chunks: []string{oversized + "\r\nnext\npar", "tial"},
want: []string{oversized, "next", "partial"}, beforeFlush: 2,
},
{
name: "buffered prefix completes over limit", chunks: []string{belowLimit, "xx\r\nnext\n"},
want: []string{atLimit + "x", "next"}, beforeFlush: 2,
},
{
name: "large chunk with multiple lines", chunks: []string{strings.Repeat(half+"\n", 3)},
want: []string{half, half, half}, beforeFlush: 3,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
// Capture emitted lines
var got []string
output := newScriptOutput(func(line string) { got = append(got, line) })
// Write chunks and simulate input buffer reuse
for _, chunk := range testCase.chunks {
data := []byte(chunk)
output.write(data)
clear(data)
}
// Ensure complete lines are emitted before flushing the final tail
require.Len(t, got, testCase.beforeFlush)
require.Equal(t, testCase.stopped, output.stopped)
// Ensure stopped output ignores further writes
if testCase.stopped {
buffered, capacity := output.pending.Len(), output.pending.Cap()
output.write([]byte("ignored\n"))
require.Equal(t, buffered, output.pending.Len())
require.Equal(t, capacity, output.pending.Cap())
}
// Flush the final tail and verify the result
output.flush()
require.Equal(t, testCase.want, got)
require.Zero(t, output.pending.Len())
require.Equal(t, testCase.stopped, output.stopped)
// Ensure flushing again does not duplicate output
output.flush()
require.Equal(t, testCase.want, got, "flushing twice must not duplicate output")
})
}
}
+31
View File
@@ -144,6 +144,18 @@ func (vm *VM) IsScheduled() bool {
}
func (vm *VM) Validate() error {
if vm.StartupScript != nil {
if err := vm.StartupScript.Transport.Validate(); err != nil {
return err
}
// Tart Guest Agent transport is only supported on the Tart runtime
if vm.StartupScript.Transport == VMScriptTransportTartGuestAgent && vm.Runtime != RuntimeTart {
return fmt.Errorf("runtime %q does not support startup script transport %q",
vm.Runtime, vm.StartupScript.Transport)
}
}
unsupportedFieldError := func(field string) error {
return fmt.Errorf("runtime %q does not support field %q", vm.Runtime, field)
}
@@ -381,6 +393,25 @@ const (
type VMScript struct {
ScriptContent string `json:"script_content,omitempty"`
Env map[string]string `json:"env,omitempty"`
// Transport defaults to SSH when omitted.
Transport VMScriptTransport `json:"transport,omitempty"`
}
type VMScriptTransport string
const (
VMScriptTransportSSH VMScriptTransport = "ssh"
VMScriptTransportTartGuestAgent VMScriptTransport = "tart-guest-agent"
)
func (transport VMScriptTransport) Validate() error {
switch transport {
case "", VMScriptTransportSSH, VMScriptTransportTartGuestAgent:
return nil
default:
return fmt.Errorf("unsupported startup script transport %q", transport)
}
}
func (vm VM) TerminalState() bool {