diff --git a/benchmark/.editorconfig b/benchmark/.editorconfig new file mode 100644 index 0000000..78b36ca --- /dev/null +++ b/benchmark/.editorconfig @@ -0,0 +1 @@ +root = true diff --git a/benchmark/.golangci.yml b/benchmark/.golangci.yml new file mode 100644 index 0000000..4f57b87 --- /dev/null +++ b/benchmark/.golangci.yml @@ -0,0 +1,92 @@ +run: + timeout: 5m + +linters: + enable-all: true + + disable: + # Messages like "struct of size 104 bytes could be of size 96 bytes" from a package + # that was last updated 2 years ago[1] are barely helpful. + # + # After all, we're writing the code for other people, so let's trust the compiler here (that's + # constantly evolving compared to this linter) and revisit this if memory usage becomes a problem. + # + # [1]: https://github.com/mdempsky/maligned/commit/6e39bd26a8c8b58c5a22129593044655a9e25959 + - maligned + + # We don't have high-performance requirements at this moment, so sacrificing + # the code readability for marginal performance gains is not worth it. + - prealloc + + # New linters that require a lot of codebase churn and noise, but perhaps we can enable them in the future. + - nlreturn + - wrapcheck + - errorlint + + # Unfortunately, we use globals due to how spf13/cobra works. + - gochecknoglobals + + # That's fine that some Proto objects don't have all fields initialized + - exhaustivestruct + + # Style linters that are total nuts. + - wsl + - gofumpt + - goimports + - funlen + + # This conflicts with the Protocol Buffers Version 3 design, + # which is largely based on default values for struct fields. + - exhaustivestruct + + # Enough parallelism for now. + - paralleltest + + # Ill-based assumptions about identifiers like fmt.Println without taking context into account. + - forbidigo + + # Advantages of using t.Helper() are too small to waste developer's cognitive stamina on it. + - thelper + + # Too restrictive defaults, plus there's already a gocyclo linter in place. + - cyclop + + # Gives false positives for textbook examples[1][2] + # [1]: https://github.com/charithe/durationcheck/issues/7 + # [2]: https://golang.org/pkg/time/ (see "To convert an integer number of units to a Duration, multiply:") + - durationcheck + + # No way to disable the "exported" check for the whole project[1] + # [1]: https://github.com/mgechev/revive/issues/244#issuecomment-560512162 + - revive + + # Unfortunately too much false-positives, e.g. for a 0700 umask or number 10 when using strconv.FormatInt() + - gomnd + + # Needs package whitelists + - depguard + + # Generates absolutely useless errors, e.g. + # "string `.yml` has 3 occurrences, make it a constant" + - goconst + + # It's OK to not sort imports + - gci + + # It's OK to not initialize some struct fields + - exhaustruct + + # This is not a library, so it's OK to use dynamic errors + - goerr113 + + # fmt.Sprintf() looks a bit nicer than string addition + - perfsprint + + # We can control this ourselves + - varnamelen + - contextcheck + +issues: + # Don't hide multiple issues that belong to one class since GitHub annotations can handle them all nicely. + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..aa5ef10 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,39 @@ +# Benchmark + +Tart comes with a Golang-based benchmarking utility that allows one to easily compare host and guest performance. + +Currently, only Flexible I/O tester workloads are supported. To run them, [install Golang](https://go.dev/) and run the following command from this (`benchmark/`) directory: + +```shell +go run cmd/main.go fio +``` + +You can also enable the debugging output to diagnose issues: + +```shell +go run cmd/main.go fio --debug +``` + +## Results + +Host: + +* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`) +* OS: macOS Sonoma 14.4.1 + +Guest: + +* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization) +* OS: macOS Sonoma 14.4.1 + +``` +Name Executor Bandwidth I/O operations +Random writing of 1MB local 2.6 GB/s 649.35 kIOPS +Random writing of 1MB Tart 2.5 GB/s 620.22 kIOPS +Random writing of 10MB local 2.6 GB/s 651.74 kIOPS +Random writing of 10MB Tart 2.5 GB/s 615.52 kIOPS +Random writing of 100MB local 1.9 GB/s 481.51 kIOPS +Random writing of 100MB Tart 2.0 GB/s 493.31 kIOPS +Random writing of 1000MB local 1.7 GB/s 414.89 kIOPS +Random writing of 1000MB Tart 1.1 GB/s 287.4 kIOPS +``` diff --git a/benchmark/cmd/main.go b/benchmark/cmd/main.go new file mode 100644 index 0000000..61c1d09 --- /dev/null +++ b/benchmark/cmd/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "context" + "github.com/cirruslabs/tart/benchmark/internal/command" + "log" + "os" + "os/signal" +) + +func main() { + // Set up a signal-interruptible context + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + + // Run the root command + if err := command.NewCommand().ExecuteContext(ctx); err != nil { + cancel() + + log.Fatal(err) + } + + cancel() +} diff --git a/benchmark/go.mod b/benchmark/go.mod new file mode 100644 index 0000000..d9fb314 --- /dev/null +++ b/benchmark/go.mod @@ -0,0 +1,29 @@ +module github.com/cirruslabs/tart/benchmark + +go 1.22.1 + +require ( + github.com/avast/retry-go/v4 v4.5.1 + github.com/dustin/go-humanize v1.0.1 + github.com/google/uuid v1.6.0 + github.com/gosuri/uitable v0.0.4 + github.com/spf13/cobra v1.8.0 + github.com/stretchr/testify v1.9.0 + go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.21.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/sys v0.18.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/benchmark/go.sum b/benchmark/go.sum new file mode 100644 index 0000000..6a61a34 --- /dev/null +++ b/benchmark/go.sum @@ -0,0 +1,52 @@ +github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= +github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= +github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/benchmark/internal/command/fio/benchmark.go b/benchmark/internal/command/fio/benchmark.go new file mode 100644 index 0000000..6561ce0 --- /dev/null +++ b/benchmark/internal/command/fio/benchmark.go @@ -0,0 +1,29 @@ +package fio + +type Benchmark struct { + Name string + Command string +} + +var benchmarks = []Benchmark{ + { + Name: "Random writing of 1MB", + Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " + + "--size 1MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1", + }, + { + Name: "Random writing of 10MB", + Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " + + "--size 10MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1", + }, + { + Name: "Random writing of 100MB", + Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " + + "--size 100MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1", + }, + { + Name: "Random writing of 1000MB", + Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " + + "--size 1000MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1", + }, +} diff --git a/benchmark/internal/command/fio/fio.go b/benchmark/internal/command/fio/fio.go new file mode 100644 index 0000000..98a8ae7 --- /dev/null +++ b/benchmark/internal/command/fio/fio.go @@ -0,0 +1,131 @@ +package fio + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/cirruslabs/tart/benchmark/internal/executor" + "github.com/cirruslabs/tart/benchmark/internal/executor/local" + "github.com/cirruslabs/tart/benchmark/internal/executor/tart" + "github.com/dustin/go-humanize" + "github.com/gosuri/uitable" + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +var debug bool + +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "fio", + Short: "run Flexible I/O tester (fio) benchmarks", + RunE: run, + } + + cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging") + + return cmd +} + +func run(cmd *cobra.Command, args []string) error { + config := zap.NewProductionConfig() + if debug { + config.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + } + logger, err := config.Build() + if err != nil { + return err + } + defer func() { + _ = logger.Sync() + }() + + executors, err := initializeExecutors(cmd.Context(), logger) + if err != nil { + return err + } + defer func() { + errs := []error{err} + + for _, executor := range executors { + if err := executor.Close(); err != nil { + errs = append(errs, fmt.Errorf("failed to close executor %s: %w", executor.Name(), err)) + } + } + + err = errors.Join(errs...) + }() + + table := uitable.New() + table.AddRow("Name", "Executor", "Bandwidth", "I/O operations") + + for _, benchmark := range benchmarks { + for _, executor := range executors { + logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name, executor.Name()) + + stdout, err := executor.Run(cmd.Context(), benchmark.Command) + if err != nil { + return err + } + + var fioResult Result + + if err := json.Unmarshal(stdout, &fioResult); err != nil { + return err + } + + if len(fioResult.Jobs) != 1 { + return fmt.Errorf("expected exactly 1 job from fio's JSON output, got %d", + len(fioResult.Jobs)) + } + + job := fioResult.Jobs[0] + + writeBandwidth := humanize.Bytes(uint64(job.Write.BW)*humanize.KByte) + "/s" + writeIOPS := humanize.SIWithDigits(job.Write.IOPS, 2, "IOPS") + + logger.Sugar().Infof("write bandwidth: %s, write IOPS: %s\n", writeBandwidth, writeIOPS) + + table.AddRow(benchmark.Name, executor.Name(), writeBandwidth, writeIOPS) + } + } + + fmt.Println(table.String()) + + return nil +} + +func initializeExecutors(ctx context.Context, logger *zap.Logger) ([]executor.Executor, error) { + var result []executor.Executor + + logger.Info("initializing local executor") + + local, err := local.New(logger) + if err != nil { + return nil, err + } + result = append(result, local) + + logger.Info("local executor initialized") + + logger.Info("initializing Tart executor") + + tart, err := tart.New(ctx, logger) + if err != nil { + return nil, err + } + result = append(result, tart) + + logger.Info("Tart executor initialized") + + for _, executor := range result { + logger.Sugar().Infof("installing Flexible I/O tester (fio) on %s executor", executor.Name()) + + if _, err := executor.Run(ctx, "brew install fio"); err != nil { + return nil, err + } + } + + return result, nil +} diff --git a/benchmark/internal/command/fio/json.go b/benchmark/internal/command/fio/json.go new file mode 100644 index 0000000..4305c60 --- /dev/null +++ b/benchmark/internal/command/fio/json.go @@ -0,0 +1,15 @@ +package fio + +type Result struct { + Jobs []Job `json:"jobs"` +} + +type Job struct { + Name string `json:"jobname"` + Write Write `json:"write"` +} + +type Write struct { + BW float64 `json:"bw"` + IOPS float64 `json:"iops"` +} diff --git a/benchmark/internal/command/root.go b/benchmark/internal/command/root.go new file mode 100644 index 0000000..d05363f --- /dev/null +++ b/benchmark/internal/command/root.go @@ -0,0 +1,20 @@ +package command + +import ( + "github.com/cirruslabs/tart/benchmark/internal/command/fio" + "github.com/spf13/cobra" +) + +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "benchmark", + SilenceUsage: true, + SilenceErrors: true, + } + + cmd.AddCommand( + fio.NewCommand(), + ) + + return cmd +} diff --git a/benchmark/internal/executor/executor.go b/benchmark/internal/executor/executor.go new file mode 100644 index 0000000..516f6a0 --- /dev/null +++ b/benchmark/internal/executor/executor.go @@ -0,0 +1,11 @@ +package executor + +import ( + "context" +) + +type Executor interface { + Name() string + Run(ctx context.Context, command string) ([]byte, error) + Close() error +} diff --git a/benchmark/internal/executor/local/local.go b/benchmark/internal/executor/local/local.go new file mode 100644 index 0000000..43161b8 --- /dev/null +++ b/benchmark/internal/executor/local/local.go @@ -0,0 +1,42 @@ +package local + +import ( + "bytes" + "context" + "go.uber.org/zap" + "go.uber.org/zap/zapio" + "io" + "os/exec" +) + +type Local struct { + logger *zap.Logger +} + +func New(logger *zap.Logger) (*Local, error) { + return &Local{ + logger: logger, + }, nil +} + +func (local *Local) Name() string { + return "local" +} + +func (local *Local) Run(ctx context.Context, command string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "zsh", "-c", command) + + loggerWriter := &zapio.Writer{Log: local.logger, Level: zap.DebugLevel} + stdoutBuf := &bytes.Buffer{} + + cmd.Stdout = io.MultiWriter(stdoutBuf, loggerWriter) + cmd.Stderr = loggerWriter + + err := cmd.Run() + + return stdoutBuf.Bytes(), err +} + +func (local *Local) Close() error { + return nil +} diff --git a/benchmark/internal/executor/local/local_test.go b/benchmark/internal/executor/local/local_test.go new file mode 100644 index 0000000..0df3c75 --- /dev/null +++ b/benchmark/internal/executor/local/local_test.go @@ -0,0 +1,20 @@ +package local_test + +import ( + "context" + "github.com/cirruslabs/tart/benchmark/internal/executor/local" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "testing" +) + +func TestLocal(t *testing.T) { + local, err := local.New(zap.NewNop()) + require.NoError(t, err) + + output, err := local.Run(context.Background(), "echo \"this is a test\"") + require.NoError(t, err) + require.Equal(t, "this is a test\n", string(output)) + + require.NoError(t, local.Close()) +} diff --git a/benchmark/internal/executor/tart/cmd.go b/benchmark/internal/executor/tart/cmd.go new file mode 100644 index 0000000..ac4da8f --- /dev/null +++ b/benchmark/internal/executor/tart/cmd.go @@ -0,0 +1,35 @@ +package tart + +import ( + "bytes" + "context" + "go.uber.org/zap" + "go.uber.org/zap/zapio" + "io" + "os/exec" + "strings" +) + +const tartBinaryName = "tart" + +func Cmd(ctx context.Context, logger *zap.Logger, args ...string) error { + _, err := CmdWithOutput(ctx, logger, args...) + + return err +} + +func CmdWithOutput(ctx context.Context, logger *zap.Logger, args ...string) (string, error) { + logger.Sugar().Debugf("running %s %s", tartBinaryName, strings.Join(args, " ")) + + cmd := exec.CommandContext(ctx, tartBinaryName, args...) + + loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel} + stdoutBuf := &bytes.Buffer{} + + cmd.Stdout = io.MultiWriter(stdoutBuf, loggerWriter) + cmd.Stderr = loggerWriter + + err := cmd.Run() + + return stdoutBuf.String(), err +} diff --git a/benchmark/internal/executor/tart/tart.go b/benchmark/internal/executor/tart/tart.go new file mode 100644 index 0000000..4ebb0fc --- /dev/null +++ b/benchmark/internal/executor/tart/tart.go @@ -0,0 +1,133 @@ +package tart + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/avast/retry-go/v4" + "github.com/google/uuid" + "go.uber.org/zap" + "golang.org/x/crypto/ssh" + "net" + "strings" + "time" +) + +const baseImage = "ghcr.io/cirruslabs/macos-sonoma-base:latest" + +type Tart struct { + vmRunCancel context.CancelFunc + vmName string + sshClient *ssh.Client + logger *zap.Logger +} + +func New(ctx context.Context, logger *zap.Logger) (*Tart, error) { + tart := &Tart{ + vmName: fmt.Sprintf("tart-benchmark-%s", uuid.NewString()), + logger: logger, + } + + if err := Cmd(ctx, tart.logger, "pull", baseImage); err != nil { + return nil, err + } + + if err := Cmd(ctx, tart.logger, "clone", baseImage, tart.vmName); err != nil { + return nil, err + } + + vmRunCtx, vmRunCancel := context.WithCancel(ctx) + tart.vmRunCancel = vmRunCancel + + go func() { + _ = Cmd(vmRunCtx, tart.logger, "run", "--no-graphics", tart.vmName) + }() + + ip, err := CmdWithOutput(ctx, tart.logger, "ip", "--wait", "60", tart.vmName) + if err != nil { + return nil, tart.Close() + } + + err = retry.Do(func() error { + dialer := net.Dialer{ + Timeout: 1 * time.Second, + } + + addr := fmt.Sprintf("%s:22", strings.TrimSpace(ip)) + + netConn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + return err + } + + sshConn, chans, reqs, err := ssh.NewClientConn(netConn, addr, &ssh.ClientConfig{ + User: "admin", + Auth: []ssh.AuthMethod{ + ssh.Password("admin"), + }, + HostKeyCallback: func(_ string, _ net.Addr, _ ssh.PublicKey) error { + return nil + }, + }) + if err != nil { + return err + } + + tart.sshClient = ssh.NewClient(sshConn, chans, reqs) + + return nil + }, retry.RetryIf(func(err error) bool { + return !errors.Is(err, context.Canceled) + })) + if err != nil { + return nil, tart.Close() + } + + return tart, nil +} + +func (tart *Tart) Name() string { + return "Tart" +} + +func (tart *Tart) Run(ctx context.Context, command string) ([]byte, error) { + sshSession, err := tart.sshClient.NewSession() + if err != nil { + return nil, err + } + + // Work around x/crypto/ssh not being context.Context-friendly (e.g. https://github.com/golang/go/issues/20288) + monitorCtx, monitorCancel := context.WithCancel(ctx) + go func() { + <-monitorCtx.Done() + _ = sshSession.Close() + }() + defer monitorCancel() + + stdoutBuf := &bytes.Buffer{} + + sshSession.Stdin = bytes.NewBufferString(command) + sshSession.Stdout = stdoutBuf + + if err := sshSession.Shell(); err != nil { + return nil, err + } + + if err := sshSession.Wait(); err != nil { + return nil, err + } + + return stdoutBuf.Bytes(), nil +} + +func (tart *Tart) Close() error { + if tart.sshClient != nil { + _ = tart.sshClient.Close() + } + + tart.vmRunCancel() + _ = Cmd(context.Background(), tart.logger, "delete", tart.vmName) + + return nil +} diff --git a/benchmark/internal/executor/tart/tart_test.go b/benchmark/internal/executor/tart/tart_test.go new file mode 100644 index 0000000..46b2136 --- /dev/null +++ b/benchmark/internal/executor/tart/tart_test.go @@ -0,0 +1,22 @@ +package tart_test + +import ( + "context" + "github.com/cirruslabs/tart/benchmark/internal/executor/tart" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "testing" +) + +func TestTart(t *testing.T) { + ctx := context.Background() + + tart, err := tart.New(ctx, zap.NewNop()) + require.NoError(t, err) + + output, err := tart.Run(ctx, "echo \"this is a test\"") + require.NoError(t, err) + require.Equal(t, "this is a test\n", string(output)) + + require.NoError(t, tart.Close()) +}