mirror of https://github.com/cirruslabs/tart.git
Introduce Golang-based benchmarking utility (#769)
* Introduce Golang-based benchmarking utility * benchmark fio: properly configure logger level * benchmark: properly terminate on Ctrl+C when initializing/running Tart * benchmark(fio): increase runtime to 30 seconds * benchmark(fio): IOPS are already per second * benchmark(fio): --numjobs 1 --iodepth 1 --end_fsync 1 * benchmark(README.md): add results
This commit is contained in:
Nikolay Edigaryev
GitHub
parent
1a3b862631
commit
5cd83c38cd
|
|
@ -0,0 +1 @@
|
|||
root = true
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
run:
|
||||
timeout: 5m
|
||||
|
||||
linters:
|
||||
enable-all: true
|
||||
|
||||
disable:
|
||||
# Messages like "struct of size 104 bytes could be of size 96 bytes" from a package
|
||||
# that was last updated 2 years ago[1] are barely helpful.
|
||||
#
|
||||
# After all, we're writing the code for other people, so let's trust the compiler here (that's
|
||||
# constantly evolving compared to this linter) and revisit this if memory usage becomes a problem.
|
||||
#
|
||||
# [1]: https://github.com/mdempsky/maligned/commit/6e39bd26a8c8b58c5a22129593044655a9e25959
|
||||
- maligned
|
||||
|
||||
# We don't have high-performance requirements at this moment, so sacrificing
|
||||
# the code readability for marginal performance gains is not worth it.
|
||||
- prealloc
|
||||
|
||||
# New linters that require a lot of codebase churn and noise, but perhaps we can enable them in the future.
|
||||
- nlreturn
|
||||
- wrapcheck
|
||||
- errorlint
|
||||
|
||||
# Unfortunately, we use globals due to how spf13/cobra works.
|
||||
- gochecknoglobals
|
||||
|
||||
# That's fine that some Proto objects don't have all fields initialized
|
||||
- exhaustivestruct
|
||||
|
||||
# Style linters that are total nuts.
|
||||
- wsl
|
||||
- gofumpt
|
||||
- goimports
|
||||
- funlen
|
||||
|
||||
# This conflicts with the Protocol Buffers Version 3 design,
|
||||
# which is largely based on default values for struct fields.
|
||||
- exhaustivestruct
|
||||
|
||||
# Enough parallelism for now.
|
||||
- paralleltest
|
||||
|
||||
# Ill-based assumptions about identifiers like fmt.Println without taking context into account.
|
||||
- forbidigo
|
||||
|
||||
# Advantages of using t.Helper() are too small to waste developer's cognitive stamina on it.
|
||||
- thelper
|
||||
|
||||
# Too restrictive defaults, plus there's already a gocyclo linter in place.
|
||||
- cyclop
|
||||
|
||||
# Gives false positives for textbook examples[1][2]
|
||||
# [1]: https://github.com/charithe/durationcheck/issues/7
|
||||
# [2]: https://golang.org/pkg/time/ (see "To convert an integer number of units to a Duration, multiply:")
|
||||
- durationcheck
|
||||
|
||||
# No way to disable the "exported" check for the whole project[1]
|
||||
# [1]: https://github.com/mgechev/revive/issues/244#issuecomment-560512162
|
||||
- revive
|
||||
|
||||
# Unfortunately too much false-positives, e.g. for a 0700 umask or number 10 when using strconv.FormatInt()
|
||||
- gomnd
|
||||
|
||||
# Needs package whitelists
|
||||
- depguard
|
||||
|
||||
# Generates absolutely useless errors, e.g.
|
||||
# "string `.yml` has 3 occurrences, make it a constant"
|
||||
- goconst
|
||||
|
||||
# It's OK to not sort imports
|
||||
- gci
|
||||
|
||||
# It's OK to not initialize some struct fields
|
||||
- exhaustruct
|
||||
|
||||
# This is not a library, so it's OK to use dynamic errors
|
||||
- goerr113
|
||||
|
||||
# fmt.Sprintf() looks a bit nicer than string addition
|
||||
- perfsprint
|
||||
|
||||
# We can control this ourselves
|
||||
- varnamelen
|
||||
- contextcheck
|
||||
|
||||
issues:
|
||||
# Don't hide multiple issues that belong to one class since GitHub annotations can handle them all nicely.
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# Benchmark
|
||||
|
||||
Tart comes with a Golang-based benchmarking utility that allows one to easily compare host and guest performance.
|
||||
|
||||
Currently, only Flexible I/O tester workloads are supported. To run them, [install Golang](https://go.dev/) and run the following command from this (`benchmark/`) directory:
|
||||
|
||||
```shell
|
||||
go run cmd/main.go fio
|
||||
```
|
||||
|
||||
You can also enable the debugging output to diagnose issues:
|
||||
|
||||
```shell
|
||||
go run cmd/main.go fio --debug
|
||||
```
|
||||
|
||||
## Results
|
||||
|
||||
Host:
|
||||
|
||||
* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
|
||||
* OS: macOS Sonoma 14.4.1
|
||||
|
||||
Guest:
|
||||
|
||||
* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
|
||||
* OS: macOS Sonoma 14.4.1
|
||||
|
||||
```
|
||||
Name Executor Bandwidth I/O operations
|
||||
Random writing of 1MB local 2.6 GB/s 649.35 kIOPS
|
||||
Random writing of 1MB Tart 2.5 GB/s 620.22 kIOPS
|
||||
Random writing of 10MB local 2.6 GB/s 651.74 kIOPS
|
||||
Random writing of 10MB Tart 2.5 GB/s 615.52 kIOPS
|
||||
Random writing of 100MB local 1.9 GB/s 481.51 kIOPS
|
||||
Random writing of 100MB Tart 2.0 GB/s 493.31 kIOPS
|
||||
Random writing of 1000MB local 1.7 GB/s 414.89 kIOPS
|
||||
Random writing of 1000MB Tart 1.1 GB/s 287.4 kIOPS
|
||||
```
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/cirruslabs/tart/benchmark/internal/command"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Set up a signal-interruptible context
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
|
||||
// Run the root command
|
||||
if err := command.NewCommand().ExecuteContext(ctx); err != nil {
|
||||
cancel()
|
||||
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
cancel()
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
module github.com/cirruslabs/tart/benchmark
|
||||
|
||||
go 1.22.1
|
||||
|
||||
require (
|
||||
github.com/avast/retry-go/v4 v4.5.1
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gosuri/uitable v0.0.4
|
||||
github.com/spf13/cobra v1.8.0
|
||||
github.com/stretchr/testify v1.9.0
|
||||
go.uber.org/zap v1.27.0
|
||||
golang.org/x/crypto v0.21.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/fatih/color v1.16.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
|
||||
github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
|
||||
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
|
||||
github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
|
||||
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
|
||||
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
|
||||
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
|
||||
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package fio
|
||||
|
||||
type Benchmark struct {
|
||||
Name string
|
||||
Command string
|
||||
}
|
||||
|
||||
var benchmarks = []Benchmark{
|
||||
{
|
||||
Name: "Random writing of 1MB",
|
||||
Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
|
||||
"--size 1MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
|
||||
},
|
||||
{
|
||||
Name: "Random writing of 10MB",
|
||||
Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
|
||||
"--size 10MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
|
||||
},
|
||||
{
|
||||
Name: "Random writing of 100MB",
|
||||
Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
|
||||
"--size 100MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
|
||||
},
|
||||
{
|
||||
Name: "Random writing of 1000MB",
|
||||
Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
|
||||
"--size 1000MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
|
||||
},
|
||||
}
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
package fio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/cirruslabs/tart/benchmark/internal/executor"
|
||||
"github.com/cirruslabs/tart/benchmark/internal/executor/local"
|
||||
"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/gosuri/uitable"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var debug bool
|
||||
|
||||
func NewCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "fio",
|
||||
Short: "run Flexible I/O tester (fio) benchmarks",
|
||||
RunE: run,
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func run(cmd *cobra.Command, args []string) error {
|
||||
config := zap.NewProductionConfig()
|
||||
if debug {
|
||||
config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
|
||||
}
|
||||
logger, err := config.Build()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = logger.Sync()
|
||||
}()
|
||||
|
||||
executors, err := initializeExecutors(cmd.Context(), logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
errs := []error{err}
|
||||
|
||||
for _, executor := range executors {
|
||||
if err := executor.Close(); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to close executor %s: %w", executor.Name(), err))
|
||||
}
|
||||
}
|
||||
|
||||
err = errors.Join(errs...)
|
||||
}()
|
||||
|
||||
table := uitable.New()
|
||||
table.AddRow("Name", "Executor", "Bandwidth", "I/O operations")
|
||||
|
||||
for _, benchmark := range benchmarks {
|
||||
for _, executor := range executors {
|
||||
logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name, executor.Name())
|
||||
|
||||
stdout, err := executor.Run(cmd.Context(), benchmark.Command)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var fioResult Result
|
||||
|
||||
if err := json.Unmarshal(stdout, &fioResult); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(fioResult.Jobs) != 1 {
|
||||
return fmt.Errorf("expected exactly 1 job from fio's JSON output, got %d",
|
||||
len(fioResult.Jobs))
|
||||
}
|
||||
|
||||
job := fioResult.Jobs[0]
|
||||
|
||||
writeBandwidth := humanize.Bytes(uint64(job.Write.BW)*humanize.KByte) + "/s"
|
||||
writeIOPS := humanize.SIWithDigits(job.Write.IOPS, 2, "IOPS")
|
||||
|
||||
logger.Sugar().Infof("write bandwidth: %s, write IOPS: %s\n", writeBandwidth, writeIOPS)
|
||||
|
||||
table.AddRow(benchmark.Name, executor.Name(), writeBandwidth, writeIOPS)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println(table.String())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func initializeExecutors(ctx context.Context, logger *zap.Logger) ([]executor.Executor, error) {
|
||||
var result []executor.Executor
|
||||
|
||||
logger.Info("initializing local executor")
|
||||
|
||||
local, err := local.New(logger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, local)
|
||||
|
||||
logger.Info("local executor initialized")
|
||||
|
||||
logger.Info("initializing Tart executor")
|
||||
|
||||
tart, err := tart.New(ctx, logger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, tart)
|
||||
|
||||
logger.Info("Tart executor initialized")
|
||||
|
||||
for _, executor := range result {
|
||||
logger.Sugar().Infof("installing Flexible I/O tester (fio) on %s executor", executor.Name())
|
||||
|
||||
if _, err := executor.Run(ctx, "brew install fio"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package fio
|
||||
|
||||
type Result struct {
|
||||
Jobs []Job `json:"jobs"`
|
||||
}
|
||||
|
||||
type Job struct {
|
||||
Name string `json:"jobname"`
|
||||
Write Write `json:"write"`
|
||||
}
|
||||
|
||||
type Write struct {
|
||||
BW float64 `json:"bw"`
|
||||
IOPS float64 `json:"iops"`
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"github.com/cirruslabs/tart/benchmark/internal/command/fio"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func NewCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "benchmark",
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
}
|
||||
|
||||
cmd.AddCommand(
|
||||
fio.NewCommand(),
|
||||
)
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
type Executor interface {
|
||||
Name() string
|
||||
Run(ctx context.Context, command string) ([]byte, error)
|
||||
Close() error
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package local
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapio"
|
||||
"io"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
type Local struct {
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
func New(logger *zap.Logger) (*Local, error) {
|
||||
return &Local{
|
||||
logger: logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (local *Local) Name() string {
|
||||
return "local"
|
||||
}
|
||||
|
||||
func (local *Local) Run(ctx context.Context, command string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, "zsh", "-c", command)
|
||||
|
||||
loggerWriter := &zapio.Writer{Log: local.logger, Level: zap.DebugLevel}
|
||||
stdoutBuf := &bytes.Buffer{}
|
||||
|
||||
cmd.Stdout = io.MultiWriter(stdoutBuf, loggerWriter)
|
||||
cmd.Stderr = loggerWriter
|
||||
|
||||
err := cmd.Run()
|
||||
|
||||
return stdoutBuf.Bytes(), err
|
||||
}
|
||||
|
||||
func (local *Local) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package local_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/cirruslabs/tart/benchmark/internal/executor/local"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocal(t *testing.T) {
|
||||
local, err := local.New(zap.NewNop())
|
||||
require.NoError(t, err)
|
||||
|
||||
output, err := local.Run(context.Background(), "echo \"this is a test\"")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "this is a test\n", string(output))
|
||||
|
||||
require.NoError(t, local.Close())
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package tart
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapio"
|
||||
"io"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const tartBinaryName = "tart"
|
||||
|
||||
func Cmd(ctx context.Context, logger *zap.Logger, args ...string) error {
|
||||
_, err := CmdWithOutput(ctx, logger, args...)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func CmdWithOutput(ctx context.Context, logger *zap.Logger, args ...string) (string, error) {
|
||||
logger.Sugar().Debugf("running %s %s", tartBinaryName, strings.Join(args, " "))
|
||||
|
||||
cmd := exec.CommandContext(ctx, tartBinaryName, args...)
|
||||
|
||||
loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
|
||||
stdoutBuf := &bytes.Buffer{}
|
||||
|
||||
cmd.Stdout = io.MultiWriter(stdoutBuf, loggerWriter)
|
||||
cmd.Stderr = loggerWriter
|
||||
|
||||
err := cmd.Run()
|
||||
|
||||
return stdoutBuf.String(), err
|
||||
}
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
package tart
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/avast/retry-go/v4"
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/crypto/ssh"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const baseImage = "ghcr.io/cirruslabs/macos-sonoma-base:latest"
|
||||
|
||||
type Tart struct {
|
||||
vmRunCancel context.CancelFunc
|
||||
vmName string
|
||||
sshClient *ssh.Client
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
func New(ctx context.Context, logger *zap.Logger) (*Tart, error) {
|
||||
tart := &Tart{
|
||||
vmName: fmt.Sprintf("tart-benchmark-%s", uuid.NewString()),
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
if err := Cmd(ctx, tart.logger, "pull", baseImage); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := Cmd(ctx, tart.logger, "clone", baseImage, tart.vmName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
vmRunCtx, vmRunCancel := context.WithCancel(ctx)
|
||||
tart.vmRunCancel = vmRunCancel
|
||||
|
||||
go func() {
|
||||
_ = Cmd(vmRunCtx, tart.logger, "run", "--no-graphics", tart.vmName)
|
||||
}()
|
||||
|
||||
ip, err := CmdWithOutput(ctx, tart.logger, "ip", "--wait", "60", tart.vmName)
|
||||
if err != nil {
|
||||
return nil, tart.Close()
|
||||
}
|
||||
|
||||
err = retry.Do(func() error {
|
||||
dialer := net.Dialer{
|
||||
Timeout: 1 * time.Second,
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("%s:22", strings.TrimSpace(ip))
|
||||
|
||||
netConn, err := dialer.DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sshConn, chans, reqs, err := ssh.NewClientConn(netConn, addr, &ssh.ClientConfig{
|
||||
User: "admin",
|
||||
Auth: []ssh.AuthMethod{
|
||||
ssh.Password("admin"),
|
||||
},
|
||||
HostKeyCallback: func(_ string, _ net.Addr, _ ssh.PublicKey) error {
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tart.sshClient = ssh.NewClient(sshConn, chans, reqs)
|
||||
|
||||
return nil
|
||||
}, retry.RetryIf(func(err error) bool {
|
||||
return !errors.Is(err, context.Canceled)
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, tart.Close()
|
||||
}
|
||||
|
||||
return tart, nil
|
||||
}
|
||||
|
||||
func (tart *Tart) Name() string {
|
||||
return "Tart"
|
||||
}
|
||||
|
||||
func (tart *Tart) Run(ctx context.Context, command string) ([]byte, error) {
|
||||
sshSession, err := tart.sshClient.NewSession()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Work around x/crypto/ssh not being context.Context-friendly (e.g. https://github.com/golang/go/issues/20288)
|
||||
monitorCtx, monitorCancel := context.WithCancel(ctx)
|
||||
go func() {
|
||||
<-monitorCtx.Done()
|
||||
_ = sshSession.Close()
|
||||
}()
|
||||
defer monitorCancel()
|
||||
|
||||
stdoutBuf := &bytes.Buffer{}
|
||||
|
||||
sshSession.Stdin = bytes.NewBufferString(command)
|
||||
sshSession.Stdout = stdoutBuf
|
||||
|
||||
if err := sshSession.Shell(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := sshSession.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return stdoutBuf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (tart *Tart) Close() error {
|
||||
if tart.sshClient != nil {
|
||||
_ = tart.sshClient.Close()
|
||||
}
|
||||
|
||||
tart.vmRunCancel()
|
||||
_ = Cmd(context.Background(), tart.logger, "delete", tart.vmName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package tart_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTart(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
tart, err := tart.New(ctx, zap.NewNop())
|
||||
require.NoError(t, err)
|
||||
|
||||
output, err := tart.Run(ctx, "echo \"this is a test\"")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "this is a test\n", string(output))
|
||||
|
||||
require.NoError(t, tart.Close())
|
||||
}
|
||||
Loading…
Reference in New Issue