From d8945503d61a86107ed5f502dfaf515cfc1d6411 Mon Sep 17 00:00:00 2001 From: Nikolay Edigaryev Date: Thu, 16 Jan 2025 23:58:26 +0400 Subject: [PATCH] Benchmark: run XcodeBenchmark with different disk settings (#1000) * Benchmark: run XcodeBenchmark with different disk settings * Add Xcode benchmark results --- benchmark/README.md | 39 +++++++ benchmark/internal/command/fio/fio.go | 49 +------- benchmark/internal/command/root.go | 2 + .../internal/command/xcode/benchmarks.go | 13 +++ benchmark/internal/command/xcode/output.go | 49 ++++++++ .../internal/command/xcode/output_test.go | 37 ++++++ benchmark/internal/command/xcode/xcode.go | 108 ++++++++++++++++++ benchmark/internal/executor/initializer.go | 57 +++++++++ 8 files changed, 306 insertions(+), 48 deletions(-) create mode 100644 benchmark/internal/command/xcode/benchmarks.go create mode 100644 benchmark/internal/command/xcode/output.go create mode 100644 benchmark/internal/command/xcode/output_test.go create mode 100644 benchmark/internal/command/xcode/xcode.go create mode 100644 benchmark/internal/executor/initializer.go diff --git a/benchmark/README.md b/benchmark/README.md index 009e98d..637d09c 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -185,3 +185,42 @@ sync test Tart (--root-disk-opts="sync=none") sync test Tart (--root-disk-opts="caching=cached") 0 B/s 35 MB/s 0 IOPS 15.67 kIOPS 0s ± 0s 11.319µs ± 24.771µs 51.731µs ± 42.208µs sync test Tart (--root-disk-opts="sync=none,caching=cached") 0 B/s 17 MB/s 0 IOPS 7.39 kIOPS 0s ± 0s 21.23µs ± 81.749µs 113.239µs ± 191.266µs ``` + +### Jan 16, 2025 + +Host: + +* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`) +* OS: macOS Sequoia 15.2 + +Guest: + +* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization) +* OS: macOS Sonoma 14.6 + +``` +Name Executor Time +XcodeBenchmark (d869315) local 2m15s +XcodeBenchmark (d869315) Tart 4m22s +XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 4m21s +XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 4m15s +XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 4m16s +``` + +``` +Name Executor Time +XcodeBenchmark (d869315) local 2m7s +XcodeBenchmark (d869315) Tart 4m37s +XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 4m35s +XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 4m19s +XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 4m16s +``` + +``` +Name Executor Time +XcodeBenchmark (d869315) local 2m6s +XcodeBenchmark (d869315) Tart 4m24s +XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 4m22s +XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 4m18s +XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 4m17s +``` diff --git a/benchmark/internal/command/fio/fio.go b/benchmark/internal/command/fio/fio.go index 2c43c4f..82eb99f 100644 --- a/benchmark/internal/command/fio/fio.go +++ b/benchmark/internal/command/fio/fio.go @@ -4,8 +4,6 @@ import ( "encoding/json" "fmt" executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor" - "github.com/cirruslabs/tart/benchmark/internal/executor/local" - "github.com/cirruslabs/tart/benchmark/internal/executor/tart" "github.com/dustin/go-humanize" "github.com/gosuri/uitable" "github.com/spf13/cobra" @@ -46,57 +44,12 @@ func run(cmd *cobra.Command, args []string) error { _ = logger.Sync() }() - var executorInitializers = []struct { - Name string - Fn func() (executorpkg.Executor, error) - }{ - { - Name: "local", - Fn: func() (executorpkg.Executor, error) { - return local.New(logger) - }, - }, - { - Name: "Tart", - Fn: func() (executorpkg.Executor, error) { - return tart.New(cmd.Context(), image, nil, logger) - }, - }, - { - Name: "Tart (--root-disk-opts=\"sync=none\")", - Fn: func() (executorpkg.Executor, error) { - return tart.New(cmd.Context(), image, []string{ - "--root-disk-opts", - "sync=none", - }, logger) - }, - }, - { - Name: "Tart (--root-disk-opts=\"caching=cached\")", - Fn: func() (executorpkg.Executor, error) { - return tart.New(cmd.Context(), image, []string{ - "--root-disk-opts", - "caching=cached", - }, logger) - }, - }, - { - Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")", - Fn: func() (executorpkg.Executor, error) { - return tart.New(cmd.Context(), image, []string{ - "--root-disk-opts", - "sync=none,caching=cached", - }, logger) - }, - }, - } - table := uitable.New() table.AddRow("Name", "Executor", "B/W (read)", "B/W (write)", "I/O (read)", "I/O (write)", "Latency (read)", "Latency (write)", "Latency (sync)") for _, benchmark := range benchmarks { - for _, executorInitializer := range executorInitializers { + for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) { if prepare != "" { shell := "/bin/sh" diff --git a/benchmark/internal/command/root.go b/benchmark/internal/command/root.go index d05363f..77a33bc 100644 --- a/benchmark/internal/command/root.go +++ b/benchmark/internal/command/root.go @@ -2,6 +2,7 @@ package command import ( "github.com/cirruslabs/tart/benchmark/internal/command/fio" + "github.com/cirruslabs/tart/benchmark/internal/command/xcode" "github.com/spf13/cobra" ) @@ -14,6 +15,7 @@ func NewCommand() *cobra.Command { cmd.AddCommand( fio.NewCommand(), + xcode.NewCommand(), ) return cmd diff --git a/benchmark/internal/command/xcode/benchmarks.go b/benchmark/internal/command/xcode/benchmarks.go new file mode 100644 index 0000000..f7991ae --- /dev/null +++ b/benchmark/internal/command/xcode/benchmarks.go @@ -0,0 +1,13 @@ +package xcode + +type Benchmark struct { + Name string + Command string +} + +var benchmarks = []Benchmark{ + { + Name: "XcodeBenchmark (d869315)", + Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && sh benchmark.sh", + }, +} diff --git a/benchmark/internal/command/xcode/output.go b/benchmark/internal/command/xcode/output.go new file mode 100644 index 0000000..8ba5faa --- /dev/null +++ b/benchmark/internal/command/xcode/output.go @@ -0,0 +1,49 @@ +package xcode + +import ( + "fmt" + "regexp" + "time" +) + +type Output struct { + Started time.Time + Ended time.Time +} + +func ParseOutput(s string) (*Output, error) { + // Ensure that the build has succeeded + matched, err := regexp.MatchString("(?m)^\\*\\* BUILD SUCCEEDED \\*\\*.*$", s) + if err != nil { + return nil, fmt.Errorf("failed to parse output: regexp failed: %v", err) + } + if !matched { + return nil, fmt.Errorf("failed to parse output: \"** BUILD SUCCEEDED **\" string " + + "not found on a separate line, make sure you have Xcode installed") + } + + re := regexp.MustCompile("Started\\s+(?P.*)\\n.*Ended\\s+(?P.*)\\n") + + matches := re.FindStringSubmatch(s) + + if len(matches) != re.NumSubexp()+1 { + return nil, fmt.Errorf("failed to parse output: cannot find Started and Ended times") + } + + startedRaw := matches[re.SubexpIndex("started")] + started, err := time.Parse(time.TimeOnly, startedRaw) + if err != nil { + return nil, fmt.Errorf("failed to parse started time %q: unsupported format", startedRaw) + } + + endedRaw := matches[re.SubexpIndex("ended")] + ended, err := time.Parse(time.TimeOnly, endedRaw) + if err != nil { + return nil, fmt.Errorf("failed to parse ended time %q: unsupported format", startedRaw) + } + + return &Output{ + Started: started, + Ended: ended, + }, nil +} diff --git a/benchmark/internal/command/xcode/output_test.go b/benchmark/internal/command/xcode/output_test.go new file mode 100644 index 0000000..f56fd5b --- /dev/null +++ b/benchmark/internal/command/xcode/output_test.go @@ -0,0 +1,37 @@ +package xcode_test + +import ( + "fmt" + "github.com/cirruslabs/tart/benchmark/internal/command/xcode" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestParseOutput(t *testing.T) { + result, err := xcode.ParseOutput(`** BUILD SUCCEEDED ** [219.713 sec] + +System Version: 14.6 +Xcode 15.4 +Hardware Overview + Model Name: Apple Virtual Machine 1 + Model Identifier: VirtualMac2,1 + Total Number of Cores: 4 + Memory: 8 GB + +✅ XcodeBenchmark has completed +1️⃣ Take a screenshot of this window (Cmd + Shift + 4 + Space) and resize to include: + - Build Time (See ** BUILD SUCCEEDED ** [XYZ sec]) + - System Version + - Xcode Version + - Hardware Overview + - Started 13:46:20 + - Ended 13:50:02 + - Date Thu Jan 16 13:50:02 UTC 2025 + +2️⃣ Share your results at https://github.com/devMEremenko/XcodeBenchmark +`) + require.NoError(t, err) + fmt.Println(result) + require.Equal(t, 222*time.Second, result.Ended.Sub(result.Started)) +} diff --git a/benchmark/internal/command/xcode/xcode.go b/benchmark/internal/command/xcode/xcode.go new file mode 100644 index 0000000..0e06625 --- /dev/null +++ b/benchmark/internal/command/xcode/xcode.go @@ -0,0 +1,108 @@ +package xcode + +import ( + "fmt" + executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor" + "github.com/gosuri/uitable" + "github.com/spf13/cobra" + "go.uber.org/zap" + "go.uber.org/zap/zapio" + "os" + "os/exec" +) + +var debug bool +var image string +var prepare string + +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "xcode", + Short: "run XCode benchmarks", + RunE: run, + } + + cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging") + cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-xcode:latest", "image to use for testing") + cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark") + + return cmd +} + +func run(cmd *cobra.Command, args []string) error { + config := zap.NewProductionConfig() + if debug { + config.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + } + logger, err := config.Build() + if err != nil { + return err + } + defer func() { + _ = logger.Sync() + }() + + table := uitable.New() + table.AddRow("Name", "Executor", "Time") + + for _, benchmark := range benchmarks { + for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) { + if prepare != "" { + shell := "/bin/sh" + + if shellFromEnv, ok := os.LookupEnv("SHELL"); ok { + shell = shellFromEnv + } + + logger.Sugar().Infof("running prepare command %q using shell %q", + prepare, shell) + + cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare) + + loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel} + + cmd.Stdout = loggerWriter + cmd.Stderr = loggerWriter + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to run prepare command %q: %v", prepare, err) + } + } + + logger.Sugar().Infof("initializing executor %s", executorInitializer.Name) + + executor, err := executorInitializer.Fn() + if err != nil { + return err + } + + logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name, + executorInitializer.Name) + + stdout, err := executor.Run(cmd.Context(), benchmark.Command) + if err != nil { + return err + } + + output, err := ParseOutput(string(stdout)) + if err != nil { + return err + } + + duration := output.Ended.Sub(output.Started) + + logger.Sugar().Infof("Xcode benchmark duration: %s", duration) + + table.AddRow(benchmark.Name, executorInitializer.Name, duration) + + if err := executor.Close(); err != nil { + return fmt.Errorf("failed to close executor %s: %w", + executorInitializer.Name, err) + } + } + } + + fmt.Println(table.String()) + + return nil +} diff --git a/benchmark/internal/executor/initializer.go b/benchmark/internal/executor/initializer.go new file mode 100644 index 0000000..dd2d03f --- /dev/null +++ b/benchmark/internal/executor/initializer.go @@ -0,0 +1,57 @@ +package executor + +import ( + "context" + "github.com/cirruslabs/tart/benchmark/internal/executor/local" + "github.com/cirruslabs/tart/benchmark/internal/executor/tart" + "go.uber.org/zap" +) + +type Initializer struct { + Name string + Fn func() (Executor, error) +} + +func DefaultInitializers(ctx context.Context, image string, logger *zap.Logger) []Initializer { + return []Initializer{ + { + Name: "local", + Fn: func() (Executor, error) { + return local.New(logger) + }, + }, + { + Name: "Tart", + Fn: func() (Executor, error) { + return tart.New(ctx, image, nil, logger) + }, + }, + { + Name: "Tart (--root-disk-opts=\"sync=none\")", + Fn: func() (Executor, error) { + return tart.New(ctx, image, []string{ + "--root-disk-opts", + "sync=none", + }, logger) + }, + }, + { + Name: "Tart (--root-disk-opts=\"caching=cached\")", + Fn: func() (Executor, error) { + return tart.New(ctx, image, []string{ + "--root-disk-opts", + "caching=cached", + }, logger) + }, + }, + { + Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")", + Fn: func() (Executor, error) { + return tart.New(ctx, image, []string{ + "--root-disk-opts", + "sync=none,caching=cached", + }, logger) + }, + }, + } +}