Benchmark: run XcodeBenchmark with different disk settings (#1000)

* Benchmark: run XcodeBenchmark with different disk settings

* Add Xcode benchmark results

benchmark/README.md

@@ -185,3 +185,42 @@ sync test                               	Tart (--root-disk-opts="sync=none")
 sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	35 MB/s    	0 IOPS     	15.67 kIOPS 	0s ± 0s                   	11.319µs ± 24.771µs       	51.731µs ± 42.208µs    
 sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	17 MB/s    	0 IOPS     	7.39 kIOPS  	0s ± 0s                   	21.23µs ± 81.749µs        	113.239µs ± 191.266µs  
 ```
+
+### Jan 16, 2025
+
+Host:
+
+* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
+* OS: macOS Sequoia 15.2
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.6
+
+```
+Name                    	Executor                                          	Time
+XcodeBenchmark (d869315)	local                                             	2m15s
+XcodeBenchmark (d869315)	Tart                                              	4m22s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	4m21s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	4m15s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	4m16s
+```
+
+```
+Name                    	Executor                                          	Time
+XcodeBenchmark (d869315)	local                                             	2m7s
+XcodeBenchmark (d869315)	Tart                                              	4m37s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	4m35s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	4m19s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	4m16s
+```
+
+```
+Name                    	Executor                                          	Time
+XcodeBenchmark (d869315)	local                                             	2m6s
+XcodeBenchmark (d869315)	Tart                                              	4m24s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	4m22s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	4m18s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	4m17s
+```

benchmark/internal/command/fio/fio.go

@@ -4,8 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
 	"github.com/dustin/go-humanize"
 	"github.com/gosuri/uitable"
 	"github.com/spf13/cobra"
@@ -46,57 +44,12 @@ func run(cmd *cobra.Command, args []string) error {
 		_ = logger.Sync()
 	}()
 
-	var executorInitializers = []struct {
-		Name string
-		Fn   func() (executorpkg.Executor, error)
-	}{
-		{
-			Name: "local",
-			Fn: func() (executorpkg.Executor, error) {
-				return local.New(logger)
-			},
-		},
-		{
-			Name: "Tart",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, nil, logger)
-			},
-		},
-		{
-			Name: "Tart (--root-disk-opts=\"sync=none\")",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, []string{
-					"--root-disk-opts",
-					"sync=none",
-				}, logger)
-			},
-		},
-		{
-			Name: "Tart (--root-disk-opts=\"caching=cached\")",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, []string{
-					"--root-disk-opts",
-					"caching=cached",
-				}, logger)
-			},
-		},
-		{
-			Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, []string{
-					"--root-disk-opts",
-					"sync=none,caching=cached",
-				}, logger)
-			},
-		},
-	}
-
 	table := uitable.New()
 	table.AddRow("Name", "Executor", "B/W (read)", "B/W (write)", "I/O (read)", "I/O (write)",
 		"Latency (read)", "Latency (write)", "Latency (sync)")
 
 	for _, benchmark := range benchmarks {
-		for _, executorInitializer := range executorInitializers {
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
 			if prepare != "" {
 				shell := "/bin/sh"
 

benchmark/internal/command/root.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"github.com/cirruslabs/tart/benchmark/internal/command/fio"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
 	"github.com/spf13/cobra"
 )
 
@@ -14,6 +15,7 @@ func NewCommand() *cobra.Command {
 
 	cmd.AddCommand(
 		fio.NewCommand(),
+		xcode.NewCommand(),
 	)
 
 	return cmd

benchmark/internal/command/xcode/benchmarks.go

@@ -0,0 +1,13 @@
+package xcode
+
+type Benchmark struct {
+	Name    string
+	Command string
+}
+
+var benchmarks = []Benchmark{
+	{
+		Name:    "XcodeBenchmark (d869315)",
+		Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && sh benchmark.sh",
+	},
+}

benchmark/internal/command/xcode/output.go

@@ -0,0 +1,49 @@
+package xcode
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+)
+
+type Output struct {
+	Started time.Time
+	Ended   time.Time
+}
+
+func ParseOutput(s string) (*Output, error) {
+	// Ensure that the build has succeeded
+	matched, err := regexp.MatchString("(?m)^\\*\\* BUILD SUCCEEDED \\*\\*.*$", s)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse output: regexp failed: %v", err)
+	}
+	if !matched {
+		return nil, fmt.Errorf("failed to parse output: \"** BUILD SUCCEEDED **\" string " +
+			"not found on a separate line, make sure you have Xcode installed")
+	}
+
+	re := regexp.MustCompile("Started\\s+(?P<started>.*)\\n.*Ended\\s+(?P<ended>.*)\\n")
+
+	matches := re.FindStringSubmatch(s)
+
+	if len(matches) != re.NumSubexp()+1 {
+		return nil, fmt.Errorf("failed to parse output: cannot find Started and Ended times")
+	}
+
+	startedRaw := matches[re.SubexpIndex("started")]
+	started, err := time.Parse(time.TimeOnly, startedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse started time %q: unsupported format", startedRaw)
+	}
+
+	endedRaw := matches[re.SubexpIndex("ended")]
+	ended, err := time.Parse(time.TimeOnly, endedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse ended time %q: unsupported format", startedRaw)
+	}
+
+	return &Output{
+		Started: started,
+		Ended:   ended,
+	}, nil
+}

benchmark/internal/command/xcode/output_test.go

@@ -0,0 +1,37 @@
+package xcode_test
+
+import (
+	"fmt"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func TestParseOutput(t *testing.T) {
+	result, err := xcode.ParseOutput(`** BUILD SUCCEEDED ** [219.713 sec]
+
+System Version: 14.6
+Xcode 15.4
+Hardware Overview
+      Model Name: Apple Virtual Machine 1
+      Model Identifier: VirtualMac2,1
+      Total Number of Cores: 4
+      Memory: 8 GB
+
+✅ XcodeBenchmark has completed
+1️⃣  Take a screenshot of this window (Cmd + Shift + 4 + Space) and resize to include:
+	- Build Time (See ** BUILD SUCCEEDED ** [XYZ sec])
+	- System Version
+	- Xcode Version
+	- Hardware Overview
+	- Started 13:46:20
+	- Ended   13:50:02
+	- Date Thu Jan 16 13:50:02 UTC 2025
+
+2️⃣  Share your results at https://github.com/devMEremenko/XcodeBenchmark
+`)
+	require.NoError(t, err)
+	fmt.Println(result)
+	require.Equal(t, 222*time.Second, result.Ended.Sub(result.Started))
+}

benchmark/internal/command/xcode/xcode.go

@@ -0,0 +1,108 @@
+package xcode
+
+import (
+	"fmt"
+	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
+	"github.com/gosuri/uitable"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"os"
+	"os/exec"
+)
+
+var debug bool
+var image string
+var prepare string
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "xcode",
+		Short: "run XCode benchmarks",
+		RunE:  run,
+	}
+
+	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+	cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-xcode:latest", "image to use for testing")
+	cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark")
+
+	return cmd
+}
+
+func run(cmd *cobra.Command, args []string) error {
+	config := zap.NewProductionConfig()
+	if debug {
+		config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+	logger, err := config.Build()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = logger.Sync()
+	}()
+
+	table := uitable.New()
+	table.AddRow("Name", "Executor", "Time")
+
+	for _, benchmark := range benchmarks {
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
+			if prepare != "" {
+				shell := "/bin/sh"
+
+				if shellFromEnv, ok := os.LookupEnv("SHELL"); ok {
+					shell = shellFromEnv
+				}
+
+				logger.Sugar().Infof("running prepare command %q using shell %q",
+					prepare, shell)
+
+				cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare)
+
+				loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+
+				cmd.Stdout = loggerWriter
+				cmd.Stderr = loggerWriter
+
+				if err := cmd.Run(); err != nil {
+					return fmt.Errorf("failed to run prepare command %q: %v", prepare, err)
+				}
+			}
+
+			logger.Sugar().Infof("initializing executor %s", executorInitializer.Name)
+
+			executor, err := executorInitializer.Fn()
+			if err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name,
+				executorInitializer.Name)
+
+			stdout, err := executor.Run(cmd.Context(), benchmark.Command)
+			if err != nil {
+				return err
+			}
+
+			output, err := ParseOutput(string(stdout))
+			if err != nil {
+				return err
+			}
+
+			duration := output.Ended.Sub(output.Started)
+
+			logger.Sugar().Infof("Xcode benchmark duration: %s", duration)
+
+			table.AddRow(benchmark.Name, executorInitializer.Name, duration)
+
+			if err := executor.Close(); err != nil {
+				return fmt.Errorf("failed to close executor %s: %w",
+					executorInitializer.Name, err)
+			}
+		}
+	}
+
+	fmt.Println(table.String())
+
+	return nil
+}

benchmark/internal/executor/initializer.go

@@ -0,0 +1,57 @@
+package executor
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	"go.uber.org/zap"
+)
+
+type Initializer struct {
+	Name string
+	Fn   func() (Executor, error)
+}
+
+func DefaultInitializers(ctx context.Context, image string, logger *zap.Logger) []Initializer {
+	return []Initializer{
+		{
+			Name: "local",
+			Fn: func() (Executor, error) {
+				return local.New(logger)
+			},
+		},
+		{
+			Name: "Tart",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, nil, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"caching=cached",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none,caching=cached",
+				}, logger)
+			},
+		},
+	}
+}