From cb92a3fa67d2cf3c1bbc8ad79c1f7a36d5cb33ab Mon Sep 17 00:00:00 2001 From: Fedor Korotkov Date: Sun, 23 Mar 2025 16:55:32 -0400 Subject: [PATCH] Use full host resources for Xcode benchmarks (#1045) --- benchmark/README.md | 40 ++++++------------- benchmark/go.mod | 1 + benchmark/go.sum | 2 + .../internal/command/xcode/benchmarks.go | 2 +- benchmark/internal/command/xcode/xcode.go | 2 +- benchmark/internal/executor/tart/tart.go | 20 ++++++++++ 6 files changed, 37 insertions(+), 30 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 637d09c..597802f 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -13,7 +13,7 @@ brew install go Finally, run the following command from this (`benchmark/`) directory: ```shell -go run cmd/main.go fio --image ghcr.io/cirruslabs/macos-sonoma-base:latest --prepare 'sudo purge && sync' +go run cmd/main.go fio --image ghcr.io/cirruslabs/macos-sequoia-base:latest --prepare 'sudo purge && sync' ``` You can also enable the debugging output to diagnose issues: @@ -186,41 +186,25 @@ sync test Tart (--root-disk-opts="caching=cached" sync test Tart (--root-disk-opts="sync=none,caching=cached") 0 B/s 17 MB/s 0 IOPS 7.39 kIOPS 0s ± 0s 21.23µs ± 81.749µs 113.239µs ± 191.266µs ``` -### Jan 16, 2025 +### March 23, 2025 Host: * Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`) -* OS: macOS Sequoia 15.2 +* OS: macOS Sequoia 15.3.2 +* Xcode: 16.2 Guest: * Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization) -* OS: macOS Sonoma 14.6 +* OS: macOS Sonoma 15.3.2 +* Xcode: 16.2 ``` -Name Executor Time -XcodeBenchmark (d869315) local 2m15s -XcodeBenchmark (d869315) Tart 4m22s -XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 4m21s -XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 4m15s -XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 4m16s -``` - -``` -Name Executor Time -XcodeBenchmark (d869315) local 2m7s -XcodeBenchmark (d869315) Tart 4m37s -XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 4m35s -XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 4m19s -XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 4m16s -``` - -``` -Name Executor Time -XcodeBenchmark (d869315) local 2m6s -XcodeBenchmark (d869315) Tart 4m24s -XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 4m22s -XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 4m18s -XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 4m17s +Name Executor Time +XcodeBenchmark (d869315) local 2m19s +XcodeBenchmark (d869315) Tart 3m59s +XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 3m48s +XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 3m35s +XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 3m14s ``` diff --git a/benchmark/go.mod b/benchmark/go.mod index d9fb314..b50a4e6 100644 --- a/benchmark/go.mod +++ b/benchmark/go.mod @@ -22,6 +22,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/benchmark/go.sum b/benchmark/go.sum index 6a61a34..e353254 100644 --- a/benchmark/go.sum +++ b/benchmark/go.sum @@ -26,6 +26,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/benchmark/internal/command/xcode/benchmarks.go b/benchmark/internal/command/xcode/benchmarks.go index f7991ae..1732882 100644 --- a/benchmark/internal/command/xcode/benchmarks.go +++ b/benchmark/internal/command/xcode/benchmarks.go @@ -8,6 +8,6 @@ type Benchmark struct { var benchmarks = []Benchmark{ { Name: "XcodeBenchmark (d869315)", - Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && sh benchmark.sh", + Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && xcrun simctl list && sh benchmark.sh", }, } diff --git a/benchmark/internal/command/xcode/xcode.go b/benchmark/internal/command/xcode/xcode.go index 0e06625..6186923 100644 --- a/benchmark/internal/command/xcode/xcode.go +++ b/benchmark/internal/command/xcode/xcode.go @@ -23,7 +23,7 @@ func NewCommand() *cobra.Command { } cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging") - cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-xcode:latest", "image to use for testing") + cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sequoia-xcode:latest", "image to use for testing") cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark") return cmd diff --git a/benchmark/internal/executor/tart/tart.go b/benchmark/internal/executor/tart/tart.go index bcf3733..3bf216e 100644 --- a/benchmark/internal/executor/tart/tart.go +++ b/benchmark/internal/executor/tart/tart.go @@ -7,11 +7,14 @@ import ( "fmt" "github.com/avast/retry-go/v4" "github.com/google/uuid" + "github.com/shirou/gopsutil/mem" "go.uber.org/zap" "go.uber.org/zap/zapio" "golang.org/x/crypto/ssh" "io" "net" + "runtime" + "strconv" "strings" "time" ) @@ -37,6 +40,23 @@ func New(ctx context.Context, image string, runArgsExtra []string, logger *zap.L return nil, err } + vmStat, err := mem.VirtualMemory() + if err != nil { + return nil, err + } + + cpus := strconv.Itoa(runtime.NumCPU()) + memory := strconv.FormatUint(vmStat.Total/1024/1024, 10) + logger.Info("Setting resources", zap.String("cpus", cpus), zap.String("memory", memory)) + setResourcesArguments := []string{ + "set", tart.vmName, + "--cpu", cpus, + "--memory", memory, + } + if err := Cmd(ctx, tart.logger, setResourcesArguments...); err != nil { + return nil, err + } + vmRunCtx, vmRunCancel := context.WithCancel(ctx) tart.vmRunCancel = vmRunCancel