public_git
/
tart
mirror of https://github.com/cirruslabs/tart.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use full host resources for Xcode benchmarks (#1045)

This commit is contained in:
Fedor Korotkov 2025-03-23 16:55:32 -04:00 committed by GitHub
parent 9c30638079
commit cb92a3fa67
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 37 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
benchmark/README.md
View File

@ -13,7 +13,7 @@ brew install go
Finally, run the following command from this (`benchmark/`) directory:
```shell
go run cmd/main.go fio --image ghcr.io/cirruslabs/macos-sonoma-base:latest --prepare 'sudo purge && sync'
go run cmd/main.go fio --image ghcr.io/cirruslabs/macos-sequoia-base:latest --prepare 'sudo purge && sync'
```
You can also enable the debugging output to diagnose issues:
@ -186,41 +186,25 @@ sync test Tart (--root-disk-opts="caching=cached"
sync test Tart (--root-disk-opts="sync=none,caching=cached") 0 B/s 17 MB/s 0 IOPS 7.39 kIOPS 0s ± 0s 21.23µs ± 81.749µs 113.239µs ± 191.266µs
```
### Jan 16, 2025
### March 23, 2025
Host:
* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
* OS: macOS Sequoia 15.2
* OS: macOS Sequoia 15.3.2
* Xcode: 16.2
Guest:
* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
* OS: macOS Sonoma 14.6
* OS: macOS Sonoma 15.3.2
* Xcode: 16.2
```
Name Executor Time
XcodeBenchmark (d869315) local 2m15s
XcodeBenchmark (d869315) Tart 4m22s
XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 4m21s
XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 4m15s
XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 4m16s
```
```
Name Executor Time
XcodeBenchmark (d869315) local 2m7s
XcodeBenchmark (d869315) Tart 4m37s
XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 4m35s
XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 4m19s
XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 4m16s
```
```
Name Executor Time
XcodeBenchmark (d869315) local 2m6s
XcodeBenchmark (d869315) Tart 4m24s
XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 4m22s
XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 4m18s
XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 4m17s
Name Executor Time
XcodeBenchmark (d869315) local 2m19s
XcodeBenchmark (d869315) Tart 3m59s
XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none") 3m48s
XcodeBenchmark (d869315) Tart (--root-disk-opts="caching=cached") 3m35s
XcodeBenchmark (d869315) Tart (--root-disk-opts="sync=none,caching=cached") 3m14s
```

1
benchmark/go.mod
View File

@ -22,6 +22,7 @@ require (
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sys v0.18.0 // indirect

2
benchmark/go.sum
View File

@ -26,6 +26,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

2
benchmark/internal/command/xcode/benchmarks.go
View File

@ -8,6 +8,6 @@ type Benchmark struct {
var benchmarks = []Benchmark{
{
Name: "XcodeBenchmark (d869315)",
Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && sh benchmark.sh",
Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && xcrun simctl list && sh benchmark.sh",
},
}

2
benchmark/internal/command/xcode/xcode.go
View File

@ -23,7 +23,7 @@ func NewCommand() *cobra.Command {
}
cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-xcode:latest", "image to use for testing")
cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sequoia-xcode:latest", "image to use for testing")
cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark")
return cmd

20
benchmark/internal/executor/tart/tart.go
View File

@ -7,11 +7,14 @@ import (
"fmt"
"github.com/avast/retry-go/v4"
"github.com/google/uuid"
"github.com/shirou/gopsutil/mem"
"go.uber.org/zap"
"go.uber.org/zap/zapio"
"golang.org/x/crypto/ssh"
"io"
"net"
"runtime"
"strconv"
"strings"
"time"
)
@ -37,6 +40,23 @@ func New(ctx context.Context, image string, runArgsExtra []string, logger *zap.L
return nil, err
}
vmStat, err := mem.VirtualMemory()
if err != nil {
return nil, err
}
cpus := strconv.Itoa(runtime.NumCPU())
memory := strconv.FormatUint(vmStat.Total/1024/1024, 10)
logger.Info("Setting resources", zap.String("cpus", cpus), zap.String("memory", memory))
setResourcesArguments := []string{
"set", tart.vmName,
"--cpu", cpus,
"--memory", memory,
}
if err := Cmd(ctx, tart.logger, setResourcesArguments...); err != nil {
return nil, err
}
vmRunCtx, vmRunCancel := context.WithCancel(ctx)
tart.vmRunCancel = vmRunCancel