Fix layerizer tests by providing a default maxRetries value

* docs(FAQ): document unsupported DHCP client identifiers

* New section "Resolving the VMs IP with bridged networking"

And a more clearer explanation of what "tart ip" does.

* Remove extraneous space in ` --resolver=arp`

* Better section name

* Add a note about Linux talkativeness

* Explain "talkativeness" a bit better

.cirrus.yml

@@ -17,8 +17,7 @@ task:
     - export PATH=$(pwd)/.build/arm64-apple-macosx/debug:$PATH
     # Run integration tests
     - cd integration-tests
-    - HOMEBREW_NO_AUTO_UPDATE=1 brew install virtualenv
-    - virtualenv venv
+    - python3 -m venv --symlinks venv
     - source venv/bin/activate
     - pip install -r requirements.txt
     - pytest --verbose --junit-xml=pytest-junit.xml
@@ -121,7 +120,7 @@ task:
     - security import certificate.p12 -k build.keychain -P password101 -T /usr/bin/codesign -T /usr/bin/pkgbuild
     - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password101 build.keychain
     - xcrun notarytool store-credentials "notarytool" --apple-id "hello@cirruslabs.org" --team-id "9M2P8L4D89" --password $AC_PASSWORD
-  install_script: 
+  install_script:
     - brew install go goreleaser/tap/goreleaser-pro getsentry/tools/sentry-cli
     - brew install mitchellh/gon/gon
   info_script:
@@ -153,7 +152,7 @@ task:
     registry_config: ENCRYPTED[!cf1a0f25325aa75bad3ce6ebc890bc53eb0044c02efa70d8cefb83ba9766275a994b4831706c52630a0692b2fa9cfb9e!]
   env:
     DEPLOY_TOKEN: ENCRYPTED[!45ed45666558902ed1c2400add734ec063103bec31841847e8c8764802fca229bfa6d85c690e16ad159e047574b48793!]
-  deploy_script: 
+  deploy_script:
     - git config --global user.name "Cirrus CI"
     - git config --global user.name "hello@cirruslabs.org"
     - git remote set-url origin https://$DEPLOY_TOKEN@github.com/cirruslabs/tart/

.goreleaser.yml

@@ -1,11 +1,12 @@
+version: 2
+
 project_name: tart
 
 before:
   hooks:
     - .ci/set-version.sh
-    - swift build --arch x86_64 -c release --product tart
-    - swift build --arch arm64 -c release --product tart
-    - gon gon.hcl
+    - swift build --arch arm64 --configuration release --product tart
+    - swift build --arch x86_64 --configuration release --product tart
 
 builds:
   - id: tart
@@ -20,8 +21,14 @@ builds:
     prebuilt:
       path: '.build/{{- if eq .Arch "arm64" }}arm64{{- else }}x86_64{{ end }}-apple-macosx/release/tart'
 
+universal_binaries:
+  - name_template: tart.app/Contents/MacOS/tart
+    replace: true
+    hooks:
+      post: gon gon.hcl
+
 archives:
-  - name_template: "{{ .ProjectName }}-{{ .Arch }}"
+  - name_template: "{{ .ProjectName }}"
     files:
       - src: Resources/embedded.provisionprofile
         dst: tart.app/Contents
@@ -42,7 +49,13 @@ brews:
     repository:
       owner: cirruslabs
       name: homebrew-cli
-    caveats: See the GitHub repository for more information
+    caveats: |
+      Tart has been installed. You might want to reduce the default DHCP lease time
+      from 86,400 to 600 seconds to avoid DHCP shortage when running lots of VMs daily:
+
+        sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.InternetSharing.default.plist bootpd -dict DHCPLeaseTimeSecs -int 600
+
+      See https://tart.run/faq/#changing-the-default-dhcp-lease-time for more details.
     homepage: https://github.com/cirruslabs/tart
     license: "Fair Source"
     description: Run macOS and Linux VMs on Apple Hardware

Sources/tart/Commands/Clone.swift

@@ -31,6 +31,15 @@ struct Clone: AsyncParsableCommand {
   @Flag(help: .hidden)
   var deduplicate: Bool = false
 
+  @Option(help: .hidden)
+  var proxy: String?
+
+  @Option(help: .hidden)
+  var caCert: String?
+
+  @Option(help: .hidden)
+  var maxRetries: UInt = 5
+
   func validate() throws {
     if newName.contains("/") {
       throw ValidationError("<new-name> should be a local name")
@@ -47,8 +56,8 @@ struct Clone: AsyncParsableCommand {
 
     if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
       // Pull the VM in case it's OCI-based and doesn't exist locally yet
-      let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
-      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
+      let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure, proxy: proxy, caCert: caCert)
+      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate, maxRetries: maxRetries)
     }
 
     let sourceVM = try VMStorageHelper.open(sourceName)

Sources/tart/Commands/Pull.swift

@@ -26,6 +26,15 @@ struct Pull: AsyncParsableCommand {
   @Flag(help: .hidden)
   var deduplicate: Bool = false
 
+  @Option(help: .hidden)
+  var proxy: String?
+
+  @Option(help: .hidden)
+  var caCert: String?
+
+  @Option(help: .hidden)
+  var maxRetries: UInt = 5
+
   func validate() throws {
     if concurrency < 1 {
       throw ValidationError("network concurrency cannot be less than 1")
@@ -42,10 +51,10 @@ struct Pull: AsyncParsableCommand {
     }
 
     let remoteName = try RemoteName(remoteName)
-    let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
+    let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure, proxy: proxy, caCert: caCert)
 
     defaultLogger.appendNewLine("pulling \(remoteName)...")
 
-    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
+    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate, maxRetries: maxRetries)
   }
 }

Sources/tart/Commands/Run.swift

@@ -183,13 +183,49 @@ struct Run: AsyncParsableCommand {
   """, valueName: "interface name"))
   var netBridged: [String] = []
 
-  @Flag(help: ArgumentHelp("Use software networking instead of the default shared (NAT) networking",
-                           discussion: "Learn how to configure Softnet for use with Tart here: https://github.com/cirruslabs/softnet"))
+  @Flag(help: ArgumentHelp("Use software networking provided by Softnet instead of the default shared (NAT) networking",
+                           discussion: """
+                           Softnet provides better network isolation and alleviates DHCP shortage on production systems. Tart invokes Softnet when this option is specified as a sub-process and communicates with it over socketpair(2).
+
+                           It is essentially a userspace packet filter which restricts the VM networking and prevents a class of security issues, such as ARP spoofing. By default, the VM will only be able to:
+
+                           * send traffic from its own MAC-address
+                           * send traffic from the IP-address assigned to it by the DHCP
+                           * send traffic to globally routable IPv4 addresses
+                           * send traffic to gateway IP of the vmnet bridge (this would normally be \"bridge100\" interface)
+                           * receive any incoming traffic
+
+                           In addition, Softnet tunes macOS built-in DHCP server to decrease its lease time from the default 86,400 seconds (one day) to 600 seconds (10 minutes). This is especially important when you use Tart to clone and run a lot of ephemeral VMs over a period of one day.
+
+                           More on Softnet here: https://github.com/cirruslabs/softnet
+                           """))
   var netSoftnet: Bool = false
 
-  @Option(help: ArgumentHelp("Comma-separated list of CIDRs to allow the traffic to when using Softnet isolation\n(e.g. --net-softnet-allow=192.168.0.0/24)", valueName: "comma-separated CIDRs"))
+  @Option(help: ArgumentHelp("Comma-separated list of CIDRs to allow the traffic to when using Softnet isolation\n(e.g. --net-softnet-allow=192.168.0.0/24)", discussion: """
+  This option allows you bypass the private IPv4 address space restrictions imposed by --net-softnet.
+
+  For example, you can allow the VM to communicate with the local network with e.g. --net-softnet-allow=10.0.0.0/16 or to completely disable the destination based restrictions with --net-softnet-allow=0.0.0.0/0.
+
+  Implies --net-softnet.
+  """, valueName: "comma-separated CIDRs"))
   var netSoftnetAllow: String?
 
+  @Option(help: ArgumentHelp("Comma-separated list of TCP ports to expose (e.g. --net-softnet-expose 2222:22,8080:80)", discussion: """
+  Options are comma-separated and are as follows:
+
+  * EXTERNAL_PORT:INTERNAL_PORT — forward TCP traffic from the EXTERNAL_PORT on a host's egress interface (automatically detected and could be Wi-Fi, Ethernet and a VPN interface) to the INTERNAL_PORT on guest's IP (as reported by "tart ip")
+
+  Note that for the port forwarding to work correctly:
+
+  * the software in guest listening on INTERNAL_PORT should either listen on 0.0.0.0 or on an IP address assigned to that guest
+  * connection to the EXTERNAL_PORT should be performed from the local network that the host is attached to or from the internet, it's not possible to connect to that forwarded port from the host itself
+
+  Another thing to keep in mind is that regular Softnet restrictions will still apply even to port forwarding. So if you're planning to access your VM from local network, and your local network is 192.168.0.0/24, for example, then add --net-softnet-allow=192.168.0.0/24. If you only need port forwarding, to completely disable Softnet restrictions you can use --net-softnet-allow=0.0.0.0/0.
+
+  Implies --net-softnet.
+  """, valueName: "comma-separated port specifications"))
+  var netSoftnetExpose: String?
+
   @Flag(help: ArgumentHelp("Restrict network access to the host-only network"))
   var netHost: Bool = false
 
@@ -229,7 +265,12 @@ struct Run: AsyncParsableCommand {
       throw ValidationError("--vnc and --vnc-experimental are mutually exclusive")
     }
 
-    // check that not more than one network option is specified
+    // Automatically enable --net-softnet when any of its related options are specified
+    if netSoftnetAllow != nil || netSoftnetExpose != nil {
+      netSoftnet = true
+    }
+
+    // Check that no more than one network option is specified
     var netFlags = 0
     if netBridged.count > 0 { netFlags += 1 }
     if netSoftnet { netFlags += 1 }
@@ -527,6 +568,10 @@ struct Run: AsyncParsableCommand {
       softnetExtraArguments += ["--allow", netSoftnetAllow]
     }
 
+    if let netSoftnetExpose = netSoftnetExpose {
+      softnetExtraArguments += ["--expose", netSoftnetExpose]
+    }
+
     if netSoftnet {
       let config = try VMConfig.init(fromURL: vmDir.configURL)
 
@@ -1029,7 +1074,7 @@ struct DirectoryShare {
     process.standardInput = inPipe
     process.launch()
 
-    inPipe.fileHandleForWriting.write(response!.data)
+    try inPipe.fileHandleForWriting.write(contentsOf: response!.data)
     try inPipe.fileHandleForWriting.close()
     process.waitUntilExit()
 

Sources/tart/Commands/Set.swift

@@ -73,8 +73,7 @@ struct Set: AsyncParsableCommand {
     }
 
     #if arch(arm64)
-      if randomSerial {
-        let oldPlatform = vmConfig.platform as! Darwin
+      if randomSerial, let oldPlatform = vmConfig.platform as? Darwin {
         vmConfig.platform = Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: oldPlatform.hardwareModel)
       }
     #endif

Sources/tart/Credentials/DockerConfigCredentialsProvider.swift

@@ -36,7 +36,11 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
 
     process.launch()
 
-    inPipe.fileHandleForWriting.write("\(host)\n".data(using: .utf8)!)
+    do {
+      try inPipe.fileHandleForWriting.write(contentsOf: "\(host)\n".data(using: .utf8)!)
+    } catch {
+      throw CredentialsProviderError.Failed(message: "Failed to write host to Docker helper!")
+    }
     inPipe.fileHandleForWriting.closeFile()
 
     let outputData = try outPipe.fileHandleForReading.readToEnd()

Sources/tart/Fetcher.swift

@@ -1,26 +1,61 @@
 import Foundation
 
-fileprivate var urlSession: URLSession = {
-  let config = URLSessionConfiguration.default
-
-  // Harbor expects a CSRF token to be present if the HTTP client
-  // carries a session cookie between its requests[1] and fails if
-  // it was not present[2].
-  //
-  // To fix that, we disable the automatic cookies carry in URLSession.
-  //
-  // [1]: https://github.com/goharbor/harbor/blob/a4c577f9ec4f18396207a5e686433a6ba203d4ef/src/server/middleware/csrf/csrf.go#L78
-  // [2]: https://github.com/cirruslabs/tart/issues/295
-  config.httpShouldSetCookies = false
-
-  return URLSession(configuration: config)
-}()
-
 class Fetcher {
-  static func fetch(_ request: URLRequest, viaFile: Bool = false, progress: Progress? = nil) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
-    let task = urlSession.dataTask(with: request)
+  let urlSession: URLSession
+  let caCert: SecCertificate?
 
-    let delegate = Delegate()
+  init(proxy: String? = nil, caCert: String? = nil) throws {
+    // Configure URLSession
+    let config = URLSessionConfiguration.default
+
+    // Harbor expects a CSRF token to be present if the HTTP client
+    // carries a session cookie between its requests[1] and fails if
+    // it was not present[2].
+    //
+    // To fix that, we disable the automatic cookies carry in URLSession.
+    //
+    // [1]: https://github.com/goharbor/harbor/blob/a4c577f9ec4f18396207a5e686433a6ba203d4ef/src/server/middleware/csrf/csrf.go#L78
+    // [2]: https://github.com/cirruslabs/tart/issues/295
+    config.httpShouldSetCookies = false
+
+    if let proxy {
+      let (host, port) = try Self.parseProxy(proxy)
+
+      config.connectionProxyDictionary = [
+        kCFNetworkProxiesHTTPEnable: true,
+        kCFNetworkProxiesHTTPProxy: host,
+        kCFNetworkProxiesHTTPPort: port,
+
+        kCFNetworkProxiesHTTPSEnable: true,
+        kCFNetworkProxiesHTTPSProxy: host,
+        kCFNetworkProxiesHTTPSPort: port,
+      ]
+    }
+
+    self.urlSession = URLSession(configuration: config)
+
+    // Load CA certificate, if any
+    if let caCert {
+      let caCertString = try String(contentsOf: URL(filePath: caCert), encoding:. utf8)
+
+      let caCertBase64Lines = caCertString.components(separatedBy: .newlines).filter { line in
+        !line.hasPrefix("-----BEGIN") && !line.hasPrefix("-----END")
+      }
+
+      guard let caCertData = Data(base64Encoded: caCertBase64Lines.joined()) else {
+        throw RuntimeError.FailedToLoadCACertificate("failed to parse Base64-encoded PEM data")
+      }
+
+      self.caCert = SecCertificateCreateWithData(nil, caCertData as CFData)!
+    } else {
+      self.caCert = nil
+    }
+  }
+
+  func fetch(_ request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
+    let task = self.urlSession.dataTask(with: request)
+
+    let delegate = Delegate(caCert: self.caCert)
     task.delegate = delegate
 
     let stream = AsyncThrowingStream<Data, Error> { continuation in
@@ -34,15 +69,78 @@ class Fetcher {
 
     return (stream, response as! HTTPURLResponse)
   }
+
+  private static func parseProxy(_ proxy: String) throws -> (String, Int) {
+    // Assume that the scheme is specified
+    var url = URL(string: proxy)
+
+    // Fall back to HTTP scheme when not specified
+    if url?.scheme == nil {
+      url = URL(string: "http://\(proxy)")
+    }
+
+    guard let url else {
+      throw RuntimeError.InvalidProxyString
+    }
+
+    guard let host = url.host() else {
+      throw RuntimeError.InvalidProxyString
+    }
+
+    guard let port = url.port else {
+      throw RuntimeError.InvalidProxyString
+    }
+
+    return (host, port)
+  }
 }
 
-fileprivate class Delegate: NSObject, URLSessionDataDelegate {
+fileprivate class Delegate: NSObject, URLSessionDelegate, URLSessionDataDelegate {
+  let caCert: SecCertificate?
   var responseContinuation: CheckedContinuation<URLResponse, Error>?
   var streamContinuation: AsyncThrowingStream<Data, Error>.Continuation?
 
   private var buffer: Data = Data()
   private let bufferFlushSize = 16 * 1024 * 1024
 
+  init(caCert: SecCertificate?) {
+    self.caCert = caCert
+  }
+
+  func urlSession(
+    _ session: URLSession,
+    didReceive challenge: URLAuthenticationChallenge,
+    completionHandler: @escaping @Sendable (URLSession.AuthChallengeDisposition, URLCredential?) -> Void
+  ) {
+    if let caCert {
+      // Ensure that we're performing server trust authentication
+      guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust,
+            let serverTrust = challenge.protectionSpace.serverTrust else {
+        completionHandler(.performDefaultHandling, nil)
+
+        return
+      }
+
+      // Set the provided CA certificate as the only anchor
+      if SecTrustSetAnchorCertificates(serverTrust, [caCert] as CFArray) != errSecSuccess {
+        completionHandler(.cancelAuthenticationChallenge, nil)
+
+        return
+      }
+
+      // Evaluate the trust
+      if SecTrustEvaluateWithError(serverTrust, nil) {
+        completionHandler(.useCredential, URLCredential(trust: serverTrust))
+      } else {
+        completionHandler(.rejectProtectionSpace, nil)
+      }
+
+      return
+    }
+
+    completionHandler(.performDefaultHandling, nil)
+  }
+
   func urlSession(
     _ session: URLSession,
     dataTask: URLSessionDataTask,

Sources/tart/OCI/Layerizer/Disk.swift

@@ -2,5 +2,5 @@ import Foundation
 
 protocol Disk {
   static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool, maxRetries: UInt) async throws
 }

Sources/tart/OCI/Layerizer/DiskV1.swift

@@ -45,7 +45,7 @@ class DiskV1: Disk {
     return pushedLayers
   }
 
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false, maxRetries: UInt = 5) async throws {
     if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
       throw OCIError.FailedToCreateVmFile
     }
@@ -57,7 +57,7 @@ class DiskV1: Disk {
     // Decompress the layers onto the disk in a single stream
     let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
       if let data = data {
-        disk.write(data)
+        try disk.write(contentsOf: data)
       }
     }
 

Sources/tart/OCI/Layerizer/DiskV2.swift

@@ -44,7 +44,7 @@ class DiskV2: Disk {
           let compressedData = try (data as NSData).compressed(using: .lz4) as Data
           let compressedDataDigest = Digest.hash(compressedData)
 
-          try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) {
+          try await retry(maxAttempts: 5) {
             if try await !registry.blobExists(compressedDataDigest) {
               _ = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb, digest: compressedDataDigest)
             }
@@ -84,7 +84,7 @@ class DiskV2: Disk {
     }
   }
 
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false, maxRetries: UInt = 5) async throws {
     // Support resumable pulls
     let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)
 
@@ -208,11 +208,26 @@ class DiskV2: Disk {
             diskWritingOffset = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
           }
 
-          try await registry.pullBlob(diskLayer.digest) { data in
-            try filter.write(data)
+          var rangeStart: Int64 = 0
 
-            // Update the progress
-            progress.completedUnitCount += Int64(data.count)
+          try await retry(maxAttempts: Int(maxRetries)) {
+            try await registry.pullBlob(diskLayer.digest, rangeStart: rangeStart) { data in
+              try filter.write(data)
+
+              // Update the progress
+              progress.completedUnitCount += Int64(data.count)
+
+              // Update the current range start
+              rangeStart += Int64(data.count)
+            }
+          } recoverFromFailure: { error in
+            if error is URLError {
+              print("Error pulling disk layer \(index + 1): \"\(error.localizedDescription)\", attempting to re-try...")
+
+              return .retry
+            }
+
+            return .throw
           }
 
           try filter.finalize()
@@ -253,7 +268,7 @@ class DiskV2: Disk {
 
           if chunk != actualContentsOnDisk {
             try disk.seek(toOffset: offset)
-            disk.write(chunk)
+            try disk.write(contentsOf: chunk)
           }
         }
 
@@ -267,7 +282,7 @@ class DiskV2: Disk {
       // is zeroed via truncate(2)
       if chunk != zeroChunk {
         try disk.seek(toOffset: offset)
-        disk.write(chunk)
+        try disk.write(contentsOf: chunk)
       }
 
       offset += UInt64(chunk.count)

Sources/tart/OCI/Registry.swift

@@ -20,6 +20,7 @@ enum HTTPCode: Int {
   case Ok = 200
   case Created = 201
   case Accepted = 202
+  case PartialContent = 206
   case Unauthorized = 401
   case NotFound = 404
 }
@@ -28,14 +29,26 @@ extension Data {
   func asText() -> String {
     String(decoding: self, as: UTF8.self)
   }
+
+  func asTextPreview(limit: Int = 1000) -> String {
+    guard count > limit else {
+      return asText()
+    }
+
+    return "\(asText().prefix(limit))..."
+  }
 }
 
 extension AsyncThrowingStream<Data, Error> {
-  func asData() async throws -> Data {
+  func asData(limitBytes: Int64? = nil) async throws -> Data {
     var result = Data()
 
     for try await chunk in self {
       result += chunk
+
+      if let limitBytes, result.count > limitBytes {
+        return result
+      }
     }
 
     return result
@@ -102,6 +115,7 @@ class Registry {
   let namespace: String
   let credentialsProviders: [CredentialsProvider]
   let authenticationKeeper = AuthenticationKeeper()
+  let fetcher: Fetcher
 
   var host: String? {
     guard let host = baseURL.host else { return nil }
@@ -115,17 +129,22 @@ class Registry {
 
   init(baseURL: URL,
        namespace: String,
-       credentialsProviders: [CredentialsProvider] = [EnvironmentCredentialsProvider(), DockerConfigCredentialsProvider(), KeychainCredentialsProvider()]
+       credentialsProviders: [CredentialsProvider] = [EnvironmentCredentialsProvider(), DockerConfigCredentialsProvider(), KeychainCredentialsProvider()],
+       proxy: String? = nil,
+       caCert: String? = nil
   ) throws {
     self.baseURL = baseURL
     self.namespace = namespace
     self.credentialsProviders = credentialsProviders
+    self.fetcher = try Fetcher(proxy: proxy, caCert: caCert)
   }
 
   convenience init(
     host: String,
     namespace: String,
     insecure: Bool = false,
+    proxy: String? = nil,
+    caCert: String? = nil,
     credentialsProviders: [CredentialsProvider] = [EnvironmentCredentialsProvider(), DockerConfigCredentialsProvider(), KeychainCredentialsProvider()]
   ) throws {
     let proto = insecure ? "http" : "https"
@@ -141,7 +160,7 @@ class Registry {
       throw RuntimeError.ImproperlyFormattedHost(host, hint)
     }
 
-    try self.init(baseURL: baseURL, namespace: namespace, credentialsProviders: credentialsProviders)
+    try self.init(baseURL: baseURL, namespace: namespace, credentialsProviders: credentialsProviders, proxy: proxy, caCert: caCert)
   }
 
   func ping() async throws {
@@ -159,7 +178,7 @@ class Registry {
                                                  body: manifestJSON)
     if response.statusCode != HTTPCode.Created.rawValue {
       throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing manifest", code: response.statusCode,
-                                                   details: data.asText())
+                                                   details: data.asTextPreview())
     }
 
     return Digest.hash(manifestJSON)
@@ -170,7 +189,7 @@ class Registry {
                                                  headers: ["Accept": ociManifestMediaType])
     if response.statusCode != HTTPCode.Ok.rawValue {
       throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling manifest", code: response.statusCode,
-                                                   details: data.asText())
+                                                   details: data.asTextPreview())
     }
 
     let manifest = try OCIManifest(fromJSON: data)
@@ -196,7 +215,7 @@ class Registry {
                                                      headers: ["Content-Length": "0"])
     if postResponse.statusCode != HTTPCode.Accepted.rawValue {
       throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (POST)", code: postResponse.statusCode,
-                                                   details: data.asText())
+                                                   details: data.asTextPreview())
     }
 
     // Figure out where to upload the blob
@@ -217,7 +236,7 @@ class Registry {
       )
       if response.statusCode != HTTPCode.Created.rawValue {
         throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (PUT) to \(uploadLocation)",
-                                                     code: response.statusCode, details: data.asText())
+                                                     code: response.statusCode, details: data.asTextPreview())
       }
       return digest
     }
@@ -240,7 +259,7 @@ class Registry {
       // always accept both statuses since AWS ECR is not following specification
       if response.statusCode != HTTPCode.Created.rawValue && response.statusCode != HTTPCode.Accepted.rawValue {
         throw RegistryError.UnexpectedHTTPStatusCode(when: "streaming blob to \(uploadLocation)",
-                                                     code: response.statusCode, details: data.asText())
+                                                     code: response.statusCode, details: data.asTextPreview())
       }
       uploadedBytes += chunk.count
       // Update location for the next chunk
@@ -259,14 +278,26 @@ class Registry {
     case HTTPCode.NotFound.rawValue:
       return false
     default:
-      throw RegistryError.UnexpectedHTTPStatusCode(when: "checking blob", code: response.statusCode, details: data.asText())
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "checking blob", code: response.statusCode, details: data.asTextPreview())
     }
   }
 
-  public func pullBlob(_ digest: String, handler: (Data) async throws -> Void) async throws {
-    let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), viaFile: true)
-    if response.statusCode != HTTPCode.Ok.rawValue {
-      let body = try await channel.asData().asText()
+  public func pullBlob(_ digest: String, rangeStart: Int64 = 0, handler: (Data) async throws -> Void) async throws {
+    var expectedStatusCode = HTTPCode.Ok
+    var headers: [String: String] = [:]
+
+    // Send Range header and expect HTTP 206 in return
+    //
+    // However, do not send Range header at all when rangeStart is 0,
+    // because it makes no sense and we might get HTTP 200 in return
+    if rangeStart != 0 {
+      expectedStatusCode = HTTPCode.PartialContent
+      headers["Range"] = "bytes=\(rangeStart)-"
+    }
+
+    let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), headers: headers, viaFile: true)
+    if response.statusCode != expectedStatusCode.rawValue {
+      let body = try await channel.asData(limitBytes: 4096).asTextPreview()
       throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling blob", code: response.statusCode,
                                                    details: body)
     }
@@ -329,7 +360,6 @@ class Registry {
     var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)
 
     if doAuth && response.statusCode == HTTPCode.Unauthorized.rawValue {
-      _ = try await channel.asData()
       try await auth(response: response)
       (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)
     }
@@ -391,7 +421,7 @@ class Registry {
     let (data, response) = try await dataRequest(.GET, authenticateURL, headers: headers, doAuth: false)
     if response.statusCode != HTTPCode.Ok.rawValue {
       throw RegistryError.AuthFailed(why: "received unexpected HTTP status code \(response.statusCode) "
-        + "while retrieving an authentication token", details: data.asText())
+        + "while retrieving an authentication token", details: data.asTextPreview())
     }
 
     await authenticationKeeper.set(try TokenResponse.parse(fromData: data))
@@ -424,6 +454,6 @@ class Registry {
     request.setValue("Tart/\(CI.version) (\(DeviceInfo.os); \(DeviceInfo.model))",
                      forHTTPHeaderField: "User-Agent")
 
-    return try await Fetcher.fetch(request, viaFile: viaFile)
+    return try await self.fetcher.fetch(request, viaFile: viaFile)
   }
 }

Sources/tart/VM.swift

@@ -83,7 +83,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Check if we already have this IPSW in cache
     var headRequest = URLRequest(url: remoteURL)
     headRequest.httpMethod = "HEAD"
-    let (_, headResponse) = try await Fetcher.fetch(headRequest, viaFile: false)
+    let (_, headResponse) = try await Fetcher().fetch(headRequest, viaFile: false)
 
     if let hash = headResponse.value(forHTTPHeaderField: "x-amz-meta-digest-sha256") {
       let ipswLocation = try IPSWCache().locationFor(fileName: "sha256:\(hash).ipsw")
@@ -99,16 +99,13 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Download the IPSW
     defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")
 
-    let downloadProgress = Progress(totalUnitCount: 100)
-    ProgressObserver(downloadProgress).log(defaultLogger)
-
     let request = URLRequest(url: remoteURL)
-    let (channel, response) = try await Fetcher.fetch(request, viaFile: true, progress: downloadProgress)
+    let (channel, response) = try await Fetcher().fetch(request, viaFile: true)
 
     let temporaryLocation = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString + ".ipsw")
-    defaultLogger.appendNewLine("Computing digest for \(temporaryLocation.path)...")
-    let digestProgress = Progress(totalUnitCount: response.expectedContentLength)
-    ProgressObserver(digestProgress).log(defaultLogger)
+
+    let progress = Progress(totalUnitCount: response.expectedContentLength)
+    ProgressObserver(progress).log(defaultLogger)
 
     FileManager.default.createFile(atPath: temporaryLocation.path, contents: nil)
     let lock = try FileLock(lockURL: temporaryLocation)
@@ -118,10 +115,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     let digest = Digest()
 
     for try await chunk in channel {
-      let chunkAsData = Data(chunk)
-      fileHandle.write(chunkAsData)
-      digest.update(chunkAsData)
-      digestProgress.completedUnitCount += Int64(chunk.count)
+      try fileHandle.write(contentsOf: chunk)
+      digest.update(chunk)
+      progress.completedUnitCount += Int64(chunk.count)
     }
 
     try fileHandle.close()

Sources/tart/VMDirectory+OCI.swift

@@ -11,7 +11,14 @@ enum OCIError: Error {
 }
 
 extension VMDirectory {
-  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws {
+  func pullFromRegistry(
+    registry: Registry,
+    manifest: OCIManifest,
+    concurrency: UInt,
+    localLayerCache: LocalLayerCache?,
+    deduplicate: Bool,
+    maxRetries: UInt
+  ) async throws {
     // Pull VM's config file layer and re-serialize it into a config file
     let configLayers = manifest.layers.filter {
       $0.mediaType == configMediaType
@@ -24,7 +31,7 @@ extension VMDirectory {
     }
     let configFile = try FileHandle(forWritingTo: configURL)
     try await registry.pullBlob(configLayers.first!.digest) { data in
-      configFile.write(data)
+      try configFile.write(contentsOf: data)
     }
     try configFile.close()
 
@@ -55,7 +62,8 @@ extension VMDirectory {
       try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL,
                                   concurrency: concurrency, progress: progress,
                                   localLayerCache: localLayerCache,
-                                  deduplicate: deduplicate)
+                                  deduplicate: deduplicate,
+                                  maxRetries: maxRetries)
     } catch let error where error is FilterError {
       throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)")
     }
@@ -79,7 +87,7 @@ extension VMDirectory {
     }
     let nvram = try FileHandle(forWritingTo: nvramURL)
     try await registry.pullBlob(nvramLayers.first!.digest) { data in
-      nvram.write(data)
+      try nvram.write(contentsOf: data)
     }
     try nvram.close()
 

Sources/tart/VMStorageHelper.swift

@@ -75,6 +75,8 @@ enum RuntimeError : Error {
   case SuspendFailed(_ message: String)
   case PullFailed(_ message: String)
   case VirtualMachineLimitExceeded(_ hint: String)
+  case InvalidProxyString
+  case FailedToLoadCACertificate(_ message: String)
 }
 
 protocol HasExitCode {
@@ -136,6 +138,10 @@ extension RuntimeError : CustomStringConvertible {
       return message
     case .VirtualMachineLimitExceeded(let hint):
       return "The number of VMs exceeds the system limit\(hint)"
+    case .InvalidProxyString:
+      return "Invalid proxy string, should be in the form of host:port"
+    case .FailedToLoadCACertificate(let message):
+      return "Failed to load CA certificate: \(message)"
     }
   }
 }

Sources/tart/VMStorageOCI.swift

@@ -140,7 +140,7 @@ class VMStorageOCI: PrunableStorage {
     try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
   }
 
-  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws {
+  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool, maxRetries: UInt) async throws {
     SentrySDK.configureScope { scope in
       scope.setContext(value: ["imageName": name.description], key: "OCI")
     }
@@ -196,7 +196,7 @@ class VMStorageOCI: PrunableStorage {
       }
 
       try await withTaskCancellationHandler(operation: {
-        try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) {
+        try await retry(maxAttempts: Int(maxRetries)) {
           // Choose the best base image which has the most deduplication ratio
           let localLayerCache = try await chooseLocalLayerCache(name, manifest, registry)
 
@@ -210,11 +210,10 @@ class VMStorageOCI: PrunableStorage {
             }
           }
 
-          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache, deduplicate: deduplicate)
+          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache, deduplicate: deduplicate, maxRetries: maxRetries)
         } recoverFromFailure: { error in
           if error is URLError {
-            print("Error: \(error.localizedDescription)")
-            print("Attempting to re-try...")
+            print("Error pulling image: \"\(error.localizedDescription)\", attempting to re-try...")
 
             return .retry
           }

benchmark/README.md

@@ -185,3 +185,42 @@ sync test                               	Tart (--root-disk-opts="sync=none")
 sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	35 MB/s    	0 IOPS     	15.67 kIOPS 	0s ± 0s                   	11.319µs ± 24.771µs       	51.731µs ± 42.208µs    
 sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	17 MB/s    	0 IOPS     	7.39 kIOPS  	0s ± 0s                   	21.23µs ± 81.749µs        	113.239µs ± 191.266µs  
 ```
+
+### Jan 16, 2025
+
+Host:
+
+* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
+* OS: macOS Sequoia 15.2
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.6
+
+```
+Name                    	Executor                                          	Time
+XcodeBenchmark (d869315)	local                                             	2m15s
+XcodeBenchmark (d869315)	Tart                                              	4m22s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	4m21s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	4m15s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	4m16s
+```
+
+```
+Name                    	Executor                                          	Time
+XcodeBenchmark (d869315)	local                                             	2m7s
+XcodeBenchmark (d869315)	Tart                                              	4m37s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	4m35s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	4m19s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	4m16s
+```
+
+```
+Name                    	Executor                                          	Time
+XcodeBenchmark (d869315)	local                                             	2m6s
+XcodeBenchmark (d869315)	Tart                                              	4m24s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	4m22s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	4m18s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	4m17s
+```

benchmark/internal/command/fio/fio.go

@@ -4,8 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
 	"github.com/dustin/go-humanize"
 	"github.com/gosuri/uitable"
 	"github.com/spf13/cobra"
@@ -46,57 +44,12 @@ func run(cmd *cobra.Command, args []string) error {
 		_ = logger.Sync()
 	}()
 
-	var executorInitializers = []struct {
-		Name string
-		Fn   func() (executorpkg.Executor, error)
-	}{
-		{
-			Name: "local",
-			Fn: func() (executorpkg.Executor, error) {
-				return local.New(logger)
-			},
-		},
-		{
-			Name: "Tart",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, nil, logger)
-			},
-		},
-		{
-			Name: "Tart (--root-disk-opts=\"sync=none\")",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, []string{
-					"--root-disk-opts",
-					"sync=none",
-				}, logger)
-			},
-		},
-		{
-			Name: "Tart (--root-disk-opts=\"caching=cached\")",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, []string{
-					"--root-disk-opts",
-					"caching=cached",
-				}, logger)
-			},
-		},
-		{
-			Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, []string{
-					"--root-disk-opts",
-					"sync=none,caching=cached",
-				}, logger)
-			},
-		},
-	}
-
 	table := uitable.New()
 	table.AddRow("Name", "Executor", "B/W (read)", "B/W (write)", "I/O (read)", "I/O (write)",
 		"Latency (read)", "Latency (write)", "Latency (sync)")
 
 	for _, benchmark := range benchmarks {
-		for _, executorInitializer := range executorInitializers {
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
 			if prepare != "" {
 				shell := "/bin/sh"
 

benchmark/internal/command/root.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"github.com/cirruslabs/tart/benchmark/internal/command/fio"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
 	"github.com/spf13/cobra"
 )
 
@@ -14,6 +15,7 @@ func NewCommand() *cobra.Command {
 
 	cmd.AddCommand(
 		fio.NewCommand(),
+		xcode.NewCommand(),
 	)
 
 	return cmd

benchmark/internal/command/xcode/benchmarks.go

@@ -0,0 +1,13 @@
+package xcode
+
+type Benchmark struct {
+	Name    string
+	Command string
+}
+
+var benchmarks = []Benchmark{
+	{
+		Name:    "XcodeBenchmark (d869315)",
+		Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && sh benchmark.sh",
+	},
+}

benchmark/internal/command/xcode/output.go

@@ -0,0 +1,49 @@
+package xcode
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+)
+
+type Output struct {
+	Started time.Time
+	Ended   time.Time
+}
+
+func ParseOutput(s string) (*Output, error) {
+	// Ensure that the build has succeeded
+	matched, err := regexp.MatchString("(?m)^\\*\\* BUILD SUCCEEDED \\*\\*.*$", s)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse output: regexp failed: %v", err)
+	}
+	if !matched {
+		return nil, fmt.Errorf("failed to parse output: \"** BUILD SUCCEEDED **\" string " +
+			"not found on a separate line, make sure you have Xcode installed")
+	}
+
+	re := regexp.MustCompile("Started\\s+(?P<started>.*)\\n.*Ended\\s+(?P<ended>.*)\\n")
+
+	matches := re.FindStringSubmatch(s)
+
+	if len(matches) != re.NumSubexp()+1 {
+		return nil, fmt.Errorf("failed to parse output: cannot find Started and Ended times")
+	}
+
+	startedRaw := matches[re.SubexpIndex("started")]
+	started, err := time.Parse(time.TimeOnly, startedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse started time %q: unsupported format", startedRaw)
+	}
+
+	endedRaw := matches[re.SubexpIndex("ended")]
+	ended, err := time.Parse(time.TimeOnly, endedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse ended time %q: unsupported format", startedRaw)
+	}
+
+	return &Output{
+		Started: started,
+		Ended:   ended,
+	}, nil
+}

benchmark/internal/command/xcode/output_test.go

@@ -0,0 +1,37 @@
+package xcode_test
+
+import (
+	"fmt"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func TestParseOutput(t *testing.T) {
+	result, err := xcode.ParseOutput(`** BUILD SUCCEEDED ** [219.713 sec]
+
+System Version: 14.6
+Xcode 15.4
+Hardware Overview
+      Model Name: Apple Virtual Machine 1
+      Model Identifier: VirtualMac2,1
+      Total Number of Cores: 4
+      Memory: 8 GB
+
+✅ XcodeBenchmark has completed
+1️⃣  Take a screenshot of this window (Cmd + Shift + 4 + Space) and resize to include:
+	- Build Time (See ** BUILD SUCCEEDED ** [XYZ sec])
+	- System Version
+	- Xcode Version
+	- Hardware Overview
+	- Started 13:46:20
+	- Ended   13:50:02
+	- Date Thu Jan 16 13:50:02 UTC 2025
+
+2️⃣  Share your results at https://github.com/devMEremenko/XcodeBenchmark
+`)
+	require.NoError(t, err)
+	fmt.Println(result)
+	require.Equal(t, 222*time.Second, result.Ended.Sub(result.Started))
+}

benchmark/internal/command/xcode/xcode.go

@@ -0,0 +1,108 @@
+package xcode
+
+import (
+	"fmt"
+	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
+	"github.com/gosuri/uitable"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"os"
+	"os/exec"
+)
+
+var debug bool
+var image string
+var prepare string
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "xcode",
+		Short: "run XCode benchmarks",
+		RunE:  run,
+	}
+
+	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+	cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-xcode:latest", "image to use for testing")
+	cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark")
+
+	return cmd
+}
+
+func run(cmd *cobra.Command, args []string) error {
+	config := zap.NewProductionConfig()
+	if debug {
+		config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+	logger, err := config.Build()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = logger.Sync()
+	}()
+
+	table := uitable.New()
+	table.AddRow("Name", "Executor", "Time")
+
+	for _, benchmark := range benchmarks {
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
+			if prepare != "" {
+				shell := "/bin/sh"
+
+				if shellFromEnv, ok := os.LookupEnv("SHELL"); ok {
+					shell = shellFromEnv
+				}
+
+				logger.Sugar().Infof("running prepare command %q using shell %q",
+					prepare, shell)
+
+				cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare)
+
+				loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+
+				cmd.Stdout = loggerWriter
+				cmd.Stderr = loggerWriter
+
+				if err := cmd.Run(); err != nil {
+					return fmt.Errorf("failed to run prepare command %q: %v", prepare, err)
+				}
+			}
+
+			logger.Sugar().Infof("initializing executor %s", executorInitializer.Name)
+
+			executor, err := executorInitializer.Fn()
+			if err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name,
+				executorInitializer.Name)
+
+			stdout, err := executor.Run(cmd.Context(), benchmark.Command)
+			if err != nil {
+				return err
+			}
+
+			output, err := ParseOutput(string(stdout))
+			if err != nil {
+				return err
+			}
+
+			duration := output.Ended.Sub(output.Started)
+
+			logger.Sugar().Infof("Xcode benchmark duration: %s", duration)
+
+			table.AddRow(benchmark.Name, executorInitializer.Name, duration)
+
+			if err := executor.Close(); err != nil {
+				return fmt.Errorf("failed to close executor %s: %w",
+					executorInitializer.Name, err)
+			}
+		}
+	}
+
+	fmt.Println(table.String())
+
+	return nil
+}

benchmark/internal/executor/initializer.go

@@ -0,0 +1,57 @@
+package executor
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	"go.uber.org/zap"
+)
+
+type Initializer struct {
+	Name string
+	Fn   func() (Executor, error)
+}
+
+func DefaultInitializers(ctx context.Context, image string, logger *zap.Logger) []Initializer {
+	return []Initializer{
+		{
+			Name: "local",
+			Fn: func() (Executor, error) {
+				return local.New(logger)
+			},
+		},
+		{
+			Name: "Tart",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, nil, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"caching=cached",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none,caching=cached",
+				}, logger)
+			},
+		},
+	}
+}

docs/faq.md

@@ -5,6 +5,16 @@ title: Frequently Asked Questions
 description: Advanced configuration and troubleshooting tips for advanced configurations.
 ---
 
+## Troubleshooting crashes
+
+If you experience a crash or encounter another error while using the tart executable, you can collect debug information to assist with troubleshooting. Run the following command in a separate terminal window to gather logs from the Tart process and the macOS Virtualization subsystem:
+
+```shell
+log stream --predicate='process=="tart" OR process CONTAINS "Virtualization"' > tart.log
+```
+
+While the events are being streamed, attempt to reproduce the issue. Once the issue is reproduced, stop the streaming by pressing Ctrl+C. Then, attach the tart.log file to your report.
+
 ## VM location on disk
 
 Tart stores all its files in `~/.tart/` directory. Local images that you can run are stored in `~/.tart/vms/`.
@@ -13,12 +23,12 @@ Remote images are pulled into `~/.tart/cache/OCIs/`.
 ## Nested virtualization support?
 
 Tart is limited by functionality of Apple's `Virtualization.Framework`. At the moment `Virtualization.Framework`
-doesn't support nested virtualization.
+supports nested virtualization only on M3 or M4 chips running macOS 15 (Sequoia). By default, it is disabled, but can be enabled by passing the `--nested` flag to `tart run`.
 
 ## Connecting to a service running on host
 
 To connect from within a virtual machine to a service running on the host machine
-please first make sure that the service is binded to `0.0.0.0`.
+please first make sure that the service is bound to `0.0.0.0`.
 
 Then from within a virtual machine you can access the service using the router's IP address that you can get either from `Preferences -> Network`
 or by running the following command in the Terminal:
@@ -29,7 +39,7 @@ netstat -nr | grep default | head -n 1 | awk '{print $2}'
 
 Note: that accessing host is only possible with the default NAT network. If you are running your virtual machines with
 [Softnet](https://github.com/cirruslabs/softnet) (via `tart run --net-softnet <VM NAME>)`, then the network isolation
-is stricter and it's not only possible to access the host.
+is stricter and it's not possible to access the host.
 
 ## Changing the default NAT subnet
 
@@ -67,6 +77,57 @@ sudo rm /var/db/dhcpd_leases
 
 And no worries, this file will be re-created on the next `tart run`.
 
+## Unsupported DHCP client identifiers
+
+Due to the limitations of the macOS built-in DHCP server, `tart ip` is unable to correctly report the IP addresses for VMs using DHCP client identifiers that are not based on VMs link-layer addresses (MAC addresses).
+
+By default, when [no `--resolver=arp` is specified](#resolving-the-vms-ip-when-using-bridged-networking), `tart ip` reads the `/var/db/dhcpd_leases` file and tries to find the freshest entry that matches the VM's MAC address (based on the `hw_address` field).
+
+However, things starts to break when the VM uses a [DUID-EN](https://metebalci.com/blog/a-note-on-dhcpv6-duid-and-prefix-delegation#duid-types) identifier, for example. One of the notorious examples of this being Ubuntu, using this type of identifier by default on latest versions.  This results in the `/var/db/dhcpd_leases` entry for Ubuntu appearing as follows:
+
+```ini
+{
+    name=ubuntu
+    ip_address=192.168.64.3
+    hw_address=ff,f1:f5:dd:7f:0:2:0:0:ab:11:cb:fb:30:b0:97:b6:3a:67
+    identifier=ff,f1:f5:dd:7f:0:2:0:0:ab:11:cb:fb:30:b0:97:b6:3a:67
+    lease=0x678e2ce7
+}
+```
+
+Because the macOS built-in DHCP server overwrites the `hw_address` with the `identifier`, it leaves no information about the VM's MAC address to the `tart ip`.
+
+To avoid this issue, make sure that your VM only sends a DHCP client identifier (option 61) with link-layer address (MAC address) or that it doesn't send this option at all.
+
+For the aforementioned Ubuntu, the solution is outlined in the section [How to integrate with Windows DHCP Server](https://netplan.readthedocs.io/en/stable/examples/#how-to-integrate-with-windows-dhcp-server) of Canonical Netplan's documentation:
+
+```yaml
+network:
+  version: 2
+  ethernets:
+    enp3s0:
+      dhcp4: yes
+      dhcp-identifier: mac
+```
+
+## Resolving the VM's IP when using bridged networking
+
+When running `tart run` with `--net-bridged`, you need to invoke `tart ip` differently, because the macOS built-in DHCP server won't have any information about the VM's IP-address:
+
+```shell
+tart ip --resolver=arp <VM>
+```
+
+This causes the `tart ip` to consult the host's ARP table instead of the `/var/db/dhcpd_leases` file.
+
+Note that this method of resolving the IP heavily relies on the level of VM's activity on the network, namely, exchanging ARP requests between the guest and the host.
+
+This is normally not an issue for macOS VMs, but on Linux VMs you might need to install Samba, which includes a [NetBIOS name server](https://www.samba.org/samba/docs/current/man-html/nmbd.8.html) and exhibits the same behavior as macOS, resulting in the population of the ARP table of the host OS:
+
+```shell
+sudo apt-get install samba
+```
+
 ## Running login/clone/pull/push commands over SSH
 
 When invoking the Tart in an SSH session, you might get error like this:
@@ -97,7 +158,7 @@ or features supported. If there is some feature missing please don't hesitate to
 Instead of Anka Registry, Tart can work with any OCI-compatible container registry. This provides a much more consistent
 and scalable experience for distributing virtual machines.
 
-Tart doesn't yet have an analogue of Anka Controller for managing long living VMs but [soon will be](https://github.com/cirruslabs/tart/issues/372).
+Tart does have an analogue of Anka Controller for managing VMs across a cluster of Mac hosts called [Orchard](orchard/quick-start.md).
 
 ## Automatic pruning
 

docs/integrations/vm-management.md

@@ -7,7 +7,7 @@ description: Use Packer to build custom VM images, configure VMs and work with r
 
 ## Creating from scratch
 
-Tart supports macOS and Linux virtual machines. All commands like `run` and `pull` work the same way regarding of the underlying OS a particular VM image has.
+Tart supports macOS and Linux virtual machines. All commands like `run` and `pull` work the same way regardless of the underlying OS a particular VM image has.
 The only difference is how such VM images are created. Please check sections below for [macOS](#creating-a-macos-vm-image-from-scratch) and [Linux](#creating-a-linux-vm-image-from-scratch) instructions.
 
 ### Creating a macOS VM image from scratch

docs/orchard/deploying-controller.md

@@ -31,7 +31,13 @@ gcloud compute addresses create orchard-ip --region=us-central1
 export ORCHARD_IP=$(gcloud compute addresses describe orchard-ip --format='value(address)' --region=us-central1)
 ```
 
-Once we have the IP address, we can create a new instance with Orchard Controller running inside a container:
+Then, ensure that there exist a firewall rule targeting `https-server` tag and allowing access to TCP port 443. If that's not the case, create one:
+
+```shell
+gcloud compute firewall-rules create default-allow-https --direction=INGRESS --priority=1000 --network=default --action=ALLOW --rules=tcp:443 --source-ranges=0.0.0.0/0 --target-tags=https-server
+```
+
+Once we have the IP address and the firewall rule set up, we can create a new instance with Orchard Controller running inside a container:
 
 ```bash
 gcloud compute instances create-with-container orchard-controller \

docs/quick-start.md

@@ -17,8 +17,8 @@ tart run sonoma-base
     It's also possible to manually install `tart` binary from the latest released archive:
 
     ```bash
-    curl -LO https://github.com/cirruslabs/tart/releases/latest/download/tart-arm64.tar.gz
-    tar -xzvf tart-arm64.tar.gz
+    curl -LO https://github.com/cirruslabs/tart/releases/latest/download/tart.tar.gz
+    tar -xzvf tart.tar.gz
     ./tart.app/Contents/MacOS/tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
     ./tart.app/Contents/MacOS/tart run sonoma-base
     ```

gon.hcl

@@ -1,7 +1,4 @@
-source = [
-  ".build/x86_64-apple-macosx/release/tart",
-  ".build/arm64-apple-macosx/release/tart"
-]
+source = [ "dist/tart_darwin_all/tart.app/Contents/MacOS/tart" ]
 bundle_id = "com.github.cirruslabs.tart"
 
 apple_id {