Fix layerizer tests by providing a default maxRetries value

* docs(FAQ): document unsupported DHCP client identifiers

* New section "Resolving the VMs IP with bridged networking"

And a more clearer explanation of what "tart ip" does.

* Remove extraneous space in ` --resolver=arp`

* Better section name

* Add a note about Linux talkativeness

* Explain "talkativeness" a bit better

.cirrus.yml

@@ -17,8 +17,7 @@ task:
     - export PATH=$(pwd)/.build/arm64-apple-macosx/debug:$PATH
     # Run integration tests
     - cd integration-tests
-    - HOMEBREW_NO_AUTO_UPDATE=1 brew install virtualenv
-    - virtualenv venv
+    - python3 -m venv --symlinks venv
     - source venv/bin/activate
     - pip install -r requirements.txt
     - pytest --verbose --junit-xml=pytest-junit.xml
@@ -121,7 +120,7 @@ task:
     - security import certificate.p12 -k build.keychain -P password101 -T /usr/bin/codesign -T /usr/bin/pkgbuild
     - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password101 build.keychain
     - xcrun notarytool store-credentials "notarytool" --apple-id "hello@cirruslabs.org" --team-id "9M2P8L4D89" --password $AC_PASSWORD
-  install_script: 
+  install_script:
     - brew install go goreleaser/tap/goreleaser-pro getsentry/tools/sentry-cli
     - brew install mitchellh/gon/gon
   info_script:
@@ -153,7 +152,7 @@ task:
     registry_config: ENCRYPTED[!cf1a0f25325aa75bad3ce6ebc890bc53eb0044c02efa70d8cefb83ba9766275a994b4831706c52630a0692b2fa9cfb9e!]
   env:
     DEPLOY_TOKEN: ENCRYPTED[!45ed45666558902ed1c2400add734ec063103bec31841847e8c8764802fca229bfa6d85c690e16ad159e047574b48793!]
-  deploy_script: 
+  deploy_script:
     - git config --global user.name "Cirrus CI"
     - git config --global user.name "hello@cirruslabs.org"
     - git remote set-url origin https://$DEPLOY_TOKEN@github.com/cirruslabs/tart/

.goreleaser.yml

@@ -1,11 +1,12 @@
+version: 2
+
 project_name: tart
 
 before:
   hooks:
     - .ci/set-version.sh
-    - swift build --arch x86_64 -c release --product tart
-    - swift build --arch arm64 -c release --product tart
-    - gon gon.hcl
+    - swift build --arch arm64 --configuration release --product tart
+    - swift build --arch x86_64 --configuration release --product tart
 
 builds:
   - id: tart
@@ -20,8 +21,14 @@ builds:
     prebuilt:
       path: '.build/{{- if eq .Arch "arm64" }}arm64{{- else }}x86_64{{ end }}-apple-macosx/release/tart'
 
+universal_binaries:
+  - name_template: tart.app/Contents/MacOS/tart
+    replace: true
+    hooks:
+      post: gon gon.hcl
+
 archives:
-  - name_template: "{{ .ProjectName }}-{{ .Arch }}"
+  - name_template: "{{ .ProjectName }}"
     files:
       - src: Resources/embedded.provisionprofile
         dst: tart.app/Contents
@@ -42,7 +49,13 @@ brews:
     repository:
       owner: cirruslabs
       name: homebrew-cli
-    caveats: See the GitHub repository for more information
+    caveats: |
+      Tart has been installed. You might want to reduce the default DHCP lease time
+      from 86,400 to 600 seconds to avoid DHCP shortage when running lots of VMs daily:
+
+        sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.InternetSharing.default.plist bootpd -dict DHCPLeaseTimeSecs -int 600
+
+      See https://tart.run/faq/#changing-the-default-dhcp-lease-time for more details.
     homepage: https://github.com/cirruslabs/tart
     license: "Fair Source"
     description: Run macOS and Linux VMs on Apple Hardware

Sources/tart/Commands/Clone.swift

@@ -31,6 +31,15 @@ struct Clone: AsyncParsableCommand {
   @Flag(help: .hidden)
   var deduplicate: Bool = false
 
+  @Option(help: .hidden)
+  var proxy: String?
+
+  @Option(help: .hidden)
+  var caCert: String?
+
+  @Option(help: .hidden)
+  var maxRetries: UInt = 5
+
   func validate() throws {
     if newName.contains("/") {
       throw ValidationError("<new-name> should be a local name")
@@ -47,8 +56,8 @@ struct Clone: AsyncParsableCommand {
 
     if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
       // Pull the VM in case it's OCI-based and doesn't exist locally yet
-      let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
-      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
+      let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure, proxy: proxy, caCert: caCert)
+      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate, maxRetries: maxRetries)
     }
 
     let sourceVM = try VMStorageHelper.open(sourceName)

Sources/tart/Commands/Pull.swift

@@ -26,6 +26,15 @@ struct Pull: AsyncParsableCommand {
   @Flag(help: .hidden)
   var deduplicate: Bool = false
 
+  @Option(help: .hidden)
+  var proxy: String?
+
+  @Option(help: .hidden)
+  var caCert: String?
+
+  @Option(help: .hidden)
+  var maxRetries: UInt = 5
+
   func validate() throws {
     if concurrency < 1 {
       throw ValidationError("network concurrency cannot be less than 1")
@@ -42,10 +51,10 @@ struct Pull: AsyncParsableCommand {
     }
 
     let remoteName = try RemoteName(remoteName)
-    let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
+    let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure, proxy: proxy, caCert: caCert)
 
     defaultLogger.appendNewLine("pulling \(remoteName)...")
 
-    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
+    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate, maxRetries: maxRetries)
   }
 }

Sources/tart/Commands/Run.swift

@@ -202,9 +202,11 @@ struct Run: AsyncParsableCommand {
   var netSoftnet: Bool = false
 
   @Option(help: ArgumentHelp("Comma-separated list of CIDRs to allow the traffic to when using Softnet isolation\n(e.g. --net-softnet-allow=192.168.0.0/24)", discussion: """
-  This option allows you bypass the private IPv4 address space restrctions imposed by --net-softnet.
+  This option allows you bypass the private IPv4 address space restrictions imposed by --net-softnet.
 
   For example, you can allow the VM to communicate with the local network with e.g. --net-softnet-allow=10.0.0.0/16 or to completely disable the destination based restrictions with --net-softnet-allow=0.0.0.0/0.
+
+  Implies --net-softnet.
   """, valueName: "comma-separated CIDRs"))
   var netSoftnetAllow: String?
 
@@ -213,9 +215,14 @@ struct Run: AsyncParsableCommand {
 
   * EXTERNAL_PORT:INTERNAL_PORT — forward TCP traffic from the EXTERNAL_PORT on a host's egress interface (automatically detected and could be Wi-Fi, Ethernet and a VPN interface) to the INTERNAL_PORT on guest's IP (as reported by "tart ip")
 
-  Note that your software should either listen on 0.0.0.0 inside of a VM or on an IP address assigned to that VM for the port forwarding to work correctly.
+  Note that for the port forwarding to work correctly:
+
+  * the software in guest listening on INTERNAL_PORT should either listen on 0.0.0.0 or on an IP address assigned to that guest
+  * connection to the EXTERNAL_PORT should be performed from the local network that the host is attached to or from the internet, it's not possible to connect to that forwarded port from the host itself
 
   Another thing to keep in mind is that regular Softnet restrictions will still apply even to port forwarding. So if you're planning to access your VM from local network, and your local network is 192.168.0.0/24, for example, then add --net-softnet-allow=192.168.0.0/24. If you only need port forwarding, to completely disable Softnet restrictions you can use --net-softnet-allow=0.0.0.0/0.
+
+  Implies --net-softnet.
   """, valueName: "comma-separated port specifications"))
   var netSoftnetExpose: String?
 
@@ -258,7 +265,12 @@ struct Run: AsyncParsableCommand {
       throw ValidationError("--vnc and --vnc-experimental are mutually exclusive")
     }
 
-    // check that not more than one network option is specified
+    // Automatically enable --net-softnet when any of its related options are specified
+    if netSoftnetAllow != nil || netSoftnetExpose != nil {
+      netSoftnet = true
+    }
+
+    // Check that no more than one network option is specified
     var netFlags = 0
     if netBridged.count > 0 { netFlags += 1 }
     if netSoftnet { netFlags += 1 }
@@ -1062,7 +1074,7 @@ struct DirectoryShare {
     process.standardInput = inPipe
     process.launch()
 
-    inPipe.fileHandleForWriting.write(response!.data)
+    try inPipe.fileHandleForWriting.write(contentsOf: response!.data)
     try inPipe.fileHandleForWriting.close()
     process.waitUntilExit()
 

Sources/tart/Commands/Set.swift

@@ -73,8 +73,7 @@ struct Set: AsyncParsableCommand {
     }
 
     #if arch(arm64)
-      if randomSerial {
-        let oldPlatform = vmConfig.platform as! Darwin
+      if randomSerial, let oldPlatform = vmConfig.platform as? Darwin {
         vmConfig.platform = Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: oldPlatform.hardwareModel)
       }
     #endif

Sources/tart/Credentials/DockerConfigCredentialsProvider.swift

@@ -36,7 +36,11 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
 
     process.launch()
 
-    inPipe.fileHandleForWriting.write("\(host)\n".data(using: .utf8)!)
+    do {
+      try inPipe.fileHandleForWriting.write(contentsOf: "\(host)\n".data(using: .utf8)!)
+    } catch {
+      throw CredentialsProviderError.Failed(message: "Failed to write host to Docker helper!")
+    }
     inPipe.fileHandleForWriting.closeFile()
 
     let outputData = try outPipe.fileHandleForReading.readToEnd()

Sources/tart/Fetcher.swift

@@ -1,26 +1,61 @@
 import Foundation
 
-fileprivate var urlSession: URLSession = {
-  let config = URLSessionConfiguration.default
-
-  // Harbor expects a CSRF token to be present if the HTTP client
-  // carries a session cookie between its requests[1] and fails if
-  // it was not present[2].
-  //
-  // To fix that, we disable the automatic cookies carry in URLSession.
-  //
-  // [1]: https://github.com/goharbor/harbor/blob/a4c577f9ec4f18396207a5e686433a6ba203d4ef/src/server/middleware/csrf/csrf.go#L78
-  // [2]: https://github.com/cirruslabs/tart/issues/295
-  config.httpShouldSetCookies = false
-
-  return URLSession(configuration: config)
-}()
-
 class Fetcher {
-  static func fetch(_ request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
-    let task = urlSession.dataTask(with: request)
+  let urlSession: URLSession
+  let caCert: SecCertificate?
 
-    let delegate = Delegate()
+  init(proxy: String? = nil, caCert: String? = nil) throws {
+    // Configure URLSession
+    let config = URLSessionConfiguration.default
+
+    // Harbor expects a CSRF token to be present if the HTTP client
+    // carries a session cookie between its requests[1] and fails if
+    // it was not present[2].
+    //
+    // To fix that, we disable the automatic cookies carry in URLSession.
+    //
+    // [1]: https://github.com/goharbor/harbor/blob/a4c577f9ec4f18396207a5e686433a6ba203d4ef/src/server/middleware/csrf/csrf.go#L78
+    // [2]: https://github.com/cirruslabs/tart/issues/295
+    config.httpShouldSetCookies = false
+
+    if let proxy {
+      let (host, port) = try Self.parseProxy(proxy)
+
+      config.connectionProxyDictionary = [
+        kCFNetworkProxiesHTTPEnable: true,
+        kCFNetworkProxiesHTTPProxy: host,
+        kCFNetworkProxiesHTTPPort: port,
+
+        kCFNetworkProxiesHTTPSEnable: true,
+        kCFNetworkProxiesHTTPSProxy: host,
+        kCFNetworkProxiesHTTPSPort: port,
+      ]
+    }
+
+    self.urlSession = URLSession(configuration: config)
+
+    // Load CA certificate, if any
+    if let caCert {
+      let caCertString = try String(contentsOf: URL(filePath: caCert), encoding:. utf8)
+
+      let caCertBase64Lines = caCertString.components(separatedBy: .newlines).filter { line in
+        !line.hasPrefix("-----BEGIN") && !line.hasPrefix("-----END")
+      }
+
+      guard let caCertData = Data(base64Encoded: caCertBase64Lines.joined()) else {
+        throw RuntimeError.FailedToLoadCACertificate("failed to parse Base64-encoded PEM data")
+      }
+
+      self.caCert = SecCertificateCreateWithData(nil, caCertData as CFData)!
+    } else {
+      self.caCert = nil
+    }
+  }
+
+  func fetch(_ request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
+    let task = self.urlSession.dataTask(with: request)
+
+    let delegate = Delegate(caCert: self.caCert)
     task.delegate = delegate
 
     let stream = AsyncThrowingStream<Data, Error> { continuation in
@@ -34,15 +69,78 @@ class Fetcher {
 
     return (stream, response as! HTTPURLResponse)
   }
+
+  private static func parseProxy(_ proxy: String) throws -> (String, Int) {
+    // Assume that the scheme is specified
+    var url = URL(string: proxy)
+
+    // Fall back to HTTP scheme when not specified
+    if url?.scheme == nil {
+      url = URL(string: "http://\(proxy)")
+    }
+
+    guard let url else {
+      throw RuntimeError.InvalidProxyString
+    }
+
+    guard let host = url.host() else {
+      throw RuntimeError.InvalidProxyString
+    }
+
+    guard let port = url.port else {
+      throw RuntimeError.InvalidProxyString
+    }
+
+    return (host, port)
+  }
 }
 
-fileprivate class Delegate: NSObject, URLSessionDataDelegate {
+fileprivate class Delegate: NSObject, URLSessionDelegate, URLSessionDataDelegate {
+  let caCert: SecCertificate?
   var responseContinuation: CheckedContinuation<URLResponse, Error>?
   var streamContinuation: AsyncThrowingStream<Data, Error>.Continuation?
 
   private var buffer: Data = Data()
   private let bufferFlushSize = 16 * 1024 * 1024
 
+  init(caCert: SecCertificate?) {
+    self.caCert = caCert
+  }
+
+  func urlSession(
+    _ session: URLSession,
+    didReceive challenge: URLAuthenticationChallenge,
+    completionHandler: @escaping @Sendable (URLSession.AuthChallengeDisposition, URLCredential?) -> Void
+  ) {
+    if let caCert {
+      // Ensure that we're performing server trust authentication
+      guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust,
+            let serverTrust = challenge.protectionSpace.serverTrust else {
+        completionHandler(.performDefaultHandling, nil)
+
+        return
+      }
+
+      // Set the provided CA certificate as the only anchor
+      if SecTrustSetAnchorCertificates(serverTrust, [caCert] as CFArray) != errSecSuccess {
+        completionHandler(.cancelAuthenticationChallenge, nil)
+
+        return
+      }
+
+      // Evaluate the trust
+      if SecTrustEvaluateWithError(serverTrust, nil) {
+        completionHandler(.useCredential, URLCredential(trust: serverTrust))
+      } else {
+        completionHandler(.rejectProtectionSpace, nil)
+      }
+
+      return
+    }
+
+    completionHandler(.performDefaultHandling, nil)
+  }
+
   func urlSession(
     _ session: URLSession,
     dataTask: URLSessionDataTask,

Sources/tart/OCI/Layerizer/Disk.swift

@@ -2,5 +2,5 @@ import Foundation
 
 protocol Disk {
   static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool, maxRetries: UInt) async throws
 }

Sources/tart/OCI/Layerizer/DiskV1.swift

@@ -45,7 +45,7 @@ class DiskV1: Disk {
     return pushedLayers
   }
 
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false, maxRetries: UInt = 5) async throws {
     if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
       throw OCIError.FailedToCreateVmFile
     }
@@ -57,7 +57,7 @@ class DiskV1: Disk {
     // Decompress the layers onto the disk in a single stream
     let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
       if let data = data {
-        disk.write(data)
+        try disk.write(contentsOf: data)
       }
     }
 

Sources/tart/OCI/Layerizer/DiskV2.swift

@@ -84,7 +84,7 @@ class DiskV2: Disk {
     }
   }
 
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false, maxRetries: UInt = 5) async throws {
     // Support resumable pulls
     let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)
 
@@ -210,7 +210,7 @@ class DiskV2: Disk {
 
           var rangeStart: Int64 = 0
 
-          try await retry(maxAttempts: 5) {
+          try await retry(maxAttempts: Int(maxRetries)) {
             try await registry.pullBlob(diskLayer.digest, rangeStart: rangeStart) { data in
               try filter.write(data)
 
@@ -268,7 +268,7 @@ class DiskV2: Disk {
 
           if chunk != actualContentsOnDisk {
             try disk.seek(toOffset: offset)
-            disk.write(chunk)
+            try disk.write(contentsOf: chunk)
           }
         }
 
@@ -282,7 +282,7 @@ class DiskV2: Disk {
       // is zeroed via truncate(2)
       if chunk != zeroChunk {
         try disk.seek(toOffset: offset)
-        disk.write(chunk)
+        try disk.write(contentsOf: chunk)
       }
 
       offset += UInt64(chunk.count)

Sources/tart/OCI/Registry.swift

@@ -115,6 +115,7 @@ class Registry {
   let namespace: String
   let credentialsProviders: [CredentialsProvider]
   let authenticationKeeper = AuthenticationKeeper()
+  let fetcher: Fetcher
 
   var host: String? {
     guard let host = baseURL.host else { return nil }
@@ -128,17 +129,22 @@ class Registry {
 
   init(baseURL: URL,
        namespace: String,
-       credentialsProviders: [CredentialsProvider] = [EnvironmentCredentialsProvider(), DockerConfigCredentialsProvider(), KeychainCredentialsProvider()]
+       credentialsProviders: [CredentialsProvider] = [EnvironmentCredentialsProvider(), DockerConfigCredentialsProvider(), KeychainCredentialsProvider()],
+       proxy: String? = nil,
+       caCert: String? = nil
   ) throws {
     self.baseURL = baseURL
     self.namespace = namespace
     self.credentialsProviders = credentialsProviders
+    self.fetcher = try Fetcher(proxy: proxy, caCert: caCert)
   }
 
   convenience init(
     host: String,
     namespace: String,
     insecure: Bool = false,
+    proxy: String? = nil,
+    caCert: String? = nil,
     credentialsProviders: [CredentialsProvider] = [EnvironmentCredentialsProvider(), DockerConfigCredentialsProvider(), KeychainCredentialsProvider()]
   ) throws {
     let proto = insecure ? "http" : "https"
@@ -154,7 +160,7 @@ class Registry {
       throw RuntimeError.ImproperlyFormattedHost(host, hint)
     }
 
-    try self.init(baseURL: baseURL, namespace: namespace, credentialsProviders: credentialsProviders)
+    try self.init(baseURL: baseURL, namespace: namespace, credentialsProviders: credentialsProviders, proxy: proxy, caCert: caCert)
   }
 
   func ping() async throws {
@@ -448,6 +454,6 @@ class Registry {
     request.setValue("Tart/\(CI.version) (\(DeviceInfo.os); \(DeviceInfo.model))",
                      forHTTPHeaderField: "User-Agent")
 
-    return try await Fetcher.fetch(request, viaFile: viaFile)
+    return try await self.fetcher.fetch(request, viaFile: viaFile)
   }
 }

Sources/tart/VM.swift

@@ -83,7 +83,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Check if we already have this IPSW in cache
     var headRequest = URLRequest(url: remoteURL)
     headRequest.httpMethod = "HEAD"
-    let (_, headResponse) = try await Fetcher.fetch(headRequest, viaFile: false)
+    let (_, headResponse) = try await Fetcher().fetch(headRequest, viaFile: false)
 
     if let hash = headResponse.value(forHTTPHeaderField: "x-amz-meta-digest-sha256") {
       let ipswLocation = try IPSWCache().locationFor(fileName: "sha256:\(hash).ipsw")
@@ -100,7 +100,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")
 
     let request = URLRequest(url: remoteURL)
-    let (channel, response) = try await Fetcher.fetch(request, viaFile: true)
+    let (channel, response) = try await Fetcher().fetch(request, viaFile: true)
 
     let temporaryLocation = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString + ".ipsw")
 
@@ -115,7 +115,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     let digest = Digest()
 
     for try await chunk in channel {
-      fileHandle.write(chunk)
+      try fileHandle.write(contentsOf: chunk)
       digest.update(chunk)
       progress.completedUnitCount += Int64(chunk.count)
     }

Sources/tart/VMDirectory+OCI.swift

@@ -11,7 +11,14 @@ enum OCIError: Error {
 }
 
 extension VMDirectory {
-  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws {
+  func pullFromRegistry(
+    registry: Registry,
+    manifest: OCIManifest,
+    concurrency: UInt,
+    localLayerCache: LocalLayerCache?,
+    deduplicate: Bool,
+    maxRetries: UInt
+  ) async throws {
     // Pull VM's config file layer and re-serialize it into a config file
     let configLayers = manifest.layers.filter {
       $0.mediaType == configMediaType
@@ -24,7 +31,7 @@ extension VMDirectory {
     }
     let configFile = try FileHandle(forWritingTo: configURL)
     try await registry.pullBlob(configLayers.first!.digest) { data in
-      configFile.write(data)
+      try configFile.write(contentsOf: data)
     }
     try configFile.close()
 
@@ -55,7 +62,8 @@ extension VMDirectory {
       try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL,
                                   concurrency: concurrency, progress: progress,
                                   localLayerCache: localLayerCache,
-                                  deduplicate: deduplicate)
+                                  deduplicate: deduplicate,
+                                  maxRetries: maxRetries)
     } catch let error where error is FilterError {
       throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)")
     }
@@ -79,7 +87,7 @@ extension VMDirectory {
     }
     let nvram = try FileHandle(forWritingTo: nvramURL)
     try await registry.pullBlob(nvramLayers.first!.digest) { data in
-      nvram.write(data)
+      try nvram.write(contentsOf: data)
     }
     try nvram.close()
 

Sources/tart/VMStorageHelper.swift

@@ -75,6 +75,8 @@ enum RuntimeError : Error {
   case SuspendFailed(_ message: String)
   case PullFailed(_ message: String)
   case VirtualMachineLimitExceeded(_ hint: String)
+  case InvalidProxyString
+  case FailedToLoadCACertificate(_ message: String)
 }
 
 protocol HasExitCode {
@@ -136,6 +138,10 @@ extension RuntimeError : CustomStringConvertible {
       return message
     case .VirtualMachineLimitExceeded(let hint):
       return "The number of VMs exceeds the system limit\(hint)"
+    case .InvalidProxyString:
+      return "Invalid proxy string, should be in the form of host:port"
+    case .FailedToLoadCACertificate(let message):
+      return "Failed to load CA certificate: \(message)"
     }
   }
 }

Sources/tart/VMStorageOCI.swift

@@ -140,7 +140,7 @@ class VMStorageOCI: PrunableStorage {
     try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
   }
 
-  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws {
+  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool, maxRetries: UInt) async throws {
     SentrySDK.configureScope { scope in
       scope.setContext(value: ["imageName": name.description], key: "OCI")
     }
@@ -196,7 +196,7 @@ class VMStorageOCI: PrunableStorage {
       }
 
       try await withTaskCancellationHandler(operation: {
-        try await retry(maxAttempts: 5) {
+        try await retry(maxAttempts: Int(maxRetries)) {
           // Choose the best base image which has the most deduplication ratio
           let localLayerCache = try await chooseLocalLayerCache(name, manifest, registry)
 
@@ -210,7 +210,7 @@ class VMStorageOCI: PrunableStorage {
             }
           }
 
-          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache, deduplicate: deduplicate)
+          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache, deduplicate: deduplicate, maxRetries: maxRetries)
         } recoverFromFailure: { error in
           if error is URLError {
             print("Error pulling image: \"\(error.localizedDescription)\", attempting to re-try...")

benchmark/README.md

@@ -185,3 +185,42 @@ sync test                               	Tart (--root-disk-opts="sync=none")
 sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	35 MB/s    	0 IOPS     	15.67 kIOPS 	0s ± 0s                   	11.319µs ± 24.771µs       	51.731µs ± 42.208µs    
 sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	17 MB/s    	0 IOPS     	7.39 kIOPS  	0s ± 0s                   	21.23µs ± 81.749µs        	113.239µs ± 191.266µs  
 ```
+
+### Jan 16, 2025
+
+Host:
+
+* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
+* OS: macOS Sequoia 15.2
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.6
+
+```
+Name                    	Executor                                          	Time
+XcodeBenchmark (d869315)	local                                             	2m15s
+XcodeBenchmark (d869315)	Tart                                              	4m22s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	4m21s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	4m15s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	4m16s
+```
+
+```
+Name                    	Executor                                          	Time
+XcodeBenchmark (d869315)	local                                             	2m7s
+XcodeBenchmark (d869315)	Tart                                              	4m37s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	4m35s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	4m19s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	4m16s
+```
+
+```
+Name                    	Executor                                          	Time
+XcodeBenchmark (d869315)	local                                             	2m6s
+XcodeBenchmark (d869315)	Tart                                              	4m24s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	4m22s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	4m18s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	4m17s
+```

benchmark/internal/command/fio/fio.go

@@ -4,8 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
 	"github.com/dustin/go-humanize"
 	"github.com/gosuri/uitable"
 	"github.com/spf13/cobra"
@@ -46,57 +44,12 @@ func run(cmd *cobra.Command, args []string) error {
 		_ = logger.Sync()
 	}()
 
-	var executorInitializers = []struct {
-		Name string
-		Fn   func() (executorpkg.Executor, error)
-	}{
-		{
-			Name: "local",
-			Fn: func() (executorpkg.Executor, error) {
-				return local.New(logger)
-			},
-		},
-		{
-			Name: "Tart",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, nil, logger)
-			},
-		},
-		{
-			Name: "Tart (--root-disk-opts=\"sync=none\")",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, []string{
-					"--root-disk-opts",
-					"sync=none",
-				}, logger)
-			},
-		},
-		{
-			Name: "Tart (--root-disk-opts=\"caching=cached\")",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, []string{
-					"--root-disk-opts",
-					"caching=cached",
-				}, logger)
-			},
-		},
-		{
-			Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")",
-			Fn: func() (executorpkg.Executor, error) {
-				return tart.New(cmd.Context(), image, []string{
-					"--root-disk-opts",
-					"sync=none,caching=cached",
-				}, logger)
-			},
-		},
-	}
-
 	table := uitable.New()
 	table.AddRow("Name", "Executor", "B/W (read)", "B/W (write)", "I/O (read)", "I/O (write)",
 		"Latency (read)", "Latency (write)", "Latency (sync)")
 
 	for _, benchmark := range benchmarks {
-		for _, executorInitializer := range executorInitializers {
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
 			if prepare != "" {
 				shell := "/bin/sh"
 

benchmark/internal/command/root.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"github.com/cirruslabs/tart/benchmark/internal/command/fio"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
 	"github.com/spf13/cobra"
 )
 
@@ -14,6 +15,7 @@ func NewCommand() *cobra.Command {
 
 	cmd.AddCommand(
 		fio.NewCommand(),
+		xcode.NewCommand(),
 	)
 
 	return cmd

benchmark/internal/command/xcode/benchmarks.go

@@ -0,0 +1,13 @@
+package xcode
+
+type Benchmark struct {
+	Name    string
+	Command string
+}
+
+var benchmarks = []Benchmark{
+	{
+		Name:    "XcodeBenchmark (d869315)",
+		Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && sh benchmark.sh",
+	},
+}

benchmark/internal/command/xcode/output.go

@@ -0,0 +1,49 @@
+package xcode
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+)
+
+type Output struct {
+	Started time.Time
+	Ended   time.Time
+}
+
+func ParseOutput(s string) (*Output, error) {
+	// Ensure that the build has succeeded
+	matched, err := regexp.MatchString("(?m)^\\*\\* BUILD SUCCEEDED \\*\\*.*$", s)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse output: regexp failed: %v", err)
+	}
+	if !matched {
+		return nil, fmt.Errorf("failed to parse output: \"** BUILD SUCCEEDED **\" string " +
+			"not found on a separate line, make sure you have Xcode installed")
+	}
+
+	re := regexp.MustCompile("Started\\s+(?P<started>.*)\\n.*Ended\\s+(?P<ended>.*)\\n")
+
+	matches := re.FindStringSubmatch(s)
+
+	if len(matches) != re.NumSubexp()+1 {
+		return nil, fmt.Errorf("failed to parse output: cannot find Started and Ended times")
+	}
+
+	startedRaw := matches[re.SubexpIndex("started")]
+	started, err := time.Parse(time.TimeOnly, startedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse started time %q: unsupported format", startedRaw)
+	}
+
+	endedRaw := matches[re.SubexpIndex("ended")]
+	ended, err := time.Parse(time.TimeOnly, endedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse ended time %q: unsupported format", startedRaw)
+	}
+
+	return &Output{
+		Started: started,
+		Ended:   ended,
+	}, nil
+}

benchmark/internal/command/xcode/output_test.go

@@ -0,0 +1,37 @@
+package xcode_test
+
+import (
+	"fmt"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func TestParseOutput(t *testing.T) {
+	result, err := xcode.ParseOutput(`** BUILD SUCCEEDED ** [219.713 sec]
+
+System Version: 14.6
+Xcode 15.4
+Hardware Overview
+      Model Name: Apple Virtual Machine 1
+      Model Identifier: VirtualMac2,1
+      Total Number of Cores: 4
+      Memory: 8 GB
+
+✅ XcodeBenchmark has completed
+1️⃣  Take a screenshot of this window (Cmd + Shift + 4 + Space) and resize to include:
+	- Build Time (See ** BUILD SUCCEEDED ** [XYZ sec])
+	- System Version
+	- Xcode Version
+	- Hardware Overview
+	- Started 13:46:20
+	- Ended   13:50:02
+	- Date Thu Jan 16 13:50:02 UTC 2025
+
+2️⃣  Share your results at https://github.com/devMEremenko/XcodeBenchmark
+`)
+	require.NoError(t, err)
+	fmt.Println(result)
+	require.Equal(t, 222*time.Second, result.Ended.Sub(result.Started))
+}

benchmark/internal/command/xcode/xcode.go

@@ -0,0 +1,108 @@
+package xcode
+
+import (
+	"fmt"
+	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
+	"github.com/gosuri/uitable"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"os"
+	"os/exec"
+)
+
+var debug bool
+var image string
+var prepare string
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "xcode",
+		Short: "run XCode benchmarks",
+		RunE:  run,
+	}
+
+	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+	cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-xcode:latest", "image to use for testing")
+	cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark")
+
+	return cmd
+}
+
+func run(cmd *cobra.Command, args []string) error {
+	config := zap.NewProductionConfig()
+	if debug {
+		config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+	logger, err := config.Build()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = logger.Sync()
+	}()
+
+	table := uitable.New()
+	table.AddRow("Name", "Executor", "Time")
+
+	for _, benchmark := range benchmarks {
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
+			if prepare != "" {
+				shell := "/bin/sh"
+
+				if shellFromEnv, ok := os.LookupEnv("SHELL"); ok {
+					shell = shellFromEnv
+				}
+
+				logger.Sugar().Infof("running prepare command %q using shell %q",
+					prepare, shell)
+
+				cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare)
+
+				loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+
+				cmd.Stdout = loggerWriter
+				cmd.Stderr = loggerWriter
+
+				if err := cmd.Run(); err != nil {
+					return fmt.Errorf("failed to run prepare command %q: %v", prepare, err)
+				}
+			}
+
+			logger.Sugar().Infof("initializing executor %s", executorInitializer.Name)
+
+			executor, err := executorInitializer.Fn()
+			if err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name,
+				executorInitializer.Name)
+
+			stdout, err := executor.Run(cmd.Context(), benchmark.Command)
+			if err != nil {
+				return err
+			}
+
+			output, err := ParseOutput(string(stdout))
+			if err != nil {
+				return err
+			}
+
+			duration := output.Ended.Sub(output.Started)
+
+			logger.Sugar().Infof("Xcode benchmark duration: %s", duration)
+
+			table.AddRow(benchmark.Name, executorInitializer.Name, duration)
+
+			if err := executor.Close(); err != nil {
+				return fmt.Errorf("failed to close executor %s: %w",
+					executorInitializer.Name, err)
+			}
+		}
+	}
+
+	fmt.Println(table.String())
+
+	return nil
+}

benchmark/internal/executor/initializer.go

@@ -0,0 +1,57 @@
+package executor
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	"go.uber.org/zap"
+)
+
+type Initializer struct {
+	Name string
+	Fn   func() (Executor, error)
+}
+
+func DefaultInitializers(ctx context.Context, image string, logger *zap.Logger) []Initializer {
+	return []Initializer{
+		{
+			Name: "local",
+			Fn: func() (Executor, error) {
+				return local.New(logger)
+			},
+		},
+		{
+			Name: "Tart",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, nil, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"caching=cached",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none,caching=cached",
+				}, logger)
+			},
+		},
+	}
+}

docs/faq.md

@@ -77,6 +77,57 @@ sudo rm /var/db/dhcpd_leases
 
 And no worries, this file will be re-created on the next `tart run`.
 
+## Unsupported DHCP client identifiers
+
+Due to the limitations of the macOS built-in DHCP server, `tart ip` is unable to correctly report the IP addresses for VMs using DHCP client identifiers that are not based on VMs link-layer addresses (MAC addresses).
+
+By default, when [no `--resolver=arp` is specified](#resolving-the-vms-ip-when-using-bridged-networking), `tart ip` reads the `/var/db/dhcpd_leases` file and tries to find the freshest entry that matches the VM's MAC address (based on the `hw_address` field).
+
+However, things starts to break when the VM uses a [DUID-EN](https://metebalci.com/blog/a-note-on-dhcpv6-duid-and-prefix-delegation#duid-types) identifier, for example. One of the notorious examples of this being Ubuntu, using this type of identifier by default on latest versions.  This results in the `/var/db/dhcpd_leases` entry for Ubuntu appearing as follows:
+
+```ini
+{
+    name=ubuntu
+    ip_address=192.168.64.3
+    hw_address=ff,f1:f5:dd:7f:0:2:0:0:ab:11:cb:fb:30:b0:97:b6:3a:67
+    identifier=ff,f1:f5:dd:7f:0:2:0:0:ab:11:cb:fb:30:b0:97:b6:3a:67
+    lease=0x678e2ce7
+}
+```
+
+Because the macOS built-in DHCP server overwrites the `hw_address` with the `identifier`, it leaves no information about the VM's MAC address to the `tart ip`.
+
+To avoid this issue, make sure that your VM only sends a DHCP client identifier (option 61) with link-layer address (MAC address) or that it doesn't send this option at all.
+
+For the aforementioned Ubuntu, the solution is outlined in the section [How to integrate with Windows DHCP Server](https://netplan.readthedocs.io/en/stable/examples/#how-to-integrate-with-windows-dhcp-server) of Canonical Netplan's documentation:
+
+```yaml
+network:
+  version: 2
+  ethernets:
+    enp3s0:
+      dhcp4: yes
+      dhcp-identifier: mac
+```
+
+## Resolving the VM's IP when using bridged networking
+
+When running `tart run` with `--net-bridged`, you need to invoke `tart ip` differently, because the macOS built-in DHCP server won't have any information about the VM's IP-address:
+
+```shell
+tart ip --resolver=arp <VM>
+```
+
+This causes the `tart ip` to consult the host's ARP table instead of the `/var/db/dhcpd_leases` file.
+
+Note that this method of resolving the IP heavily relies on the level of VM's activity on the network, namely, exchanging ARP requests between the guest and the host.
+
+This is normally not an issue for macOS VMs, but on Linux VMs you might need to install Samba, which includes a [NetBIOS name server](https://www.samba.org/samba/docs/current/man-html/nmbd.8.html) and exhibits the same behavior as macOS, resulting in the population of the ARP table of the host OS:
+
+```shell
+sudo apt-get install samba
+```
+
 ## Running login/clone/pull/push commands over SSH
 
 When invoking the Tart in an SSH session, you might get error like this:

docs/quick-start.md

@@ -17,8 +17,8 @@ tart run sonoma-base
     It's also possible to manually install `tart` binary from the latest released archive:
 
     ```bash
-    curl -LO https://github.com/cirruslabs/tart/releases/latest/download/tart-arm64.tar.gz
-    tar -xzvf tart-arm64.tar.gz
+    curl -LO https://github.com/cirruslabs/tart/releases/latest/download/tart.tar.gz
+    tar -xzvf tart.tar.gz
     ./tart.app/Contents/MacOS/tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
     ./tart.app/Contents/MacOS/tart run sonoma-base
     ```

gon.hcl

@@ -1,7 +1,4 @@
-source = [
-  ".build/x86_64-apple-macosx/release/tart",
-  ".build/arm64-apple-macosx/release/tart"
-]
+source = [ "dist/tart_darwin_all/tart.app/Contents/MacOS/tart" ]
 bundle_id = "com.github.cirruslabs.tart"
 
 apple_id {