Ditch AsyncHTTPClient in favor of URLSession (#260)

* tart create: support fetching URLs specified in the --from-ipsw option

* Use x-amz-meta-digest-sha256 header to cache IPSWs

.ci/create-pkg.sh

@@ -6,6 +6,6 @@ export VERSION="${CIRRUS_TAG:-0}"
 
 mkdir -p .ci/pkg/
 cp .build/arm64-apple-macosx/debug/tart .ci/pkg/
-pkgbuild --root .ci/pkg --version $VERSION --install-location /usr/local/bin/ --identifier com.github.cirruslabs.tart --sign "Developer ID Installer: Fedor Korotkov (9M2P8L4D89)" "./dist/Tart-$VERSION.pkg"
+pkgbuild --root .ci/pkg --version $VERSION --install-location /usr/local/bin/ --identifier com.github.cirruslabs.tart --sign "Developer ID Installer: Cirrus Labs, Inc. (9M2P8L4D89)" "./dist/Tart-$VERSION.pkg"
 xcrun notarytool submit "./dist/Tart-$VERSION.pkg" --keychain-profile "notarytool" --wait
 xcrun stapler staple "./dist/Tart-$VERSION.pkg"

.cirrus.yml

@@ -22,7 +22,7 @@ task:
   macos_instance:
     image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
   env:
-    MACOS_CERTIFICATE: ENCRYPTED[8a6930a8c1286e7e536ea41b7647ea40e99174ad15e9cfcc753754fea55a619b355415629dff515b54a8921643e314e5]
+    MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
     AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
     GITHUB_TOKEN: ENCRYPTED[!98ace8259c6024da912c14d5a3c5c6aac186890a8d4819fad78f3e0c41a4e0cd3a2537dd6e91493952fb056fa434be7c!]
     GORELEASER_KEY: ENCRYPTED[!9b80b6ef684ceaf40edd4c7af93014ee156c8aba7e6e5795f41c482729887b5c31f36b651491d790f1f668670888d9fd!]

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [cirruslabs]

Package.resolved

@@ -1,14 +1,5 @@
 {
   "pins" : [
-    {
-      "identity" : "async-http-client",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/swift-server/async-http-client",
-      "state" : {
-        "revision" : "df87a860fdc41a595d5ca67f74cde9adbccc099a",
-        "version" : "1.11.4"
-      }
-    },
     {
       "identity" : "dynamic",
       "kind" : "remoteSourceControl",
@@ -37,12 +28,12 @@
       }
     },
     {
-      "identity" : "swift-atomics",
+      "identity" : "swift-async-algorithms",
       "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-atomics.git",
+      "location" : "https://github.com/apple/swift-async-algorithms",
       "state" : {
-        "revision" : "919eb1d83e02121cdb434c7bfc1f0c66ef17febe",
-        "version" : "1.0.2"
+        "branch" : "main",
+        "revision" : "f05e450f0b909c0e80670a47516c4b9700b9e5da"
       }
     },
     {
@@ -55,57 +46,12 @@
       }
     },
     {
-      "identity" : "swift-log",
+      "identity" : "swift-collections",
       "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-log.git",
+      "location" : "https://github.com/apple/swift-collections.git",
       "state" : {
-        "revision" : "5d66f7ba25daf4f94100e7022febf3c75e37a6c7",
-        "version" : "1.4.2"
-      }
-    },
-    {
-      "identity" : "swift-nio",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-nio.git",
-      "state" : {
-        "revision" : "124119f0bb12384cef35aa041d7c3a686108722d",
-        "version" : "2.40.0"
-      }
-    },
-    {
-      "identity" : "swift-nio-extras",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-nio-extras.git",
-      "state" : {
-        "revision" : "8eea84ec6144167354387ef9244b0939f5852dc8",
-        "version" : "1.11.0"
-      }
-    },
-    {
-      "identity" : "swift-nio-http2",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-nio-http2.git",
-      "state" : {
-        "revision" : "108ac15087ea9b79abb6f6742699cf31de0e8772",
-        "version" : "1.22.0"
-      }
-    },
-    {
-      "identity" : "swift-nio-ssl",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-nio-ssl.git",
-      "state" : {
-        "revision" : "1750873bce84b4129b5303655cce2c3d35b9ed3a",
-        "version" : "2.19.0"
-      }
-    },
-    {
-      "identity" : "swift-nio-transport-services",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-nio-transport-services.git",
-      "state" : {
-        "revision" : "1a4692acb88156e3da1b0c6732a8a38b2a744166",
-        "version" : "1.12.0"
+        "revision" : "f504716c27d2e5d4144fa4794b12129301d17729",
+        "version" : "1.0.3"
       }
     },
     {

Package.swift

@@ -13,15 +13,15 @@ let package = Package(
     .package(url: "https://github.com/apple/swift-argument-parser", from: "1.1.2"),
     .package(url: "https://github.com/mhdhejazi/Dynamic", branch: "master"),
     .package(url: "https://github.com/pointfreeco/swift-parsing", from: "0.9.2"),
-    .package(url: "https://github.com/swift-server/async-http-client", from: "1.11.4"),
     .package(url: "https://github.com/apple/swift-algorithms", from: "1.0.0"),
+    .package(url: "https://github.com/apple/swift-async-algorithms", branch: "main"),
     .package(url: "https://github.com/malcommac/SwiftDate", from: "6.3.1")
   ],
   targets: [
     .executableTarget(name: "tart", dependencies: [
       .product(name: "Algorithms", package: "swift-algorithms"),
+      .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"),
       .product(name: "ArgumentParser", package: "swift-argument-parser"),
-      .product(name: "AsyncHTTPClient", package: "async-http-client"),
       .product(name: "Dynamic", package: "Dynamic"),
       .product(name: "Parsing", package: "swift-parsing"),
       .product(name: "SwiftDate", package: "SwiftDate"),

README.md

@@ -8,6 +8,22 @@ Built by CI engineers for your automation needs. Here are some highlights of Tar
 * Use Tart Packer Plugin to automate VM creation.
 * Built-in CI integration.
 
+*Tart* is already adopted by several automation services:
+
+<p align="center">
+  <a href="https://cirrus-ci.org/guide/macOS/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/CirrusCI.png" height="65"/>
+  </a>
+  <a href="https://codemagic.io/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Codemagic.png" height="65"/>
+  </a>
+  <a href="https://testingbot.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/TestingBot.png" height="65"/>
+  </a>
+</p>
+
+## Usage
+
 Try running a Tart VM on your Apple Silicon device running macOS Monterey or later (will download a 25 GB image):
 
 ```shell
@@ -222,8 +238,8 @@ This invocation calls the `tart pull` implicitly (if the image is not being pres
 <details>
   <summary>Why Tart is free and open sourced?</summary>
 
-  Tart is a relatively small project, and it didn't feel right to try to monetize it.
-  Apple did all the heavy lifting with their `Virtualization.Framework`.
+  Apple did all the heavy lifting with their `Virtualization.Framework` and it just felt right to develop Tart in the open.
+  Please consider [becoming a sponsor](https://github.com/sponsors/cirruslabs) if you find Tart saving a substantial amount of money on licensing and engineering hours for your company.
 </details>
 
 <details>
@@ -252,3 +268,21 @@ This invocation calls the `tart pull` implicitly (if the image is not being pres
   Tart is limited by functionality of Apple's `Virtualization.Framework`. At the moment `Virtualization.Framework`
   doesn't support nested virtualization.
 </details>
+
+<details>
+  <summary>Changing the default NAT subnet</summary>
+
+  To change the default network to `192.168.77.1`:
+
+  ```
+  sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.vmnet.plist Shared_Net_Address -string 192.168.77.1
+  ```
+
+  Note that even through a network would normally be specified as `192.168.77.0`, the [vmnet framework](https://developer.apple.com/documentation/vmnet) seems to treat this as a starting address too and refuses to pick up such network-like values.
+
+  The default subnet mask `255.255.255.0` should suffice for most use-cases, however, you can also change it to `255.255.0.0`, for example:
+
+  ```
+  sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.vmnet.plist Shared_Net_Mask -string 255.255.0.0
+  ```
+</details>

Resources/Users/CirrusCI.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbcf6f65f26c0e17b2fe55a9712fc783315cd64905a0febce758b31d7e4e923b
+size 6787

Resources/Users/Codemagic.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4239996f30145992936cfc8faa0232fa8904aedf4de61e30ef4c4fc86cad587f
+size 14588

Resources/Users/TestingBot.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:227192fcd215c9cc05b1ebcd2616bd1f8c95be184726f64d91fc4507a88c66e1
+size 18393

Resources/tart.entitlements

@@ -5,4 +5,4 @@
 	<key>com.apple.security.virtualization</key>
 	<true/>
 </dict>
-</plist>
+</plist>

Sources/tart/Commands/Clone.swift

@@ -32,12 +32,22 @@ struct Clone: AsyncParsableCommand {
       }
 
       let sourceVM = try VMStorageHelper.open(sourceName)
-      let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress())
 
       let tmpVMDir = try VMDirectory.temporary()
+
+      // Lock the temporary VM directory to prevent it's garbage collection
+      let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
+      try tmpVMDirLock.lock()
+
       try await withTaskCancellationHandler(operation: {
+        let lock = try FileLock(lockURL: Config().tartHomeDir)
+        try lock.lock()
+
+        let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress())
         try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC)
         try localStorage.move(newName, from: tmpVMDir)
+
+        try lock.unlock()
       }, onCancel: {
         try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
       })

Sources/tart/Commands/Create.swift

@@ -9,7 +9,7 @@ struct Create: AsyncParsableCommand {
   @Argument(help: "VM name")
   var name: String
 
-  @Option(help: ArgumentHelp("create a macOS VM using path to the IPSW file (or \"latest\") to fetch the latest appropriate IPSW", valueName: "path"))
+  @Option(help: ArgumentHelp("create a macOS VM using path to the IPSW file or URL (or \"latest\", to fetch the latest supported IPSW automatically)", valueName: "path"))
   var fromIPSW: String?
 
   @Flag(help: "create a Linux VM")
@@ -27,13 +27,24 @@ struct Create: AsyncParsableCommand {
   func run() async throws {
     do {
       let tmpVMDir = try VMDirectory.temporary()
+
+      // Lock the temporary VM directory to prevent it's garbage collection
+      let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
+      try tmpVMDirLock.lock()
+
       try await withTaskCancellationHandler(operation: {
         if let fromIPSW = fromIPSW {
+          let ipswURL: URL
+
           if fromIPSW == "latest" {
-            _ = try await VM(vmDir: tmpVMDir, ipswURL: nil, diskSizeGB: diskSize)
+            ipswURL = try await VM.latestIPSWURL()
+          } else if fromIPSW.starts(with: "http://") || fromIPSW.starts(with: "https://") {
+            ipswURL = URL(string: fromIPSW)!
           } else {
-            _ = try await VM(vmDir: tmpVMDir, ipswURL: URL(fileURLWithPath: fromIPSW), diskSizeGB: diskSize)
+            ipswURL = URL(fileURLWithPath: fromIPSW)
           }
+
+          _ = try await VM(vmDir: tmpVMDir, ipswURL: ipswURL, diskSizeGB: diskSize)
         }
 
         if linux {

Sources/tart/Commands/Delete.swift

@@ -6,11 +6,13 @@ struct Delete: AsyncParsableCommand {
   static var configuration = CommandConfiguration(abstract: "Delete a VM")
 
   @Argument(help: "VM name")
-  var name: String
+  var name: [String]
 
   func run() async throws {
     do {
-      try VMStorageHelper.delete(name)
+      for it in name {
+        try VMStorageHelper.delete(it)
+      }
 
       Foundation.exit(0)
     } catch {

Sources/tart/Commands/IP.swift

@@ -44,11 +44,12 @@ struct IP: AsyncParsableCommand {
     let waitUntil = Calendar.current.date(byAdding: .second, value: Int(secondsToWait), to: Date.now)!
 
     repeat {
-      if let ip = try Leases().resolveMACAddress(macAddress: vmMACAddress) {
+      if let leases = try Leases(), let ip = try leases.resolveMACAddress(macAddress: vmMACAddress) {
         return ip
       }
 
-      try await Task.sleep(nanoseconds: 1_000_000)
+      // wait a second
+      try await Task.sleep(nanoseconds: 1_000_000_000)
     } while Date.now < waitUntil
 
     return nil

Sources/tart/Commands/Prune.swift

@@ -101,7 +101,7 @@ struct Prune: AsyncParsableCommand {
         break
       }
 
-      cacheReclaimedBytes -= try prunable.sizeBytes()
+      cacheReclaimedBytes += try prunable.sizeBytes()
       try prunable.delete()
     }
   }

Sources/tart/Commands/Rename.swift

@@ -0,0 +1,40 @@
+import ArgumentParser
+import Foundation
+
+struct Rename: AsyncParsableCommand {
+    static var configuration = CommandConfiguration(abstract: "Rename a VM")
+
+    @Argument(help: "VM name")
+    var name: String
+
+    @Argument(help: "new VM name")
+    var newName: String
+
+    func validate() throws {
+        if newName.contains("/") {
+            throw ValidationError("<new-name> should be a local name")
+        }
+    }
+
+    func run() async throws {
+        do {
+            let localStorage = VMStorageLocal()
+
+            if !localStorage.exists(name) {
+                throw ValidationError("failed to rename a non-existent VM: \(name)")
+            }
+
+            if localStorage.exists(newName) {
+                throw ValidationError("failed to rename VM \(name), target VM \(name) already exists, delete it first!")
+            }
+
+            try localStorage.rename(name, newName)
+
+            Foundation.exit(0)
+        } catch {
+            print(error)
+
+            Foundation.exit(1)
+        }
+    }
+}

Sources/tart/Commands/Run.swift

@@ -19,6 +19,9 @@ struct Run: AsyncParsableCommand {
           discussion: "Useful for integrating Tart VMs into other tools.\nUse `tart ip` in order to get an IP for SSHing or VNCing into the VM.")) 
   var noGraphics: Bool = false
 
+  @Flag(help: "Force open a UI window, even when VNC is enabled.")
+  var graphics: Bool = false
+
   @Flag(help: "Boot into recovery mode") 
   var recovery: Bool = false
   
@@ -47,15 +50,32 @@ struct Run: AsyncParsableCommand {
   @Option(help: ArgumentHelp("""
                              Additional directory shares with an optional read-only specifier\n(e.g. --dir=\"build:~/src/build\" --dir=\"sources:~/src/sources:ro\")
                              """, discussion: """
+                                              Requires host to be macOS 13.0 (Ventura) or newer.
                                               All shared directories are automatically mounted to "/Volumes/My Shared Files" directory on macOS,
                                               while on Linux you have to do it manually: "mount -t virtiofs com.apple.virtio-fs.automount /mount/point".
+                                              For macOS guests, they must be running macOS 13.0 (Ventura) or newer.
                                               """, valueName: "name:path[:ro]"))
   var dir: [String] = []
 
+  @Option(help: ArgumentHelp("""
+                             Use bridged networking instead of the default shared (NAT) networking \n(e.g. --net-bridged=en0 or --net-bridged=\"Wi-Fi\")
+                             """, discussion: """
+                                              Specify "list" as an interface name (--net-bridged=list) to list the available bridged interfaces.
+                                              """, valueName: "interface name"))
+  var netBridged: String?
+
   func validate() throws {
     if vnc && vncExperimental {
       throw ValidationError("--vnc and --vnc-experimental are mutually exclusive")
     }
+
+    if withSoftnet && netBridged != nil {
+      throw ValidationError("--with-softnet and --net-bridged are mutually exclusive")
+    }
+
+    if graphics && noGraphics {
+      throw ValidationError("--graphics and --no-graphics are mutually exclusive")
+    }
   }
 
   @MainActor
@@ -63,7 +83,7 @@ struct Run: AsyncParsableCommand {
     let vmDir = try VMStorageLocal().open(name)
     vm = try VM(
       vmDir: vmDir,
-      withSoftnet: withSoftnet,
+      network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
       additionalDiskAttachments: additionalDiskAttachments(),
       directoryShares: directoryShares()
     )
@@ -116,18 +136,61 @@ struct Run: AsyncParsableCommand {
     }
     sigintSrc.activate()
 
-    if noGraphics || vnc || vncExperimental {
+    let useVNCWithoutGraphics = (vnc || vncExperimental) && !graphics
+    if noGraphics || useVNCWithoutGraphics {
       dispatchMain()
     } else {
       runUI()
     }
   }
 
+  func userSpecifiedNetwork(vmDir: VMDirectory) throws -> Network? {
+    if withSoftnet {
+      let config = try VMConfig.init(fromURL: vmDir.configURL)
+
+      return try Softnet(vmMACAddress: config.macAddress.string)
+    }
+
+    if let netBridged = netBridged {
+      let matchingInterfaces = VZBridgedNetworkInterface.networkInterfaces.filter { interface in
+        interface.identifier == netBridged || interface.localizedDisplayName == netBridged
+      }
+
+      if matchingInterfaces.isEmpty {
+        let available = bridgeInterfaces().joined(separator: ", ")
+        throw ValidationError("no bridge interfaces matched \"\(netBridged)\", "
+          + "available interfaces: \(available)")
+      }
+
+      if matchingInterfaces.count > 1 {
+        throw ValidationError("more than one bridge interface matched \"\(netBridged)\", "
+          + "consider refining the search criteria")
+      }
+
+      return NetworkBridged(interface: matchingInterfaces.first!)
+    }
+
+    return nil
+  }
+
+  func bridgeInterfaces() -> [String] {
+    VZBridgedNetworkInterface.networkInterfaces.map { interface in
+      var bridgeDescription = interface.identifier
+
+      if let localizedDisplayName = interface.localizedDisplayName {
+        bridgeDescription += " (or \"\(localizedDisplayName)\")"
+      }
+
+      return bridgeDescription
+    }
+  }
+
   func additionalDiskAttachments() throws -> [VZDiskImageStorageDeviceAttachment] {
     var result: [VZDiskImageStorageDeviceAttachment] = []
     let readOnlySuffix = ":ro"
+    let expandedDiskPaths = disk.map { NSString(string:$0).expandingTildeInPath }
 
-    for rawDisk in disk {
+    for rawDisk in expandedDiskPaths {
       if rawDisk.hasSuffix(readOnlySuffix) {
         result.append(try VZDiskImageStorageDeviceAttachment(
           url: URL(fileURLWithPath: String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count))),
@@ -166,7 +229,7 @@ struct Run: AsyncParsableCommand {
 
       let (name, path) = (String(splits[0]), String(splits[1]))
 
-      result.append(DirectoryShare(name: name, path: URL(fileURLWithPath: path), readOnly: readOnly))
+      result.append(DirectoryShare(name: name, path: URL(fileURLWithPath: NSString(string: path).expandingTildeInPath), readOnly: readOnly))
     }
 
     return result

Sources/tart/Config.swift

@@ -3,8 +3,9 @@ import Foundation
 struct Config {
   let tartHomeDir: URL
   let tartCacheDir: URL
+  let tartTmpDir: URL
 
-  init() {
+  init() throws {
     var tartHomeDir: URL
 
     if let customTartHome = ProcessInfo.processInfo.environment["TART_HOME"] {
@@ -14,9 +15,27 @@ struct Config {
               .homeDirectoryForCurrentUser
               .appendingPathComponent(".tart", isDirectory: true)
     }
-
     self.tartHomeDir = tartHomeDir
+
     tartCacheDir = tartHomeDir.appendingPathComponent("cache", isDirectory: true)
+    try FileManager.default.createDirectory(at: tartCacheDir, withIntermediateDirectories: true)
+
+    tartTmpDir = tartHomeDir.appendingPathComponent("tmp", isDirectory: true)
+    try FileManager.default.createDirectory(at: tartTmpDir, withIntermediateDirectories: true)
+  }
+
+  func gc() throws {
+    for entry in try FileManager.default.contentsOfDirectory(at: tartTmpDir,
+        includingPropertiesForKeys: [], options: []) {
+      let lock = try FileLock(lockURL: entry)
+      if try !lock.trylock() {
+        continue
+      }
+
+      try FileManager.default.removeItem(at: entry)
+
+      try lock.unlock()
+    }
   }
 
   static func jsonEncoder() -> JSONEncoder {

Sources/tart/Credentials/DockerConfigCredentialsProvider.swift

@@ -1,6 +1,6 @@
 import Foundation
 
-class HelperProgramCredentialsProvider: CredentialsProvider {
+class DockerConfigCredentialsProvider: CredentialsProvider {
   func retrieve(host: String) throws -> (String, String)? {
     let dockerConfigURL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".docker").appendingPathComponent("config.json")
     if !FileManager.default.fileExists(atPath: dockerConfigURL.path) {
@@ -8,6 +8,9 @@ class HelperProgramCredentialsProvider: CredentialsProvider {
     }
     let config = try JSONDecoder().decode(DockerConfig.self, from: Data(contentsOf: dockerConfigURL))
 
+    if let credentialsFromAuth = config.auths?[host]?.decodeCredentials() {
+      return credentialsFromAuth
+    }
     if let helperProgram = config.credHelpers?[host] {
       return try executeHelper(binaryName: "docker-credential-\(helperProgram)", host: host)
     }
@@ -54,9 +57,31 @@ class HelperProgramCredentialsProvider: CredentialsProvider {
 }
 
 struct DockerConfig: Codable {
+  var auths: Dictionary<String, DockerAuthConfig>? = Dictionary()
   var credHelpers: Dictionary<String, String>? = Dictionary()
 }
 
+struct DockerAuthConfig: Codable {
+  var auth: String? = nil
+
+  func decodeCredentials() -> (String, String)? {
+    // auth is a base64("username:password")
+    guard let authBase64 = auth else {
+      return nil
+    }
+    guard let data = Data(base64Encoded: authBase64) else {
+      return nil
+    }
+    guard let components = String(data: data, encoding: .utf8)?.components(separatedBy: ":") else {
+      return nil
+    }
+    if components.count != 2 {
+      return nil
+    }
+    return (components[0], components[1])
+  }
+}
+
 struct DockerGetOutput: Codable {
   var Username: String
   var Secret: String

Sources/tart/FileLock.swift

@@ -0,0 +1,48 @@
+import Foundation
+import System
+
+enum FileLockError: Error, Equatable {
+    case Failed(_ message: String)
+    case AlreadyLocked
+}
+
+class FileLock {
+    let url: URL
+    let fd: Int32
+
+    init(lockURL: URL) throws {
+        url = lockURL
+        fd = open(lockURL.path, 0)
+    }
+
+    deinit {
+        close(fd)
+    }
+
+    func trylock() throws -> Bool {
+        try flockWrapper(LOCK_EX | LOCK_NB)
+    }
+
+    func lock() throws {
+        _ = try flockWrapper(LOCK_EX)
+    }
+
+    func unlock() throws {
+        _ = try flockWrapper(LOCK_UN)
+    }
+
+    func flockWrapper(_ operation: Int32) throws -> Bool {
+        let ret = flock(fd, operation)
+        if ret != 0 {
+            let details = Errno(rawValue: CInt(errno))
+
+            if (operation & LOCK_NB) != 0 && details == .wouldBlock {
+                return false
+            }
+
+            throw FileLockError.Failed("failed to lock \(url): \(details)")
+        }
+
+        return true
+    }
+}

Sources/tart/IPSWCache.swift

@@ -5,12 +5,12 @@ class IPSWCache: PrunableStorage {
   let baseURL: URL
 
   init() throws {
-    baseURL = Config().tartCacheDir.appendingPathComponent("IPSWs", isDirectory: true)
+    baseURL = try Config().tartCacheDir.appendingPathComponent("IPSWs", isDirectory: true)
     try FileManager.default.createDirectory(at: baseURL, withIntermediateDirectories: true)
   }
 
-  func locationFor(image: VZMacOSRestoreImage) -> URL {
-    baseURL.appendingPathComponent("\(image.buildVersion).ipsw", isDirectory: false)
+  func locationFor(fileName: String) -> URL {
+    baseURL.appendingPathComponent(fileName, isDirectory: false)
   }
 
   func prunables() throws -> [Prunable] {

Sources/tart/MACAddressResolver/Leases.swift

@@ -19,14 +19,21 @@ enum LeasesError: Error {
 class Leases {
   private let leases: [MACAddress : Lease]
 
-  convenience init() throws {
+  convenience init?() throws {
     try self.init(URL(fileURLWithPath: "/var/db/dhcpd_leases"))
   }
 
-  convenience init(_ fromURL: URL) throws {
-    let fileContents = try String(contentsOf: fromURL, encoding: .utf8)
+  convenience init?(_ fromURL: URL) throws {
+    do {
+      let urlContents = try String(contentsOf: fromURL, encoding: .utf8)
+      try self.init(urlContents)
+    } catch {
+      if error.isFileNotFound() {
+        return nil
+      }
 
-    try self.init(fileContents)
+      throw error
+    }
   }
 
   init(_ fromString: String) throws {

Sources/tart/Network/Network.swift

@@ -0,0 +1,7 @@
+import Virtualization
+
+protocol Network {
+  func attachment() -> VZNetworkDeviceAttachment
+  func run() throws
+  func stop() throws
+}

Sources/tart/Network/NetworkBridged.swift

@@ -0,0 +1,22 @@
+import Foundation
+import Virtualization
+
+class NetworkBridged: Network {
+    let interface: VZBridgedNetworkInterface
+
+    init(interface: VZBridgedNetworkInterface) {
+        self.interface = interface
+    }
+
+    func attachment() -> VZNetworkDeviceAttachment {
+        VZBridgedNetworkDeviceAttachment(interface: interface)
+    }
+
+    func run() throws {
+        // no-op, only used for Softnet
+    }
+
+    func stop() throws {
+        // no-op, only used for Softnet
+    }
+}

Sources/tart/Network/NetworkShared.swift

@@ -0,0 +1,16 @@
+import Foundation
+import Virtualization
+
+class NetworkShared: Network {
+    func attachment() -> VZNetworkDeviceAttachment {
+        VZNATNetworkDeviceAttachment()
+    }
+
+    func run() throws {
+        // no-op, only used for Softnet
+    }
+
+    func stop() throws {
+        // no-op, only used for Softnet
+    }
+}

Sources/tart/Network/Softnet.swift

@@ -1,10 +1,11 @@
 import Foundation
+import Virtualization
 
 enum SoftnetError: Error {
   case InitializationFailed(why: String)
 }
 
-class Softnet {
+class Softnet: Network {
   private let process = Process()
 
   let vmFD: Int32
@@ -57,4 +58,9 @@ class Softnet {
       throw SoftnetError.InitializationFailed(why: "setsockopt(SO_SNDBUF) returned \(ret)")
     }
   }
+
+  func attachment() -> VZNetworkDeviceAttachment {
+    let fh = FileHandle.init(fileDescriptor: vmFD)
+    return VZFileHandleNetworkDeviceAttachment(fileHandle: fh)
+  }
 }

Sources/tart/OCI/Registry.swift

@@ -1,28 +1,44 @@
 import Foundation
-import NIOCore
-import NIOHTTP1
-import AsyncHTTPClient
 import Algorithms
-import NIOPosix
+import AsyncAlgorithms
+
+let chunkSizeBytes = 1 * 1024 * 1024
 
 enum RegistryError: Error {
-  case UnexpectedHTTPStatusCode(when: String, code: UInt, details: String = "")
+  case UnexpectedHTTPStatusCode(when: String, code: Int, details: String = "")
   case MissingLocationHeader
   case AuthFailed(why: String, details: String = "")
   case MalformedHeader(why: String)
 }
 
-extension HTTPClientResponse.Body {
-  func readTextResponse() async throws -> String? {
-    let data = try await readResponse()
-    return String(decoding: data, as: UTF8.self)
-  }
+enum HTTPMethod: String {
+  case GET = "GET"
+  case POST = "POST"
+  case PUT = "PUT"
+  case PATCH = "PATCH"
+}
 
-  func readResponse() async throws -> Data {
+enum HTTPCode: Int {
+  case Ok = 200
+  case Created = 201
+  case Accepted = 202
+  case Unauthorized = 401
+}
+
+extension Data {
+  func asText() async throws -> String? {
+    String(decoding: self, as: UTF8.self)
+  }
+}
+
+extension URLSession.AsyncBytes {
+  func asData() async throws -> Data {
     var result = Data()
-    for try await part in self {
-      result.append(Data(buffer: part))
+
+    for try await chunk in chunks(ofCount: chunkSizeBytes) {
+      result += chunk
     }
+
     return result
   }
 }
@@ -78,14 +94,6 @@ struct TokenResponse: Decodable, Authentication {
 }
 
 class Registry {
-  private let httpClient = HTTPClient(
-    eventLoopGroupProvider: .shared(MultiThreadedEventLoopGroup(numberOfThreads: 1))
-  )
-
-  deinit {
-    try! httpClient.syncShutdown()
-  }
-
   let baseURL: URL
   let namespace: String
   let credentialsProviders: [CredentialsProvider]
@@ -94,7 +102,7 @@ class Registry {
 
   init(urlComponents: URLComponents,
        namespace: String,
-       credentialsProviders: [CredentialsProvider] = [HelperProgramCredentialsProvider(), KeychainCredentialsProvider()]
+       credentialsProviders: [CredentialsProvider] = [DockerConfigCredentialsProvider(), KeychainCredentialsProvider()]
   ) throws {
     baseURL = urlComponents.url!
     self.namespace = namespace
@@ -105,7 +113,7 @@ class Registry {
     host: String,
     namespace: String,
     insecure: Bool = false,
-    credentialsProviders: [CredentialsProvider] = [HelperProgramCredentialsProvider(), KeychainCredentialsProvider()]
+    credentialsProviders: [CredentialsProvider] = [DockerConfigCredentialsProvider(), KeychainCredentialsProvider()]
   ) throws {
     let proto = insecure ? "http" : "https"
     let baseURLComponents = URLComponents(string: proto + "://" + host + "/v2/")!
@@ -114,43 +122,43 @@ class Registry {
   }
 
   func ping() async throws {
-    let response = try await endpointRequest(.GET, "/v2/")
-    if response.status != .ok {
-      throw RegistryError.UnexpectedHTTPStatusCode(when: "doing ping", code: response.status.code)
+    let (_, response) = try await endpointRequest(.GET, "/v2/")
+    if response.statusCode != HTTPCode.Ok.rawValue {
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "doing ping", code: response.statusCode)
     }
   }
 
   func pushManifest(reference: String, manifest: OCIManifest) async throws -> String {
     let manifestJSON = try manifest.toJSON()
 
-    let response = try await endpointRequest(.PUT, "\(namespace)/manifests/\(reference)",
+    let (bytes, response) = try await endpointRequest(.PUT, "\(namespace)/manifests/\(reference)",
       headers: ["Content-Type": manifest.mediaType],
       body: manifestJSON)
-    if response.status != .created {
-      throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing manifest", code: response.status.code,
-        details: try await response.body.readTextResponse() ?? "")
+    if response.statusCode != HTTPCode.Created.rawValue {
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing manifest", code: response.statusCode,
+        details: try await bytes.asData().asText() ?? "")
     }
 
     return Digest.hash(manifestJSON)
   }
 
   public func pullManifest(reference: String) async throws -> (OCIManifest, Data) {
-    let response = try await endpointRequest(.GET, "\(namespace)/manifests/\(reference)",
+    let (bytes, response) = try await endpointRequest(.GET, "\(namespace)/manifests/\(reference)",
       headers: ["Accept": ociManifestMediaType])
-    if response.status != .ok {
-      let body = try await response.body.readTextResponse()
-      throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling manifest", code: response.status.code,
+    if response.statusCode != HTTPCode.Ok.rawValue {
+      let body = try await bytes.asData().asText()
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling manifest", code: response.statusCode,
         details: body ?? "")
     }
 
-    let manifestData = try await response.body.readResponse()
+    let manifestData = try await bytes.asData()
     let manifest = try OCIManifest(fromJSON: manifestData)
 
     return (manifest, manifestData)
   }
 
-  private func uploadLocationFromResponse(_ response: HTTPClientResponse) throws -> URLComponents {
-    guard let uploadLocationRaw = response.headers.first(name: "Location") else {
+  private func uploadLocationFromResponse(_ response: HTTPURLResponse) throws -> URLComponents {
+    guard let uploadLocationRaw = response.value(forHTTPHeaderField: "Location") else {
       throw RegistryError.MissingLocationHeader
     }
 
@@ -163,11 +171,11 @@ class Registry {
 
   public func pushBlob(fromData: Data, chunkSizeMb: Int = 0) async throws -> String {
     // Initiate a blob upload
-    let postResponse = try await endpointRequest(.POST, "\(namespace)/blobs/uploads/",
+    let (bytes, postResponse) = try await endpointRequest(.POST, "\(namespace)/blobs/uploads/",
       headers: ["Content-Length": "0"])
-    if postResponse.status != .accepted {
-      let body = try await postResponse.body.readTextResponse()
-      throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (POST)", code: postResponse.status.code,
+    if postResponse.statusCode != HTTPCode.Accepted.rawValue {
+      let body = try await bytes.asData().asText()
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (POST)", code: postResponse.statusCode,
         details: body ?? "")
     }
 
@@ -178,7 +186,7 @@ class Registry {
     
     if chunkSizeMb == 0 {
       // monolithic upload
-      let response = try await rawRequest(
+      let (bytes, response) = try await rawRequest(
         .PUT,
         uploadLocation,
         headers: [
@@ -187,10 +195,10 @@ class Registry {
         parameters: ["digest": digest],
         body: fromData
       )
-      if response.status != .created {
-        let body = try await response.body.readTextResponse()
+      if response.statusCode != HTTPCode.Created.rawValue {
+        let body = try await bytes.asData().asText()
         throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (PUT) to \(uploadLocation)",
-          code: response.status.code, details: body ?? "")
+          code: response.statusCode, details: body ?? "")
       }
       return digest
     }
@@ -200,7 +208,7 @@ class Registry {
     let chunks = fromData.chunks(ofCount: chunkSizeMb == 0 ? fromData.count : chunkSizeMb * 1_000_000)
     for (index, chunk) in chunks.enumerated() {
       let lastChunk = index == (chunks.count - 1)
-      let response = try await rawRequest(
+      let (bytes, response) = try await rawRequest(
         lastChunk ? .PUT : .PATCH, 
         uploadLocation,
         headers: [
@@ -210,11 +218,11 @@ class Registry {
         parameters: lastChunk ? ["digest": digest] : [:],
         body: chunk
       )
-      let expectedStatus: HTTPResponseStatus = lastChunk ? .created : .accepted
-      if response.status != expectedStatus {
-        let body = try await response.body.readTextResponse()
+      let expectedStatus = lastChunk ? HTTPCode.Created.rawValue : HTTPCode.Accepted.rawValue
+      if response.statusCode != expectedStatus {
+        let body = try await bytes.asData().asText()
         throw RegistryError.UnexpectedHTTPStatusCode(when: "streaming blob to \(uploadLocation)",
-          code: response.status.code, details: body ?? "")
+          code: response.statusCode, details: body ?? "")
       }
       uploadedBytes += chunk.count
       // Update location for the next chunk
@@ -224,18 +232,18 @@ class Registry {
     return digest
   }
 
-  public func pullBlob(_ digest: String, handler: (ByteBuffer) throws -> Void) async throws {
-    let response = try await endpointRequest(.GET, "\(namespace)/blobs/\(digest)")
-    if response.status != .ok {
-      let body = try await response.body.readTextResponse()
-      throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling blob", code: response.status.code,
+  public func pullBlob(_ digest: String, handler: (Data) throws -> Void) async throws {
+    let (bytes, response) = try await endpointRequest(.GET, "\(namespace)/blobs/\(digest)")
+    if response.statusCode != HTTPCode.Ok.rawValue {
+      let body = try await bytes.asData().asText()
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling blob", code: response.statusCode,
         details: body ?? "")
     }
 
-    for try await part in response.body {
+    for try await part in bytes.chunks(ofCount: chunkSizeBytes) {
       try Task.checkCancellation()
 
-      try handler(part)
+      try handler(Data(part))
     }
   }
 
@@ -245,7 +253,7 @@ class Registry {
     headers: Dictionary<String, String> = Dictionary(),
     parameters: Dictionary<String, String> = Dictionary(),
     body: Data? = nil
-  ) async throws -> HTTPClientResponse {
+  ) async throws -> (URLSession.AsyncBytes, HTTPURLResponse) {
     let url = URL(string: endpoint, relativeTo: baseURL)!
     let urlComponents = URLComponents(url: url, resolvingAgainstBaseURL: true)!
 
@@ -259,7 +267,7 @@ class Registry {
     parameters: Dictionary<String, String> = Dictionary(),
     body: Data? = nil,
     doAuth: Bool = true
-  ) async throws -> HTTPClientResponse {
+  ) async throws -> (URLSession.AsyncBytes, HTTPURLResponse) {
     var urlComponents = urlComponents
 
     if urlComponents.queryItems == nil && !parameters.isEmpty {
@@ -269,14 +277,14 @@ class Registry {
       URLQueryItem(name: key, value: value)
     })
 
-    var request = HTTPClientRequest(url: urlComponents.string!)
-    request.method = method
+    var request = URLRequest(url: urlComponents.url!)
+    request.httpMethod = method.rawValue
     for (key, value) in headers {
-      request.headers.add(name: key, value: value)
+      request.addValue(value, forHTTPHeaderField: key)
     }
-    if body != nil {
-      request.headers.add(name: "Content-Length", value: "\(body!.count)")
-      request.body = HTTPClientRequest.Body.bytes(body!)
+    if let body = body {
+      request.addValue("\(body.count)", forHTTPHeaderField: "Content-Length")
+      request.httpBody = body
     }
 
     // Invalidate token if it has expired
@@ -284,19 +292,19 @@ class Registry {
       currentAuthToken = nil
     }
 
-    var response = try await authAwareRequest(request: request)
+    var (bytes, response) = try await authAwareRequest(request: request)
 
-    if doAuth && response.status == .unauthorized {
+    if doAuth && response.statusCode == HTTPCode.Unauthorized.rawValue {
       try await auth(response: response)
-      response = try await authAwareRequest(request: request)
+      (bytes, response) = try await authAwareRequest(request: request)
     }
 
-    return response
+    return (bytes, response)
   }
 
-  private func auth(response: HTTPClientResponse) async throws {
+  private func auth(response: HTTPURLResponse) async throws {
     // Process WWW-Authenticate header
-    guard let wwwAuthenticateRaw = response.headers.first(name: "WWW-Authenticate") else {
+    guard let wwwAuthenticateRaw = response.value(forHTTPHeaderField: "WWW-Authenticate") else {
       throw RegistryError.AuthFailed(why: "got HTTP 401, but WWW-Authenticate header is missing")
     }
 
@@ -345,14 +353,14 @@ class Registry {
       headers["Authorization"] = "Basic \(encodedCredentials!)"
     }
 
-    let response = try await rawRequest(.GET, authenticateURL, headers: headers, doAuth: false)
-    if response.status != .ok {
-      let body = try await response.body.readTextResponse() ?? ""
-      throw RegistryError.AuthFailed(why: "received unexpected HTTP status code \(response.status.code) "
+    let (bytes, response) = try await rawRequest(.GET, authenticateURL, headers: headers, doAuth: false)
+    if response.statusCode != HTTPCode.Ok.rawValue {
+      let body = try await bytes.asData() .asText() ?? ""
+      throw RegistryError.AuthFailed(why: "received unexpected HTTP status code \(response.statusCode) "
         + "while retrieving an authentication token", details: body)
     }
 
-    let bodyData = try await response.body.readResponse()
+    let bodyData = try await bytes.asData()
     currentAuthToken = try TokenResponse.parse(fromData: bodyData)
   }
 
@@ -365,14 +373,16 @@ class Registry {
     return nil
   }
 
-  private func authAwareRequest(request: HTTPClientRequest) async throws -> HTTPClientResponse {
+  private func authAwareRequest(request: URLRequest) async throws -> (URLSession.AsyncBytes, HTTPURLResponse) {
     var request = request
 
     if let token = currentAuthToken {
       let (name, value) = token.header()
-      request.headers.add(name: name, value: value)
+      request.addValue(value, forHTTPHeaderField: name)
     }
 
-    return try await httpClient.execute(request, deadline: .distantFuture)
+    let (bytes, response) = try await URLSession.shared.bytes(for: request)
+
+    return (bytes, response as! HTTPURLResponse)
   }
 }

Sources/tart/Root.swift

@@ -17,6 +17,7 @@ struct Root: AsyncParsableCommand {
       Pull.self,
       Push.self,
       Prune.self,
+      Rename.self,
       Delete.self,
     ])
 
@@ -32,10 +33,16 @@ struct Root: AsyncParsableCommand {
     }
     sigintSrc.activate()
 
+    // Set line-buffered output for stdout
+    setlinebuf(stdout)
+
     // Parse and run command
     do {
       var command = try parseAsRoot()
 
+      // Run garbage-collection before each command (shouldn't take too long)
+      try Config().gc()
+
       if var asyncCommand = command as? AsyncParsableCommand {
         try await asyncCommand.run()
       } else {

Sources/tart/VM.swift

@@ -14,7 +14,7 @@ struct UnsupportedOSError: Error, CustomStringConvertible {
   let description: String
 
   init(_ what: String, _ plural: String) {
-    description = "error: \(what) \(plural) only supported on macOS 13.0 (Ventura) or newer"
+    description = "error: \(what) \(plural) only supported on hosts running macOS 13.0 (Ventura) or newer"
   }
 }
 
@@ -34,10 +34,10 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
   // VM's config
   var config: VMConfig
 
-  var softnet: Softnet? = nil
+  var network: Network
 
   init(vmDir: VMDirectory,
-       withSoftnet: Bool = false,
+       network: Network = NetworkShared(),
        additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = [],
        directoryShares: [DirectoryShare] = []
   ) throws {
@@ -49,13 +49,10 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     }
 
     // Initialize the virtual machine and its configuration
-    if withSoftnet {
-      softnet = try Softnet(vmMACAddress: config.macAddress.string)
-    }
-
+    self.network = network
     let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL,
       nvramURL: vmDir.nvramURL, vmConfig: config,
-      softnet: softnet, additionalDiskAttachments: additionalDiskAttachments,
+      network: network, additionalDiskAttachments: additionalDiskAttachments,
             directoryShares: directoryShares)
     virtualMachine = VZVirtualMachine(configuration: configuration)
 
@@ -63,26 +60,29 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     virtualMachine.delegate = self
   }
 
-  static func retrieveLatestIPSW() async throws -> URL {
-    defaultLogger.appendNewLine("Looking up the latest supported IPSW...")
-    let image = try await withCheckedThrowingContinuation { continuation in
-      VZMacOSRestoreImage.fetchLatestSupported() { result in
-        continuation.resume(with: result)
+  static func retrieveIPSW(remoteURL: URL) async throws -> URL {
+    // Check if we already have this IPSW in cache
+    var request = URLRequest(url: remoteURL)
+    request.httpMethod = "HEAD"
+    let (_, response) = try await URLSession.shared.data(for: request)
+    let httpURLResponse = response as! HTTPURLResponse
+
+    if let hash = httpURLResponse.value(forHTTPHeaderField: "x-amz-meta-digest-sha256") {
+      let ipswLocation = try IPSWCache().locationFor(fileName: "sha256:\(hash).ipsw")
+
+      if FileManager.default.fileExists(atPath: ipswLocation.path) {
+        defaultLogger.appendNewLine("Using cached *.ipsw file...")
+        try ipswLocation.updateAccessDate()
+
+        return ipswLocation
       }
     }
 
-    let expectedIPSWLocation = try IPSWCache().locationFor(image: image)
-
-    if FileManager.default.fileExists(atPath: expectedIPSWLocation.path) {
-      defaultLogger.appendNewLine("Using cached *.ipsw file...")
-      try expectedIPSWLocation.updateAccessDate()
-      return expectedIPSWLocation
-    }
-
-    defaultLogger.appendNewLine("Fetching \(expectedIPSWLocation.lastPathComponent)...")
+    // Download the IPSW
+    defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")
 
     let data: Data = try await withCheckedThrowingContinuation { continuation in
-      let downloadedTask = URLSession.shared.dataTask(with: image.url) { data, response, error in
+      let downloadedTask = URLSession.shared.dataTask(with: remoteURL) { data, response, error in
         if error != nil {
           continuation.resume(throwing: error!)
           return
@@ -97,10 +97,24 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
       downloadedTask.resume()
     }
 
-    try data.write(to: expectedIPSWLocation, options: [.atomic])
-    return expectedIPSWLocation
+    let ipswLocation = try IPSWCache().locationFor(fileName: Digest.hash(data) + ".ipsw")
+    try data.write(to: ipswLocation, options: [.atomic])
+
+    return ipswLocation
   }
-  
+
+  static func latestIPSWURL() async throws -> URL {
+    defaultLogger.appendNewLine("Looking up the latest supported IPSW...")
+
+    let image = try await withCheckedThrowingContinuation { continuation in
+      VZMacOSRestoreImage.fetchLatestSupported() { result in
+        continuation.resume(with: result)
+      }
+    }
+
+    return image.url
+  }
+
   var inFinalState: Bool {
     get {
       virtualMachine.state == VZVirtualMachine.State.stopped ||
@@ -112,12 +126,16 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
   init(
     vmDir: VMDirectory,
-    ipswURL: URL?,
+    ipswURL: URL,
     diskSizeGB: UInt16,
-    withSoftnet: Bool = false,
+    network: Network = NetworkShared(),
     additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = []
   ) async throws {
-    let ipswURL = ipswURL != nil ? ipswURL! : try await VM.retrieveLatestIPSW();
+    var ipswURL = ipswURL
+
+    if !ipswURL.isFileURL {
+      ipswURL = try await VM.retrieveIPSW(remoteURL: ipswURL)
+    }
 
     // Load the restore image and try to get the requirements
     // that match both the image and our platform
@@ -149,12 +167,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     try config.save(toURL: vmDir.configURL)
 
     // Initialize the virtual machine and its configuration
-    if withSoftnet {
-      softnet = try Softnet(vmMACAddress: config.macAddress.string)
-    }
-
+    self.network = network
     let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL,
-      vmConfig: config, softnet: softnet,
+      vmConfig: config, network: network,
       additionalDiskAttachments: additionalDiskAttachments,
       directoryShares: [])
     virtualMachine = VZVirtualMachine(configuration: configuration)
@@ -164,7 +179,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
     // Run automated installation
     try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
-      DispatchQueue.main.async {
+      DispatchQueue.main.async { [ipswURL] in
         let installer = VZMacOSInstaller(virtualMachine: self.virtualMachine, restoringFromImageAt: ipswURL)
 
         defaultLogger.appendNewLine("Installing OS...")
@@ -193,9 +208,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
   }
 
   func run(_ recovery: Bool) async throws {
-    if let softnet = softnet {
-      try softnet.run()
-    }
+    try network.run()
 
     DispatchQueue.main.sync {
       Task {
@@ -225,16 +238,14 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
       }
     }
 
-    if let softnet = softnet {
-      try softnet.stop();
-    }
+    try network.stop()
   }
 
   static func craftConfiguration(
     diskURL: URL,
     nvramURL: URL,
     vmConfig: VMConfig,
-    softnet: Softnet? = nil,
+    network: Network = NetworkShared(),
     additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment],
     directoryShares: [DirectoryShare]
   ) throws -> VZVirtualMachineConfiguration {
@@ -268,13 +279,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
     // Networking
     let vio = VZVirtioNetworkDeviceConfiguration()
-
-    if let softnet = softnet {
-      let fh = FileHandle.init(fileDescriptor: softnet.vmFD)
-      vio.attachment = VZFileHandleNetworkDeviceAttachment(fileHandle: fh)
-    } else {
-      vio.attachment = VZNATNetworkDeviceAttachment()
-    }
+    vio.attachment = network.attachment()
     vio.macAddress = vmConfig.macAddress
     configuration.networkDevices = [vio]
 

Sources/tart/VMDirectory+OCI.swift

@@ -35,8 +35,8 @@ extension VMDirectory {
       throw OCIError.FailedToCreateVmFile
     }
     let configFile = try FileHandle(forWritingTo: configURL)
-    try await registry.pullBlob(configLayers.first!.digest) { buffer in
-      configFile.write(Data(buffer: buffer))
+    try await registry.pullBlob(configLayers.first!.digest) { data in
+      configFile.write(data)
     }
     try configFile.close()
 
@@ -70,8 +70,7 @@ extension VMDirectory {
     ProgressObserver(progress).log(defaultLogger)
 
     for diskLayer in diskLayers {
-      try await registry.pullBlob(diskLayer.digest) { buffer in
-        let data = Data(buffer: buffer)
+      try await registry.pullBlob(diskLayer.digest) { data in
         try filter.write(data)
         progress.completedUnitCount += Int64(data.count)
       }
@@ -92,8 +91,8 @@ extension VMDirectory {
       throw OCIError.FailedToCreateVmFile
     }
     let nvram = try FileHandle(forWritingTo: nvramURL)
-    try await registry.pullBlob(nvramLayers.first!.digest) { buffer in
-      nvram.write(Data(buffer: buffer))
+    try await registry.pullBlob(nvramLayers.first!.digest) { data in
+      nvram.write(data)
     }
     try nvram.close()
   }

Sources/tart/VMDirectory.swift

@@ -29,7 +29,7 @@ struct VMDirectory: Prunable {
   }
 
   static func temporary() throws -> VMDirectory {
-    let tmpDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+    let tmpDir = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString)
     try FileManager.default.createDirectory(at: tmpDir, withIntermediateDirectories: false)
 
     return VMDirectory(baseURL: tmpDir)

Sources/tart/VMStorageLocal.swift

@@ -1,7 +1,7 @@
 import Foundation
 
 class VMStorageLocal {
-  let baseURL: URL = Config().tartHomeDir.appendingPathComponent("vms", isDirectory: true)
+  let baseURL: URL = try! Config().tartHomeDir.appendingPathComponent("vms", isDirectory: true)
 
   private func vmURL(_ name: String) -> URL {
     baseURL.appendingPathComponent(name, isDirectory: true)
@@ -32,6 +32,10 @@ class VMStorageLocal {
     _ = try FileManager.default.replaceItemAt(vmURL(name), withItemAt: from.baseURL)
   }
 
+  func rename(_ name: String, _ newName: String) throws {
+    _ = try FileManager.default.replaceItemAt(vmURL(newName), withItemAt: vmURL(name))
+  }
+
   func delete(_ name: String) throws {
     try FileManager.default.removeItem(at: vmURL(name))
   }

Sources/tart/VMStorageOCI.swift

@@ -1,12 +1,16 @@
 import Foundation
 
 class VMStorageOCI: PrunableStorage {
-  let baseURL = Config().tartCacheDir.appendingPathComponent("OCIs", isDirectory: true)
+  let baseURL = try! Config().tartCacheDir.appendingPathComponent("OCIs", isDirectory: true)
 
   private func vmURL(_ name: RemoteName) -> URL {
     baseURL.appendingRemoteName(name)
   }
 
+  private func hostDirectoryURL(_ name: RemoteName) -> URL {
+    baseURL.appendingHost(name)
+  }
+
   func exists(_ name: RemoteName) -> Bool {
     VMDirectory(baseURL: vmURL(name)).initialized
   }
@@ -125,17 +129,41 @@ class VMStorageOCI: PrunableStorage {
     let digestName = RemoteName(host: name.host, namespace: name.namespace,
             reference: Reference(digest: Digest.hash(manifestData)))
 
+    // Ensure that host directory for given RemoteName exists in OCI storage
+    let hostDirectoryURL = hostDirectoryURL(digestName)
+    try FileManager.default.createDirectory(at: hostDirectoryURL, withIntermediateDirectories: true)
+
+    // Acquire a lock on it to prevent concurrent pulls for a single host
+    let lock = try FileLock(lockURL: hostDirectoryURL)
+
+    let sucessfullyLocked = try lock.trylock()
+    if !sucessfullyLocked {
+      print("waiting for lock...")
+      try lock.lock()
+    }
+    defer { try! lock.unlock() }
+
+    if Task.isCancelled {
+      throw CancellationError()
+    }
+
     if !exists(digestName) {
       let tmpVMDir = try VMDirectory.temporary()
 
+      // Lock the temporary VM directory to prevent it's garbage collection
+      let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
+      try tmpVMDirLock.lock()
+
       // Try to reclaim some cache space if we know the VM size in advance
       if let uncompressedDiskSize = manifest.uncompressedDiskSize() {
         let requiredCapacityBytes = UInt64(uncompressedDiskSize + 128 * 1024 * 1024)
 
-        let attrs = try tmpVMDir.baseURL.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey])
-        let availableCapacityBytes = UInt64(attrs.volumeAvailableCapacityForImportantUsage!)
+        let attrs = try Config().tartCacheDir.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey, .volumeAvailableCapacityKey])
+        let availableCapacityBytes = max(UInt64(attrs.volumeAvailableCapacityForImportantUsage!), UInt64(attrs.volumeAvailableCapacity!))
 
-        if availableCapacityBytes < requiredCapacityBytes {
+        // There is a suspicious that occasionally capacity is returned as zero which can't be true.
+        // Let's validate to avoid unnecessary pruning.
+        if 0 < availableCapacityBytes && availableCapacityBytes < requiredCapacityBytes {
           try Prune.pruneReclaim(reclaimBytes: requiredCapacityBytes - availableCapacityBytes)
         }
       }
@@ -181,4 +209,8 @@ extension URL {
 
     return result
   }
+
+  func appendingHost(_ name: RemoteName) -> URL {
+    self.appendingPathComponent(name.host, isDirectory: true)
+  }
 }

Tests/TartTests/FileLockTests.swift

@@ -0,0 +1,34 @@
+import XCTest
+@testable import tart
+
+final class FileLockTests: XCTestCase {
+    func testSimple() throws {
+        // Create a temporary file that will be used as a lock
+        let url = temporaryFile()
+
+        // Make sure this file can be locked and unlocked
+        let lock = try FileLock(lockURL: url)
+        try lock.lock()
+        try lock.unlock()
+    }
+
+    func testDoubleLockResultsInError() throws {
+        // Create a temporary file that will be used as a lock
+        let url = temporaryFile()
+
+        // Create two locks on a same file and ensure one of them fails
+        let firstLock = try FileLock(lockURL: url)
+        try firstLock.lock()
+
+        let secondLock = try! FileLock(lockURL: url)
+        XCTAssertFalse(try secondLock.trylock())
+    }
+
+    private func temporaryFile() -> URL {
+        let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(UUID().uuidString)
+
+        FileManager.default.createFile(atPath: url.path, contents: nil)
+
+        return url
+    }
+}

Tests/TartTests/RegistryTests.swift

@@ -34,8 +34,8 @@ final class RegistryTests: XCTestCase {
 
         // Pull it
         var pulledBlob = Data()
-        try await registry.pullBlob(pushedBlobDigest) { buffer in
-            pulledBlob.append(Data(buffer: buffer))
+        try await registry.pullBlob(pushedBlobDigest) { data in
+            pulledBlob.append(data)
         }
 
         // Ensure that both blobs are identical
@@ -52,8 +52,8 @@ final class RegistryTests: XCTestCase {
 
         // Pull it
         var pulledLargeBlob = Data()
-        try await registry.pullBlob(largeBlobDigest) { buffer in
-            pulledLargeBlob.append(Data(buffer: buffer))       
+        try await registry.pullBlob(largeBlobDigest) { data in
+            pulledLargeBlob.append(data)
         }
 
         // Ensure that both blobs are identical

gon.hcl

@@ -7,6 +7,6 @@ apple_id {
 }
 
 sign {
-  application_identity = "Developer ID Application: Fedor Korotkov"
+  application_identity = "Developer ID Application: Cirrus Labs, Inc."
   entitlements_file = "Resources/tart.entitlements"
 }