Homebrew: depend on Softnet package (#136)

* Softnet: an alternative to built-in NAT with better isolation

* Softnet: increase socketpair(2) socket buffer sizes to 1 MiB

* Pass VM's FD and MAC address to the Softnet

* Softnet: implement graceful shutdown

* Bring back the dispatchMain() and task cancellation

* tart pull: check for cancellation when pulling response body

* Don't dispatchMain() in withTaskCancellationHandler()

* Move VNC URL opening logic into VNCWrapper.open()

.cirrus.yml

@@ -1,14 +1,16 @@
-persistent_worker:
-  labels:
-    name: Mac-Mini-M1
-
 task:
-  name: Test
+  name: Test on Ventura
+  persistent_worker:
+    labels:
+      name: Mac-Mini-M1
+  build_script: swift test
   test_script: swift test
 
 task:
   name: Build
   only_if: $CIRRUS_TAG == ''
+  macos_instance:
+    image: ghcr.io/cirruslabs/macos-monterey-xcode:14
   build_script: swift build --product tart
   sign_script: codesign --sign - --entitlements Resources/tart.entitlements --force .build/debug/tart
   binary_artifacts:
@@ -17,9 +19,12 @@ task:
 task:
   name: Release
   only_if: $CIRRUS_TAG != ''
+  macos_instance:
+    image: ghcr.io/cirruslabs/macos-monterey-xcode:14
   env:
     GITHUB_TOKEN: ENCRYPTED[!98ace8259c6024da912c14d5a3c5c6aac186890a8d4819fad78f3e0c41a4e0cd3a2537dd6e91493952fb056fa434be7c!]
     GORELEASER_KEY: ENCRYPTED[!9b80b6ef684ceaf40edd4c7af93014ee156c8aba7e6e5795f41c482729887b5c31f36b651491d790f1f668670888d9fd!]
+  install_script: brew install go goreleaser/tap/goreleaser-pro
   info_script:
     - xcodebuild -version
     - swift -version

.goreleaser.yml

@@ -36,5 +36,7 @@ brews:
     homepage: https://github.com/cirruslabs/tart
     description: Run macOS VMs on Apple Silicon
     skip_upload: auto
+    dependencies:
+      - "cirruslabs/cli/softnet"
     custom_block: |
       depends_on :macos => :monterey

Sources/tart/Commands/Create.swift

@@ -13,7 +13,7 @@ struct Create: AsyncParsableCommand {
   var fromIPSW: String?
 
   @Option(help: ArgumentHelp("Disk size in Gb")) 
-  var diskSize: UInt8 = 50
+  var diskSize: UInt16 = 50
 
   func validate() throws {
     if fromIPSW == nil {

Sources/tart/Commands/Run.swift

@@ -27,55 +27,54 @@ struct Run: AsyncParsableCommand {
           discussion: "Useful since VNC supports copy/paste, drag and drop, etc.\nNote that Remote Login option should be enabled inside the VM.")) 
   var vnc: Bool = false
 
+  @Flag var withSoftnet: Bool = false
+
   @MainActor
-  func run() async throws {
-    if recovery && vnc {
-      print("You can't run in recovery and use VNC!")
-      Foundation.exit(1)      
-    }
-    
+  func run() async throws {    
     let vmDir = try VMStorageLocal().open(name)
-    vm = try VM(vmDir: vmDir)
+    vm = try VM(vmDir: vmDir, withSoftnet: withSoftnet)
 
-    await withThrowingTaskGroup(of: Void.self) { group in
-      if vnc {
-        group.addTask(operation: {
-          do {
-            print("Waiting for the VM to boot...")
-            let resolvedIP = try await IP.resolveIP(vm!.config, secondsToWait: 60)
-            guard let ip = resolvedIP else {
-              throw IPNotFound()
-            }
-            let url = URL(string: "vnc://\(ip)")!
-            print("Opening \(url)")
-            NSWorkspace.shared.open(url)
-          } catch {
-            print("Failed to get an IP for screen sharing: \(error)")
-          }
-        })
-      }
-      group.addTask {
-        do {
-          try await vm!.run(recovery)
+    var vncWrapper: VNCWrapper?
 
-          Foundation.exit(0)
-        } catch {
-          if error.localizedDescription.contains("Failed to lock auxiliary storage.") {
-            print("Virtual machine \"\(name)\" is already running!")
-          } else {
-            print(error)
-          }
+    if vnc {
+        vncWrapper = VNCWrapper(virtualMachine: vm!.virtualMachine)
+    }
 
-          Foundation.exit(1)
+    let task = Task {
+      do {
+        if let vncWrapper = vncWrapper {
+          await vncWrapper.open()
         }
-      }
 
-      if noGraphics || vnc {
-        dispatchMain()
-      } else {
-        runUI()
+        try await vm!.run(recovery)
+
+        if let vncWrapper = vncWrapper {
+          try vncWrapper.stop()
+        }
+
+        Foundation.exit(0)
+      } catch {
+        if error.localizedDescription.contains("Failed to lock auxiliary storage.") {
+          print("Virtual machine \"\(name)\" is already running!")
+        } else {
+          print(error)
+        }
+
+        Foundation.exit(1)
       }
     }
+
+    let sigintSrc = DispatchSource.makeSignalSource(signal: SIGINT)
+    sigintSrc.setEventHandler {
+      task.cancel()
+    }
+    sigintSrc.activate()
+
+    if noGraphics || vnc {
+      dispatchMain()
+    } else {
+      runUI()
+    }
   }
 
   private func runUI() {

Sources/tart/Commands/Set.swift

@@ -17,7 +17,7 @@ struct Set: AsyncParsableCommand {
   var display: VMDisplayConfig?
 
   @Option(help: .hidden)
-  var diskSize: UInt8?
+  var diskSize: UInt16?
 
   func run() async throws {
     do {

Sources/tart/OCI/Manifest.swift

@@ -3,15 +3,38 @@ import Foundation
 let ociManifestMediaType = "application/vnd.oci.image.manifest.v1+json"
 let ociConfigMediaType = "application/vnd.oci.image.config.v1+json"
 
+// Annotations
+let uncompressedDiskSizeAnnotation = "org.cirruslabs.tart.uncompressed-disk-size"
+
 struct OCIManifest: Codable, Equatable {
   var schemaVersion: Int = 2
   var mediaType: String = ociManifestMediaType
   var config: OCIManifestConfig
   var layers: [OCIManifestLayer] = Array()
+  var annotations: Dictionary<String, String>?
+
+  init(config: OCIManifestConfig, layers: [OCIManifestLayer], uncompressedDiskSize: UInt64? = nil) {
+    self.config = config
+    self.layers = layers
+
+    if let uncompressedDiskSize = uncompressedDiskSize {
+      annotations = [
+        uncompressedDiskSizeAnnotation: String(uncompressedDiskSize)
+      ]
+    }
+  }
 
   func digest() throws -> String {
     try Digest.hash(JSONEncoder().encode(self))
   }
+
+  func uncompressedDiskSize() -> UInt64? {
+    guard let value = annotations?[uncompressedDiskSizeAnnotation] else {
+      return nil
+    }
+
+    return UInt64(value)
+  }
 }
 
 struct OCIManifestConfig: Codable, Equatable {

Sources/tart/OCI/Registry.swift

@@ -184,8 +184,8 @@ class Registry {
       body: fromData)
     if putResponse.status != .created {
       let body = try await postResponse.body.readTextResponse()
-      throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (PUT)", code: putResponse.status.code,
-        details: body ?? "")
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (PUT) to \(uploadLocation)",
+              code: putResponse.status.code, details: body ?? "")
     }
 
     return digest
@@ -200,6 +200,8 @@ class Registry {
     }
 
     for try await part in response.body {
+      try Task.checkCancellation()
+
       try handler(part)
     }
   }

Sources/tart/Passphrase/PassphraseGenerator.swift

@@ -0,0 +1,13 @@
+import Foundation
+
+struct PassphraseGenerator: Sequence {
+    func makeIterator() -> PassphraseIterator {
+        PassphraseIterator()
+    }
+}
+
+struct PassphraseIterator: IteratorProtocol {
+    mutating func next() -> String? {
+        passphrases[Int(arc4random_uniform(UInt32(passphrases.count)))]
+    }
+}

Sources/tart/Root.swift

@@ -28,7 +28,6 @@ struct Root: AsyncParsableCommand {
     let sigintSrc = DispatchSource.makeSignalSource(signal: SIGINT)
     sigintSrc.setEventHandler {
       task.cancel()
-      Darwin.exit(1)
     }
     sigintSrc.activate()
 

Sources/tart/Softnet.swift

@@ -0,0 +1,77 @@
+import Foundation
+
+enum SoftnetError: Error {
+  case InitializationFailed(why: String)
+}
+
+class Softnet {
+  private let process = Process()
+
+  let vmFD: Int32
+
+  init(vmMACAddress: String) throws {
+    let binaryName = "softnet"
+
+    guard let executableURL = Self.resolveBinaryPath(binaryName) else {
+      throw SoftnetError.InitializationFailed(why: "\(binaryName) not found in PATH")
+    }
+
+    let fds = UnsafeMutablePointer<Int32>.allocate(capacity: MemoryLayout<Int>.stride * 2)
+
+    let ret = socketpair(AF_UNIX, SOCK_DGRAM, 0, fds)
+    if ret != 0 {
+      throw SoftnetError.InitializationFailed(why: "socketpair() failed with exit code \(ret)")
+    }
+
+    vmFD = fds[0]
+    let softnetFD = fds[1]
+
+    try setSocketBuffers(vmFD, 1 * 1024 * 1024);
+    try setSocketBuffers(softnetFD, 1 * 1024 * 1024);
+
+    process.executableURL = executableURL
+    process.arguments = ["--vm-fd", String(STDIN_FILENO), "--vm-mac-address", vmMACAddress]
+    process.standardInput = FileHandle(fileDescriptor: softnetFD, closeOnDealloc: false)
+  }
+
+  func run() throws {
+    try process.run()
+  }
+
+  func stop() throws {
+    process.interrupt()
+    process.waitUntilExit()
+  }
+
+  private static func resolveBinaryPath(_ name: String) -> URL? {
+    guard let path = ProcessInfo.processInfo.environment["PATH"] else {
+      return nil
+    }
+
+    for pathComponent in path.split(separator: ":") {
+      let url = URL(fileURLWithPath: String(pathComponent))
+              .appendingPathComponent(name, isDirectory: false)
+
+      if FileManager.default.fileExists(atPath: url.path) {
+        return url
+      }
+    }
+
+    return nil
+  }
+
+  private func setSocketBuffers(_ fd: Int32, _ sizeBytes: Int) throws {
+    var option_value = sizeBytes
+    let option_len = socklen_t(MemoryLayout<Int>.size)
+
+    var ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &option_value, option_len)
+    if ret != 0 {
+      throw SoftnetError.InitializationFailed(why: "setsockopt(SO_RCVBUF) returned \(ret)")
+    }
+
+    ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &option_value, option_len)
+    if ret != 0 {
+      throw SoftnetError.InitializationFailed(why: "setsockopt(SO_SNDBUF) returned \(ret)")
+    }
+  }
+}

Sources/tart/VM.swift

@@ -23,17 +23,24 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
   // VM's config
   var config: VMConfig
 
-  init(vmDir: VMDirectory) throws {
+  var softnet: Softnet? = nil
+
+  init(vmDir: VMDirectory, withSoftnet: Bool = false) throws {
     let auxStorage = VZMacAuxiliaryStorage(contentsOf: vmDir.nvramURL)
 
     name = vmDir.name
     config = try VMConfig.init(fromURL: vmDir.configURL)
 
-    let configuration = try VM.craftConfiguration(diskURL: vmDir.diskURL, auxStorage: auxStorage, vmConfig: config)
+    // Initialize the virtual machine and its configuration
+    if withSoftnet {
+      softnet = try Softnet(vmMACAddress: config.macAddress.string)
+    }
+
+    let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, auxStorage: auxStorage, vmConfig: config,
+      softnet: softnet)
     virtualMachine = VZVirtualMachine(configuration: configuration)
 
     super.init()
-
     virtualMachine.delegate = self
   }
 
@@ -77,8 +84,17 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     try data.write(to: expectedIPSWLocation, options: [.atomic])
     return expectedIPSWLocation
   }
+  
+  var inFinalState: Bool {
+    get {
+      virtualMachine.state == VZVirtualMachine.State.stopped ||
+        virtualMachine.state == VZVirtualMachine.State.paused ||
+        virtualMachine.state == VZVirtualMachine.State.error
+      
+    }
+  }
 
-  init(vmDir: VMDirectory, ipswURL: URL?, diskSizeGB: UInt8) async throws {
+  init(vmDir: VMDirectory, ipswURL: URL?, diskSizeGB: UInt16, withSoftnet: Bool = false) async throws {
     let ipswURL = ipswURL != nil ? ipswURL! : try await VM.retrieveLatestIPSW();
 
     // Load the restore image and try to get the requirements
@@ -111,11 +127,15 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     try config.save(toURL: vmDir.configURL)
 
     // Initialize the virtual machine and its configuration
-    let configuration = try VM.craftConfiguration(diskURL: vmDir.diskURL, auxStorage: auxStorage, vmConfig: config)
+    if withSoftnet {
+      softnet = try Softnet(vmMACAddress: config.macAddress.string)
+    }
+
+    let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, auxStorage: auxStorage, vmConfig: config,
+      softnet: softnet)
     virtualMachine = VZVirtualMachine(configuration: configuration)
 
     super.init()
-
     virtualMachine.delegate = self
 
     // Run automated installation
@@ -134,6 +154,10 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
   }
 
   func run(_ recovery: Bool) async throws {
+    if let softnet = softnet {
+      try softnet.run()
+    }
+
     try await virtualMachine.start(recovery)
 
     await withTaskCancellationHandler(operation: {
@@ -149,9 +173,18 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
         }
       }
     }
+
+    if let softnet = softnet {
+      try softnet.stop();
+    }
   }
 
-  static func craftConfiguration(diskURL: URL, auxStorage: VZMacAuxiliaryStorage, vmConfig: VMConfig) throws -> VZVirtualMachineConfiguration {
+  static func craftConfiguration(
+    diskURL: URL,
+    auxStorage: VZMacAuxiliaryStorage,
+    vmConfig: VMConfig,
+    softnet: Softnet? = nil
+  ) throws -> VZVirtualMachineConfiguration {
     let configuration = VZVirtualMachineConfiguration()
 
     // Boot loader
@@ -203,7 +236,13 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
     // Networking
     let vio = VZVirtioNetworkDeviceConfiguration()
-    vio.attachment = VZNATNetworkDeviceAttachment()
+
+    if let softnet = softnet {
+      let fh = FileHandle.init(fileDescriptor: softnet.vmFD)
+      vio.attachment = VZFileHandleNetworkDeviceAttachment(fileHandle: fh)
+    } else {
+      vio.attachment = VZNATNetworkDeviceAttachment()
+    }
     vio.macAddress = vmConfig.macAddress
     configuration.networkDevices = [vio]
 

Sources/tart/VMDirectory+OCI.swift

@@ -117,8 +117,10 @@ extension VMDirectory {
     // Read VM's compressed disk as chunks
     // and sequentially upload them as blobs
     let disk = try FileHandle(forReadingFrom: diskURL)
+    var diskReadBytes: UInt64 = 0
     let compressingFilter = try InputFilter<Data>(.compress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { _ in
       let data = try disk.read(upToCount: Self.bufferSizeBytes)
+      diskReadBytes += UInt64(data?.count ?? 0)
 
       progress.completedUnitCount += Int64(data?.count ?? 0)
 
@@ -146,7 +148,8 @@ extension VMDirectory {
     let ociConfigDigest = try await registry.pushBlob(fromData: ociConfigJSON)
     let manifest = OCIManifest(
             config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),
-            layers: layers
+            layers: layers,
+            uncompressedDiskSize: diskReadBytes
     )
 
     // Manifest

Sources/tart/VMDirectory.swift

@@ -68,7 +68,7 @@ struct VMDirectory {
     try newVMConfig.save(toURL: to.configURL)
   }
 
-  func resizeDisk(_ sizeGB: UInt8) throws {
+  func resizeDisk(_ sizeGB: UInt16) throws {
     if !FileManager.default.fileExists(atPath: diskURL.path) {
       FileManager.default.createFile(atPath: diskURL.path, contents: nil, attributes: nil)
     }

Sources/tart/VNCWrapper.swift

@@ -0,0 +1,53 @@
+import Foundation
+import Dynamic
+import Virtualization
+
+class VNCWrapper {
+    private let password: String
+    private let vnc: Dynamic
+
+    init(virtualMachine: VZVirtualMachine) {
+        password = Array(PassphraseGenerator().prefix(4)).joined(separator: "-")
+        let securityConfiguration = Dynamic._VZVNCAuthenticationSecurityConfiguration(password: password)
+        vnc = Dynamic._VZVNCServer(port: 0, queue: DispatchQueue.global(),
+                securityConfiguration: securityConfiguration)
+        vnc.virtualMachine = virtualMachine
+        vnc.start()
+    }
+
+    func open() async {
+        do {
+            let port = try await Self.waitForPort(vnc: vnc)
+
+            let url = URL(string: "vnc://:\(password)@127.0.0.1:\(port)")!
+            print("Opening \(url)...")
+
+            if ProcessInfo.processInfo.environment["CI"] == nil {
+                NSWorkspace.shared.open(url)
+            }
+        } catch {
+            print("Failed to retrieve a VNC server port: \(error)")
+        }
+    }
+
+    func stop() throws {
+        vnc.stop()
+    }
+
+    deinit {
+        try? stop()
+    }
+
+    private static func waitForPort(vnc: Dynamic) async throws -> UInt16 {
+        while true {
+            // Port is 0 shortly after start(),
+            // but will be initialized later
+            if let port = vnc.port.asUInt16, port != 0 {
+                return port
+            }
+
+            // Wait 50 ms.
+            try await Task.sleep(nanoseconds: 50_000_000)
+        }
+    }
+}