tart run --nested to enable nested virtualization when available (#938)

Only works for Linux VMs under Sequoia hosts.

Fixes #933
Fixes #701

.ci/set-version.sh

@@ -3,3 +3,5 @@
 TMPFILE=$(mktemp)
 envsubst < Sources/tart/CI/CI.swift > $TMPFILE
 mv $TMPFILE Sources/tart/CI/CI.swift
+
+/usr/libexec/PlistBuddy -c "Add :CFBundleShortVersionString string ${CIRRUS_TAG}" Resources/Info.plist

PROFILING.md

@@ -0,0 +1,64 @@
+# Profiling Tart
+
+## Using `time(1)`
+
+Perhaps, the easiest, but not the most comprehensive way to tell what's going on with Tart is to use the [`time(1)`](https://ss64.com/mac/time.html) command.
+
+In the example below, you will run `tart pull` via `time(1)` to gather generalized CPU, I/O and memory usage metrics:
+
+```shell
+/usr/bin/time -l tart pull ghcr.io/cirruslabs/macos-sequoia-base:latest
+```
+
+**Note:** you need to specify a full path to `time(1)` binary, otherwise the shell's built-in `time` command will be invoked, which doesn't have the `-l` command-line argument.
+
+**Note:** The `-l` command-line argument makes `time(1)` return much more useful information, for example, maximum memory usage.
+
+When running the command above, you'll see the `tart pull` output first as it pulls the image, and then the `time(1)` output, which will be printed once the Tart process finishes:
+
+```
+      172.17 real        10.29 user         8.36 sys
+           353796096  maximum resident set size
+                   0  average shared memory size
+                   0  average unshared data size
+                   0  average unshared stack size
+               23838  page reclaims
+                  35  page faults
+                   0  swaps
+                   0  block input operations
+                   0  block output operations
+                   8  messages sent
+                   8  messages received
+                   0  signals received
+                 146  voluntary context switches
+              222950  involuntary context switches
+         39683070975  instructions retired
+         27562035252  cycles elapsed
+           170920448  peak memory footprint
+```
+
+From the output above, you can tell that `tart pull` spent nearly 90% of time off-CPU (`real` > `user` + `sys`), which means that Tart was mostly waiting for the I/O (be it a network or disk), instead of decompressing disk layers or doing other useful computations.
+
+## Using `xctrace(1)`
+
+[`xctrace(1)`](https://keith.github.io/xcode-man-pages/xctrace.1.html) is a `.trace` format recorder for the [Instruments](https://en.wikipedia.org/wiki/Instruments_(software)) app, which yields much more powerful insights compared to `time(1)`. For example, it can tell which Tart functions spent the most time on the CPU, thus allowing the Tart developers to further optimize these functions.
+
+To use it, make sure that [Xcode](https://developer.apple.com/xcode/resources/) is installed. If you're installing Xcode for the first time on the machine, you'll need to launch it once and click the blue "Install" button. There's no need to choose any platforms except for the macOS.
+
+Once done, you can create a CPU profile of `tart pull`:
+
+```shell
+xctrace record --template "CPU Profiler" --target-stdout - --launch -- /opt/homebrew/bin/tart pull ghcr.io/cirruslabs/macos-sequoia-base:latest
+```
+
+Now that `xctrace(1)` is running, you'll see the `tart pull`-related output first, and once finished, the following line will appear:
+
+```
+Output file saved as: Launch_[...].trace
+```
+
+To view this trace in the Instruments app, simply find this directory in Finder and double-click it. Instruments app will appear:
+
+![](Resources/Instruments.png)
+
+To send this trace, right-click its directory in Finder and choose "Compress [...]". This will result in a similarly named file with a `.zip` at the end, which can now be conveniently sent via email or uploaded.

Resources/Info.plist

@@ -10,8 +10,6 @@
 	<string>org.cirruslabs.tart</string>
 	<key>CFBundleExecutable</key>
 	<string>tart</string>
-	<key>LSBackgroundOnly</key>
-	<string>1</string>
 	<key>CFBundleIconFiles</key>
 	<array>
 		<string>AppIcon.png</string>

Sources/tart/Commands/Clone.swift

@@ -28,6 +28,9 @@ struct Clone: AsyncParsableCommand {
   @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
   var concurrency: UInt = 4
 
+  @Flag(help: .hidden)
+  var deduplicate: Bool = false
+
   func validate() throws {
     if newName.contains("/") {
       throw ValidationError("<new-name> should be a local name")
@@ -45,7 +48,7 @@ struct Clone: AsyncParsableCommand {
     if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
       // Pull the VM in case it's OCI-based and doesn't exist locally yet
       let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
-      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency)
+      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
     }
 
     let sourceVM = try VMStorageHelper.open(sourceName)

Sources/tart/Commands/Pull.swift

@@ -23,6 +23,9 @@ struct Pull: AsyncParsableCommand {
   @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
   var concurrency: UInt = 4
 
+  @Flag(help: .hidden)
+  var deduplicate: Bool = false
+
   func validate() throws {
     if concurrency < 1 {
       throw ValidationError("network concurrency cannot be less than 1")
@@ -43,6 +46,6 @@ struct Pull: AsyncParsableCommand {
 
     defaultLogger.appendNewLine("pulling \(remoteName)...")
 
-    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency)
+    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
   }
 }

Sources/tart/Commands/Run.swift

@@ -156,6 +156,9 @@ struct Run: AsyncParsableCommand {
   """, valueName: "[name:]path[:options]"))
   var dir: [String] = []
 
+  @Flag(help: ArgumentHelp("Enable nested virtualization if possible"))
+  var nested: Bool = false
+
   @Option(help: ArgumentHelp("""
   Use bridged networking instead of the default shared (NAT) networking \n(e.g. --net-bridged=en0 or --net-bridged=\"Wi-Fi\")
   """, discussion: """
@@ -221,6 +224,14 @@ struct Run: AsyncParsableCommand {
       throw ValidationError("--captures-system-keys can only be used with the default VM view")
     }
 
+    if nested {
+      if #unavailable(macOS 15) {
+        throw ValidationError("Nested virtualization is supported on hosts starting with macOS 15 (Sequia), and later.")
+      } else if !VZGenericPlatformConfiguration.isNestedVirtualizationSupported {
+        throw ValidationError("Nested virtualization is available for Mac with the M3 chip, and later.")
+      }
+    }
+
     let localStorage = VMStorageLocal()
     let vmDir = try localStorage.open(name)
     if try vmDir.state() == .Suspended {
@@ -294,6 +305,7 @@ struct Run: AsyncParsableCommand {
       directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
       serialPorts: serialPorts,
       suspendable: suspendable,
+      nested: nested,
       audio: !noAudio,
       clipboard: !noClipboard,
       sync: VZDiskImageSynchronizationMode(diskOptions.syncModeRaw)

Sources/tart/OCI/Layerizer/Disk.swift

@@ -2,5 +2,5 @@ import Foundation
 
 protocol Disk {
   static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?) async throws
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws
 }

Sources/tart/OCI/Layerizer/DiskV1.swift

@@ -45,7 +45,7 @@ class DiskV1: Disk {
     return pushedLayers
   }
 
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
     if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
       throw OCIError.FailedToCreateVmFile
     }

Sources/tart/OCI/Layerizer/DiskV2.swift

@@ -69,12 +69,12 @@ class DiskV2: Disk {
     }
   }
 
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
     // Support resumable pulls
     let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)
 
     if !pullResumed {
-      if let localLayerCache = localLayerCache {
+      if deduplicate, let localLayerCache = localLayerCache {
         // Clone the local layer cache's disk and use it as a base, potentially
         // reducing the space usage since some blocks won't be written at all
         try FileManager.default.copyItem(at: localLayerCache.diskURL, to: diskURL)
@@ -151,26 +151,31 @@ class DiskV2: Disk {
 
           // Also open the disk file for reading and verifying
           // its contents in case the local layer cache is used
-          let rdisk: FileHandle? = if localLayerCache != nil {
+          let rdisk: FileHandle? = if deduplicate && localLayerCache != nil {
             try FileHandle(forReadingFrom: diskURL)
           } else {
             nil
           }
 
-          // Check if we already have this layer contents in the local layer cache
-          if let localLayerCache = localLayerCache, let localLayerInfo = localLayerCache.findInfo(digest: diskLayer.digest, offsetHint: diskWritingOffset) {
-            // indicates that the locally cloned disk image has the same content at the given offset
-            let localHit = localLayerInfo.uncompressedContentDigest == uncompressedLayerContentDigest
-              && localLayerInfo.range.lowerBound == diskWritingOffset
-            // doesn't seem that localHit can ever be false if the localLayerCache is not nil
-            // but let's just add extra safety here and check it
-            if !localHit {
+          // Check if we already have this layer contents in the local layer cache,
+          // or perhaps even on the cloned disk (when the deduplication is enabled)
+          if let localLayerCache = localLayerCache,
+             let localLayerInfo = localLayerCache.findInfo(digest: diskLayer.digest, offsetHint: diskWritingOffset),
+             localLayerInfo.uncompressedContentDigest == uncompressedLayerContentDigest {
+            if deduplicate && localLayerInfo.range.lowerBound == diskWritingOffset {
+              // Do nothing, because the data is already on the disk that we've inherited from
+            } else {
               // Fulfil the layer contents from the local blob cache
               let data = localLayerCache.subdata(localLayerInfo.range)
               _ = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
             }
+
             try disk.close()
 
+            if let rdisk = rdisk {
+              try rdisk.close()
+            }
+
             // Update the progress
             progress.completedUnitCount += Int64(diskLayer.size)
 
@@ -198,6 +203,10 @@ class DiskV2: Disk {
           try filter.finalize()
 
           try disk.close()
+
+          if let rdisk = rdisk {
+            try rdisk.close()
+          }
         }
 
         globalDiskWritingOffset += uncompressedLayerSize

Sources/tart/OCI/Registry.swift

@@ -327,12 +327,12 @@ class Registry {
       request.httpBody = body
     }
 
-    var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
+    var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)
 
     if doAuth && response.statusCode == HTTPCode.Unauthorized.rawValue {
       _ = try await channel.asData()
       try await auth(response: response)
-      (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
+      (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)
     }
 
     return (channel, response)
@@ -413,11 +413,13 @@ class Registry {
     return nil
   }
 
-  private func authAwareRequest(request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+  private func authAwareRequest(request: URLRequest, viaFile: Bool = false, doAuth: Bool) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
     var request = request
 
-    if let (name, value) = await authenticationKeeper.header() {
-      request.addValue(value, forHTTPHeaderField: name)
+    if doAuth {
+      if let (name, value) = await authenticationKeeper.header() {
+        request.addValue(value, forHTTPHeaderField: name)
+      }
     }
 
     request.setValue("Tart/\(CI.version) (\(DeviceInfo.os); \(DeviceInfo.model))",

Sources/tart/Platform/Darwin.swift

@@ -58,7 +58,11 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
       VZMacOSBootLoader()
     }
 
-    func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
+    func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration {
+      if needsNestedVirtualization {
+        throw RuntimeError.VMConfigurationError("macOS virtual machines do not support nested virtualization")
+      }
+
       let result = VZMacPlatformConfiguration()
 
       result.machineIdentifier = ecid

Sources/tart/Platform/Linux.swift

@@ -14,8 +14,12 @@ struct Linux: Platform {
     return result
   }
 
-  func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
-    VZGenericPlatformConfiguration()
+  func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration {
+    let config = VZGenericPlatformConfiguration()
+    if #available(macOS 15, *) {
+      config.isNestedVirtualizationEnabled = needsNestedVirtualization
+    }
+    return config
   }
 
   func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration {

Sources/tart/Platform/Platform.swift

@@ -3,7 +3,7 @@ import Virtualization
 protocol Platform: Codable {
   func os() -> OS
   func bootLoader(nvramURL: URL) throws -> VZBootLoader
-  func platform(nvramURL: URL) throws -> VZPlatformConfiguration
+  func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration
   func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration
   func keyboards() -> [VZKeyboardConfiguration]
   func pointingDevices() -> [VZPointingDeviceConfiguration]

Sources/tart/URL+AccessDate.swift

@@ -1,4 +1,5 @@
 import Foundation
+import System
 
 extension URL {
   func accessDate() throws -> Date {
@@ -13,7 +14,9 @@ extension URL {
     let times = [accessDate.asTimeval(), modificationDate.asTimeval()]
     let ret = utimes(path, times)
     if ret != 0 {
-      throw RuntimeError.FailedToUpdateAccessDate("utimes(2) failed: \(ret.explanation())")
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.FailedToUpdateAccessDate("utimes(2) failed: \(details)")
     }
   }
 }

Sources/tart/VM.swift

@@ -47,6 +47,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
        directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
        serialPorts: [VZSerialPortConfiguration] = [],
        suspendable: Bool = false,
+       nested: Bool = false,
        audio: Bool = true,
        clipboard: Bool = true,
        sync: VZDiskImageSynchronizationMode = .full
@@ -66,6 +67,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
                                                 directorySharingDevices: directorySharingDevices,
                                                 serialPorts: serialPorts,
                                                 suspendable: suspendable,
+                                                nested: nested,
                                                 audio: audio,
                                                 clipboard: clipboard,
                                                 sync: sync
@@ -295,6 +297,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     directorySharingDevices: [VZDirectorySharingDeviceConfiguration],
     serialPorts: [VZSerialPortConfiguration],
     suspendable: Bool = false,
+    nested: Bool = false,
     audio: Bool = true,
     clipboard: Bool = true,
     sync: VZDiskImageSynchronizationMode = .full
@@ -309,7 +312,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     configuration.memorySize = vmConfig.memorySize
 
     // Platform
-    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL)
+    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL, needsNestedVirtualization: nested)
 
     // Display
     configuration.graphicsDevices = [vmConfig.platform.graphicsDevice(vmConfig: vmConfig)]
@@ -365,13 +368,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
       try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: sync) :
       try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .automatic, synchronizationMode: sync)
 
-    var device: VZStorageDeviceConfiguration
-    if #available(macOS 14, *), vmConfig.os == .linux {
-      device = VZNVMExpressControllerDeviceConfiguration(attachment: attachment)
-    } else {
-      device = VZVirtioBlockDeviceConfiguration(attachment: attachment)
-    }
-    var devices: [VZStorageDeviceConfiguration] = [device]
+    var devices: [VZStorageDeviceConfiguration] = [VZVirtioBlockDeviceConfiguration(attachment: attachment)]
     devices.append(contentsOf: additionalStorageDevices)
     configuration.storageDevices = devices
 

Sources/tart/VMDirectory+OCI.swift

@@ -11,7 +11,7 @@ enum OCIError: Error {
 }
 
 extension VMDirectory {
-  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?) async throws {
+  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws {
     // Pull VM's config file layer and re-serialize it into a config file
     let configLayers = manifest.layers.filter {
       $0.mediaType == configMediaType
@@ -54,12 +54,13 @@ extension VMDirectory {
     do {
       try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL,
                                   concurrency: concurrency, progress: progress,
-                                  localLayerCache: localLayerCache)
+                                  localLayerCache: localLayerCache,
+                                  deduplicate: deduplicate)
     } catch let error where error is FilterError {
       throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)")
     }
 
-    if let llc = localLayerCache {
+    if deduplicate, let llc = localLayerCache {
       // set custom attribute to remember deduplicated bytes
       diskURL.setDeduplicatedBytes(llc.deduplicatedBytes)
     }

Sources/tart/VMStorageOCI.swift

@@ -140,7 +140,7 @@ class VMStorageOCI: PrunableStorage {
     try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
   }
 
-  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt) async throws {
+  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws {
     SentrySDK.configureScope { scope in
       scope.setContext(value: ["imageName": name.description], key: "OCI")
     }
@@ -203,10 +203,14 @@ class VMStorageOCI: PrunableStorage {
           if let llc = localLayerCache {
             let deduplicatedHuman = ByteCountFormatter.string(fromByteCount: Int64(llc.deduplicatedBytes), countStyle: .file)
 
-            defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to deduplicate \(deduplicatedHuman), using it as a base...")
+            if deduplicate {
+              defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to deduplicate \(deduplicatedHuman), using it as a base...")
+            } else {
+              defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to avoid fetching \(deduplicatedHuman), will try use it...")
+            }
           }
 
-          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache)
+          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache, deduplicate: deduplicate)
         } recoverFromFailure: { error in
           if error is Retryable {
             print("Error: \(error.localizedDescription)")
@@ -235,6 +239,9 @@ class VMStorageOCI: PrunableStorage {
       // are excluded from garbage collection
       VMDirectory(baseURL: vmURL(name)).markExplicitlyPulled()
     }
+
+    // to explicitly set the image as being accessed so it won't get pruned immediately
+    _ = try VMStorageOCI().open(name)
   }
 
   func linked(from: RemoteName, to: RemoteName) -> Bool {