Package.swift: bump Sentry SDK to 8.36.0 + upgrade other packages (#905)

* Package.swift: bump Sentry SDK to 8.36.0

* $ swift package update

.cirrus.yml

@@ -8,11 +8,11 @@ task:
       name: dev-mini
     resources:
       tart-vms: 1
+  build_script:
+    - swift build
   test_script:
     - swift test
   integration_test_script:
-    # Build Tart
-    - swift build
     - codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart
     - export PATH=$(pwd)/.build/arm64-apple-macosx/debug:$PATH
     # Run integration tests

Package.resolved

@@ -1,5 +1,5 @@
 {
-  "originHash" : "2c514a4a1d7e106713db744bee89edb40d75da63e6611990ec2f4b0da53c0455",
+  "originHash" : "a10a6363a6d2cd2761653d1587b81f5e3d55b1f981b7d6a2964b34da91addf13",
   "pins" : [
     {
       "identity" : "antlr4",
@@ -7,7 +7,7 @@
       "location" : "https://github.com/antlr/antlr4",
       "state" : {
         "branch" : "dev",
-        "revision" : "2703a8516c0fb7fe92db6b9c40e0113f577646d2"
+        "revision" : "2a7904a595479954ccfb1c78124fc3d7f5bebcb1"
       }
     },
     {
@@ -24,8 +24,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/groue/Semaphore",
       "state" : {
-        "revision" : "f1c4a0acabeb591068dea6cffdd39660b86dec28",
+        "revision" : "2543679282aa6f6c8ecf2138acd613ed20790bc2",
-        "version" : "0.0.8"
+        "version" : "0.1.0"
       }
     },
     {
@@ -33,8 +33,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/getsentry/sentry-cocoa",
       "state" : {
-        "revision" : "ef4fec9dfb8dd5027b09a4a5c9362feafd118e1a",
+        "revision" : "5575af93efb776414f243e93d6af9f6258dc539a",
-        "version" : "8.24.0"
+        "version" : "8.36.0"
       }
     },
     {
@@ -51,8 +51,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-argument-parser",
       "state" : {
-        "revision" : "46989693916f56d1186bd59ac15124caef896560",
+        "revision" : "41982a3656a71c768319979febd796c6fd111d5c",
-        "version" : "1.3.1"
+        "version" : "1.5.0"
       }
     },
     {
@@ -61,7 +61,7 @@
       "location" : "https://github.com/apple/swift-async-algorithms",
       "state" : {
         "branch" : "main",
-        "revision" : "f05e450f0b909c0e80670a47516c4b9700b9e5da"
+        "revision" : "5c8bd186f48c16af0775972700626f0b74588278"
       }
     },
     {
@@ -78,8 +78,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-collections.git",
       "state" : {
-        "revision" : "f504716c27d2e5d4144fa4794b12129301d17729",
+        "revision" : "9bf03ff58ce34478e66aaee630e491823326fd06",
-        "version" : "1.0.3"
+        "version" : "1.1.3"
       }
     },
     {
@@ -87,8 +87,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-log.git",
       "state" : {
-        "revision" : "e97a6fcb1ab07462881ac165fdbb37f067e205d5",
+        "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537",
-        "version" : "1.5.4"
+        "version" : "1.6.1"
       }
     },
     {
@@ -105,8 +105,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/fumoboy007/swift-retry",
       "state" : {
-        "revision" : "9f133487ffc2ab4539688c29efe57bb1ba31d7b0",
+        "revision" : "df9d7b185d2e433147ec0083a73c257e665eea0d",
-        "version" : "0.2.3"
+        "version" : "0.2.4"
       }
     },
     {
@@ -141,8 +141,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/nicklockwood/SwiftFormat",
       "state" : {
-        "revision" : "9df3b01f477163b33d5e63c5e2e5b9f946a49c56",
+        "revision" : "ab6844edb79a7b88dc6320e6cee0a0db7674dac3",
-        "version" : "0.53.6"
+        "version" : "0.54.5"
       }
     },
     {

Package.swift

@@ -18,7 +18,7 @@ let package = Package(
     .package(url: "https://github.com/antlr/antlr4", branch: "dev"),
     .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.2.0")),
     .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.53.6"),
-    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.24.0"),
+    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.36.0"),
     .package(url: "https://github.com/cfilipov/TextTable", branch: "master"),
     .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.8.0"),
     .package(url: "https://github.com/orchetect/SwiftRadix", from: "1.3.1"),

README.md

@@ -17,7 +17,7 @@ service — a drop-in replacement for the standard GitHub-hosted runners, offeri
   </a>
 </p>
 
-Many companies are using Tart in their internal setups. Here are a few of them:
+Many companies are using Tart in their internal setups. Here are just a few of them:
 
 <p align="center">
   <a href="https://atlassian.com/" target=_blank>
@@ -47,6 +47,9 @@ Many companies are using Tart in their internal setups. Here are a few of them:
   <a href="https://www.pitsdatarecovery.net/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/PITSGlobalDataRecoveryServices.png" height="65"/>
   </a>
+  <a href="https://expo.dev/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Expo.png" height="65"/>
+  </a>
 </p>
 
 **Note:** If your company or project is using Tart please consider [sharing with the community](https://github.com/cirruslabs/tart/discussions/857).

Sources/tart/Commands/Run.swift

@@ -12,6 +12,39 @@ var vm: VM?
 struct IPNotFound: Error {
 }
 
+@available(macOS 14, *)
+extension VZDiskSynchronizationMode {
+  public init(_ description: String) throws {
+    switch description {
+    case "none":
+      self = .none
+    case "full":
+      self = .full
+    case "":
+      self = .full
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk synchronization mode: \"\(description)\"")
+    }
+  }
+}
+
+extension VZDiskImageSynchronizationMode {
+  public init(_ description: String) throws {
+    switch description {
+    case "none":
+      self = .none
+    case "fsync":
+      self = .fsync
+    case "full":
+      self = .full
+    case "":
+      self = .full
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk image synchronization mode: \"\(description)\"")
+    }
+  }
+}
+
 struct Run: AsyncParsableCommand {
   static var configuration = CommandConfiguration(abstract: "Run a VM")
 
@@ -140,6 +173,20 @@ struct Run: AsyncParsableCommand {
   @Flag(help: ArgumentHelp("Restrict network access to the host-only network"))
   var netHost: Bool = false
 
+  @Option(help: ArgumentHelp("Set the root disk options (e.g. --root-disk-opts=\"ro\" or --root-disk-opts=\"sync=none\")",
+                             discussion: """
+                             Options are comma-separated and are as follows:
+
+                             * ro — attach the root disk in read-only mode instead of the default read-write (e.g. --root-disk-opts="ro")
+
+                             * sync=none — disable data synchronization with the permanent storage to increase performance at the cost of a higher chance of data loss (e.g. --root-disk-opts="sync=none")
+
+                             * sync=fsync — enable data synchronization with the permanent storage, but don't ensure that it was actually written (e.g. --root-disk-opts="sync=fsync")
+
+                             * sync=full — enable data synchronization with the permanent storage and ensure that it was actually written (e.g. --root-disk-opts="sync=full")
+                             """, valueName: "options"))
+  var rootDiskOpts: String = ""
+
   #if arch(arm64)
     @Flag(help: ArgumentHelp("Disables audio and entropy devices and switches to only Mac-specific input devices.", discussion: "Useful for running a VM that can be suspended via \"tart suspend\"."))
   #endif
@@ -182,7 +229,7 @@ struct Run: AsyncParsableCommand {
 
     if suspendable {
       let config = try VMConfig.init(fromURL: vmDir.configURL)
-      if (config.platform is Linux) {
+      if !(config.platform is PlatformSuspendable) {
         throw ValidationError("You can only suspend macOS VMs")
       }
       if dir.count > 0 {
@@ -219,8 +266,6 @@ struct Run: AsyncParsableCommand {
       try Softnet.configureSUIDBitIfNeeded()
     }
 
-    let additionalDiskAttachments = try additionalDiskAttachments()
-
     var serialPorts: [VZSerialPortConfiguration] = []
     if serial {
       let tty_fd = createPTY()
@@ -239,15 +284,19 @@ struct Run: AsyncParsableCommand {
       serialPorts.append(createSerialPortConfiguration(tty_read!, tty_write!))
     }
 
+    // Parse root disk options
+    let diskOptions = DiskOptions(rootDiskOpts)
+
     vm = try VM(
       vmDir: vmDir,
       network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
-      additionalStorageDevices: additionalDiskAttachments,
+      additionalStorageDevices: try additionalDiskAttachments(),
       directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
       serialPorts: serialPorts,
       suspendable: suspendable,
       audio: !noAudio,
-      clipboard: !noClipboard
+      clipboard: !noClipboard,
+      sync: VZDiskImageSynchronizationMode(diskOptions.syncModeRaw)
     )
 
     let vncImpl: VNC? = try {
@@ -297,7 +346,35 @@ struct Run: AsyncParsableCommand {
           }
         #endif
 
-        try await vm!.start(recovery: recovery, resume: resume)
+        do {
+          try await vm!.start(recovery: recovery, resume: resume)
+        } catch let error as VZError {
+          if error.code == .virtualMachineLimitExceeded {
+            var hint = ""
+
+            do {
+              let runningVMs: [String] = try localStorage.list().compactMap { (name, vmDir) in
+                if try !vmDir.running() {
+                  return nil
+                }
+
+                return name
+              }
+
+              if !runningVMs.isEmpty {
+                let runningVMsJoined = runningVMs.joined(separator: ", ")
+
+                hint = " (other running VMs: \(runningVMsJoined))"
+              }
+            } catch {
+              // we can't provide any hint
+            }
+
+            throw RuntimeError.VirtualMachineLimitExceeded(hint)
+          }
+
+          throw error
+        }
 
         if let vncImpl = vncImpl {
           let vncURL = try await vncImpl.waitForURL(netBridged: !netBridged.isEmpty)
@@ -455,15 +532,9 @@ struct Run: AsyncParsableCommand {
   }
 
   func additionalDiskAttachments() throws -> [VZStorageDeviceConfiguration] {
-    var result: [VZStorageDeviceConfiguration] = []
+    try disk.map {
-
+      try AdditionalDisk(parseFrom: $0).configuration
-    let expandedDiskPaths = disk.map { NSString(string:$0).expandingTildeInPath }
-
-    for rawDisk in expandedDiskPaths {
-      result.append(try AdditionalDisk(parseFrom: rawDisk).configuration)
     }
-
-    return result
   }
 
   func directoryShares() throws -> [VZDirectorySharingDeviceConfiguration] {
@@ -700,18 +771,20 @@ struct AdditionalDisk {
         throw UnsupportedOSError("attaching Network Block Devices", "are")
       }
 
-      let syncMode = try parseSyncMode(syncModeRaw)
-
       let nbdAttachment = try VZNetworkBlockDeviceStorageDeviceAttachment(
         url: diskURL!,
         timeout: 30,
         isForcedReadOnly: diskReadOnly,
-        synchronizationMode: syncMode
+        synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw)
       )
 
       return VZVirtioBlockDeviceConfiguration(attachment: nbdAttachment)
     }
 
+    // Expand the tilde (~) since at this point we're dealing with a local path,
+    // and "expandingTildeInPath" seems to corrupt the remote URLs like nbd://
+    let diskPath = NSString(string: diskPath).expandingTildeInPath
+
     let diskFileURL = URL(fileURLWithPath: diskPath)
 
     if pathHasMode(diskPath, mode: S_IFBLK) {
@@ -734,7 +807,7 @@ struct AdditionalDisk {
       }
 
       let blockAttachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: FileHandle(fileDescriptor: fd, closeOnDealloc: true),
-                                                                         readOnly: diskReadOnly, synchronizationMode: try parseSyncMode(syncModeRaw))
+                                                                         readOnly: diskReadOnly, synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw))
 
       return VZVirtioBlockDeviceConfiguration(attachment: blockAttachment)
     }
@@ -768,7 +841,7 @@ struct AdditionalDisk {
       url: diskFileURL,
       readOnly: diskReadOnly,
       cachingMode: .automatic,
-      synchronizationMode: try Self.parseImageSyncMode(syncModeRaw)
+      synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw)
     )
 
     return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
@@ -776,58 +849,36 @@ struct AdditionalDisk {
 
   static func parseOptions(_ parseFrom: String) -> (String, Bool, String) {
     var arguments = parseFrom.split(separator: ":")
-    let options = arguments.last!.split(separator: ",")
 
-    var readOnly: Bool = false
+    let options = DiskOptions(String(arguments.last!))
-    var syncModeRaw: String = ""
+    if options.foundAtLeastOneOption {
+      arguments.removeLast()
+    }
 
-    var foundAtLeastOneOption: Bool = false
+    return (arguments.joined(separator: ":"), options.readOnly, options.syncModeRaw)
+  }
+}
+
+struct DiskOptions {
+  var readOnly: Bool = false
+  var syncModeRaw: String = ""
+  var foundAtLeastOneOption: Bool = false
+
+  init(_ parseFrom: String) {
+    let options = parseFrom.split(separator: ",")
 
     for option in options {
       switch true {
       case option == "ro":
-        readOnly = true
+        self.readOnly = true
-        foundAtLeastOneOption = true
+        self.foundAtLeastOneOption = true
       case option.hasPrefix("sync="):
-        syncModeRaw = String(option.dropFirst("sync=".count))
+        self.syncModeRaw = String(option.dropFirst("sync=".count))
-        foundAtLeastOneOption = true
+        self.foundAtLeastOneOption = true
       default:
         continue
       }
     }
-
-    if foundAtLeastOneOption {
-      arguments.removeLast()
-    }
-
-    return (arguments.joined(separator: ":"), readOnly, syncModeRaw)
-  }
-
-  @available(macOS 14, *)
-  static func parseSyncMode(_ parseFrom: String) throws -> VZDiskSynchronizationMode {
-    switch parseFrom {
-    case "none":
-      return .none
-    case "full":
-      return .full
-    case "":
-      return .full
-    default:
-      throw RuntimeError.VMConfigurationError("unsupported disk synchronization mode: \"\(parseFrom)\"")
-    }
-  }
-
-  static func parseImageSyncMode(_ parseFrom: String) throws -> VZDiskImageSynchronizationMode {
-    switch parseFrom {
-    case "none":
-      return .none
-    case "full":
-      return .full
-    case "":
-      return .full
-    default:
-      throw RuntimeError.VMConfigurationError("unsupported disk image synchronization mode: \"\(parseFrom)\"")
-    }
   }
 }
 

Sources/tart/LocalLayerCache.swift

@@ -1,12 +1,19 @@
 import Foundation
 
 struct LocalLayerCache {
+  struct DigestInfo {
+    let range: Range<Data.Index>
+    let compressedDigest: String
+    let uncompressedContentDigest: String?
+  }
+
   let name: String
   let deduplicatedBytes: UInt64
   let diskURL: URL
 
   private let mappedDisk: Data
-  private var digestToRange: [String : Range<Data.Index>] = [:]
+  private var digestToRange: [String: DigestInfo] = [:]
+  private var offsetToRange: [UInt64: DigestInfo] = [:]
 
   init?(_ name: String, _ deduplicatedBytes: UInt64, _ diskURL: URL, _ manifest: OCIManifest) throws {
     self.name = name
@@ -24,17 +31,27 @@ struct LocalLayerCache {
         return nil
       }
 
-      self.digestToRange[layer.digest] = Int(offset)..<Int(offset+uncompressedSize)
+      let info = DigestInfo(
+        range: Int(offset)..<Int(offset + uncompressedSize),
+        compressedDigest: layer.digest,
+        uncompressedContentDigest: layer.uncompressedContentDigest()!
+      )
+      self.digestToRange[layer.digest] = info
+      self.offsetToRange[offset] = info
 
       offset += uncompressedSize
     }
   }
 
-  func find(_ digest: String) -> Data? {
+  func findInfo(digest: String, offsetHint: UInt64) -> DigestInfo? {
-    guard let foundRange = self.digestToRange[digest] else {
+    // Layers can have the same digests, for example, empty ones. Let's use the offset hint to make a better guess.
-      return nil
+    if let info = self.offsetToRange[offsetHint], info.compressedDigest == digest {
+      return info
     }
+    return self.digestToRange[digest]
+  }
 
-    return self.mappedDisk.subdata(in: foundRange)
+  func subdata(_ range: Range<Data.Index>) -> Data {
+    return self.mappedDisk.subdata(in: range)
   }
 }

Sources/tart/OCI/Layerizer/DiskV2.swift

@@ -1,6 +1,7 @@
 import Foundation
 import Compression
 import System
+import Retry
 
 class DiskV2: Disk {
   private static let bufferSizeBytes = 4 * 1024 * 1024
@@ -26,8 +27,22 @@ class DiskV2: Disk {
         // Launch a disk layer pushing task
         group.addTask {
           let compressedData = try (data as NSData).compressed(using: .lz4) as Data
+          let compressedDataDigest = Digest.hash(compressedData)
 
-          let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+          try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) {
+            if try await !registry.blobExists(compressedDataDigest) {
+              _ = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb, digest: compressedDataDigest)
+            }
+          } recoverFromFailure: { error in
+            if error is URLError {
+              print("Error: \(error.localizedDescription)")
+              print("Attempting to re-try...")
+
+              return .retry
+            }
+
+            return .throw
+          }
 
           // Update progress using a relative value
           progress.completedUnitCount += Int64(data.count)
@@ -35,7 +50,7 @@ class DiskV2: Disk {
           return (index, OCIManifestLayer(
             mediaType: diskV2MediaType,
             size: compressedData.count,
-            digest: layerDigest,
+            digest: compressedDataDigest,
             uncompressedSize: UInt64(data.count),
             uncompressedContentDigest: Digest.hash(data)
           ))
@@ -121,11 +136,14 @@ class DiskV2: Disk {
         // Launch a fetching and decompression task
         group.addTask {
           // No need to fetch and decompress anything if we've already done so
-          if try pullResumed && Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+          if pullResumed {
-            // Update the progress
+            // do not check hash in the condition above to make it lazy e.g. only do expensive calculations if needed
-            progress.completedUnitCount += Int64(diskLayer.size)
+            if try Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+              // Update the progress
+              progress.completedUnitCount += Int64(diskLayer.size)
 
-            return
+              return
+            }
           }
 
           // Open the disk file for writing
@@ -140,9 +158,17 @@ class DiskV2: Disk {
           }
 
           // Check if we already have this layer contents in the local layer cache
-          if let localLayerCache = localLayerCache, let data = localLayerCache.find(diskLayer.digest), Digest.hash(data) == uncompressedLayerContentDigest {
+          if let localLayerCache = localLayerCache, let localLayerInfo = localLayerCache.findInfo(digest: diskLayer.digest, offsetHint: diskWritingOffset) {
-            // Fulfil the layer contents from the local blob cache
+            // indicates that the locally cloned disk image has the same content at the given offset
-            _ = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
+            let localHit = localLayerInfo.uncompressedContentDigest == uncompressedLayerContentDigest
+              && localLayerInfo.range.lowerBound == diskWritingOffset
+            // doesn't seem that localHit can ever be false if the localLayerCache is not nil
+            // but let's just add extra safety here and check it
+            if !localHit {
+              // Fulfil the layer contents from the local blob cache
+              let data = localLayerCache.subdata(localLayerInfo.range)
+              _ = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
+            }
             try disk.close()
 
             // Update the progress

Sources/tart/OCI/Registry.swift

@@ -10,6 +10,7 @@ enum RegistryError: Error {
 }
 
 enum HTTPMethod: String {
+  case HEAD = "HEAD"
   case GET = "GET"
   case POST = "POST"
   case PUT = "PUT"
@@ -21,6 +22,7 @@ enum HTTPCode: Int {
   case Created = 201
   case Accepted = 202
   case Unauthorized = 401
+  case NotFound = 404
 }
 
 extension Data {
@@ -189,7 +191,7 @@ class Registry {
     return URLComponents(url: uploadLocation.absolutize(baseURL), resolvingAgainstBaseURL: true)!
   }
 
-  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0) async throws -> String {
+  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0, digest: String? = nil) async throws -> String {
     // Initiate a blob upload
     let (data, postResponse) = try await dataRequest(.POST, endpointURL("\(namespace)/blobs/uploads/"),
                                                      headers: ["Content-Length": "0"])
@@ -201,7 +203,7 @@ class Registry {
     // Figure out where to upload the blob
     var uploadLocation = try uploadLocationFromResponse(postResponse)
 
-    let digest = Digest.hash(fromData)
+    let digest = digest ?? Digest.hash(fromData)
 
     if chunkSizeMb == 0 {
       // monolithic upload
@@ -249,6 +251,19 @@ class Registry {
     return digest
   }
 
+  public func blobExists(_ digest: String) async throws -> Bool {
+    let (data, response) = try await dataRequest(.HEAD, endpointURL("\(namespace)/blobs/\(digest)"))
+
+    switch response.statusCode {
+    case HTTPCode.Ok.rawValue:
+      return true
+    case HTTPCode.NotFound.rawValue:
+      return false
+    default:
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "checking blob", code: response.statusCode, details: data.asText())
+    }
+  }
+
   public func pullBlob(_ digest: String, handler: (Data) async throws -> Void) async throws {
     let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), viaFile: true)
     if response.statusCode != HTTPCode.Ok.rawValue {

Sources/tart/Platform/Darwin.swift

@@ -8,7 +8,7 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
 
 #if arch(arm64)
 
-  struct Darwin: Platform {
+  struct Darwin: PlatformSuspendable {
     var ecid: VZMacMachineIdentifier
     var hardwareModel: VZMacHardwareModel
 
@@ -103,18 +103,32 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
     func keyboards() -> [VZKeyboardConfiguration] {
       if #available(macOS 14, *) {
         // Mac keyboard is only supported by guests starting with macOS Ventura
-        return [VZMacKeyboardConfiguration()]
+        return [VZUSBKeyboardConfiguration(), VZMacKeyboardConfiguration()]
       } else {
         return [VZUSBKeyboardConfiguration()]
       }
     }
 
+    func keyboardsSuspendable() -> [VZKeyboardConfiguration] {
+      if #available(macOS 14, *) {
+        return [VZMacKeyboardConfiguration()]
+      } else {
+        // fallback to the regular configuration
+        return keyboards()
+      }
+    }
+
     func pointingDevices() -> [VZPointingDeviceConfiguration] {
-      if #available(macOS 13, *) {
+      // Trackpad is only supported by guests starting with macOS Ventura
+      [VZUSBScreenCoordinatePointingDeviceConfiguration(), VZMacTrackpadConfiguration()]
+    }
+
+    func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration] {
+      if #available(macOS 14, *) {
         return [VZMacTrackpadConfiguration()]
       } else {
         // fallback to the regular configuration
-        return [VZUSBScreenCoordinatePointingDeviceConfiguration()]
+        return pointingDevices()
       }
     }
   }

Sources/tart/Platform/Platform.swift

@@ -8,3 +8,8 @@ protocol Platform: Codable {
   func keyboards() -> [VZKeyboardConfiguration]
   func pointingDevices() -> [VZPointingDeviceConfiguration]
 }
+
+protocol PlatformSuspendable: Platform {
+  func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration]
+  func keyboardsSuspendable() -> [VZKeyboardConfiguration]
+}

Sources/tart/URL+Prunable.swift

@@ -17,7 +17,6 @@ extension URL: Prunable {
   func deduplicatedSizeBytes() throws -> Int {
     let values = try resourceValues(forKeys: [.totalFileAllocatedSizeKey, .mayShareFileContentKey])
     // make sure the file's origin file is there and duplication works
-    var dedublicatedSize = 0
     if values.mayShareFileContent == true {
       return Int(deduplicatedBytes())
     }

Sources/tart/VM.swift

@@ -48,7 +48,8 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
        serialPorts: [VZSerialPortConfiguration] = [],
        suspendable: Bool = false,
        audio: Bool = true,
-       clipboard: Bool = true
+       clipboard: Bool = true,
+       sync: VZDiskImageSynchronizationMode = .full
   ) throws {
     name = vmDir.name
     config = try VMConfig.init(fromURL: vmDir.configURL)
@@ -66,7 +67,8 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
                                                 serialPorts: serialPorts,
                                                 suspendable: suspendable,
                                                 audio: audio,
-                                                clipboard: clipboard
+                                                clipboard: clipboard,
+                                                sync: sync
     )
     virtualMachine = VZVirtualMachine(configuration: configuration)
 
@@ -294,7 +296,8 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     serialPorts: [VZSerialPortConfiguration],
     suspendable: Bool = false,
     audio: Bool = true,
-    clipboard: Bool = true
+    clipboard: Bool = true,
+    sync: VZDiskImageSynchronizationMode = .full
   ) throws -> VZVirtualMachineConfiguration {
     let configuration = VZVirtualMachineConfiguration()
 
@@ -314,20 +317,29 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Audio
     let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
 
-    let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
-    let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
-
     if audio && !suspendable {
+      let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
+      let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
+
       inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
       outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
+
+      soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
+    } else {
+      // just a null speaker
+      soundDeviceConfiguration.streams = [VZVirtioSoundDeviceOutputStreamConfiguration()]
     }
 
-    soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
     configuration.audioDevices = [soundDeviceConfiguration]
 
     // Keyboard and mouse
-    configuration.keyboards = vmConfig.platform.keyboards()
+    if suspendable, let platformSuspendable = vmConfig.platform.self as? PlatformSuspendable {
-    configuration.pointingDevices = vmConfig.platform.pointingDevices()
+      configuration.keyboards = platformSuspendable.keyboardsSuspendable()
+      configuration.pointingDevices = platformSuspendable.pointingDevicesSuspendable()
+    } else {
+      configuration.keyboards = vmConfig.platform.keyboards()
+      configuration.pointingDevices = vmConfig.platform.pointingDevices()
+    }
 
     // Networking
     configuration.networkDevices = network.attachments().map {
@@ -350,8 +362,8 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Storage
     let attachment: VZDiskImageStorageDeviceAttachment = vmConfig.os == .linux ?
       // Use "cached" caching mode for virtio drive to prevent fs corruption on linux
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: .full) :
+      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: sync) :
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)
+      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .automatic, synchronizationMode: sync)
 
     var device: VZStorageDeviceConfiguration
     if #available(macOS 14, *), vmConfig.os == .linux {

Sources/tart/VMStorageHelper.swift

@@ -71,6 +71,7 @@ enum RuntimeError : Error {
   case OCIUnsupportedDiskFormat(_ format: String)
   case SuspendFailed(_ message: String)
   case PullFailed(_ message: String)
+  case VirtualMachineLimitExceeded(_ hint: String)
 }
 
 protocol HasExitCode {
@@ -128,6 +129,8 @@ extension RuntimeError : CustomStringConvertible {
       return "Failed to suspend the VM: \(message)"
     case .PullFailed(let message):
       return message
+    case .VirtualMachineLimitExceeded(let hint):
+      return "The number of VMs exceeds the system limit\(hint)"
     }
   }
 }

Sources/tart/VMStorageOCI.swift

@@ -208,14 +208,14 @@ class VMStorageOCI: PrunableStorage {
 
           try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache)
         } recoverFromFailure: { error in
-          if error is RuntimeError {
+          if error is Retryable {
-            return .throw
+            print("Error: \(error.localizedDescription)")
+            print("Attempting to re-try...")
+
+            return .retry
           }
 
-          print("Error: \(error.localizedDescription)")
+          return .throw
-          print("Attempting to re-try...")
-
-          return .retry
         }
         try move(digestName, from: tmpVMDir)
         transaction.finish()
@@ -297,7 +297,7 @@ class VMStorageOCI: PrunableStorage {
 
     // Now, find the best match based on how many bytes we'll deduplicate
     let choosen = candidates.filter {
-      $0.deduplicatedBytes > 0
+      $0.deduplicatedBytes > 1024 * 1024 * 1024 // save at least 1GB
     }.max { left, right in
       return left.deduplicatedBytes < right.deduplicatedBytes
     }

docs/orchard/architecture-and-security.md

@@ -0,0 +1,65 @@
+## Architecture
+
+Orchard cluster consists of two components:
+
+* Controller — responsible for managing the cluster and scheduling of resources
+* Worker — responsible for executing the VMs
+* Client — responsible for creating, modifying and removing the resources on the Controller, can either be an Orchard CLI or [an API consumer](/orchard/integration-guide)
+
+Normally you deploy a single Controller that needs to be accessible to both the Clients and Workers. Then you can deploy the Workers, which can reside anywhere and be inaccessible to Clients directly, e.g. behind a NAT.
+
+## Security
+
+When an Orchard Client or a Worker connects to the Controller, they need to establish trust and verify that they're talking to the right Controller, so that no [man-in-the-middle attack](https://en.wikipedia.org/wiki/Man-in-the-middle_attack) is possible.
+
+Similarly to web-browsers (that rely on the [public key infrastructure](https://en.wikipedia.org/wiki/Public_key_infrastructure)) and SSH (which relies on semi-automated fingerprint verification), Orchard combines these two traits in a hybrid approach by defaulting to automatic PKI verification (can be disabled by [`--no-pki`](#--no-pki-override)) and falling-back to a manual verification for self-signed certificates.
+
+This hybrid approach is needed because the Controller can be configured in two ways:
+
+* *Controller with a publicly valid certificate*
+    * can be configured manually by passing `--controller-cert` and `--controller-key` command-line arguments to `orchard controller run`
+* *Controller with a self-signed certificate*
+    * configured automatically on first Controller start-up when no `--controller-cert` and `--controller-key` command-line arguments are passed
+
+Below we'll explain how Orchard client and Worker secure the connection when accessing these two Controller types.
+
+### Client
+
+Client is associated with the Controller using a `orchard context create` command, which works as follows:
+
+* Client attempts to connect to the Controller and validate its certificate using host's root CA set (can be disabled with [`--no-pki`](#--no-pki-override))
+* if the Client encounters a  *Controller with a publicly valid certificate*, that would be the last step and the association would succeed
+* if the Client is dealing with *Controller with a self-signed certificate*, the Client will do another connection attempt to probe the Controller's certificate
+* the probed Controller's certificate fingerprint is then presented to the user, and if the user agrees to trust it, the Client then considers that certificate to be trusted for a given context
+* Client finally connects to the Controller again with a trusted CA set containing only that certificate, executes the final API sanity checks, and if everything is OK then the association succeeds
+
+Afterward, each interaction with the Controller  (e.g. `orchard create vm` command) will stick to the chosen verification method and will re-verify the presented Controller's certificate against:
+
+* *Controller with a self-signed certificate*: a trusted certificate stored in the Orchard's configuration file
+* *Controller with a publicly valid certificate*: host's root CA set
+
+### Worker
+
+To make the Worker connect to the Controller, a Bootstrap Token needs to be obtained using the `orchard get bootstrap-token` command.
+
+While this approach provides a less ad-hoc experience than that you'd have with `orchard context create`, it allows one to mass-deploy workers non-interactively, using tools such as Ansible.
+
+This resulting Bootstrap Token will either include the Controller's certificate (when the current context is with a *Controller with a self-signed certificate*) or omit it (when the current context is with a *Controller with a publicly valid certificate*).
+
+The way Worker connects to the Controller using the `orchard worker run` command is as follows:
+
+* when the Bootstrap Token contains the Controller's certificate:
+    * the Orchard Worker will try to connect to the Controller with a trusted CA set containing only that certificate
+* when the Bootstrap Token has no Controller's certificate:
+    * the Orchard Worker will try the PKI approach (can be disabled with [`--no-pki`](#--no-pki-override) to effectively prevent the Worker from connecting) and fail if certificate verification using PKI is not possible
+
+### `--no-pki` override
+
+If you only intend to access the *Controller with a self-signed certificate* and want to additionally guard yourself against [CA compromises](https://en.wikipedia.org/wiki/Certificate_authority#CA_compromise) and other PKI-specific attacks, pass a `--no-pki` command-line argument to the following commands:
+
+* `orchard context create --no-pki`
+    * this will prevent the Client from using PKI and will let you interactively verify the Controller's certificate fingerprint before connecting, thus creating a non-PKI association
+* `orchard worker run --no-pki`
+    * this will prevent the Worker from trying to use PKI when connecting to the Controller using a Bootstrap Token that has no certificate included in it, thus failing fast and letting you know that you need to create a proper Bootstrap Token
+
+We've deliberately chosen not to use environment variables (e.g. `ORCHARD_NO_PKI`) because they fail silently (e.g. due to a typo), compared to command-line arguments, which will result in an error that is much easier to detect.

docs/orchard/deploying-controller.md

@@ -0,0 +1,195 @@
+## Introduction
+
+Compared to Worker, which can only be deployed on a macOS machine, Controller can be also deployed on Linux.
+
+In fact, we've made a [container image](https://github.com/cirruslabs/orchard/pkgs/container/orchard) to ease deploying the Controller in container-native environments such as Kubernetes.
+
+Another thing to keep in mind that Orchard API is secured by default: all requests must be authenticated with the credentials of a service account. When you first run Orchard Controller, a `bootstrap-admin` service account will be created automatically and credentials will be printed to the standard output.
+
+If you already have a token in mind that you want to use for the `bootstrap-admin` service account, or you've got locked out and want this service account with a well-known password back, you can set the `ORCHARD_BOOTSTRAP_ADMIN_TOKEN` when running the controller.
+
+For example to use a secure, random value:
+
+```bash
+ORCHARD_BOOTSTRAP_ADMIN_TOKEN=$(openssl rand -hex 32) orchard controller run
+```
+
+## Deployment Methods
+
+While you can always start `orchard controller run` manually with the required arguments, this method is not recommended due to lack of persistence.
+
+In the following sections you'll find several examples of how to run Orchard Controller in various environments in a more persistent way. Feel free to submit PRs with more examples.
+
+### Google Compute Engine
+
+An example below will deploy a single instance of Orchard Controller in Google Cloud Compute Engine in `us-central1` region.
+
+First, let's create a static IP address for our instance:
+
+```bash
+gcloud compute addresses create orchard-ip --region=us-central1
+export ORCHARD_IP=$(gcloud compute addresses describe orchard-ip --format='value(address)' --region=us-central1)
+```
+
+Once we have the IP address, we can create a new instance with Orchard Controller running inside a container:
+
+```bash
+gcloud compute instances create-with-container orchard-controller \
+  --machine-type=e2-micro \
+  --zone=us-central1-a \
+  --image-family cos-stable \
+  --image-project cos-cloud \
+  --tags=https-server \
+  --address=$ORCHARD_IP \
+  --container-image=ghcr.io/cirruslabs/orchard:latest \
+  --container-env=PORT=443 \
+  --container-env=ORCHARD_BOOTSTRAP_ADMIN_TOKEN=$ORCHARD_BOOTSTRAP_ADMIN_TOKEN \
+  --container-mount-host-path=host-path=/home/orchard-data,mode=rw,mount-path=/data
+```
+
+Now you can create a new context for your local client:
+
+```bash
+orchard context create --name production \
+  --service-account-name bootstrap-admin \
+  --service-account-token $ORCHARD_BOOTSTRAP_ADMIN_TOKEN \
+  https://$ORCHARD_IP:443
+```
+
+And select it as the default context:
+
+```bash
+orchard context default production
+```
+
+### Kubernetes (GKE, EKS, etc.)
+
+The easiest way to run Orchard Controller on Kubernetes is to expose it through the `LoadBalancer` service.
+
+This way no fiddling with the TLS certificates and HTTP proxying is needed, and most cloud providers will allocate a ready-to-use IP-address that can directly used in `orchard context create` and `orchard worker run` commands, or additionally assigned to a DNS domain name for a more memorable hostname.
+
+Do deploy on Kubernetes, only three resources are needed:
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: orchard-controller
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+  # Uncomment this when deploying on Amazon's EKS and
+  # change to the desired storage class name if needed
+  # storageClassName: gp2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: orchard-controller
+spec:
+  serviceName: orchard-controller
+  replicas: 1
+  selector:
+    matchLabels:
+      app: orchard-controller
+  template:
+    metadata:
+      labels:
+        app: orchard-controller
+    spec:
+      containers:
+        - name: orchard-controller
+          image: ghcr.io/cirruslabs/orchard:latest
+          volumeMounts:
+            - mountPath: /data
+              name: orchard-controller
+      volumes:
+        - name: orchard-controller
+          persistentVolumeClaim:
+            claimName: orchard-controller
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: orchard-controller
+spec:
+  selector:
+    app: orchard-controller
+  ports:
+    - protocol: TCP
+      port: 6120
+      targetPort: 6120
+  type: LoadBalancer
+```
+
+Once deployed, the bootstrap credentials will be printed to the standard output. You can inspect them by running `kubectl logs deployment/orchard-controller`.
+
+The resources above ensure that Controller's database is stored in a persistent storage and survives restats.
+
+You can further allocate a static IP address and use it by adding annotations to the `Service` resource. Here's how to do that:
+
+* on Google's GKE: <https://cloud.google.com/kubernetes-engine/docs/concepts/service-load-balancer-parameters#spd-static-ip>
+* on Amazon's EKS: <https://kubernetes.io/docs/reference/labels-annotations-taints/#service-beta-kubernetes-io-aws-load-balancer-eip-allocations>
+
+### systemd service on Debian-based distributions
+
+This should work for most Debian-based distributions like Debian, Ubuntu, etc.
+
+Firstly, make sure that the APT transport for downloading packages via HTTPS and common X.509 certificates are installed:
+
+```shell
+sudo apt-get update && sudo apt-get -y install apt-transport-https ca-certificates
+```
+
+Then, add the Cirrus Labs repository:
+
+```shell
+echo "deb [trusted=yes] https://apt.fury.io/cirruslabs/ /" | sudo tee /etc/apt/sources.list.d/cirruslabs.list
+```
+
+Update the package index files and install the Orchard Controller:
+
+```shell
+sudo apt-get update && sudo apt-get -y install orchard-controller
+```
+
+Finally, enable and start the Orchard Controller systemd service:
+
+```shell
+sudo systemctl enable orchard-controller
+sudo systemctl start orchard-controller
+```
+
+The bootstrap credentials will be printed to the standard output. You can inspect them by running `sudo systemctl status orhcard-controller` or `journalctl -u orchard-controller`.
+
+### systemd service on RPM-based distributions
+
+This should work for most RPM-based distributions like Fedora, CentOS, etc.
+
+First, create a `/etc/yum.repos.d/cirruslabs.repo` file with the following contents:
+
+```ini
+[cirruslabs]
+name=Cirrus Labs Repo
+baseurl=https://yum.fury.io/cirruslabs/
+enabled=1
+gpgcheck=0
+```
+
+Then, install the Orchard Controller:
+
+```shell
+sudo yum -y install orchard-controller
+```
+
+Finally, enable and start the Orchard Controller systemd service:
+
+```shell
+systemctl enable orchard-controller
+systemctl start orchard-controller
+```
+
+The bootstrap credentials will be printed to the standard output. You can inspect them by running `sudo systemctl status orhcard-controller` or `journalctl -u orchard-controller`.

docs/orchard/deploying-workers.md

@@ -0,0 +1,127 @@
+## Obtain a Boostrap Token
+
+First, create a service account with a minimal set of roles (`compute:read` and `compute:write`) required for proper Worker functioning:
+
+```bash
+orchard create service-account worker-pool-m1 --roles "compute:read" --roles "compute:write"
+```
+
+Then, generate a Bootstrap Token for this service account:
+
+```shell
+orchard get bootstrap-token worker-pool-m1
+```
+
+We will reference the value of the Bootstrap Token generated here as `${BOOTSTRAP_TOKEN}` below.
+
+Further, we assume that Orchard controller is available on `orchard.example.com`
+
+## Deployment Methods
+
+While you can always run `orchard worker run` manually with the required arguments, this method of deploying the Worker is not recommended.
+
+Instead, we've listed a more persistent methods of a Worker deployment below.
+
+### launchd
+
+[launchd](https://launchd.info/) is an init system for macOS that manages daemons, agents and other background processes.
+
+In this deployment method, we'll create a new job definition file for the launchd to manage on its behalf.
+
+To begin, first install Orchard:
+
+```shell
+brew install cirruslabs/cli/orchard
+```
+
+Ensure that the following command:
+
+```shell
+which orchard
+```
+
+...yields `/opt/homebrew/bin/orchard`. If not, you'll need to replace all of the occurences of `/opt/homebrew/bin/orchard` in the job definition below.
+
+Then, create a launchd job definition in `/Library/LaunchDaemons/org.cirruslabs.orchard.worker.plist` with the following contents:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>Label</key>
+    <string>org.cirruslabs.orchard.worker</string>
+    <key>UserName</key>
+    <string>admin</string>
+    <key>Program</key>
+    <string>/opt/homebrew/bin/orchard</string>
+    <key>ProgramArguments</key>
+    <array>
+      <string>/opt/homebrew/bin/orchard</string>
+      <string>worker</string>
+      <string>run</string>
+      <string>--bootstrap-token</string>
+      <string>${BOOTSTRAP_TOKEN}</string>
+      <string>orchard.example.com</string>
+    </array>
+    <key>EnvironmentVariables</key>
+    <dict>
+      <key>PATH</key>
+      <string>/bin:/usr/bin:/usr/local/bin:/opt/homebrew/bin</string>
+    </dict>
+    <key>WorkingDirectory</key>
+    <string>/var/empty</string>
+    <key>RunAtLoad</key>
+    <true/>
+    <key>KeepAlive</key>
+    <true/>
+    <key>StandardOutPath</key>
+    <string>/Users/admin/orchard-launchd.log</string>
+    <key>StandardErrorPath</key>
+    <string>/Users/admin/orchard-launchd.log</string>
+  </dict>
+</plist>
+```
+
+This assumes that your macOS user on the host is named `admin`. If not, change all occurrences of `admin` in the job definition above to `$USER`.
+
+Finally, change the `orchard.example.com` to the FQDN or an IP-address of your Orchard Controller.
+
+Now, you can start the job:
+
+```shell
+launchctl load -w /Library/LaunchDaemons/org.cirruslabs.orchard.worker.plist
+```
+
+### Ansible
+
+If you have a set of machines that you want to use as Orchard Workers, you can use [Ansible](https://docs.ansible.com/) to configure them.
+
+We've created the [cirruslabs/ansible-orchard](https://github.com/cirruslabs/ansible-orchard) repository with a basic Ansible playbook for convenient setup.
+
+To use it, clone it locally:
+
+```shell
+git clone https://github.com/cirruslabs/ansible-orchard.git
+cd ansible-orchard/
+```
+
+Make sure that the Ansible Galaxy dependencies are installed:
+
+```shell
+ansible-galaxy install -r requirements.yml
+```
+
+Then, edit the `production-pool` file and populate the following fields:
+
+* `hosts` — replace `worker-1.hosts.internal` with your worker FQDN or IP-address and add more hosts if needed
+* `ansible_user` — set it macOS user on the host for the SSH to work
+* `orchard_worker_user` — set it macOS user on the host under which the Worker will run, e.g. `admin`
+* `orchard_worker_controller_url` — set it to FQDN or an IP-address of your Orchard Controller, for example, `orchard.example.com`
+* `orchard_worker_bootstrap_token` — set it to `${BOOTSTRAP_TOKEN}` we've generated above
+
+Deploy the playbook:
+
+```shell
+ansible-playbook --inventory-file production-pool --ask-pass playbook-workers.yml
+```

docs/orchard/integration-guide.md

@@ -0,0 +1,187 @@
+Orchard has a REST API that follows [OpenAPI specification](https://swagger.io/specification/) and is described in [`api/openapi.yaml`](https://github.com/cirruslabs/orchard/blob/main/api/openapi.yaml).
+
+You can run `orchard dev` locally and navigate to `http://127.0.0.1:6120/v1/` for interactive documentation.
+
+![](/assets/images/orchard/orchard-api-documentation-browser.png)
+
+## Using the API
+
+Below you'll find examples of using Orchard API via vanilla Python's request library and Golang package that Orchard CLI build on top of.
+
+### Authentication
+
+When running in non-development mode, Orchard API expects a [basic access authentication](https://en.wikipedia.org/wiki/Basic_access_authentication) to be provided for each API call.
+
+Below you'll find two snippets that retrieve controller's information and output its version:
+
+#### Authentication in Python
+
+```python
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+def main():
+  # Authentication
+  basic_auth = HTTPBasicAuth("service account name", "service account token")
+
+  response = requests.get("http://127.0.0.1:6120/v1/info", auth=basic_auth)
+
+  print(response.json()["version"])
+
+
+if __name__ == '__main__':
+  main()
+```
+
+#### Authentication in Golang
+
+```go
+package main
+
+import (
+  "context"
+  "fmt"
+  "github.com/cirruslabs/orchard/pkg/client"
+  "log"
+)
+
+func main() {
+  client, err := client.New()
+  if err != nil {
+    log.Fatalf("failed to initialize Orchard API client: %v", err)
+  }
+
+  controllerInfo, err := client.Controller().Info(context.Background())
+  if err != nil {
+    log.Fatalf("failed to retrieve controller's information: %v", err)
+  }
+
+  fmt.Println(controllerInfo.Version)
+}
+```
+
+Note that we don't provide any credentials for Golang's version of the snippet: this is because Orchard's Golang API client (`github.com/cirruslabs/orchard/pkg/client`) has the ability to read the current's user Orchard context automatically.
+
+### Creating a VM
+
+A more intricate example would be spinning off a VM with a startup script that outputs date, reading its logs and removing it from the controller:
+
+#### Creating a VM in Python
+
+```python
+import time
+import uuid
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+def main():
+  vm_name = str(uuid.uuid4())
+
+  basic_auth = HTTPBasicAuth("service account name", "service account token")
+
+  # Create VM
+  response = requests.post("http://127.0.0.1:6120/v1/vms", auth=basic_auth, json={
+    "name": vm_name,
+    "image": "ghcr.io/cirruslabs/macos-sonoma-base:latest",
+    "cpu": 4,
+    "memory": 4096,
+    "startup_script": {
+      "script_content": "date",
+    }
+  })
+  response.raise_for_status()
+
+  # Retrieve VM's logs
+  while True:
+    response = requests.get(f"http://127.0.0.1:6120/v1/vms/{vm_name}/events", auth=basic_auth)
+    response.raise_for_status()
+
+    result = response.json()
+
+    if isinstance(result, list) and len(result) != 0:
+      print(result[0]["payload"])
+      break
+
+    time.sleep(1)
+
+  # Delete VM
+  response = requests.delete(f"http://127.0.0.1:6120/v1/vms/{vm_name}", auth=basic_auth)
+  response.raise_for_status()
+
+
+if __name__ == '__main__':
+  main()
+```
+
+#### Creating a VM in Golang
+
+```go
+package main
+
+import (
+  "context"
+  "fmt"
+  "github.com/cirruslabs/orchard/pkg/client"
+  v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
+  "github.com/google/uuid"
+  "log"
+  "time"
+)
+
+func main() {
+  vmName := uuid.New().String()
+
+  client, err := client.New()
+  if err != nil {
+    log.Fatalf("failed to initialize Orchard API client: %v", err)
+  }
+
+  // Create VM
+  err = client.VMs().Create(context.Background(), &v1.VM{
+    Meta: v1.Meta{
+      Name: vmName,
+    },
+    Image:  "ghcr.io/cirruslabs/macos-sonoma-base:latest",
+    CPU:    4,
+    Memory: 4096,
+    StartupScript: &v1.VMScript{
+      ScriptContent: "date",
+    },
+  })
+  if err != nil {
+    log.Fatalf("failed to create VM: %v")
+  }
+
+  // Retrieve VM's logs
+  for {
+    vmLogs, err := client.VMs().Logs(context.Background(), vmName)
+    if err != nil {
+      log.Fatalf("failed to retrieve VM logs")
+    }
+
+    if len(vmLogs) != 0 {
+      fmt.Println(vmLogs[0])
+      break
+    }
+
+    time.Sleep(time.Second)
+  }
+
+  // Delete VM
+  if err := client.VMs().Delete(context.Background(), vmName); err != nil {
+    log.Fatalf("failed to delete VM: %v", err)
+  }
+}
+```
+
+## Resource management
+
+Some resources, such as `Worker` and `VM`, have a `resource` field which is a dictionary that maps between resource names and their amounts (amount requested or amount provided, depending on the resource) and is useful for scheduling.
+
+Well-known resources:
+
+* `org.cirruslabs.tart-vms` — number of Tart VM slots available on the machine or requested by the VM
+    * this number is `2` for workers and `1` for VMs by default

docs/orchard/managing-cluster.md

@@ -0,0 +1,30 @@
+## Backups
+
+In order to backup the Orchard Controller, simply copy its `ORCHARD_HOME` (which defaults to `~/.orchard/`) directory somewhere safe and restore it when needed.
+
+This directory contains a BadgerDB database that Controller uses to store state and an X.509 certificate with key.
+
+## Upgrades
+
+Since the Orchard's initial release, we've managed to maintain the backwards compatibility between versions up to this day, so generally, it doesn't matter whether you upgrade the Controller or Worker(s) first.
+
+In case a new functionality is introduced, you might be required to finish the upgrade of both the Controller and the Worker(s) to be able to use it fully.
+
+In case there will be backwards-incompatible changes introduced in the future, we will try to do our best and highlight this in the [release notes](https://github.com/cirruslabs/orchard/releases) accordingly.
+
+## Observability
+
+Both the Controller and Worker produce some useful OpenTelemetry metrics. Metrics are scoped with `org.cirruslabs.orchard` prefix and include information about resource utilization, statuses or Workers, scheduling/pull time and many more.
+
+By default, the telemetry is sent to `https://localhost:4317` using the gRPC protocol and to `http://localhost:4318` using the HTTP protocol.
+
+You can override this by setting the [standard OpenTelemetry environment variable](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) `OTEL_EXPORTER_OTLP_ENDPOINT`.
+
+Please refer to [OTEL Collector documentation](https://opentelemetry.io/docs/collector/) for instruction on how to setup a sidecar for the metrics collections or find out if your SaaS monitoring has an available OTEL endpoint (see [Honeycomb](https://docs.honeycomb.io/send-data/opentelemetry/) as an example).
+
+### Sending metrics to Google Cloud Platform
+
+There are two standard options of ingesting metrics procuded by Orchard Controller and Workers into the GCP:
+
+* [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) + [Google Cloud Exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/googlecloudexporter/README.md) — open-source solution that can be later re-purposed to send metrics to any OTLP-compatible endpoint by swapping a single [exporter](https://opentelemetry.io/docs/collector/configuration/#exporters)
+* [Ops Agent](https://cloud.google.com/monitoring/agent/ops-agent/otlp) — Google-backed solution with a syntax similar to OpenTelemetry Collector, but tied to GCP-only

docs/orchard/quick-start.md

@@ -0,0 +1,101 @@
+Tart is great for running workloads on a single machine, but what if you have more than one computer at your disposal
+and
+a couple of VMs is not enough anymore for your needs? This is where [Orchard](https://github.com/cirruslabs/orchard)
+comes in to play!
+
+It allows you to orchestrate multiple Tart-capable hosts from either an Orchard CLI (which we demonstrate below)
+or [through the API](/orchard/integration-guide).
+
+The easiest way to start is to run Orchard in local development mode:
+
+```shell
+brew install cirruslabs/cli/orchard
+orchard dev
+```
+
+This will run an Orchard Controller and an Orchard Worker in a single process on your local machine, allowing you to
+test both the CLI functionality and the API from a tool like cURL or programming language of choice, without the need to
+authenticate requests.
+
+Note that in production deployments, these two components are started separately and enable security by default. Please
+refer to [Deploying Controller](/orchard/deploying-controller) and [Deploying Workers](/orchard/deploying-workers) for
+more information.
+
+## Creating Virtual Machines
+
+Now, let's create a Virtual Machine:
+
+```shell
+orchard create vm --image ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
+```
+
+You can check a list of VM resources to see if the Virtual Machine we've created above is already running:
+
+```shell
+orchard list vms
+```
+
+## Accessing Virtual Machines
+
+Orchard has an ability to do port forwarding that `ssh` and `vnc` commands are built on top of. All port forwarding
+connections are done via the Orchard Controller instance which "proxies" a secure connection to the Orchard Workers.
+
+Therefore, your workers can be located under a stricter firewall that only allows connections to the Orchard Controller
+instance. Orchard Controller instance is secured by default and all API calls are authenticated and authorized.
+
+### SSH
+
+To SSH into a VM, use the `orchard ssh` command:
+
+```shell
+orchard ssh vm sonoma-base
+```
+
+You can specify the `--username` and `--password` flags to specify the username/password pair to use for the SSH
+protocol. By default, `admin`/`admin` is used.
+
+You can also execute remote commands instead of spawning a login shell, similarly to how OpenSSH's `ssh` command accepts
+a command argument:
+
+```shell
+orchard ssh vm sonoma-base "uname -a"
+```
+
+You can execute scripts remotely this way, by telling the remote command-line interpreter to read from the standard
+input and using the redirection operator as follows:
+
+```shell
+orchard ssh vm sonoma-base "bash -s" < script.sh
+```
+
+### VNC
+
+Similarly to `ssh` command, you can use `vnc` command to open Screen Sharing into a remote VM:
+
+```shell
+orchard vnc vm sonoma-base
+```
+
+You can specify the `--username` and `--password` flags to specify the username/password pair to use for the VNC
+protocol. By default, `admin`/`admin` is used.
+
+## Deleting Virtual Machines
+
+The following command will delete the VM we've created above and clean-up the resources associated with it:
+
+```shell
+orchard delete vm sonoma-base
+```
+
+## Environment variables
+
+In addition to controlling the Orchard via the CLI arguments, there are environment variables that may be beneficial
+both when automating Orchard and in daily use:
+
+| Variable name                   | Description                                                                                                                                                                                                                                                                                                                                                                  |
+|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `ORCHARD_HOME`                  | Override Orchard's home directory. Useful when running multiple Orchard instances on the same host and when testing.                                                                                                                                                                                                                                                         |
+| `ORCHARD_LICENSE_TIER`          | The default license limit only allows connecting 4 Orchard Workers to the Orchard Controller. If you've purchased a [Gold Tier License](/licensing/), set this variable to `gold` to increase the limit to 20 Orchard Workers. And if you've purchased a [Platinum Tier License](/licensing/), set this variable to `platinum` to increase the limit to 200 Orchard Workers. |
+| `ORCHARD_URL`                   | Override controller URL on per-command basis.                                                                                                                                                                                                                                                                                                                                |
+| `ORCHARD_SERVICE_ACCOUNT_NAME`  | Override service account name (used for controller API auth) on per-command basis.                                                                                                                                                                                                                                                                                           |
+| `ORCHARD_SERVICE_ACCOUNT_TOKEN` | Override service account token (used for controller API auth) on per-command basis.                                                                                                                                                                                                                                                                                          |

docs/theme/overrides/home.html

@@ -85,10 +85,10 @@
       <!-- landing image -->
       <div class="tx-landing__image">
         <dotlottie-player
-          src="/assets/animations/TartLogo.lottie"
+            src="/assets/animations/TartLogo.lottie"
-          mode="normal"
+            mode="normal"
-          style="width: 75%; margin: auto"
+            style="width: 75%; margin: auto"
-          autoplay
+            autoplay
         />
       </div>
 
@@ -114,10 +114,9 @@
       <header class="md-typeset">
         <h1 id="virtualization-and-beyond">
           Virtualization and beyond
-          <a
+          <a href="#virtualization-and-beyond"
-            href="#virtualization-and-beyond"
+             class="headerlink"
-            class="headerlink"
+             title="Permanent link"
-            title="Permanent link"
           >
             ¶
           </a>
@@ -125,12 +124,11 @@
       </header>
       <div class="mdx-spotlight">
         <figure class="mdx-spotlight__feature">
-          <img
+          <img src="assets/images/spotlight/virtualization-framework.webp"
-            src="assets/images/spotlight/virtualization-framework.webp"
+               alt="Apple’s native Virtualization.Framework"
-            alt="Apple’s native Virtualization.Framework"
+               loading="lazy"
-            loading="lazy"
+               width="500"
-            width="500"
+               height="212"
-            height="212"
           />
           <figcaption class="md-typeset">
             <h2>Native performance</h2>
@@ -144,12 +142,11 @@
           </figcaption>
         </figure>
         <figure class="mdx-spotlight__feature">
-          <img
+          <img src="assets/images/spotlight/supported-registries.webp"
-            src="assets/images/spotlight/supported-registries.webp"
+               alt="OCI-compatible container registries"
-            alt="OCI-compatible container registries"
+               loading="lazy"
-            loading="lazy"
+               width="500"
-            width="500"
+               height="160"
-            height="160"
           />
           <figcaption class="md-typeset">
             <h2>Remote storage for Virtual Machines</h2>
@@ -161,39 +158,39 @@
           </figcaption>
         </figure>
         <figure class="mdx-spotlight__feature">
-          <img
+          <img src="assets/images/spotlight/github-actions-runners.webp"
-            src="assets/images/spotlight/github-actions-runners.webp"
+               alt="GitHub Actions Runners"
-            alt="GitHub Actions Runners"
+               loading="lazy"
-            loading="lazy"
+               width="500"
-            width="500"
+               height="280"
-            height="280"
           />
           <figcaption class="md-typeset">
             <h2>Seamless integration with your existing automations</h2>
             <p>
               Tart integrates with many continuous integration systems, including a dedicated
               service of on-demand GitHub Actions Runners. With a single line change, you can cut your
-              CI/CD costs by up to <b>30 times</b> by using <a href="https://cirrus-runners.app/">Cirrus Runners</a>
+              CI/CD costs by up to <b>30 times</b> by using <a href="https://cirrus-runners.app/">Cirrus
+              Runners</a>
               to run your workflows.
             </p>
           </figcaption>
         </figure>
         <figure class="mdx-spotlight__feature">
           <div id="lottie-player">
-            <dotlottie-player
+            <dotlottie-player src="/assets/animations/Orchard.lottie"
-              src="/assets/animations/Orchard.lottie"
+                              mode="normal"
-              mode="normal"
+                              style="height: 280px; margin: auto"
-              style="height: 280px; margin: auto"
+                              autoplay
-              autoplay
+                              loop
-              loop
             />
           </div>
           <figcaption class="md-typeset">
             <h2>Run at scale with <a href="https://github.com/cirruslabs/orchard">Orchard</a></h2>
             <p>
-              Tart toolset includes Orchard Orchestration &mdash; tool to run and manage Tart virtual machines
+              Tart toolset includes Orchard Orchestration &mdash; tool to run and manage Tart virtual
-              at scale on a cluster of Apple Silicon hosts. An Orchard Cluster exposes a simple REST API to manage
+              machines at scale on a cluster of Apple Silicon hosts. An Orchard Cluster exposes a simple REST API to
-              thousands virtual machines. Orchard CLI allows accessing remote virtual machines like they run locally.
+              manage thousands virtual machines. Orchard CLI allows accessing remote virtual machines like they run
+              locally.
             </p>
           </figcaption>
         </figure>
@@ -208,38 +205,38 @@
       <header class="md-typeset">
         <h1 id="powerhouse">
           Automation Powerhouse
-          <a
+          <a href="#powerhouse"
-            href="#powerhouse"
+             class="headerlink"
-            class="headerlink"
+             title="Permanent link"
-            title="Permanent link"
           >
             ¶
           </a>
         </h1>
       </header>
       <script>
-        fetch("https://api.github.com/repos/cirruslabs/tart/releases?per_page=100")
+          fetch("https://api.github.com/repos/cirruslabs/tart/releases?per_page=100")
-          .then((response) => response.json())
+              .then((response) => response.json())
-          .then((releases) => {
+              .then((releases) => {
-            let allDownloads = 0;
+                  let allDownloads = 0;
-            for (let release of releases) {
+                  for (let release of releases) {
-              for (let asset of release.assets) {
+                      for (let asset of release.assets) {
-                if (asset && asset.content_type === "application/octet-stream") {
+                          if (asset && asset.content_type === "application/octet-stream") {
-                  allDownloads += asset.download_count || 0
+                              allDownloads += asset.download_count || 0
-                }
+                          }
-              }
+                      }
-            }
+                  }
-            let counterElement = document.getElementById('installation-counter');
+                  let counterElement = document.getElementById('installation-counter');
-            if (counterElement) {
+                  if (counterElement) {
-              // Live installation count is available starting version 1.0.0
+                      // Live installation count is available starting version 1.0.0
-              // Prior Tart was installed a little over 14,000 times, let's count them too
+                      // Prior Tart was installed a little over 14,000 times, let's count them too
-              let installationPriorV1 = 14
+                      let installationPriorV1 = 14
-              counterElement.textContent = (installationPriorV1 + Math.round(allDownloads / 1000)) + ",000"
+                      counterElement.textContent = (installationPriorV1 + Math.round(allDownloads / 1000)) + ",000"
-            }
+                  }
-          })
+              })
       </script>
       <h2>
-        With more than <strong id="installation-counter">25,000</strong> installations to date, Tart has been adopted for various scenarios.
+        With more than <strong id="installation-counter">25,000</strong> installations to date, Tart has been
+        adopted for various scenarios.
         Its applications range from powering CI/CD pipelines and reproducible local development environments,
         to helping in the testing of device management systems without actual physical devices.
       </h2>
@@ -254,10 +251,9 @@
       <header class="md-typeset">
         <h1 id="what-our-users-say">
           What our users say
-          <a
+          <a href="#what-our-users-say"
-            href="#what-our-users-say"
+             class="headerlink"
-            class="headerlink"
+             title="Permanent link"
-            title="Permanent link"
           >
             ¶
           </a>
@@ -265,70 +261,60 @@
       </header>
       <div class="mdx-users">
         <figure class="mdx-users__testimonial">
-          <img
+          <img src="assets/images/users/mikhail-tokarev.webp"
-            src="assets/images/users/mitchell-hashimoto.webp"
+               alt="Mikhail Tokarev"
-            alt="Mitchell Hashimoto"
+               loading="lazy"
-            loading="lazy"
+               width="200"
-            width="200"
+               height="200"
-            height="200"
           />
           <figcaption class="md-typeset">
-            <h2>Mitchell Hashimoto</h2>
             <h3>
-              <a href="https://www.hashicorp.com/" target="_blank">HashiCorp</a> co-founder
+              Mikhail Tokarev, CTO at <a href="https://codemagic.io/start/" target="_blank">Codemagic</a>
             </h3>
             <hr/>
             <cite>
-              I've been using "Cirrus Runners" since <a href="https://x.com/mitchellh/status/1731071326201561194" target="_blank">that tweet</a> and
+              Thanks to the minimal overhead of using the Apple Virtualization
-              it has been fantastic. Huge speed increase, huge cost decrease, zero maintenance, exactly what I wanted.
+              API, we’ve seen some performance improvements in booting new
+              virtual machines compared with Anka.
             </cite>
           </figcaption>
         </figure>
         <figure class="mdx-users__testimonial">
-          <img
+          <img src="assets/images/users/expo.webp"
-            src="assets/images/users/seb-jachec.webp"
+               alt="Expo"
-            alt="Sebastian Jachec"
+               loading="lazy"
-            loading="lazy"
+               width="200"
-            width="200"
+               height="200"
-            height="200"
           />
           <figcaption class="md-typeset">
-            <h2>Sebastian Jachec</h2>
             <h3>
-              Mobile Engineer at
+              Infrastructure Team at <a href="https://expo.dev/" target="_blank">Expo</a>
-              <a href="https://daybridge.com/" target="_blank">Daybridge</a>
             </h3>
             <hr/>
             <cite>
-              It&rsquo;s been plain-sailing with the
+              Tart was the practical way for us to use the Virtualization framework. Cirrus Labs’
-              <a href="/integrations/github-actions">Cirrus Runners</a>&nbsp;&mdash;
+              continued maintenance and support gives us confidence, and it is also important for us
-              they&rsquo;ve been great! They&rsquo;re consistently&nbsp;60+%
+              to be able to read the source code when we need to understand an abstraction layer below.
-              faster on&nbsp;workflows that we&nbsp;previously used Github
-              Actions&rsquo; macOS runners for.
             </cite>
           </figcaption>
         </figure>
         <figure class="mdx-users__testimonial">
-          <img
+          <img src="assets/images/users/snowflake.webp"
-            src="assets/images/users/max-lapides.webp"
+               alt="Snowflake"
-            alt="Max Lapides"
+               loading="lazy"
-            loading="lazy"
+               width="200"
-            width="200"
+               height="200"
-            height="200"
           />
           <figcaption class="md-typeset">
-            <h2>Max Lapides</h2>
             <h3>
-              Senior Mobile Engineer at
+              Red Team at <a href="https://www.snowflake.com/" target="_blank">Snowflake</a>
-              <a href="https://www.tonal.com/" target="_blank">Tonal</a>
             </h3>
             <hr/>
             <cite>
-              Previously, we were using the GitHub&#8209;hosted macOS runners
+              The Snowflake Red Team had a need for macOS CI/CD and a segmented macOS development
-              and our iOS build took ~30&nbsp;minutes. Now with
+              environment. We solved this problem and shared our implementation with macOS EC2 and Tart.
-              <a href="/integrations/github-actions">Cirrus Runners</a>, the iOS build only
+              We also automated this process with Terraform/Packer to simplify the deployment of our
-              takes ~12&nbsp;minutes. That’s a huge boost to our productivity,
+              infrastructure and machine images.
-              and for only $150/month per runner it is much less expensive too.
             </cite>
           </figcaption>
         </figure>

mkdocs.yml

@@ -91,13 +91,19 @@ nav:
   - "Home": index.md
   - "Quick Start": quick-start.md
   - "Integrations":
-    - "Self-hosted CI": integrations/cirrus-cli.md
+      - "Self-hosted CI": integrations/cirrus-cli.md
-    - "GitHub Actions": https://cirrus-runners.app/
+      - "GitHub Actions": https://cirrus-runners.app/
-    - "GitLab Runner": integrations/gitlab-runner.md
+      - "GitLab Runner": integrations/gitlab-runner.md
-    - "Buildkite": integrations/buildkite.md
+      - "Buildkite": integrations/buildkite.md
-    - "Managing VMs": integrations/vm-management.md
+      - "Managing VMs": integrations/vm-management.md
   - "Support & Licensing": licensing.md
-  - "Orchestration": https://github.com/cirruslabs/orchard
+  - "Orchestration":
+      - "Quick Start": orchard/quick-start.md
+      - "Architecture and Security": orchard/architecture-and-security.md
+      - "Deploying Controller": orchard/deploying-controller.md
+      - "Deploying Workers": orchard/deploying-workers.md
+      - "Managing the Cluster": orchard/managing-cluster.md
+      - "Integrating with the API": orchard/integration-guide.md
   - "FAQ": faq.md
   - "Legal":
       - 'Terms of Service': legal/terms.md