Package.swift: bump Sentry SDK to 8.36.0 + upgrade other packages (#905 )

* Package.swift: bump Sentry SDK to 8.36.0 * $ swift package update
Completely disable audio devices in case of --no-audio (#904 )
2026-10-01 19:51:10 +02:00 · 2024-09-19 19:06:37 +00:00 · 2024-09-17 09:40:37 +00:00 · 2024-09-09 20:45:59 +04:00 · 2024-09-09 16:40:17 +04:00 · 2024-08-28 00:09:57 +04:00
97 changed files with 3012 additions and 650 deletions
@@ -1,8 +1,5 @@
 use_compute_credits: true

-env: 
-  XCODE_TAG: 15.2
-
 task:
  name: Test on Sonoma
  alias: test
@@ -11,11 +8,11 @@ task:
      name: dev-mini
    resources:
      tart-vms: 1
+  build_script:
+    - swift build
  test_script:
    - swift test
  integration_test_script:
-    # Build Tart
-    - swift build
    - codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart
    - export PATH=$(pwd)/.build/arm64-apple-macosx/debug:$PATH
    # Run integration tests
@@ -41,7 +38,7 @@ task:
  name: Lint
  alias: lint
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-runner:sonoma
  lint_script:
    - swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore --lint --report swiftformat.json .
  always:
@@ -58,7 +55,7 @@ task:
  name: Build ($BUILD_ARCH)
  alias: build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-runner:sonoma
  build_script: swift build --arch $BUILD_ARCH --product tart
  sign_script: codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/$BUILD_ARCH-apple-macosx/debug/tart
  binary_artifacts:
@@ -71,7 +68,7 @@ task:
    - lint
    - build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-runner:sonoma
  env:
    MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
    AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -93,7 +90,7 @@ task:
    - security find-identity -v
    - xcodebuild -version
    - swift -version
-  goreleaser_script: goreleaser release --skip-publish --snapshot --clean
+  goreleaser_script: goreleaser release --skip=publish --snapshot --clean
  always:
    dist_artifacts:
      path: "dist/*"
@@ -106,7 +103,7 @@ task:
    - test
    - build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-runner:sonoma
  env:
    MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
    AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -114,7 +111,7 @@ task:
    GORELEASER_KEY: ENCRYPTED[!9b80b6ef684ceaf40edd4c7af93014ee156c8aba7e6e5795f41c482729887b5c31f36b651491d790f1f668670888d9fd!]
    SENTRY_ORG: cirrus-labs
    SENTRY_PROJECT: persistent-workers
-    SENTRY_AUTH_TOKEN: ENCRYPTED[!c16a5cf7da5f856b4bc2f21fe8cb7aa2a6c981f851c094ed4d3025fd02ea59a58a86cee8b193a69a1fc20fa217e56ac3!]
+    SENTRY_AUTH_TOKEN: ENCRYPTED[!9eaf2875d51b113e2f68598441ff8e6b2e53242e48fcb93633bd75a373fbe2e7caa900d837cc92f0b142b65579731644!]
  setup_script:
    - cd $HOME
    - echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
@@ -16,3 +16,6 @@ dist/

 # mkdocs
 .cache
+
+# mkdocs-material
+site
@@ -26,6 +26,12 @@ archives:
      - src: Resources/embedded.provisionprofile
        dst: tart.app/Contents
        strip_parent: true
+      - src: Resources/Info.plist
+        dst: tart.app/Contents
+        strip_parent: true
+      - src: Resources/AppIcon.png
+        dst: tart.app/Contents/Resources
+        strip_parent: true
      - LICENSE

 release:
@@ -46,5 +52,6 @@ brews:
    install: |
      libexec.install Dir["*"]
      bin.write_exec_script "#{libexec}/tart.app/Contents/MacOS/tart"
+      generate_completions_from_executable(libexec/"tart.app/Contents/MacOS/tart", "--generate-completion-script")
    custom_block: |
      depends_on :macos => :ventura
@@ -1,4 +1,5 @@
 {
+  "originHash" : "a10a6363a6d2cd2761653d1587b81f5e3d55b1f981b7d6a2964b34da91addf13",
  "pins" : [
    {
      "identity" : "antlr4",
@@ -6,7 +7,7 @@
      "location" : "https://github.com/antlr/antlr4",
      "state" : {
        "branch" : "dev",
-        "revision" : "2703a8516c0fb7fe92db6b9c40e0113f577646d2"
+        "revision" : "2a7904a595479954ccfb1c78124fc3d7f5bebcb1"
      }
    },
    {
@@ -18,13 +19,22 @@
        "revision" : "772883073d044bc754d401cabb6574624eb3778f"
      }
    },
+    {
+      "identity" : "semaphore",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/groue/Semaphore",
+      "state" : {
+        "revision" : "2543679282aa6f6c8ecf2138acd613ed20790bc2",
+        "version" : "0.1.0"
+      }
+    },
    {
      "identity" : "sentry-cocoa",
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/getsentry/sentry-cocoa",
      "state" : {
-        "revision" : "d277532e1c8af813981ba01f591b15bbdd735615",
-        "version" : "8.8.0"
+        "revision" : "5575af93efb776414f243e93d6af9f6258dc539a",
+        "version" : "8.36.0"
      }
    },
    {
@@ -32,8 +42,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/apple/swift-algorithms",
      "state" : {
-        "revision" : "b14b7f4c528c942f121c8b860b9410b2bf57825e",
-        "version" : "1.0.0"
+        "revision" : "f6919dfc309e7f1b56224378b11e28bab5bccc42",
+        "version" : "1.2.0"
      }
    },
    {
@@ -41,8 +51,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/apple/swift-argument-parser",
      "state" : {
-        "revision" : "f3c9084a71ef4376f2fabbdf1d3d90a49f1fabdb",
-        "version" : "1.1.2"
+        "revision" : "41982a3656a71c768319979febd796c6fd111d5c",
+        "version" : "1.5.0"
      }
    },
    {
@@ -51,7 +61,7 @@
      "location" : "https://github.com/apple/swift-async-algorithms",
      "state" : {
        "branch" : "main",
-        "revision" : "f05e450f0b909c0e80670a47516c4b9700b9e5da"
+        "revision" : "5c8bd186f48c16af0775972700626f0b74588278"
      }
    },
    {
@@ -59,8 +69,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/apple/swift-atomics.git",
      "state" : {
-        "revision" : "919eb1d83e02121cdb434c7bfc1f0c66ef17febe",
-        "version" : "1.0.2"
+        "revision" : "cd142fd2f64be2100422d658e7411e39489da985",
+        "version" : "1.2.0"
      }
    },
    {
@@ -68,8 +78,17 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/apple/swift-collections.git",
      "state" : {
-        "revision" : "f504716c27d2e5d4144fa4794b12129301d17729",
-        "version" : "1.0.3"
+        "revision" : "9bf03ff58ce34478e66aaee630e491823326fd06",
+        "version" : "1.1.3"
+      }
+    },
+    {
+      "identity" : "swift-log",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-log.git",
+      "state" : {
+        "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537",
+        "version" : "1.6.1"
      }
    },
    {
@@ -81,13 +100,31 @@
        "version" : "1.0.2"
      }
    },
+    {
+      "identity" : "swift-retry",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/fumoboy007/swift-retry",
+      "state" : {
+        "revision" : "df9d7b185d2e433147ec0083a73c257e665eea0d",
+        "version" : "0.2.4"
+      }
+    },
    {
      "identity" : "swift-sysctl",
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/sersoft-gmbh/swift-sysctl.git",
      "state" : {
-        "revision" : "71fd64ee84819bb19fbecfb36d5a4503726b6fb7",
-        "version" : "1.6.0"
+        "revision" : "a91be36de6803ebe48f678699dfd0694c2200d2f",
+        "version" : "1.8.0"
+      }
+    },
+    {
+      "identity" : "swift-xattr",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/jozefizso/swift-xattr",
+      "state" : {
+        "revision" : "f8605af7b3290dbb235fb182ec6e9035d0c8c3ac",
+        "version" : "3.0.0"
      }
    },
    {
@@ -95,8 +132,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/malcommac/SwiftDate",
      "state" : {
-        "revision" : "6190d0cefff3013e77ed567e6b074f324e5c5bf5",
-        "version" : "6.3.1"
+        "revision" : "5d943224c3bb173e6ecf27295611615eba90c80e",
+        "version" : "7.0.0"
      }
    },
    {
@@ -104,8 +141,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/nicklockwood/SwiftFormat",
      "state" : {
-        "revision" : "da637c398c5d08896521b737f2868ddc2e7996ae",
-        "version" : "0.50.6"
+        "revision" : "ab6844edb79a7b88dc6320e6cee0a0db7674dac3",
+        "version" : "0.54.5"
      }
    },
    {
@@ -127,5 +164,5 @@
      }
    }
  ],
-  "version" : 2
+  "version" : 3
 }
@@ -1,4 +1,4 @@
-// swift-tools-version:5.7
+// swift-tools-version:5.10

 import PackageDescription
 let package = Package(
@@ -10,18 +10,21 @@ let package = Package(
    .executable(name: "tart", targets: ["tart"])
  ],
  dependencies: [
-    .package(url: "https://github.com/apple/swift-argument-parser", from: "1.1.2"),
+    .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.1"),
    .package(url: "https://github.com/mhdhejazi/Dynamic", branch: "master"),
-    .package(url: "https://github.com/apple/swift-algorithms", from: "1.0.0"),
+    .package(url: "https://github.com/apple/swift-algorithms", from: "1.2.0"),
    .package(url: "https://github.com/apple/swift-async-algorithms", branch: "main"),
-    .package(url: "https://github.com/malcommac/SwiftDate", from: "6.3.1"),
+    .package(url: "https://github.com/malcommac/SwiftDate", from: "7.0.0"),
    .package(url: "https://github.com/antlr/antlr4", branch: "dev"),
-    .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.0.0")),
-    .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.50.6"),
-    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.8.0"),
+    .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.2.0")),
+    .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.53.6"),
+    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.36.0"),
    .package(url: "https://github.com/cfilipov/TextTable", branch: "master"),
-    .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.0.0"),
-    .package(url: "https://github.com/orchetect/SwiftRadix", from: "1.3.0")
+    .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.8.0"),
+    .package(url: "https://github.com/orchetect/SwiftRadix", from: "1.3.1"),
+    .package(url: "https://github.com/groue/Semaphore", from: "0.0.8"),
+    .package(url: "https://github.com/fumoboy007/swift-retry", from: "0.2.3"),
+    .package(url: "https://github.com/jozefizso/swift-xattr", from: "3.0.0"),
  ],
  targets: [
    .executableTarget(name: "tart", dependencies: [
@@ -36,6 +39,9 @@ let package = Package(
      .product(name: "TextTable", package: "TextTable"),
      .product(name: "Sysctl", package: "swift-sysctl"),
      .product(name: "SwiftRadix", package: "SwiftRadix"),
+      .product(name: "Semaphore", package: "Semaphore"),
+      .product(name: "DMRetry", package: "swift-retry"),
+      .product(name: "XAttr", package: "swift-xattr"),
    ], exclude: [
      "OCI/Reference/Makefile",
      "OCI/Reference/Reference.g4",
@@ -12,28 +12,28 @@ Tart powers [Cirrus Runners](https://cirrus-runners.app/)
 service — a drop-in replacement for the standard GitHub-hosted runners, offering 2-3 times better performance for a fraction of the price.

 <p align="center">
-  <a href="https://tart.run/integrations/github-actions/?utm_source=github&utm_medium=referral" target=_blank>
+  <a href="https://cirrus-runners.app/?utm_source=github&utm_medium=referral" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/CirrusRunnersForGHA.png" height="65"/>
  </a>
 </p>

-Many companies are using Tart in their internal setups. Here are a few of them:
+Many companies are using Tart in their internal setups. Here are just a few of them:

 <p align="center">
-  <a href="https://ahrefs.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/ahrefs.png" height="65"/>
+  <a href="https://atlassian.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Atlassian.png" height="65"/>
  </a>
-  <a href="https://krisp.ai/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Krisp.png" height="65"/>
+  <a href="https://www.figma.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Figma.png" height="65"/>
  </a>
  <a href="https://mullvad.net/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Mullvad.png" height="65"/>
  </a>
-  <a href="https://shape.dk/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/shape.png" height="65"/>
+  <a href="https://krisp.ai/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Krisp.png" height="65"/>
  </a>
-  <a href="https://suran.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Suran.png" height="65"/>
+  <a href="https://testingbot.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/TestingBot.png" height="65"/>
  </a>
  <a href="https://symflower.com/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Symflower.png" height="65"/>
@@ -41,15 +41,18 @@ Many companies are using Tart in their internal setups. Here are a few of them:
  <a href="https://transloadit.com/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Transloadit.png" height="65"/>
  </a>
-  <a href="https://uphold.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Uphold.png" height="65"/>
+  <a href="https://cirrus-ci.org/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/CirrusCI.png" height="65"/>
  </a>
  <a href="https://www.pitsdatarecovery.net/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/PITSGlobalDataRecoveryServices.png" height="65"/>
  </a>
+  <a href="https://expo.dev/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Expo.png" height="65"/>
+  </a>
 </p>

-**Note:** If your company or project is using Tart please consider [adding yourself to the list above](/Resources/Users/HowToAddYourself.md).
+**Note:** If your company or project is using Tart please consider [sharing with the community](https://github.com/cirruslabs/tart/discussions/857).

 <p align="center">
  <a href="https://aws.amazon.com/marketplace/pp/prodview-qczco34wlkdws?utm_source=github&utm_medium=referral" target=_blank>
@@ -14,5 +14,10 @@
    <array>
        <string>AppIcon.png</string>
    </array>
+    <key>NSAppTransportSecurity</key>
+    <dict>
+        <key>NSAllowsArbitraryLoads</key>
+        <true/>
+    </dict>
 </dict>
 </plist>
@@ -1,4 +0,0 @@
-If you'd like to highlight your use of Tart, please create a `456px` by `130px` logo and create a PR
-that adds it to `README.md` in alphabetical order. Don't forget to include a small description of your usage pattern.
-
-You can refer to `Background.png` as a base for your logo.
@@ -8,7 +8,7 @@ struct Clone: AsyncParsableCommand {
    discussion: """
    Creates a local virtual machine by cloning either a remote or another local virtual machine.

-    Due to copy-on-write magic in Apple File System a cloned VM won't actually claim all the space right away.
+    Due to copy-on-write magic in Apple File System, a cloned VM won't actually claim all the space right away.
    Only changes to a cloned disk will be written and claim new space. By default, Tart checks available capacity
    in Tart's home directory and checks if there is enough space for the worst possible scenario: when the whole disk
    will be modified.
@@ -18,7 +18,7 @@ struct Clone: AsyncParsableCommand {
    """
  )

-  @Argument(help: "source VM name")
+  @Argument(help: "source VM name", completion: .custom(completeMachines))
  var sourceName: String

  @Argument(help: "new VM name")
@@ -63,7 +63,7 @@ struct Clone: AsyncParsableCommand {
      try lock.lock()

      let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress())
-        && sourceVM.state() != "suspended"
+        && sourceVM.state() != .Suspended
      try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC)

      try localStorage.move(newName, from: tmpVMDir)
@@ -16,7 +16,7 @@ struct Create: AsyncParsableCommand {
  @Flag(help: "create a Linux VM")
  var linux: Bool = false

-  @Option(help: ArgumentHelp("Disk size in Gb")) 
+  @Option(help: ArgumentHelp("Disk size in GB"))
  var diskSize: UInt16 = 50

  func validate() throws {
@@ -5,7 +5,7 @@ import SwiftUI
 struct Delete: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Delete a VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeMachines))
  var name: [String]

  func run() async throws {
@@ -4,7 +4,7 @@ import Foundation
 struct Export: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Export VM to a compressed .tvm file")

-  @Argument(help: "Source VM name.")
+  @Argument(help: "Source VM name.", completion: .custom(completeMachines))
  var name: String

  @Argument(help: "Path to the destination file.")
@@ -5,7 +5,7 @@ import SystemConfiguration
 struct FQN: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Get a fully-qualified VM name", shouldDisplay: false)

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeMachines))
  var name: String

  func run() async throws {
@@ -2,6 +2,7 @@ import ArgumentParser
 import Foundation

 fileprivate struct VMInfo: Encodable {
+  let OS: OS
  let CPU: Int
  let Memory: UInt64
  let Disk: Int
@@ -14,7 +15,7 @@ fileprivate struct VMInfo: Encodable {
 struct Get: AsyncParsableCommand {
  static var configuration = CommandConfiguration(commandName: "get", abstract: "Get a VM's configuration")

-  @Argument(help: "VM name.")
+  @Argument(help: "VM name.", completion: .custom(completeLocalMachines))
  var name: String

  @Option(help: "Output format: text or json")
@@ -25,8 +26,7 @@ struct Get: AsyncParsableCommand {
    let vmConfig = try VMConfig(fromURL: vmDir.configURL)
    let memorySizeInMb = vmConfig.memorySize / 1024 / 1024

-    let info = VMInfo(CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: try vmDir.sizeGB(), Size: String(format: "%.3f", Float(try vmDir.allocatedSizeBytes()) / 1000 / 1000 / 1000),
-                      Display: vmConfig.display.description, Running: try vmDir.running(), State: try vmDir.state())
+    let info = VMInfo(OS: vmConfig.os, CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: try vmDir.sizeGB(), Size: String(format: "%.3f", Float(try vmDir.allocatedSizeBytes()) / 1000 / 1000 / 1000), Display: vmConfig.display.description, Running: try vmDir.running(), State: try vmDir.state().rawValue)
    print(format.renderSingle(info))
  }
 }
@@ -13,7 +13,7 @@ enum IPResolutionStrategy: String, ExpressibleByArgument, CaseIterable {
 struct IP: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Get VM's IP address")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeLocalMachines))
  var name: String

  @Option(help: "Number of seconds to wait for a potential VM booting")
@@ -61,7 +61,7 @@ struct IP: AsyncParsableCommand {
          return ip
        }
      case .dhcp:
-        if let leases = try Leases(), let ip = try leases.ResolveMACAddress(macAddress: vmMACAddress) {
+        if let leases = try Leases(), let ip = leases.ResolveMACAddress(macAddress: vmMACAddress) {
          return ip
        }
      }
@@ -7,6 +7,7 @@ fileprivate struct VMInfo: Encodable {
  let Name: String
  let Disk: Int
  let Size: Int
+  let SizeOnDisk: Int
  let Running: Bool
  let State: String
 }
@@ -38,13 +39,13 @@ struct List: AsyncParsableCommand {

    if source == nil || source == "local" {
      infos += sortedInfos(try VMStorageLocal().list().map { (name, vmDir) in
-        try VMInfo(Source: "local", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), Running: vmDir.running(), State: vmDir.state())
+        try VMInfo(Source: "local", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), SizeOnDisk: vmDir.allocatedSizeGB() - vmDir.deduplicatedSizeGB(), Running: vmDir.running(), State: vmDir.state().rawValue)
      })
    }

    if source == nil || source == "oci" {
      infos += sortedInfos(try VMStorageOCI().list().map { (name, vmDir, _) in
-        try VMInfo(Source: "oci", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), Running: vmDir.running(), State: vmDir.state())
+        try VMInfo(Source: "OCI", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), SizeOnDisk: vmDir.allocatedSizeGB() - vmDir.deduplicatedSizeGB(), Running: vmDir.running(), State: vmDir.state().rawValue)
      })
    }

@@ -49,9 +49,10 @@ struct Login: AsyncParsableCommand {
    ])

    if !noValidate {
+      let registry = try Registry(host: host, namespace: "", insecure: insecure,
+                                  credentialsProviders: [credentialsProvider])
+
      do {
-        let registry = try Registry(host: host, namespace: "", insecure: insecure,
-                                    credentialsProviders: [credentialsProvider])
        try await registry.ping()
      } catch {
        throw RuntimeError.InvalidCredentials("invalid credentials: \(error)")
@@ -6,7 +6,7 @@ import Compression
 struct Push: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Push a VM to a registry")

-  @Argument(help: "local or remote VM name")
+  @Argument(help: "local or remote VM name", completion: .custom(completeMachines))
  var localName: String

  @Argument(help: "remote VM name(s)")
@@ -15,6 +15,9 @@ struct Push: AsyncParsableCommand {
  @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
  var insecure: Bool = false

+  @Option(help: "network concurrency to use when pushing a local VM to the OCI-compatible registry")
+  var concurrency: UInt = 4
+
  @Option(help: ArgumentHelp("chunk size in MB if registry supports chunked uploads",
                             discussion: """
                             By default monolithic method is used for uploading blobs to the registry but some registries support a more efficient chunked method.
@@ -33,6 +36,10 @@ struct Push: AsyncParsableCommand {
  func run() async throws {
    let ociStorage = VMStorageOCI()
    let localVMDir = try VMStorageHelper.open(localName)
+    let lock = try localVMDir.lock()
+    if try !lock.trylock() {
+      throw RuntimeError.VMIsRunning(localName)
+    }

    // Parse remote names supplied by the user
    let remoteNames = try remoteNames.map{
@@ -73,7 +80,8 @@ struct Push: AsyncParsableCommand {
          registry: registry,
          references: references,
          chunkSizeMb: chunkSize,
-          diskFormat: diskFormat
+          diskFormat: diskFormat,
+          concurrency: concurrency
        )
        // Populate the local cache (if requested)
        if populateCache {
@@ -2,9 +2,9 @@ import ArgumentParser
 import Foundation

 struct Rename: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Rename a VM")
+  static var configuration = CommandConfiguration(abstract: "Rename a local VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeLocalMachines))
  var name: String

  @Argument(help: "new VM name")
@@ -20,7 +20,7 @@ struct Rename: AsyncParsableCommand {
    let localStorage = VMStorageLocal()

    if !localStorage.exists(name) {
-      throw ValidationError("failed to rename a non-existent VM: \(name)")
+      throw ValidationError("failed to rename a non-existent local VM: \(name)")
    }

    if localStorage.exists(newName) {
@@ -5,16 +5,50 @@ import Dispatch
 import SwiftUI
 import Virtualization
 import Sentry
+import System

 var vm: VM?

 struct IPNotFound: Error {
 }

+@available(macOS 14, *)
+extension VZDiskSynchronizationMode {
+  public init(_ description: String) throws {
+    switch description {
+    case "none":
+      self = .none
+    case "full":
+      self = .full
+    case "":
+      self = .full
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk synchronization mode: \"\(description)\"")
+    }
+  }
+}
+
+extension VZDiskImageSynchronizationMode {
+  public init(_ description: String) throws {
+    switch description {
+    case "none":
+      self = .none
+    case "fsync":
+      self = .fsync
+    case "full":
+      self = .full
+    case "":
+      self = .full
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk image synchronization mode: \"\(description)\"")
+    }
+  }
+}
+
 struct Run: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Run a VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeLocalMachines))
  var name: String

  @Flag(help: ArgumentHelp(
@@ -33,12 +67,17 @@ struct Run: AsyncParsableCommand {
  ))
  var serialPath: String?

-  @Flag(help: ArgumentHelp("Force open a UI window, even when VNC is enabled.", visibility: .hidden))
+  @Flag(help: ArgumentHelp("Force open a UI window, even when VNC is enabled.", visibility: .private))
  var graphics: Bool = false

  @Flag(help: "Disable audio pass-through to host.")
  var noAudio: Bool = false

+  @Flag(help: ArgumentHelp(
+    "Disable clipboard sharing between host and guest.",
+    discussion: "Only works with Linux-based guest operating systems."))
+  var noClipboard: Bool = false
+
  #if arch(arm64)
    @Flag(help: "Boot into recovery mode")
  #endif
@@ -54,25 +93,36 @@ struct Run: AsyncParsableCommand {

  #if arch(arm64)
    @Flag(help: ArgumentHelp(
-      "Use Virtualization.Framework's VNC server instead of the build-in UI.",
+      "Use Virtualization.Framework's VNC server instead of the built-in UI.",
      discussion: "Useful since this type of VNC is available in recovery mode and in macOS installation.\n"
        + "Note that this feature is experimental and there may be bugs present when using VNC."))
  #endif
  var vncExperimental: Bool = false

  @Option(help: ArgumentHelp("""
-  Additional disk attachments with an optional read-only specifier\n(e.g. --disk=\"disk.bin\" --disk=\"ubuntu.iso:ro\" --disk=\"/dev/disk0\")
+  Additional disk attachments with an optional read-only and synchronization options (e.g. --disk="disk.bin", --disk="ubuntu.iso:ro", --disk="/dev/disk0", --disk "ghcr.io/cirruslabs/xcode:16.0:ro" or --disk="nbd://localhost:10809/myDisk:sync=none")
  """, discussion: """
-  Can be either a disk image file or a block device like a local SSD on AWS EC2 Mac instances.
+  The disk attachment can be a:

-  Learn how to create a disk image using Disk Utility here:
-  https://support.apple.com/en-gb/guide/disk-utility/dskutl11888/mac
+  * path to a disk image file
+  * path to a block device (for example, a local SSD on AWS EC2 Mac instances)
+  * remote VM name whose disk will be mounted
+  * Network Block Device (NBD) URL

-  To work with block devices 'tart' binary must be executed as root which affects locating Tart VMs.
-  To workaround this issue pass TART_HOME explicitly:
+  Options are comma-separated and are as follows:
+
+  * ro — attach the specified disk in read-only mode instead of the default read-write (e.g. --disk="disk.img:ro")
+
+  * sync=none — disable data synchronization with the permanent storage to increase performance at the cost of a higher chance of data loss (e.g. --disk="disk.img:sync=none")
+
+  Learn how to create a disk image using Disk Utility here: https://support.apple.com/en-gb/guide/disk-utility/dskutl11888/mac
+
+  To work with block devices, the easiest way is to modify their permissions (e.g. by using "sudo chown $USER /dev/diskX") or to run the Tart binary as root, which affects locating Tart VMs.
+
+  To work around this pass TART_HOME explicitly:

  sudo TART_HOME="$HOME/.tart" tart run sonoma --disk=/dev/disk0
-  """, valueName: "path[:ro]"))
+  """, valueName: "path[:options]"))
  var disk: [String] = []

  #if arch(arm64)
@@ -117,9 +167,26 @@ struct Run: AsyncParsableCommand {
                           discussion: "Learn how to configure Softnet for use with Tart here: https://github.com/cirruslabs/softnet"))
  var netSoftnet: Bool = false

+  @Option(help: ArgumentHelp("Comma-separated list of CIDRs to allow the traffic to when using Softnet isolation\n(e.g. --net-softnet-allow=192.168.0.0/24)", valueName: "comma-separated CIDRs"))
+  var netSoftnetAllow: String?
+
  @Flag(help: ArgumentHelp("Restrict network access to the host-only network"))
  var netHost: Bool = false

+  @Option(help: ArgumentHelp("Set the root disk options (e.g. --root-disk-opts=\"ro\" or --root-disk-opts=\"sync=none\")",
+                             discussion: """
+                             Options are comma-separated and are as follows:
+
+                             * ro — attach the root disk in read-only mode instead of the default read-write (e.g. --root-disk-opts="ro")
+
+                             * sync=none — disable data synchronization with the permanent storage to increase performance at the cost of a higher chance of data loss (e.g. --root-disk-opts="sync=none")
+
+                             * sync=fsync — enable data synchronization with the permanent storage, but don't ensure that it was actually written (e.g. --root-disk-opts="sync=fsync")
+
+                             * sync=full — enable data synchronization with the permanent storage and ensure that it was actually written (e.g. --root-disk-opts="sync=full")
+                             """, valueName: "options"))
+  var rootDiskOpts: String = ""
+
  #if arch(arm64)
    @Flag(help: ArgumentHelp("Disables audio and entropy devices and switches to only Mac-specific input devices.", discussion: "Useful for running a VM that can be suspended via \"tart suspend\"."))
  #endif
@@ -146,8 +213,8 @@ struct Run: AsyncParsableCommand {
      throw ValidationError("--net-bridged, --net-softnet and --net-host are mutually exclusive")
    }

-    if graphics {
-      print("--graphics is deprecated and will be removed in the future.\n")
+    if graphics && noGraphics {
+      throw ValidationError("--graphics and --no-graphics are mutually exclusive")
    }

    if (noGraphics || vnc || vncExperimental) && captureSystemKeys {
@@ -156,11 +223,15 @@ struct Run: AsyncParsableCommand {

    let localStorage = VMStorageLocal()
    let vmDir = try localStorage.open(name)
-    if try vmDir.state() == "suspended" {
+    if try vmDir.state() == .Suspended {
      suspendable = true
    }

    if suspendable {
+      let config = try VMConfig.init(fromURL: vmDir.configURL)
+      if !(config.platform is PlatformSuspendable) {
+        throw ValidationError("You can only suspend macOS VMs")
+      }
      if dir.count > 0 {
        throw ValidationError("Suspending VMs with shared directories is not supported")
      }
@@ -179,26 +250,22 @@ struct Run: AsyncParsableCommand {
    let vmDir = try localStorage.open(name)

    let storageLock = try FileLock(lockURL: Config().tartHomeDir)
-    if try vmDir.state() == "suspended" {
-      try storageLock.lock() // lock before checking
-      let needToGenerateNewMac = try localStorage.list().contains {
-        // check if there is a running VM with the same MAC but different name
-        try $1.running() && $1.macAddress() == vmDir.macAddress() && $1.name != vmDir.name
-      }
-
-      if needToGenerateNewMac {
-        print("There is already a running VM with the same MAC address!")
-        print("Resetting VM to assign a new MAC address...")
-        try vmDir.regenerateMACAddress()
-      }
+    try storageLock.lock()
+    // check if there is a running VM with the same MAC address
+    let hasRunningMACCollision = try localStorage.list().contains {
+      // check if there is a running VM with the same MAC but different name
+      try $1.running() && $1.macAddress() == vmDir.macAddress() && $1.name != vmDir.name
+    }
+    if hasRunningMACCollision {
+      print("There is already a running VM with the same MAC address!")
+      print("Resetting VM to assign a new MAC address...")
+      try vmDir.regenerateMACAddress()
    }

    if (netSoftnet || netHost) && isInteractiveSession() {
      try Softnet.configureSUIDBitIfNeeded()
    }

-    let additionalDiskAttachments = try additionalDiskAttachments()
-
    var serialPorts: [VZSerialPortConfiguration] = []
    if serial {
      let tty_fd = createPTY()
@@ -217,14 +284,19 @@ struct Run: AsyncParsableCommand {
      serialPorts.append(createSerialPortConfiguration(tty_read!, tty_write!))
    }

+    // Parse root disk options
+    let diskOptions = DiskOptions(rootDiskOpts)
+
    vm = try VM(
      vmDir: vmDir,
      network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
-      additionalStorageDevices: additionalDiskAttachments,
+      additionalStorageDevices: try additionalDiskAttachments(),
      directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
      serialPorts: serialPorts,
      suspendable: suspendable,
-      audio: !noAudio
+      audio: !noAudio,
+      clipboard: !noClipboard,
+      sync: VZDiskImageSynchronizationMode(diskOptions.syncModeRaw)
    )

    let vncImpl: VNC? = try {
@@ -274,10 +346,38 @@ struct Run: AsyncParsableCommand {
          }
        #endif

-        try await vm!.start(recovery: recovery, resume: resume)
+        do {
+          try await vm!.start(recovery: recovery, resume: resume)
+        } catch let error as VZError {
+          if error.code == .virtualMachineLimitExceeded {
+            var hint = ""
+
+            do {
+              let runningVMs: [String] = try localStorage.list().compactMap { (name, vmDir) in
+                if try !vmDir.running() {
+                  return nil
+                }
+
+                return name
+              }
+
+              if !runningVMs.isEmpty {
+                let runningVMsJoined = runningVMs.joined(separator: ", ")
+
+                hint = " (other running VMs: \(runningVMsJoined))"
+              }
+            } catch {
+              // we can't provide any hint
+            }
+
+            throw RuntimeError.VirtualMachineLimitExceeded(hint)
+          }
+
+          throw error
+        }

        if let vncImpl = vncImpl {
-          let vncURL = try await vncImpl.waitForURL()
+          let vncURL = try await vncImpl.waitForURL(netBridged: !netBridged.isEmpty)

          if noGraphics || ProcessInfo.processInfo.environment["CI"] != nil {
            print("VNC server is running at \(vncURL)")
@@ -346,10 +446,23 @@ struct Run: AsyncParsableCommand {
    }
    sigusr1Src.activate()

-    if noGraphics {
+    // Gracefull shutdown support. For macOS this brings up a dialog,
+    // asking the user if they are sure they want to shut down.
+    signal(SIGUSR2, SIG_IGN)
+    let sigusr2Src = DispatchSource.makeSignalSource(signal: SIGUSR2)
+    sigusr2Src.setEventHandler {
+      Task {
+        print("Requesting guest OS to stop...")
+        try vm!.virtualMachine.requestStop()
+      }
+    }
+    sigusr2Src.activate()
+
+    let useVNCWithoutGraphics = (vnc || vncExperimental) && !graphics
+    if noGraphics || useVNCWithoutGraphics {
      // enter the main even loop, without bringing up any UI,
      // and just wait for the VM to exit.
-      NSApp.run()
+      NSApplication.shared.run()
    } else {
      runUI(suspendable, captureSystemKeys)
    }
@@ -370,14 +483,22 @@ struct Run: AsyncParsableCommand {
  }

  func userSpecifiedNetwork(vmDir: VMDirectory) throws -> Network? {
+    var softnetExtraArguments: [String] = []
+
+    if let netSoftnetAllow = netSoftnetAllow {
+      softnetExtraArguments += ["--allow", netSoftnetAllow]
+    }
+
    if netSoftnet {
      let config = try VMConfig.init(fromURL: vmDir.configURL)
-      return try Softnet(vmMACAddress: config.macAddress.string)
+
+      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: softnetExtraArguments)
    }

    if netHost {
      let config = try VMConfig.init(fromURL: vmDir.configURL)
-      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: ["--vm-net-type", "host"])
+
+      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: ["--vm-net-type", "host"] + softnetExtraArguments)
    }

    if netBridged.count > 0 {
@@ -411,46 +532,9 @@ struct Run: AsyncParsableCommand {
  }

  func additionalDiskAttachments() throws -> [VZStorageDeviceConfiguration] {
-    var result: [VZStorageDeviceConfiguration] = []
-    let readOnlySuffix = ":ro"
-    let expandedDiskPaths = disk.map { NSString(string:$0).expandingTildeInPath }
-
-    for rawDisk in expandedDiskPaths {
-      let diskReadOnly = rawDisk.hasSuffix(readOnlySuffix)
-      let diskPath = diskReadOnly ? String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count)) : rawDisk
-      let diskURL = URL(fileURLWithPath: diskPath)
-
-      // check if `diskPath` is a block device or a directory
-      if pathHasMode(diskPath, mode: S_IFBLK) || pathHasMode(diskPath, mode: S_IFDIR) {
-        print("Using block device\n")
-        guard #available(macOS 14, *) else {
-          throw UnsupportedOSError("attaching block devices", "are")
-        }
-        let fileHandle = FileHandle(forUpdatingAtPath: diskPath)
-        guard fileHandle != nil else {
-          if ProcessInfo.processInfo.userName != "root" {
-            throw RuntimeError.VMConfigurationError("need to run as root to work with block devices")
-          }
-          throw RuntimeError.VMConfigurationError("block device \(diskURL.url.path) seems to be already in use, unmount it first via 'diskutil unmount'")
-        }
-        let attachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: fileHandle!, readOnly: diskReadOnly, synchronizationMode: .full)
-        result.append(VZVirtioBlockDeviceConfiguration(attachment: attachment))
-      } else {
-        // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
-        // see https://github.com/cirruslabs/tart/issues/323 for more details.
-        if try !diskReadOnly && !FileLock(lockURL: diskURL).trylock() {
-          throw RuntimeError.DiskAlreadyInUse("disk \(diskURL.url.path) seems to be already in use, unmount it first in Finder")
-        }
-
-        let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
-          url: diskURL,
-          readOnly: diskReadOnly
-        )
-        result.append(VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment))
-      }
+    try disk.map {
+      try AdditionalDisk(parseFrom: $0).configuration
    }
-
-    return result
  }

  func directoryShares() throws -> [VZDirectorySharingDeviceConfiguration] {
@@ -523,71 +607,69 @@ struct Run: AsyncParsableCommand {
  }

  private func runUI(_ suspendable: Bool, _ captureSystemKeys: Bool) {
-    let nsApp = NSApplication.shared
-    nsApp.setActivationPolicy(.regular)
-    nsApp.activate(ignoringOtherApps: true)
+    MainApp.suspendable = suspendable
+    MainApp.capturesSystemKeys = captureSystemKeys
+    MainApp.main()
+  }
+}

-    struct MainApp: App {
-      static var disappearSignal: Int32 = SIGINT
-      static var capturesSystemKeys: Bool = false
+struct MainApp: App {
+  static var suspendable: Bool = false
+  static var capturesSystemKeys: Bool = false

-      @NSApplicationDelegateAdaptor private var appDelegate: MinimalMenuAppDelegate
+  @NSApplicationDelegateAdaptor private var appDelegate: MinimalMenuAppDelegate

-      var body: some Scene {
-        WindowGroup(vm!.name) {
-          Group {
-            VMView(vm: vm!, capturesSystemKeys: MainApp.capturesSystemKeys).onAppear {
-              NSWindow.allowsAutomaticWindowTabbing = false
-            }.onDisappear {
-              let ret = kill(getpid(), MainApp.disappearSignal)
-              if ret != 0 {
-                // Fallback to the old termination method that doesn't
-                // propagate the cancellation to Task's in case graceful
-                // termination via kill(2) is not successful
-                NSApplication.shared.terminate(self)
-              }
-            }
-          }.frame(
-            minWidth: CGFloat(vm!.config.display.width),
-            idealWidth: CGFloat(vm!.config.display.width),
-            maxWidth: .infinity,
-            minHeight: CGFloat(vm!.config.display.height),
-            idealHeight: CGFloat(vm!.config.display.height),
-            maxHeight: .infinity
-          )
-        }.commands {
-          // Remove some standard menu options
-          CommandGroup(replacing: .help, addition: {})
-          CommandGroup(replacing: .newItem, addition: {})
-          CommandGroup(replacing: .pasteboard, addition: {})
-          CommandGroup(replacing: .textEditing, addition: {})
-          CommandGroup(replacing: .undoRedo, addition: {})
-          CommandGroup(replacing: .windowSize, addition: {})
-          // Replace some standard menu options
-          CommandGroup(replacing: .appInfo) { AboutTart(config: vm!.config) }
-          CommandMenu("Control") {
-            Button("Start") {
-              Task { try await vm!.virtualMachine.start() }
-            }
-            Button("Stop") {
-              Task { try await vm!.virtualMachine.stop() }
-            }
-            Button("Request Stop") {
-              Task { try vm!.virtualMachine.requestStop() }
-            }
-            if #available(macOS 14, *) {
-              Button("Suspend") {
-                kill(getpid(), SIGUSR1)
-              }
+  var body: some Scene {
+    WindowGroup(vm!.name) {
+      Group {
+        VMView(vm: vm!, capturesSystemKeys: MainApp.capturesSystemKeys).onAppear {
+          NSWindow.allowsAutomaticWindowTabbing = false
+        }.onDisappear {
+          let ret = kill(getpid(), MainApp.suspendable ? SIGUSR1 : SIGINT)
+          if ret != 0 {
+            // Fallback to the old termination method that doesn't
+            // propagate the cancellation to Task's in case graceful
+            // termination via kill(2) is not successful
+            NSApplication.shared.terminate(self)
+          }
+        }
+      }.frame(
+        minWidth: CGFloat(vm!.config.display.width),
+        idealWidth: CGFloat(vm!.config.display.width),
+        maxWidth: .infinity,
+        minHeight: CGFloat(vm!.config.display.height),
+        idealHeight: CGFloat(vm!.config.display.height),
+        maxHeight: .infinity
+      )
+    }.commands {
+      // Remove some standard menu options
+      CommandGroup(replacing: .help, addition: {})
+      CommandGroup(replacing: .newItem, addition: {})
+      CommandGroup(replacing: .pasteboard, addition: {})
+      CommandGroup(replacing: .textEditing, addition: {})
+      CommandGroup(replacing: .undoRedo, addition: {})
+      CommandGroup(replacing: .windowSize, addition: {})
+      // Replace some standard menu options
+      CommandGroup(replacing: .appInfo) { AboutTart(config: vm!.config) }
+      CommandMenu("Control") {
+        Button("Start") {
+          Task { try await vm!.virtualMachine.start() }
+        }
+        Button("Stop") {
+          Task { try await vm!.virtualMachine.stop() }
+        }
+        Button("Request Stop") {
+          Task { try vm!.virtualMachine.requestStop() }
+        }
+        if #available(macOS 14, *) {
+          if (MainApp.suspendable) {
+            Button("Suspend") {
+              kill(getpid(), SIGUSR1)
            }
          }
        }
      }
    }
-
-    MainApp.disappearSignal = suspendable ? SIGUSR1 : SIGINT
-    MainApp.capturesSystemKeys = captureSystemKeys
-    MainApp.main()
  }
 }

@@ -597,6 +679,18 @@ class MinimalMenuAppDelegate: NSObject, NSApplicationDelegate, ObservableObject

  func applicationDidFinishLaunching(_ : Notification) {
    NSApplication.shared.mainMenu?.removeItem(at: indexOfEditMenu)
+
+    let nsApp = NSApplication.shared
+    nsApp.setActivationPolicy(.regular)
+    nsApp.activate(ignoringOtherApps: true)
+  }
+
+  func applicationShouldTerminate(_ sender: NSApplication) -> NSApplication.TerminateReply {
+    if (kill(getpid(), MainApp.suspendable ? SIGUSR1 : SIGINT) == 0) {
+      return .terminateLater
+    } else {
+      return .terminateNow
+    }
  }
 }

@@ -660,6 +754,134 @@ struct VMView: NSViewRepresentable {
  }
 }

+struct AdditionalDisk {
+  let configuration: VZStorageDeviceConfiguration
+
+  init(parseFrom: String) throws {
+    let (diskPath, readOnly, syncModeRaw) = Self.parseOptions(parseFrom)
+
+    self.configuration = try Self.craft(diskPath, readOnly: readOnly, syncModeRaw: syncModeRaw)
+  }
+
+  static func craft(_ diskPath: String, readOnly diskReadOnly: Bool, syncModeRaw: String) throws -> VZStorageDeviceConfiguration {
+    let diskURL = URL(string: diskPath)
+
+    if (["nbd", "nbds", "nbd+unix", "nbds+unix"].contains(diskURL?.scheme)) {
+      guard #available(macOS 14, *) else {
+        throw UnsupportedOSError("attaching Network Block Devices", "are")
+      }
+
+      let nbdAttachment = try VZNetworkBlockDeviceStorageDeviceAttachment(
+        url: diskURL!,
+        timeout: 30,
+        isForcedReadOnly: diskReadOnly,
+        synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw)
+      )
+
+      return VZVirtioBlockDeviceConfiguration(attachment: nbdAttachment)
+    }
+
+    // Expand the tilde (~) since at this point we're dealing with a local path,
+    // and "expandingTildeInPath" seems to corrupt the remote URLs like nbd://
+    let diskPath = NSString(string: diskPath).expandingTildeInPath
+
+    let diskFileURL = URL(fileURLWithPath: diskPath)
+
+    if pathHasMode(diskPath, mode: S_IFBLK) {
+      guard #available(macOS 14, *) else {
+        throw UnsupportedOSError("attaching block devices", "are")
+      }
+
+      let fd = open(diskPath, diskReadOnly ? O_RDONLY : O_RDWR)
+      if fd == -1 {
+        let details = Errno(rawValue: CInt(errno))
+
+        switch details.rawValue {
+        case EBUSY:
+          throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "already in use, try umounting it via \"diskutil unmountDisk\" (when the whole disk) or \"diskutil umount\" (when mounting a single partition)")
+        case EACCES:
+          throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "permission denied, consider changing the disk's owner using \"sudo chown $USER \(diskFileURL.url.path)\" or run Tart as a superuser (see --disk help for more details on how to do that correctly)")
+        default:
+          throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "\(details)")
+        }
+      }
+
+      let blockAttachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: FileHandle(fileDescriptor: fd, closeOnDealloc: true),
+                                                                         readOnly: diskReadOnly, synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw))
+
+      return VZVirtioBlockDeviceConfiguration(attachment: blockAttachment)
+    }
+
+    // Support remote VM names in --disk command-line argument
+    if let remoteName = try? RemoteName(diskPath) {
+      let vmDir = try VMStorageOCI().open(remoteName)
+
+      // Unfortunately, VZDiskImageStorageDeviceAttachment does not support
+      // FileHandle, so we can't easily clone the disk, open it and unlink(2)
+      // to simplify the garbage collection, so use an intermediate directory.
+      let clonedDiskURL = try Config().tartTmpDir.appendingPathComponent("run-disk-\(UUID().uuidString)")
+
+      try FileManager.default.copyItem(at: vmDir.diskURL, to: clonedDiskURL)
+
+      let lock = try FileLock(lockURL: clonedDiskURL)
+      try lock.lock()
+
+      let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(url: clonedDiskURL, readOnly: diskReadOnly)
+
+      return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
+    }
+
+    // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
+    // see https://github.com/cirruslabs/tart/issues/323 for more details.
+    if try !diskReadOnly && !FileLock(lockURL: diskFileURL).trylock() {
+      throw RuntimeError.DiskAlreadyInUse("disk \(diskFileURL.url.path) seems to be already in use, unmount it first in Finder")
+    }
+
+    let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
+      url: diskFileURL,
+      readOnly: diskReadOnly,
+      cachingMode: .automatic,
+      synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw)
+    )
+
+    return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
+  }
+
+  static func parseOptions(_ parseFrom: String) -> (String, Bool, String) {
+    var arguments = parseFrom.split(separator: ":")
+
+    let options = DiskOptions(String(arguments.last!))
+    if options.foundAtLeastOneOption {
+      arguments.removeLast()
+    }
+
+    return (arguments.joined(separator: ":"), options.readOnly, options.syncModeRaw)
+  }
+}
+
+struct DiskOptions {
+  var readOnly: Bool = false
+  var syncModeRaw: String = ""
+  var foundAtLeastOneOption: Bool = false
+
+  init(_ parseFrom: String) {
+    let options = parseFrom.split(separator: ",")
+
+    for option in options {
+      switch true {
+      case option == "ro":
+        self.readOnly = true
+        self.foundAtLeastOneOption = true
+      case option.hasPrefix("sync="):
+        self.syncModeRaw = String(option.dropFirst("sync=".count))
+        self.foundAtLeastOneOption = true
+      default:
+        continue
+      }
+    }
+  }
+}
+
 struct DirectoryShare {
  let name: String?
  let path: URL
@@ -806,5 +1028,5 @@ func pathHasMode(_ path: String, mode: mode_t) -> Bool {
  guard statRes != -1 else {
    return false
  }
-  return (Int32(st.st_mode) & Int32(mode)) == Int32(mode)
+  return (st.st_mode & S_IFMT) == mode
 }
@@ -1,10 +1,11 @@
 import ArgumentParser
 import Foundation
+import Virtualization

 struct Set: AsyncParsableCommand {
  static var configuration = CommandConfiguration(commandName: "set", abstract: "Modify VM's configuration")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeLocalMachines))
  var name: String

  @Option(help: "Number of VM CPUs")
@@ -16,6 +17,17 @@ struct Set: AsyncParsableCommand {
  @Option(help: "VM display resolution in a format of <width>x<height>. For example, 1200x800")
  var display: VMDisplayConfig?

+  @Flag(help: ArgumentHelp("Generate a new random MAC address for the VM."))
+  var randomMAC: Bool = false
+
+  #if arch(arm64)
+    @Flag(help: ArgumentHelp("Generate a new random serial number for the macOS VM."))
+  #endif
+  var randomSerial: Bool = false
+
+  @Option(help: ArgumentHelp("Replace the VM's disk contents with the disk contents at path.", valueName: "path"))
+  var disk: String?
+
  @Option(help: ArgumentHelp("Resize the VMs disk to the specified size in GB (note that the disk size can only be increased to avoid losing data)",
                             discussion: """
                             Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images
@@ -51,8 +63,27 @@ struct Set: AsyncParsableCommand {
      }
    }

+    if randomMAC {
+      vmConfig.macAddress = VZMACAddress.randomLocallyAdministered()
+    }
+
+    #if arch(arm64)
+      if randomSerial {
+        let oldPlatform = vmConfig.platform as! Darwin
+        vmConfig.platform = Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: oldPlatform.hardwareModel)
+      }
+    #endif
+
    try vmConfig.save(toURL: vmDir.configURL)

+    if let disk = disk {
+      let temporaryDiskURL = try Config().tartTmpDir.appendingPathComponent("set-disk-\(UUID().uuidString)")
+
+      try FileManager.default.copyItem(atPath: disk, toPath: temporaryDiskURL.path())
+
+      _ = try FileManager.default.replaceItemAt(vmDir.diskURL, withItemAt: temporaryDiskURL)
+    }
+
    if diskSize != nil {
      try vmDir.resizeDisk(diskSize!)
    }
@@ -6,7 +6,7 @@ import SwiftDate
 struct Stop: AsyncParsableCommand {
  static var configuration = CommandConfiguration(commandName: "stop", abstract: "Stop a VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeRunningMachines))
  var name: String

  @Option(name: [.short, .long], help: "Seconds to wait for graceful termination before forcefully terminating the VM")
@@ -15,12 +15,12 @@ struct Stop: AsyncParsableCommand {
  func run() async throws {
    let vmDir = try VMStorageLocal().open(name)
    switch try vmDir.state() {
-    case "suspended":
+    case .Suspended:
      try stopSuspended(vmDir)
-    case "running":
+    case .Running:
      try await stopRunning(vmDir)
-    default:
-      return
+    case .Stopped:
+      throw RuntimeError.VMNotRunning(name)
    }
  }

@@ -6,7 +6,7 @@ import SwiftDate
 struct Suspend: AsyncParsableCommand {
  static var configuration = CommandConfiguration(commandName: "suspend", abstract: "Suspend a VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeRunningMachines))
  var name: String

  func run() async throws {
@@ -14,7 +14,7 @@ struct Suspend: AsyncParsableCommand {
    let lock = try vmDir.lock()

    // Find the VM's PID
-    var pid = try lock.pid()
+    let pid = try lock.pid()
    if pid == 0 {
      throw RuntimeError.VMNotRunning("VM \"\(name)\" is not running")
    }
@@ -41,7 +41,7 @@ class KeychainCredentialsProvider: CredentialsProvider {
                              kSecAttrLabel as String: "Tart Credentials",
    ]
    let value: [String: Any] = [kSecAttrAccount as String: user,
-                                kSecValueData as String: passwordData,
+                                kSecValueData as String: passwordData as Any,
    ]

    let status = SecItemCopyMatching(key as CFDictionary, nil)
@@ -3,19 +3,32 @@ import AsyncAlgorithms

 fileprivate let urlSession = createURLSession()

-class Fetcher {
-  static func fetch(_ request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    if viaFile {
-      return try await fetchViaFile(request)
-    }
-
-    return try await fetchViaMemory(request)
+class DownloadDelegate: NSObject, URLSessionTaskDelegate {
+  let progress: Progress
+  init(_ progress: Progress) throws {
+    self.progress = progress
  }

-  private static func fetchViaMemory(_ request: URLRequest) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+  func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) {
+    self.progress.addChild(task.progress, withPendingUnitCount: self.progress.totalUnitCount)
+  }
+}
+
+class Fetcher {
+  static func fetch(_ request: URLRequest, viaFile: Bool = false, progress: Progress? = nil) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+    let delegate = progress != nil ? try DownloadDelegate(progress!) : nil
+
+    if viaFile {
+      return try await fetchViaFile(request, delegate: delegate)
+    }
+
+    return try await fetchViaMemory(request, delegate: delegate)
+  }
+
+  private static func fetchViaMemory(_ request: URLRequest, delegate: URLSessionTaskDelegate? = nil) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
    let dataCh = AsyncThrowingChannel<Data, Error>()

-    let (data, response) = try await urlSession.data(for: request)
+    let (data, response) = try await urlSession.data(for: request, delegate: delegate)

    Task {
      await dataCh.send(data)
@@ -26,10 +39,10 @@ class Fetcher {
    return (dataCh, response as! HTTPURLResponse)
  }

-  private static func fetchViaFile(_ request: URLRequest) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+  private static func fetchViaFile(_ request: URLRequest, delegate: URLSessionTaskDelegate? = nil) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
    let dataCh = AsyncThrowingChannel<Data, Error>()

-    let (fileURL, response) = try await urlSession.download(for: request)
+    let (fileURL, response) = try await urlSession.download(for: request, delegate: delegate)

    // Acquire a handle to the downloaded file and then remove it.
    //
@@ -0,0 +1,57 @@
+import Foundation
+
+struct LocalLayerCache {
+  struct DigestInfo {
+    let range: Range<Data.Index>
+    let compressedDigest: String
+    let uncompressedContentDigest: String?
+  }
+
+  let name: String
+  let deduplicatedBytes: UInt64
+  let diskURL: URL
+
+  private let mappedDisk: Data
+  private var digestToRange: [String: DigestInfo] = [:]
+  private var offsetToRange: [UInt64: DigestInfo] = [:]
+
+  init?(_ name: String, _ deduplicatedBytes: UInt64, _ diskURL: URL, _ manifest: OCIManifest) throws {
+    self.name = name
+    self.deduplicatedBytes = deduplicatedBytes
+    self.diskURL = diskURL
+
+    // mmap(2) the disk that contains the layers from the manifest
+    self.mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+
+    // Record the ranges of the disk layers listed in the manifest
+    var offset: UInt64 = 0
+
+    for layer in manifest.layers.filter({ $0.mediaType == diskV2MediaType }) {
+      guard let uncompressedSize = layer.uncompressedSize() else {
+        return nil
+      }
+
+      let info = DigestInfo(
+        range: Int(offset)..<Int(offset + uncompressedSize),
+        compressedDigest: layer.digest,
+        uncompressedContentDigest: layer.uncompressedContentDigest()!
+      )
+      self.digestToRange[layer.digest] = info
+      self.offsetToRange[offset] = info
+
+      offset += uncompressedSize
+    }
+  }
+
+  func findInfo(digest: String, offsetHint: UInt64) -> DigestInfo? {
+    // Layers can have the same digests, for example, empty ones. Let's use the offset hint to make a better guess.
+    if let info = self.offsetToRange[offsetHint], info.compressedDigest == digest {
+      return info
+    }
+    return self.digestToRange[digest]
+  }
+
+  func subdata(_ range: Range<Data.Index>) -> Data {
+    return self.mappedDisk.subdata(in: range)
+  }
+}
@@ -1,7 +1,8 @@
 import Virtualization
+import Semaphore

 protocol Network {
  func attachments() -> [VZNetworkDeviceAttachment]
-  func run(_ sema: DispatchSemaphore) throws
+  func run(_ sema: AsyncSemaphore) throws
  func stop() async throws
 }
@@ -1,4 +1,5 @@
 import Foundation
+import Semaphore
 import Virtualization

 class NetworkBridged: Network {
@@ -12,7 +13,7 @@ class NetworkBridged: Network {
    interfaces.map { VZBridgedNetworkDeviceAttachment(interface: $0) }
  }

-  func run(_ sema: DispatchSemaphore) throws {
+  func run(_ sema: AsyncSemaphore) throws {
    // no-op, only used for Softnet
  }

@@ -1,4 +1,5 @@
 import Foundation
+import Semaphore
 import Virtualization

 class NetworkShared: Network {
@@ -6,7 +7,7 @@ class NetworkShared: Network {
    [VZNATNetworkDeviceAttachment()]
  }

-  func run(_ sema: DispatchSemaphore) throws {
+  func run(_ sema: AsyncSemaphore) throws {
    // no-op, only used for Softnet
  }

@@ -1,7 +1,8 @@
-import Foundation
-import Virtualization
 import Atomics
+import Foundation
+import Semaphore
 import System
+import Virtualization

 enum SoftnetError: Error {
  case InitializationFailed(why: String)
@@ -44,7 +45,7 @@ class Softnet: Network {
    return executableURL
  }

-  func run(_ sema: DispatchSemaphore) throws {
+  func run(_ sema: AsyncSemaphore) throws {
    try process.run()

    monitorTask = Task {
@@ -1,6 +1,6 @@
 import Foundation

 protocol Disk {
-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer]
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?) async throws
 }
@@ -5,7 +5,7 @@ class DiskV1: Disk {
  private static let bufferSizeBytes = 4 * 1024 * 1024
  private static let layerLimitBytes = 500 * 1000 * 1000

-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
    var pushedLayers: [OCIManifestLayer] = []

    // Open the disk file
@@ -45,7 +45,7 @@ class DiskV1: Disk {
    return pushedLayers
  }

-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil) async throws {
    if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
      throw OCIError.FailedToCreateVmFile
    }
@@ -1,48 +1,89 @@
 import Foundation
 import Compression
+import System
+import Retry

 class DiskV2: Disk {
  private static let bufferSizeBytes = 4 * 1024 * 1024
-  private static let layerLimitBytes = 500 * 1000 * 1000
-  private static let holeGranularityBytes = 64 * 1024
+  private static let layerLimitBytes = 512 * 1024 * 1024

-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
-    var pushedLayers: [OCIManifestLayer] = []
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
+    var pushedLayers: [(index: Int, pushedLayer: OCIManifestLayer)] = []

    // Open the disk file
-    var mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+    let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])

    // Compress the disk file as multiple individually decompressible streams,
-    // each equal ``Self.layerLimitBytes`` bytes or slightly larger due to the
-    // internal compressor's buffer
-    var offset: UInt64 = 0
+    // each equal ``Self.layerLimitBytes`` bytes or less due to LZ4 compression
+    try await withThrowingTaskGroup(of: (Int, OCIManifestLayer).self) { group in
+      for (index, data) in mappedDisk.chunks(ofCount: layerLimitBytes).enumerated() {
+        // Respect the concurrency limit
+        if index >= concurrency {
+          if let (index, pushedLayer) = try await group.next() {
+            pushedLayers.append((index, pushedLayer))
+          }
+        }

-    while let (compressedData, uncompressedSize, uncompressedDigest) = try compressNextLayerOfLimitBytesOrMore(mappedDisk: mappedDisk, offset: offset) {
-      offset += uncompressedSize
+        // Launch a disk layer pushing task
+        group.addTask {
+          let compressedData = try (data as NSData).compressed(using: .lz4) as Data
+          let compressedDataDigest = Digest.hash(compressedData)

-      let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+          try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) {
+            if try await !registry.blobExists(compressedDataDigest) {
+              _ = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb, digest: compressedDataDigest)
+            }
+          } recoverFromFailure: { error in
+            if error is URLError {
+              print("Error: \(error.localizedDescription)")
+              print("Attempting to re-try...")

-      pushedLayers.append(OCIManifestLayer(
-        mediaType: diskV2MediaType,
-        size: compressedData.count,
-        digest: layerDigest,
-        uncompressedSize: uncompressedSize,
-        uncompressedContentDigest: uncompressedDigest
-      ))
+              return .retry
+            }

-      // Update progress using a relative value
-      progress.completedUnitCount += Int64(uncompressedSize)
+            return .throw
+          }
+
+          // Update progress using a relative value
+          progress.completedUnitCount += Int64(data.count)
+
+          return (index, OCIManifestLayer(
+            mediaType: diskV2MediaType,
+            size: compressedData.count,
+            digest: compressedDataDigest,
+            uncompressedSize: UInt64(data.count),
+            uncompressedContentDigest: Digest.hash(data)
+          ))
+        }
+      }
+
+      for try await pushedLayer in group {
+        pushedLayers.append(pushedLayer)
+      }
    }

-    return pushedLayers
+    return pushedLayers.sorted {
+      $0.index < $1.index
+    }.map {
+      $0.pushedLayer
+    }
  }

-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil) async throws {
    // Support resumable pulls
    let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)

-    if !pullResumed && !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
-      throw OCIError.FailedToCreateVmFile
+    if !pullResumed {
+      if let localLayerCache = localLayerCache {
+        // Clone the local layer cache's disk and use it as a base, potentially
+        // reducing the space usage since some blocks won't be written at all
+        try FileManager.default.copyItem(at: localLayerCache.diskURL, to: diskURL)
+      } else {
+        // Otherwise create an empty disk
+        if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
+          throw OCIError.FailedToCreateVmFile
+        }
+      }
    }

    // Calculate the uncompressed disk size
@@ -62,6 +103,15 @@ class DiskV2: Disk {
    try disk.truncate(atOffset: uncompressedDiskSize)
    try disk.close()

+    // Determine the file system block size
+    var st = stat()
+    if stat(diskURL.path, &st) == -1 {
+      let details = Errno(rawValue: errno)
+
+      throw RuntimeError.PullFailed("failed to stat(2) disk \(diskURL.path): \(details)")
+    }
+    let fsBlockSize = UInt64(st.st_blksize)
+
    // Concurrently fetch and decompress layers
    try await withThrowingTaskGroup(of: Void.self) { group in
      var globalDiskWritingOffset: UInt64 = 0
@@ -86,29 +136,47 @@ class DiskV2: Disk {
        // Launch a fetching and decompression task
        group.addTask {
          // No need to fetch and decompress anything if we've already done so
-          if try pullResumed && Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+          if pullResumed {
+            // do not check hash in the condition above to make it lazy e.g. only do expensive calculations if needed
+            if try Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+              // Update the progress
+              progress.completedUnitCount += Int64(diskLayer.size)
+
+              return
+            }
+          }
+
+          // Open the disk file for writing
+          let disk = try FileHandle(forWritingTo: diskURL)
+
+          // Also open the disk file for reading and verifying
+          // its contents in case the local layer cache is used
+          let rdisk: FileHandle? = if localLayerCache != nil {
+            try FileHandle(forReadingFrom: diskURL)
+          } else {
+            nil
+          }
+
+          // Check if we already have this layer contents in the local layer cache
+          if let localLayerCache = localLayerCache, let localLayerInfo = localLayerCache.findInfo(digest: diskLayer.digest, offsetHint: diskWritingOffset) {
+            // indicates that the locally cloned disk image has the same content at the given offset
+            let localHit = localLayerInfo.uncompressedContentDigest == uncompressedLayerContentDigest
+              && localLayerInfo.range.lowerBound == diskWritingOffset
+            // doesn't seem that localHit can ever be false if the localLayerCache is not nil
+            // but let's just add extra safety here and check it
+            if !localHit {
+              // Fulfil the layer contents from the local blob cache
+              let data = localLayerCache.subdata(localLayerInfo.range)
+              _ = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
+            }
+            try disk.close()
+
            // Update the progress
            progress.completedUnitCount += Int64(diskLayer.size)

            return
          }

-          // Open the disk file
-          let disk = try FileHandle(forWritingTo: diskURL)
-
-          // A zero chunk for faster than byte-by-byte comparisons
-          //
-          // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
-          // on the last block since it costs only 64 KiB of excess data per 500 MB layer.
-          //
-          // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
-          // +--------------------------------------+---------------------------------------------------+
-          // | Operation                            | time(1) result                                    |
-          // +--------------------------------------+---------------------------------------------------+
-          // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
-          // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
-          // +--------------------------------------+---------------------------------------------------+
-          let zeroChunk = Data(count: holeGranularityBytes)
          var diskWritingOffset = diskWritingOffset

          // Pull and decompress a single layer into the specific offset on disk
@@ -117,15 +185,7 @@ class DiskV2: Disk {
              return
            }

-            for chunk in data.chunks(ofCount: holeGranularityBytes) {
-              // Only write chunks that are not zero
-              if chunk != zeroChunk {
-                try disk.seek(toOffset: diskWritingOffset)
-                disk.write(chunk)
-              }
-
-              diskWritingOffset += UInt64(chunk.count)
-            }
+            diskWritingOffset = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
          }

          try await registry.pullBlob(diskLayer.digest) { data in
@@ -145,44 +205,66 @@ class DiskV2: Disk {
    }
  }

-  private static func compressNextLayerOfLimitBytesOrMore(mappedDisk: Data, offset: UInt64) throws -> (Data, UInt64, String)? {
-    var compressedData = Data()
-    var bytesRead: UInt64 = 0
-    let digest = Digest()
+  private static func zeroSkippingWrite(_ disk: FileHandle, _ rdisk: FileHandle?, _ fsBlockSize: UInt64, _ offset: UInt64, _ data: Data) throws -> UInt64 {
+    let holeGranularityBytes = 64 * 1024

-    // Create a compressing filter that we will terminate upon
-    // reaching ``Self.layerLimitBytes`` of compressed data
-    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: bufferSizeBytes) { (length: Int) -> Data? in
-      if compressedData.count >= Self.layerLimitBytes {
-        return nil
+    // A zero chunk for faster than byte-by-byte comparisons
+    //
+    // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
+    // on the last block since it costs only 64 KiB of excess data per 500 MB layer.
+    //
+    // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
+    // +--------------------------------------+---------------------------------------------------+
+    // | Operation                            | time(1) result                                    |
+    // +--------------------------------------+---------------------------------------------------+
+    // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
+    // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
+    // +--------------------------------------+---------------------------------------------------+
+    let zeroChunk = Data(count: holeGranularityBytes)
+
+    var offset = offset
+
+    for chunk in data.chunks(ofCount: holeGranularityBytes) {
+      // If the local layer cache is used, only write chunks that differ
+      // since the base disk can contain anything at any position
+      if let rdisk = rdisk {
+        // F_PUNCHHOLE requires the holes to be aligned to file system block boundaries
+        let isHoleAligned = (offset % fsBlockSize) == 0 && (UInt64(chunk.count) % fsBlockSize) == 0
+
+        if isHoleAligned && chunk == zeroChunk {
+          var arg = fpunchhole_t(fp_flags: 0, reserved: 0, fp_offset: off_t(offset), fp_length: off_t(chunk.count))
+
+          if fcntl(disk.fileDescriptor, F_PUNCHHOLE, &arg) == -1 {
+            let details = Errno(rawValue: errno)
+
+            throw RuntimeError.PullFailed("failed to punch hole: \(details)")
+          }
+        } else {
+          try rdisk.seek(toOffset: offset)
+          let actualContentsOnDisk = try rdisk.read(upToCount: chunk.count)
+
+          if chunk != actualContentsOnDisk {
+            try disk.seek(toOffset: offset)
+            disk.write(chunk)
+          }
+        }
+
+        offset += UInt64(chunk.count)
+
+        continue
      }

-      let readFromByte = Int(offset + bytesRead)
-
-      let numBytesToRead = min(mappedDisk.count - readFromByte, bufferSizeBytes)
-      if numBytesToRead == 0 {
-        return nil
+      // Otherwise, only write chunks that are not zero
+      // since the base disk is created from scratch and
+      // is zeroed via truncate(2)
+      if chunk != zeroChunk {
+        try disk.seek(toOffset: offset)
+        disk.write(chunk)
      }

-      let uncompressedChunk = mappedDisk.subdata(in: readFromByte ..< (readFromByte + numBytesToRead))
-
-      bytesRead += UInt64(uncompressedChunk.count)
-      digest.update(uncompressedChunk)
-
-      return uncompressedChunk
+      offset += UInt64(chunk.count)
    }

-    // Retrieve compressed data chunks, but normally no more than ``Self.layerLimitBytes`` bytes
-    while let compressedChunk = try compressingFilter.readData(ofLength: Self.bufferSizeBytes) {
-      compressedData.append(compressedChunk)
-    }
-
-    // Nothing was read this time from the disk,
-    // signal that to the consumer
-    if bytesRead == 0 {
-      return nil
-    }
-
-    return (compressedData, bytesRead, digest.finalize())
+    return offset
  }
 }
@@ -78,7 +78,7 @@ struct OCIManifestConfig: Codable, Equatable {
  var digest: String
 }

-struct OCIManifestLayer: Codable, Equatable {
+struct OCIManifestLayer: Codable, Equatable, Hashable {
  var mediaType: String
  var size: Int
  var digest: String
@@ -113,6 +113,14 @@ struct OCIManifestLayer: Codable, Equatable {
  func uncompressedContentDigest() -> String? {
    annotations?[uncompressedContentDigestAnnotation]
  }
+
+  static func == (lhs: Self, rhs: Self) -> Bool {
+    return lhs.digest == rhs.digest
+  }
+
+  func hash(into hasher: inout Hasher) {
+    hasher.combine(digest)
+  }
 }

 struct Descriptor: Equatable {
@@ -10,6 +10,7 @@ enum RegistryError: Error {
 }

 enum HTTPMethod: String {
+  case HEAD = "HEAD"
  case GET = "GET"
  case POST = "POST"
  case PUT = "PUT"
@@ -21,6 +22,7 @@ enum HTTPCode: Int {
  case Created = 201
  case Accepted = 202
  case Unauthorized = 401
+  case NotFound = 404
 }

 extension Data {
@@ -42,9 +44,6 @@ extension AsyncThrowingChannel<Data, Error> {
 }

 struct TokenResponse: Decodable, Authentication {
-  let defaultIssuedAt = Date()
-  let defaultExpiresIn = 60
-
  var token: String?
  var accessToken: String?
  var expiresIn: Int?
@@ -66,7 +65,8 @@ struct TokenResponse: Decodable, Authentication {
      return dateFormatter.date(from: dateString) ?? Date()
    }

-    let response = try decoder.decode(TokenResponse.self, from: fromData)
+    var response = try decoder.decode(TokenResponse.self, from: fromData)
+    response.issuedAt = response.issuedAt ?? Date()

    guard response.token != nil || response.accessToken != nil else {
      throw DecodingError.keyNotFound(CodingKeys.token, .init(codingPath: [], debugDescription: "Missing token or access_token. One must be present."))
@@ -85,7 +85,7 @@ struct TokenResponse: Decodable, Authentication {
      //
      // [1]: https://docs.docker.com/registry/spec/auth/token/#requesting-a-token

-      (issuedAt ?? defaultIssuedAt) + TimeInterval(expiresIn ?? defaultExpiresIn)
+      (issuedAt ?? Date()) + TimeInterval(expiresIn ?? 60)
    }
  }

@@ -114,11 +114,11 @@ class Registry {
    return host
  }

-  init(urlComponents: URLComponents,
+  init(baseURL: URL,
       namespace: String,
       credentialsProviders: [CredentialsProvider] = [EnvironmentCredentialsProvider(), DockerConfigCredentialsProvider(), KeychainCredentialsProvider()]
  ) throws {
-    baseURL = urlComponents.url!
+    self.baseURL = baseURL
    self.namespace = namespace
    self.credentialsProviders = credentialsProviders
  }
@@ -132,7 +132,17 @@ class Registry {
    let proto = insecure ? "http" : "https"
    let baseURLComponents = URLComponents(string: proto + "://" + host + "/v2/")!

-    try self.init(urlComponents: baseURLComponents, namespace: namespace, credentialsProviders: credentialsProviders)
+    guard let baseURL = baseURLComponents.url else {
+      var hint = ""
+
+      if host.hasPrefix("http://") || host.hasPrefix("https://") {
+        hint += ", make sure that it doesn't start with http:// or https://"
+      }
+
+      throw RuntimeError.ImproperlyFormattedHost(host, hint)
+    }
+
+    try self.init(baseURL: baseURL, namespace: namespace, credentialsProviders: credentialsProviders)
  }

  func ping() async throws {
@@ -181,7 +191,7 @@ class Registry {
    return URLComponents(url: uploadLocation.absolutize(baseURL), resolvingAgainstBaseURL: true)!
  }

-  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0) async throws -> String {
+  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0, digest: String? = nil) async throws -> String {
    // Initiate a blob upload
    let (data, postResponse) = try await dataRequest(.POST, endpointURL("\(namespace)/blobs/uploads/"),
                                                     headers: ["Content-Length": "0"])
@@ -193,7 +203,7 @@ class Registry {
    // Figure out where to upload the blob
    var uploadLocation = try uploadLocationFromResponse(postResponse)

-    let digest = Digest.hash(fromData)
+    let digest = digest ?? Digest.hash(fromData)

    if chunkSizeMb == 0 {
      // monolithic upload
@@ -241,6 +251,19 @@ class Registry {
    return digest
  }

+  public func blobExists(_ digest: String) async throws -> Bool {
+    let (data, response) = try await dataRequest(.HEAD, endpointURL("\(namespace)/blobs/\(digest)"))
+
+    switch response.statusCode {
+    case HTTPCode.Ok.rawValue:
+      return true
+    case HTTPCode.NotFound.rawValue:
+      return false
+    default:
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "checking blob", code: response.statusCode, details: data.asText())
+    }
+  }
+
  public func pullBlob(_ digest: String, handler: (Data) async throws -> Void) async throws {
    let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), viaFile: true)
    if response.statusCode != HTTPCode.Ok.rawValue {
@@ -50,6 +50,10 @@ struct Root: AsyncParsableCommand {
    }
    defer { SentrySDK.flush(timeout: 2.seconds.timeInterval) }

+    SentrySDK.configureScope { scope in
+      scope.setExtra(value: ProcessInfo.processInfo.arguments, key: "Command-line arguments")
+    }
+
    // Enrich future events with Cirrus CI-specific tags
    if let tags = ProcessInfo.processInfo.environment["CIRRUS_SENTRY_TAGS"] {
      SentrySDK.configureScope { scope in
@@ -0,0 +1,28 @@
+import Foundation
+
+fileprivate func normalizeName(_ name: String) -> String {
+  // Colons are misinterpreted by Zsh completion
+  return name.replacingOccurrences(of: ":", with: "\\:")
+}
+
+func completeMachines(_ arguments: [String]) -> [String] {
+  let localVMs = (try? VMStorageLocal().list().map { name, _ in
+    normalizeName(name)
+  }) ?? []
+  let ociVMs = (try? VMStorageOCI().list().map { name, _, _ in
+    normalizeName(name)
+  }) ?? []
+  return (localVMs + ociVMs)
+}
+
+func completeLocalMachines(_ arguments: [String]) -> [String] {
+  let localVMs = (try? VMStorageLocal().list()) ?? []
+  return localVMs.map { name, _ in normalizeName(name) }
+}
+
+func completeRunningMachines(_ arguments: [String]) -> [String] {
+  let localVMs = (try? VMStorageLocal().list()) ?? []
+  return localVMs
+    .filter { _, vmDir in (try? vmDir.state() == .Running) ?? false}
+    .map { name, _ in normalizeName(name) }
+}
@@ -1,4 +1,5 @@
 import Foundation
+import XAttr

 extension URL: Prunable {
  var url: URL {
@@ -13,7 +14,31 @@ extension URL: Prunable {
    try resourceValues(forKeys: [.totalFileAllocatedSizeKey]).totalFileAllocatedSize!
  }

+  func deduplicatedSizeBytes() throws -> Int {
+    let values = try resourceValues(forKeys: [.totalFileAllocatedSizeKey, .mayShareFileContentKey])
+    // make sure the file's origin file is there and duplication works
+    if values.mayShareFileContent == true {
+      return Int(deduplicatedBytes())
+    }
+    return 0
+  }
+
  func sizeBytes() throws -> Int {
    try resourceValues(forKeys: [.totalFileSizeKey]).totalFileSize!
  }
+
+  func setDeduplicatedBytes(_ size: UInt64) {
+    let data = "\(size)".data(using: .utf8)!
+    try! self.setExtendedAttribute(name: "run.tart.deduplicated-bytes", value: data)
+  }
+
+  func deduplicatedBytes() -> UInt64 {
+    guard let data = try? self.extendedAttributeValue(forName: "run.tart.deduplicated-bytes") else {
+      return 0
+    }
+    if let strValue = String(data: data, encoding: .utf8) {
+      return UInt64(strValue) ?? 0
+    }
+    return 0
+  }
 }
@@ -1,6 +1,7 @@
 import Foundation
 import Virtualization
 import AsyncAlgorithms
+import Semaphore

 struct UnsupportedRestoreImageError: Error {
 }
@@ -30,7 +31,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
  var configuration: VZVirtualMachineConfiguration

  // Semaphore used to communicate with the VZVirtualMachineDelegate
-  var sema = DispatchSemaphore(value: 0)
+  var sema = AsyncSemaphore(value: 0)

  // VM's config
  var name: String
@@ -46,7 +47,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
       directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
       serialPorts: [VZSerialPortConfiguration] = [],
       suspendable: Bool = false,
-       audio: Bool = true
+       audio: Bool = true,
+       clipboard: Bool = true,
+       sync: VZDiskImageSynchronizationMode = .full
  ) throws {
    name = vmDir.name
    config = try VMConfig.init(fromURL: vmDir.configURL)
@@ -63,7 +66,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
                                                directorySharingDevices: directorySharingDevices,
                                                serialPorts: serialPorts,
                                                suspendable: suspendable,
-                                                audio: audio
+                                                audio: audio,
+                                                clipboard: clipboard,
+                                                sync: sync
    )
    virtualMachine = VZVirtualMachine(configuration: configuration)

@@ -91,12 +96,17 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    // Download the IPSW
    defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")

-    let (channel, response) = try await Fetcher.fetch(URLRequest(url: remoteURL), viaFile: true)
+    let downloadProgress = Progress(totalUnitCount: 100)
+    ProgressObserver(downloadProgress).log(defaultLogger)

-    let progress = Progress(totalUnitCount: response.expectedContentLength)
-    ProgressObserver(progress).log(defaultLogger)
+    let request = URLRequest(url: remoteURL)
+    let (channel, response) = try await Fetcher.fetch(request, viaFile: true, progress: downloadProgress)

    let temporaryLocation = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString + ".ipsw")
+    defaultLogger.appendNewLine("Computing digest for \(temporaryLocation.path)...")
+    let digestProgress = Progress(totalUnitCount: response.expectedContentLength)
+    ProgressObserver(digestProgress).log(defaultLogger)
+
    FileManager.default.createFile(atPath: temporaryLocation.path, contents: nil)
    let lock = try FileLock(lockURL: temporaryLocation)
    try lock.lock()
@@ -108,7 +118,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
      let chunkAsData = Data(chunk)
      fileHandle.write(chunkAsData)
      digest.update(chunkAsData)
-      progress.completedUnitCount += Int64(chunk.count)
+      digestProgress.completedUnitCount += Int64(chunk.count)
    }

    try fileHandle.close()
@@ -191,18 +201,24 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
      virtualMachine.delegate = self

      // Run automated installation
-      try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
-        DispatchQueue.main.async { [ipswURL] in
-          let installer = VZMacOSInstaller(virtualMachine: self.virtualMachine, restoringFromImageAt: ipswURL)
+      try await install(ipswURL)
+    }

-          defaultLogger.appendNewLine("Installing OS...")
-          ProgressObserver(installer.progress).log(defaultLogger)
+    @MainActor
+    private func install(_ url: URL) async throws {
+      let installer = VZMacOSInstaller(virtualMachine: self.virtualMachine, restoringFromImageAt: url)
+      defaultLogger.appendNewLine("Installing OS...")
+      ProgressObserver(installer.progress).log(defaultLogger)

+      try await withTaskCancellationHandler(operation: {
+        try await withCheckedThrowingContinuation { continuation in
          installer.install { result in
            continuation.resume(with: result)
          }
        }
-      }
+      }, onCancel: {
+        installer.progress.cancel()
+      })
    }
  #endif

@@ -232,16 +248,18 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
  }

  func run() async throws {
-    await withTaskCancellationHandler(operation: {
-      // Wait for the VM to finish running
-      // or for the exit condition
-      sema.wait()
-    }, onCancel: {
-      sema.signal()
-    })
+    do {
+      try await sema.waitUnlessCancelled()
+    } catch is CancellationError {
+      // Triggered by "tart stop", Ctrl+C, or closing the
+      // VM window, so shut down the VM gracefully below.
+    }

    if Task.isCancelled {
-      try await stop()
+      if (self.virtualMachine.state == VZVirtualMachine.State.running) {
+        print("Stopping VM...")
+        try await stop()
+      }
    }

    try await network.stop()
@@ -277,7 +295,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    directorySharingDevices: [VZDirectorySharingDeviceConfiguration],
    serialPorts: [VZSerialPortConfiguration],
    suspendable: Bool = false,
-    audio: Bool = true
+    audio: Bool = true,
+    clipboard: Bool = true,
+    sync: VZDiskImageSynchronizationMode = .full
  ) throws -> VZVirtualMachineConfiguration {
    let configuration = VZVirtualMachineConfiguration()

@@ -295,16 +315,23 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    configuration.graphicsDevices = [vmConfig.platform.graphicsDevice(vmConfig: vmConfig)]

    // Audio
+    let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
+
    if audio && !suspendable {
-      let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
      let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
-      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
      let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
+
+      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
      outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
+
      soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
-      configuration.audioDevices = [soundDeviceConfiguration]
+    } else {
+      // just a null speaker
+      soundDeviceConfiguration.streams = [VZVirtioSoundDeviceOutputStreamConfiguration()]
    }

+    configuration.audioDevices = [soundDeviceConfiguration]
+
    // Keyboard and mouse
    if suspendable, let platformSuspendable = vmConfig.platform.self as? PlatformSuspendable {
      configuration.keyboards = platformSuspendable.keyboardsSuspendable()
@@ -322,11 +349,21 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
      return vio
    }

+    // Clipboard sharing via Spice agent
+    if clipboard && vmConfig.os == .linux {
+      let spiceAgentConsoleDevice = VZVirtioConsoleDeviceConfiguration()
+      let spiceAgentPort = VZVirtioConsolePortConfiguration()
+      spiceAgentPort.name = VZSpiceAgentPortAttachment.spiceAgentPortName
+      spiceAgentPort.attachment = VZSpiceAgentPortAttachment()
+      spiceAgentConsoleDevice.ports[0] = spiceAgentPort
+      configuration.consoleDevices.append(spiceAgentConsoleDevice)
+    }
+
    // Storage
    let attachment: VZDiskImageStorageDeviceAttachment = vmConfig.os == .linux ?
      // Use "cached" caching mode for virtio drive to prevent fs corruption on linux
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: .full) :
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)
+      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: sync) :
+      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .automatic, synchronizationMode: sync)

    var device: VZStorageDeviceConfiguration
    if #available(macOS 14, *), vmConfig.os == .linux {
@@ -1,3 +1,4 @@
+import Compression
 import Foundation
 import Sentry

@@ -10,18 +11,7 @@ enum OCIError: Error {
 }

 extension VMDirectory {
-  private static let bufferSizeBytes = 64 * 1024 * 1024
-  private static let layerLimitBytes = 500 * 1000 * 1000
-
-  func pullFromRegistry(registry: Registry, reference: String, concurrency: UInt) async throws {
-    defaultLogger.appendNewLine("pulling manifest...")
-
-    let (manifest, _) = try await registry.pullManifest(reference: reference)
-
-    return try await pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
-  }
-
-  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt) async throws {
+  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?) async throws {
    // Pull VM's config file layer and re-serialize it into a config file
    let configLayers = manifest.layers.filter {
      $0.mediaType == configMediaType
@@ -61,7 +51,18 @@ extension VMDirectory {
    let progress = Progress(totalUnitCount: diskCompressedSize)
    ProgressObserver(progress).log(defaultLogger)

-    try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL, concurrency: concurrency, progress: progress)
+    do {
+      try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL,
+                                  concurrency: concurrency, progress: progress,
+                                  localLayerCache: localLayerCache)
+    } catch let error where error is FilterError {
+      throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)")
+    }
+
+    if let llc = localLayerCache {
+      // set custom attribute to remember deduplicated bytes
+      diskURL.setDeduplicatedBytes(llc.deduplicatedBytes)
+    }

    // Pull VM's NVRAM file layer and store it in an NVRAM file
    defaultLogger.appendNewLine("pulling NVRAM...")
@@ -80,9 +81,12 @@ extension VMDirectory {
      nvram.write(data)
    }
    try nvram.close()
+
+    // Serialize VM's manifest to enable better deduplication on subsequent "tart pull"'s
+    try manifest.toJSON().write(to: manifestURL)
  }

-  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String) async throws -> RemoteName {
+  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String, concurrency: UInt) async throws -> RemoteName {
    var layers = Array<OCIManifestLayer>()

    // Read VM's config and push it as blob
@@ -101,9 +105,9 @@ extension VMDirectory {

    switch diskFormat {
    case "v1":
-      layers.append(contentsOf: try await DiskV1.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+      layers.append(contentsOf: try await DiskV1.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, concurrency: concurrency, progress: progress))
    case "v2":
-      layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+      layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, concurrency: concurrency, progress: progress))
    default:
      throw RuntimeError.OCIUnsupportedDiskFormat(diskFormat)
    }
@@ -3,6 +3,12 @@ import Virtualization
 import CryptoKit

 struct VMDirectory: Prunable {
+  enum State: String {
+    case Running = "running"
+    case Suspended = "suspended"
+    case Stopped = "stopped"
+  }
+
  var baseURL: URL

  var configURL: URL {
@@ -17,6 +23,9 @@ struct VMDirectory: Prunable {
  var stateURL: URL {
    baseURL.appendingPathComponent("state.vzvmsave")
  }
+  var manifestURL: URL {
+    baseURL.appendingPathComponent("manifest.json")
+  }

  var explicitlyPulledMark: URL {
    baseURL.appendingPathComponent(".explicitly-pulled")
@@ -47,13 +56,13 @@ struct VMDirectory: Prunable {
    return try lock.pid() != 0
  }

-  func state() throws -> String {
+  func state() throws -> State {
    if try running() {
-      return "running"
+      return State.Running
    } else if FileManager.default.fileExists(atPath: stateURL.path) {
-      return "suspended"
+      return State.Suspended
    } else {
-      return "stopped"
+      return State.Stopped
    }
  }

@@ -173,6 +182,14 @@ struct VMDirectory: Prunable {
    try allocatedSizeBytes() / 1000 / 1000 / 1000
  }

+  func deduplicatedSizeBytes() throws -> Int {
+    try configURL.deduplicatedSizeBytes() + diskURL.deduplicatedSizeBytes() + nvramURL.deduplicatedSizeBytes()
+  }
+
+  func deduplicatedSizeGB() throws -> Int {
+    try deduplicatedSizeBytes() / 1000 / 1000 / 1000
+  }
+
  func sizeBytes() throws -> Int {
    try configURL.sizeBytes() + diskURL.sizeBytes() + nvramURL.sizeBytes()
  }
@@ -55,11 +55,13 @@ enum RuntimeError : Error {
  case VMAlreadyRunning(_ message: String)
  case NoIPAddressFound(_ message: String)
  case DiskAlreadyInUse(_ message: String)
+  case FailedToOpenBlockDevice(_ path: String, _ explanation: String)
  case InvalidDiskSize(_ message: String)
  case FailedToUpdateAccessDate(_ message: String)
  case PIDLockFailed(_ message: String)
  case FailedToParseRemoteName(_ message: String)
  case VMTerminationFailed(_ message: String)
+  case ImproperlyFormattedHost(_ host: String, _ hint: String)
  case InvalidCredentials(_ message: String)
  case VMDirectoryAlreadyInitialized(_ message: String)
  case ExportFailed(_ message: String)
@@ -68,6 +70,8 @@ enum RuntimeError : Error {
  case OCIStorageError(_ message: String)
  case OCIUnsupportedDiskFormat(_ format: String)
  case SuspendFailed(_ message: String)
+  case PullFailed(_ message: String)
+  case VirtualMachineLimitExceeded(_ hint: String)
 }

 protocol HasExitCode {
@@ -93,6 +97,8 @@ extension RuntimeError : CustomStringConvertible {
      return message
    case .DiskAlreadyInUse(let message):
      return message
+    case .FailedToOpenBlockDevice(let path, let explanation):
+      return "failed to open block device \(path): \(explanation)"
    case .InvalidDiskSize(let message):
      return message
    case .FailedToUpdateAccessDate(let message):
@@ -103,6 +109,8 @@ extension RuntimeError : CustomStringConvertible {
      return "failed to parse remote name: \(cause)"
    case .VMTerminationFailed(let message):
      return message
+    case .ImproperlyFormattedHost(let host, let hint):
+      return "improperly formatted host \"\(host)\" was provided\(hint)"
    case .InvalidCredentials(let message):
      return message
    case .VMDirectoryAlreadyInitialized(let message):
@@ -119,6 +127,10 @@ extension RuntimeError : CustomStringConvertible {
      return "OCI disk format \(format) is not supported by this version of Tart"
    case .SuspendFailed(let message):
      return "Failed to suspend the VM: \(message)"
+    case .PullFailed(let message):
+      return message
+    case .VirtualMachineLimitExceeded(let hint):
+      return "The number of VMs exceeds the system limit\(hint)"
    }
  }
 }
@@ -66,7 +66,7 @@ class VMStorageLocal: PrunableStorage {
  }

  func prunables() throws -> [Prunable] {
-    try list().map { (_, vmDir) in vmDir }
+    try list().map { (_, vmDir) in vmDir }.filter { try !$0.running() }
  }

  func hasVMsWithMACAddress(macAddress: String) throws -> Bool {
@@ -1,5 +1,6 @@
 import Foundation
 import Sentry
+import Retry

 class VMStorageOCI: PrunableStorage {
  let baseURL = try! Config().tartCacheDir.appendingPathComponent("OCIs", isDirectory: true)
@@ -112,6 +113,10 @@ class VMStorageOCI: PrunableStorage {
        continue
      }

+      // Split the relative VM's path at the last component
+      // and figure out which character should be used
+      // to join them together, either ":" for tags or
+      // "@" for hashes
      let parts = [foundURL.deletingLastPathComponent().relativePath, foundURL.lastPathComponent]
      var name: String

@@ -122,6 +127,9 @@ class VMStorageOCI: PrunableStorage {
        name = parts.joined(separator: "@")
      }

+      // Remove the percent-encoding, if any
+      name = percentDecode(name)
+
      result.append((name, vmDir, isSymlink))
    }

@@ -134,7 +142,7 @@ class VMStorageOCI: PrunableStorage {

  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt) async throws {
    SentrySDK.configureScope { scope in
-      scope.setContext(value: ["imageName": name], key: "OCI")
+      scope.setContext(value: ["imageName": name.description], key: "OCI")
    }

    defaultLogger.appendNewLine("pulling manifest...")
@@ -188,7 +196,27 @@ class VMStorageOCI: PrunableStorage {
      }

      try await withTaskCancellationHandler(operation: {
-        try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
+        try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) {
+          // Choose the best base image which has the most deduplication ratio
+          let localLayerCache = try await chooseLocalLayerCache(name, manifest, registry)
+
+          if let llc = localLayerCache {
+            let deduplicatedHuman = ByteCountFormatter.string(fromByteCount: Int64(llc.deduplicatedBytes), countStyle: .file)
+
+            defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to deduplicate \(deduplicatedHuman), using it as a base...")
+          }
+
+          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache)
+        } recoverFromFailure: { error in
+          if error is Retryable {
+            print("Error: \(error.localizedDescription)")
+            print("Attempting to re-try...")
+
+            return .retry
+          }
+
+          return .throw
+        }
        try move(digestName, from: tmpVMDir)
        transaction.finish()
      }, onCancel: {
@@ -225,13 +253,66 @@ class VMStorageOCI: PrunableStorage {

    try gc()
  }
+
+  func chooseLocalLayerCache(_ name: RemoteName, _ manifest: OCIManifest, _ registry: Registry) async throws -> LocalLayerCache? {
+    // Establish a closure that will calculate how much bytes
+    // we'll deduplicate if we re-use the given manifest
+    let target = Swift.Set(manifest.layers)
+
+    let calculateDeduplicatedBytes = { (manifest: OCIManifest) -> UInt64 in
+      target.intersection(manifest.layers).map({ UInt64($0.size) }).reduce(0, +)
+    }
+
+    // Load OCI VM images and their manifests (if present)
+    var candidates: [(name: String, vmDir: VMDirectory, manifest: OCIManifest, deduplicatedBytes: UInt64)] = []
+
+    for (name, vmDir, isSymlink) in try list() {
+      if isSymlink {
+        continue
+      }
+
+      guard let manifestJSON = try? Data(contentsOf: vmDir.manifestURL) else {
+        continue
+      }
+
+      guard let manifest = try? OCIManifest(fromJSON: manifestJSON) else {
+        continue
+      }
+
+      candidates.append((name, vmDir, manifest, calculateDeduplicatedBytes(manifest)))
+    }
+
+    // Previously we haven't stored the OCI VM image manifests, but still fetched the VM image manifest if
+    // what the user was trying to pull was a tagged image, and we already had that image in the OCI VM cache
+    //
+    // Keep supporting this behavior for backwards comaptibility, but only communicate
+    // with the registry if we haven't already retrieved the manifest for that OCI VM image.
+    if name.reference.type == .Tag,
+       let vmDir = try? open(name),
+       let digest = try? digest(name),
+       try !candidates.contains(where: {try $0.manifest.digest() == digest}),
+       let (manifest, _) = try? await registry.pullManifest(reference: digest) {
+      candidates.append((name.description, vmDir, manifest, calculateDeduplicatedBytes(manifest)))
+    }
+
+    // Now, find the best match based on how many bytes we'll deduplicate
+    let choosen = candidates.filter {
+      $0.deduplicatedBytes > 1024 * 1024 * 1024 // save at least 1GB
+    }.max { left, right in
+      return left.deduplicatedBytes < right.deduplicatedBytes
+    }
+
+    return try choosen.flatMap({ choosen in
+      try LocalLayerCache(choosen.name, choosen.deduplicatedBytes, choosen.vmDir.diskURL, choosen.manifest)
+    })
+  }
 }

 extension URL {
  func appendingRemoteName(_ name: RemoteName) -> URL {
    var result: URL = self

-    for pathComponent in (name.host + "/" + name.namespace + "/" + name.reference.value).split(separator: "/") {
+    for pathComponent in (percentEncode(name.host) + "/" + name.namespace + "/" + name.reference.value).split(separator: "/") {
      result = result.appendingPathComponent(String(pathComponent))
    }

@@ -239,6 +320,23 @@ extension URL {
  }

  func appendingHost(_ name: RemoteName) -> URL {
-    self.appendingPathComponent(name.host, isDirectory: true)
+    self.appendingPathComponent(percentEncode(name.host), isDirectory: true)
  }
 }
+
+// Work around a pretty inane Swift's URL behavior where calling
+// appendingPathComponent() or deletingLastPathComponent() on a
+// URL like URL(filePath: "example.com:8080") (note the "filePath")
+// will flip its isFileURL from "true" to "false" and discard its
+// absolute path infromation (if any).
+//
+// The same kind of operations won't do anything to a URL like
+// URL(filePath: "127.0.0.1:8080"), which makes things even more
+// ridiculous.
+private func percentEncode(_ s: String) -> String {
+  return s.addingPercentEncoding(withAllowedCharacters: CharacterSet(charactersIn: ":").inverted)!
+}
+
+private func percentDecode(_ s: String) -> String {
+  s.removingPercentEncoding!
+}
@@ -15,7 +15,7 @@ class FullFledgedVNC: VNC {
    vnc.start()
  }

-  func waitForURL() async throws -> URL {
+  func waitForURL(netBridged: Bool) async throws -> URL {
    while true {
      // Port is 0 shortly after start(),
      // but will be initialized later
@@ -9,9 +9,9 @@ class ScreenSharingVNC: VNC {
    self.vmConfig = vmConfig
  }

-  func waitForURL() async throws -> URL {
+  func waitForURL(netBridged: Bool) async throws -> URL {
    let vmMACAddress = MACAddress(fromString: vmConfig.macAddress.string)!
-    let ip = try await IP.resolveIP(vmMACAddress, secondsToWait: 60)
+    let ip = try await IP.resolveIP(vmMACAddress, resolutionStrategy: netBridged ? .arp : .dhcp, secondsToWait: 60)

    if let ip = ip {
      return URL(string: "vnc://\(ip)")!
@@ -1,6 +1,6 @@
 import Foundation

 protocol VNC {
-  func waitForURL() async throws -> URL
+  func waitForURL(netBridged: Bool) async throws -> URL
  func stop() throws
 }
@@ -36,7 +36,7 @@ final class LayerizerTests: XCTestCase {
    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)

    print("pushing disk...")
-    let diskLayers = try await DiskV1.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+    let diskLayers = try await DiskV1.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, concurrency: 4, progress: Progress())

    print("pulling disk...")
    try await DiskV1.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
@@ -57,7 +57,7 @@ final class LayerizerTests: XCTestCase {
    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)

    print("pushing disk...")
-    let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+    let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, concurrency: 4, progress: Progress())

    print("pulling disk...")
    try await DiskV2.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
@@ -3,24 +3,26 @@ import XCTest

 final class TokenResponseTests: XCTestCase {
  func testBasic() throws {
+    var expectedTokenExpiresAtRange = DateInterval()
    let tokenResponseRaw = Data("{\"token\":\"some token\"}".utf8)
    let tokenResponse = try TokenResponse.parse(fromData: tokenResponseRaw)

    XCTAssertEqual(tokenResponse.token, "some token")

-    let expectedTokenExpiresAtRange = Date()...Date().addingTimeInterval(60)
+    expectedTokenExpiresAtRange.end = Date().addingTimeInterval(60)
    XCTAssertTrue(expectedTokenExpiresAtRange.contains(tokenResponse.tokenExpiresAt))

    XCTAssertTrue(tokenResponse.isValid())
  }

  func testExpirationBasic() throws {
+    var expectedTokenExpiresAtRange = DateInterval()
    let tokenResponseRaw = Data("{\"token\":\"some token\",\"expires_in\":2}".utf8)
    let tokenResponse = try TokenResponse.parse(fromData: tokenResponseRaw)

    XCTAssertEqual(tokenResponse.expiresIn, 2)

-    let expectedTokenExpiresAtRange = Date()...Date().addingTimeInterval(2)
+    expectedTokenExpiresAtRange.end = Date().addingTimeInterval(2)
    XCTAssertTrue(expectedTokenExpiresAtRange.contains(tokenResponse.tokenExpiresAt))

    XCTAssertTrue(tokenResponse.isValid())
@@ -39,7 +39,7 @@ class RegistryRunner {
    let port = try Self.dockerCmd("inspect", containerID, "--format", "{{(index (index .NetworkSettings.Ports \"5000/tcp\") 0).HostPort}}")
      .trimmingCharacters(in: CharacterSet.newlines)

-    registry = try Registry(urlComponents: URLComponents(string: "http://127.0.0.1:\(port)/v2/")!,
+    registry = try Registry(baseURL: URL(string: "http://127.0.0.1:\(port)/v2/")!,
                            namespace: "vm-image")

    // Wait for the Docker Registry to start
@@ -0,0 +1 @@
+root = true
@@ -0,0 +1,92 @@
+run:
+  timeout: 5m
+
+linters:
+  enable-all: true
+
+  disable:
+    # Messages like "struct of size 104 bytes could be of size 96 bytes" from a package
+    # that was last updated 2 years ago[1] are barely helpful.
+    #
+    # After all, we're writing the code for other people, so let's trust the compiler here (that's
+    # constantly evolving compared to this linter) and revisit this if memory usage becomes a problem.
+    #
+    # [1]: https://github.com/mdempsky/maligned/commit/6e39bd26a8c8b58c5a22129593044655a9e25959
+    - maligned
+
+    # We don't have high-performance requirements at this moment, so sacrificing
+    # the code readability for marginal performance gains is not worth it.
+    - prealloc
+
+    # New linters that require a lot of codebase churn and noise, but perhaps we can enable them in the future.
+    - nlreturn
+    - wrapcheck
+    - errorlint
+
+    # Unfortunately, we use globals due to how spf13/cobra works.
+    - gochecknoglobals
+
+    # That's fine that some Proto objects don't have all fields initialized
+    - exhaustivestruct
+
+    # Style linters that are total nuts.
+    - wsl
+    - gofumpt
+    - goimports
+    - funlen
+
+    # This conflicts with the Protocol Buffers Version 3 design,
+    # which is largely based on default values for struct fields.
+    - exhaustivestruct
+
+    # Enough parallelism for now.
+    - paralleltest
+
+    # Ill-based assumptions about identifiers like fmt.Println without taking context into account.
+    - forbidigo
+
+    # Advantages of using t.Helper() are too small to waste developer's cognitive stamina on it.
+    - thelper
+
+    # Too restrictive defaults, plus there's already a gocyclo linter in place.
+    - cyclop
+
+    # Gives false positives for textbook examples[1][2]
+    # [1]: https://github.com/charithe/durationcheck/issues/7
+    # [2]: https://golang.org/pkg/time/ (see "To convert an integer number of units to a Duration, multiply:")
+    - durationcheck
+
+    # No way to disable the "exported" check for the whole project[1]
+    # [1]: https://github.com/mgechev/revive/issues/244#issuecomment-560512162
+    - revive
+
+    # Unfortunately too much false-positives, e.g. for a 0700 umask or number 10 when using strconv.FormatInt()
+    - gomnd
+
+    # Needs package whitelists
+    - depguard
+
+    # Generates absolutely useless errors, e.g.
+    # "string `.yml` has 3 occurrences, make it a constant"
+    - goconst
+
+    # It's OK to not sort imports
+    - gci
+
+    # It's OK to not initialize some struct fields
+    - exhaustruct
+
+    # This is not a library, so it's OK to use dynamic errors
+    - goerr113
+
+    # fmt.Sprintf() looks a bit nicer than string addition
+    - perfsprint
+
+    # We can control this ourselves
+    - varnamelen
+    - contextcheck
+
+issues:
+  # Don't hide multiple issues that belong to one class since GitHub annotations can handle them all nicely.
+  max-issues-per-linter: 0
+  max-same-issues: 0
@@ -0,0 +1,39 @@
+# Benchmark
+
+Tart comes with a Golang-based benchmarking utility that allows one to easily compare host and guest performance.
+
+Currently, only Flexible I/O tester workloads are supported. To run them, [install Golang](https://go.dev/) and run the following command from this (`benchmark/`) directory:
+
+```shell
+go run cmd/main.go fio
+```
+
+You can also enable the debugging output to diagnose issues:
+
+```shell
+go run cmd/main.go fio --debug
+```
+
+## Results
+
+Host:
+
+* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
+* OS: macOS Sonoma 14.4.1
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.4.1
+
+```
+Name                    	Executor	Bandwidth	I/O operations
+Random writing of 1MB   	local   	2.6 GB/s 	649.35 kIOPS  
+Random writing of 1MB   	Tart    	2.5 GB/s 	620.22 kIOPS  
+Random writing of 10MB  	local   	2.6 GB/s 	651.74 kIOPS  
+Random writing of 10MB  	Tart    	2.5 GB/s 	615.52 kIOPS  
+Random writing of 100MB 	local   	1.9 GB/s 	481.51 kIOPS  
+Random writing of 100MB 	Tart    	2.0 GB/s 	493.31 kIOPS  
+Random writing of 1000MB	local   	1.7 GB/s 	414.89 kIOPS  
+Random writing of 1000MB	Tart    	1.1 GB/s 	287.4 kIOPS  
+```
@@ -0,0 +1,23 @@
+package main
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/command"
+	"log"
+	"os"
+	"os/signal"
+)
+
+func main() {
+	// Set up a signal-interruptible context
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+
+	// Run the root command
+	if err := command.NewCommand().ExecuteContext(ctx); err != nil {
+		cancel()
+
+		log.Fatal(err)
+	}
+
+	cancel()
+}
@@ -0,0 +1,29 @@
+module github.com/cirruslabs/tart/benchmark
+
+go 1.22.1
+
+require (
+	github.com/avast/retry-go/v4 v4.5.1
+	github.com/dustin/go-humanize v1.0.1
+	github.com/google/uuid v1.6.0
+	github.com/gosuri/uitable v0.0.4
+	github.com/spf13/cobra v1.8.0
+	github.com/stretchr/testify v1.9.0
+	go.uber.org/zap v1.27.0
+	golang.org/x/crypto v0.21.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fatih/color v1.16.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
@@ -0,0 +1,52 @@
+github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
+github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
+github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -0,0 +1,29 @@
+package fio
+
+type Benchmark struct {
+	Name    string
+	Command string
+}
+
+var benchmarks = []Benchmark{
+	{
+		Name: "Random writing of 1MB",
+		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
+			"--size 1MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+	},
+	{
+		Name: "Random writing of 10MB",
+		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
+			"--size 10MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+	},
+	{
+		Name: "Random writing of 100MB",
+		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
+			"--size 100MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+	},
+	{
+		Name: "Random writing of 1000MB",
+		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
+			"--size 1000MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+	},
+}
@@ -0,0 +1,131 @@
+package fio
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/cirruslabs/tart/benchmark/internal/executor"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	"github.com/dustin/go-humanize"
+	"github.com/gosuri/uitable"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+)
+
+var debug bool
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "fio",
+		Short: "run Flexible I/O tester (fio) benchmarks",
+		RunE:  run,
+	}
+
+	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+
+	return cmd
+}
+
+func run(cmd *cobra.Command, args []string) error {
+	config := zap.NewProductionConfig()
+	if debug {
+		config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+	logger, err := config.Build()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = logger.Sync()
+	}()
+
+	executors, err := initializeExecutors(cmd.Context(), logger)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		errs := []error{err}
+
+		for _, executor := range executors {
+			if err := executor.Close(); err != nil {
+				errs = append(errs, fmt.Errorf("failed to close executor %s: %w", executor.Name(), err))
+			}
+		}
+
+		err = errors.Join(errs...)
+	}()
+
+	table := uitable.New()
+	table.AddRow("Name", "Executor", "Bandwidth", "I/O operations")
+
+	for _, benchmark := range benchmarks {
+		for _, executor := range executors {
+			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name, executor.Name())
+
+			stdout, err := executor.Run(cmd.Context(), benchmark.Command)
+			if err != nil {
+				return err
+			}
+
+			var fioResult Result
+
+			if err := json.Unmarshal(stdout, &fioResult); err != nil {
+				return err
+			}
+
+			if len(fioResult.Jobs) != 1 {
+				return fmt.Errorf("expected exactly 1 job from fio's JSON output, got %d",
+					len(fioResult.Jobs))
+			}
+
+			job := fioResult.Jobs[0]
+
+			writeBandwidth := humanize.Bytes(uint64(job.Write.BW)*humanize.KByte) + "/s"
+			writeIOPS := humanize.SIWithDigits(job.Write.IOPS, 2, "IOPS")
+
+			logger.Sugar().Infof("write bandwidth: %s, write IOPS: %s\n", writeBandwidth, writeIOPS)
+
+			table.AddRow(benchmark.Name, executor.Name(), writeBandwidth, writeIOPS)
+		}
+	}
+
+	fmt.Println(table.String())
+
+	return nil
+}
+
+func initializeExecutors(ctx context.Context, logger *zap.Logger) ([]executor.Executor, error) {
+	var result []executor.Executor
+
+	logger.Info("initializing local executor")
+
+	local, err := local.New(logger)
+	if err != nil {
+		return nil, err
+	}
+	result = append(result, local)
+
+	logger.Info("local executor initialized")
+
+	logger.Info("initializing Tart executor")
+
+	tart, err := tart.New(ctx, logger)
+	if err != nil {
+		return nil, err
+	}
+	result = append(result, tart)
+
+	logger.Info("Tart executor initialized")
+
+	for _, executor := range result {
+		logger.Sugar().Infof("installing Flexible I/O tester (fio) on %s executor", executor.Name())
+
+		if _, err := executor.Run(ctx, "brew install fio"); err != nil {
+			return nil, err
+		}
+	}
+
+	return result, nil
+}
@@ -0,0 +1,15 @@
+package fio
+
+type Result struct {
+	Jobs []Job `json:"jobs"`
+}
+
+type Job struct {
+	Name  string `json:"jobname"`
+	Write Write  `json:"write"`
+}
+
+type Write struct {
+	BW   float64 `json:"bw"`
+	IOPS float64 `json:"iops"`
+}
@@ -0,0 +1,20 @@
+package command
+
+import (
+	"github.com/cirruslabs/tart/benchmark/internal/command/fio"
+	"github.com/spf13/cobra"
+)
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:           "benchmark",
+		SilenceUsage:  true,
+		SilenceErrors: true,
+	}
+
+	cmd.AddCommand(
+		fio.NewCommand(),
+	)
+
+	return cmd
+}
@@ -0,0 +1,11 @@
+package executor
+
+import (
+	"context"
+)
+
+type Executor interface {
+	Name() string
+	Run(ctx context.Context, command string) ([]byte, error)
+	Close() error
+}
@@ -0,0 +1,42 @@
+package local
+
+import (
+	"bytes"
+	"context"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"io"
+	"os/exec"
+)
+
+type Local struct {
+	logger *zap.Logger
+}
+
+func New(logger *zap.Logger) (*Local, error) {
+	return &Local{
+		logger: logger,
+	}, nil
+}
+
+func (local *Local) Name() string {
+	return "local"
+}
+
+func (local *Local) Run(ctx context.Context, command string) ([]byte, error) {
+	cmd := exec.CommandContext(ctx, "zsh", "-c", command)
+
+	loggerWriter := &zapio.Writer{Log: local.logger, Level: zap.DebugLevel}
+	stdoutBuf := &bytes.Buffer{}
+
+	cmd.Stdout = io.MultiWriter(stdoutBuf, loggerWriter)
+	cmd.Stderr = loggerWriter
+
+	err := cmd.Run()
+
+	return stdoutBuf.Bytes(), err
+}
+
+func (local *Local) Close() error {
+	return nil
+}
@@ -0,0 +1,20 @@
+package local_test
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"testing"
+)
+
+func TestLocal(t *testing.T) {
+	local, err := local.New(zap.NewNop())
+	require.NoError(t, err)
+
+	output, err := local.Run(context.Background(), "echo \"this is a test\"")
+	require.NoError(t, err)
+	require.Equal(t, "this is a test\n", string(output))
+
+	require.NoError(t, local.Close())
+}
@@ -0,0 +1,35 @@
+package tart
+
+import (
+	"bytes"
+	"context"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"io"
+	"os/exec"
+	"strings"
+)
+
+const tartBinaryName = "tart"
+
+func Cmd(ctx context.Context, logger *zap.Logger, args ...string) error {
+	_, err := CmdWithOutput(ctx, logger, args...)
+
+	return err
+}
+
+func CmdWithOutput(ctx context.Context, logger *zap.Logger, args ...string) (string, error) {
+	logger.Sugar().Debugf("running %s %s", tartBinaryName, strings.Join(args, " "))
+
+	cmd := exec.CommandContext(ctx, tartBinaryName, args...)
+
+	loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+	stdoutBuf := &bytes.Buffer{}
+
+	cmd.Stdout = io.MultiWriter(stdoutBuf, loggerWriter)
+	cmd.Stderr = loggerWriter
+
+	err := cmd.Run()
+
+	return stdoutBuf.String(), err
+}
@@ -0,0 +1,133 @@
+package tart
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"github.com/avast/retry-go/v4"
+	"github.com/google/uuid"
+	"go.uber.org/zap"
+	"golang.org/x/crypto/ssh"
+	"net"
+	"strings"
+	"time"
+)
+
+const baseImage = "ghcr.io/cirruslabs/macos-sonoma-base:latest"
+
+type Tart struct {
+	vmRunCancel context.CancelFunc
+	vmName      string
+	sshClient   *ssh.Client
+	logger      *zap.Logger
+}
+
+func New(ctx context.Context, logger *zap.Logger) (*Tart, error) {
+	tart := &Tart{
+		vmName: fmt.Sprintf("tart-benchmark-%s", uuid.NewString()),
+		logger: logger,
+	}
+
+	if err := Cmd(ctx, tart.logger, "pull", baseImage); err != nil {
+		return nil, err
+	}
+
+	if err := Cmd(ctx, tart.logger, "clone", baseImage, tart.vmName); err != nil {
+		return nil, err
+	}
+
+	vmRunCtx, vmRunCancel := context.WithCancel(ctx)
+	tart.vmRunCancel = vmRunCancel
+
+	go func() {
+		_ = Cmd(vmRunCtx, tart.logger, "run", "--no-graphics", tart.vmName)
+	}()
+
+	ip, err := CmdWithOutput(ctx, tart.logger, "ip", "--wait", "60", tart.vmName)
+	if err != nil {
+		return nil, tart.Close()
+	}
+
+	err = retry.Do(func() error {
+		dialer := net.Dialer{
+			Timeout: 1 * time.Second,
+		}
+
+		addr := fmt.Sprintf("%s:22", strings.TrimSpace(ip))
+
+		netConn, err := dialer.DialContext(ctx, "tcp", addr)
+		if err != nil {
+			return err
+		}
+
+		sshConn, chans, reqs, err := ssh.NewClientConn(netConn, addr, &ssh.ClientConfig{
+			User: "admin",
+			Auth: []ssh.AuthMethod{
+				ssh.Password("admin"),
+			},
+			HostKeyCallback: func(_ string, _ net.Addr, _ ssh.PublicKey) error {
+				return nil
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+		tart.sshClient = ssh.NewClient(sshConn, chans, reqs)
+
+		return nil
+	}, retry.RetryIf(func(err error) bool {
+		return !errors.Is(err, context.Canceled)
+	}))
+	if err != nil {
+		return nil, tart.Close()
+	}
+
+	return tart, nil
+}
+
+func (tart *Tart) Name() string {
+	return "Tart"
+}
+
+func (tart *Tart) Run(ctx context.Context, command string) ([]byte, error) {
+	sshSession, err := tart.sshClient.NewSession()
+	if err != nil {
+		return nil, err
+	}
+
+	// Work around x/crypto/ssh not being context.Context-friendly (e.g. https://github.com/golang/go/issues/20288)
+	monitorCtx, monitorCancel := context.WithCancel(ctx)
+	go func() {
+		<-monitorCtx.Done()
+		_ = sshSession.Close()
+	}()
+	defer monitorCancel()
+
+	stdoutBuf := &bytes.Buffer{}
+
+	sshSession.Stdin = bytes.NewBufferString(command)
+	sshSession.Stdout = stdoutBuf
+
+	if err := sshSession.Shell(); err != nil {
+		return nil, err
+	}
+
+	if err := sshSession.Wait(); err != nil {
+		return nil, err
+	}
+
+	return stdoutBuf.Bytes(), nil
+}
+
+func (tart *Tart) Close() error {
+	if tart.sshClient != nil {
+		_ = tart.sshClient.Close()
+	}
+
+	tart.vmRunCancel()
+	_ = Cmd(context.Background(), tart.logger, "delete", tart.vmName)
+
+	return nil
+}
@@ -0,0 +1,22 @@
+package tart_test
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"testing"
+)
+
+func TestTart(t *testing.T) {
+	ctx := context.Background()
+
+	tart, err := tart.New(ctx, zap.NewNop())
+	require.NoError(t, err)
+
+	output, err := tart.Run(ctx, "echo \"this is a test\"")
+	require.NoError(t, err)
+	require.Equal(t, "this is a test\n", string(output))
+
+	require.NoError(t, tart.Close())
+}
@@ -0,0 +1,62 @@
+---
+draft: false
+date: 2024-06-20
+search:
+  exclude: true
+authors:
+  - edigaryev
+categories:
+  - orchard
+---
+
+# Jumping through the hoops: SSH jump host functionality in Orchard
+
+Almost a year ago, when we started building [Orchard](https://github.com/cirruslabs/orchard), an orchestration system for Tart, we quickly realized that most worker machines will be in a private network, and that VMs will be only reachable from the worker machines themselves. Thus, one of our goals became to simplify accessing the compute resources in a cluster through a centralized controller host.
+
+This effort resulted in commands like `orchard port-forward` and `orchard ssh`, which were later improved to support connecting not just to the VMs, but to the worker machines themselves.
+
+Today, we’re making an even further step in this effort: with a trivial configuration, an Orchard controller can act as an SSH jump host to allow connecting to the VMs using just the `ssh` command like `ssh -J <service account name>@orchard-controller.example.com <VM name>`!
+
+<!-- more -->
+
+## Implementation
+
+In a typical cluster there’s one controller, to which workers connect by calling various REST API endpoints to synchronize the worker & VMs state. Each worker also maintains a persistent bi-directional gRPC connection with the controller, with the goal of improving the overall reactivity and making the port-forwarding work.
+
+The gRPC service definition that the controller offers is pretty minimalistic:
+
+```protobuf
+service Controller {
+  rpc Watch(google.protobuf.Empty) returns (stream WatchInstruction);
+  rpc PortForward(stream PortForwardData) returns (stream PortForwardData);
+}
+```
+
+Each watch instruction corresponds a single action to be done by the worker, which can either be a request for establishing a port-forwarding stream or a request for VMs re-syncing:
+
+```protobuf
+oneof action {
+  PortForward port_forward_action = 1;
+  SyncVMs sync_vms_action = 2;
+}
+```
+
+Now, when the user invokes `orchard port-forward` or `orchard ssh`, controller effectively becomes a rendezvous point by accepting the WebSocket connection from the user, and then asking the worker associated with the requested VM to establish a port-forwarding stream, and finally proxying the two streams together.
+
+![An illustration showing the Orchard controller and worker proxying the SSH connection](../images/jumping-through-the-hoops.png)
+
+SSH protocol works the same way, multiplexing multiple channels in a single transport connection, where each channel can be upgraded either to an interactive session (that’s what you get when you `ssh` to the server) or X11 channel (for X11 forwarding using `-X`), direct or forward TCP/IP channels (these are used for local and remote port-forwarding when using `-L` and `-R` options correspondingly) and so on.
+
+In fact, `ssh -J` jump host functionality also uses the direct TCP/IP channel, which is [just a single port-forwarding request](https://datatracker.ietf.org/doc/html/rfc4254#section-7.2) that needs to be implemented. We’ve used [Golang's SSH library](https://pkg.go.dev/golang.org/x/crypto/ssh) as the most mature choice for this task, and it’s been pleasant to work with so far.
+
+The support for `ssh -J` has landed in Orchard version 0.19.0. To configure the SSH jump host, simply add the `--listen-ssh` command-line argument to your `orchard controller run` invocation.
+
+Once running, you can connect to any VM in the cluster using the `ssh -J <service account name>@orchard-controller.example.com <VM name>`. The password for the jump host is the corresponding service account’s token.
+
+## Future plans
+
+First of all, we’d like to thank our paid clients, without which this feature wouldn’t be possible. [Become one now](../../licensing.md) and get the benefit of higher Tart VMs and Orchard workers allowances and making sure that the roadmap for Tart and Orchard is aligned with your company's needs.
+
+In the near future we plan to implement a mechanism similar to `authorized_keys` file that will allow attaching public SSH keys to the Orchard controller’s service accounts, and thus avoid the need to type the passwords.
+
+Stay tuned and don’t hesitate to send us your feedback on [GitHub](https://github.com/cirruslabs/orchard) and [Twitter](https://x.com/cirrus_labs)!
@@ -1,6 +1,8 @@
 ---
 hide:
  - navigation
+title: Frequently Asked Questions
+description: Advanced configuration and troubleshooting tips for advanced configurations.
 ---

 ## VM location on disk
@@ -1,4 +1,5 @@
 ---
 template: overrides/home.html
-title: Tart
+title: Toolset to build, run and manage macOS and Linux VMs
+description: Native performance. Remote storage for Virtual Machines. Many integrations including GitHub, GitLab and more.
 ---
@@ -1,3 +1,8 @@
+---
+title: Buildkite Integration
+description: Run pipeline steps in isolated ephemeral Tart Virtual Machines.
+---
+
 # Buildkite

 It is possible to run [Buildkite](https://buildkite.com/) pipeline steps in isolated ephemeral Tart Virtual Machines with the help of [Tart Buildkite Plugin](https://github.com/cirruslabs/tart-buildkite-plugin):
@@ -1,3 +1,8 @@
+---
+title: Cirrus CLI
+description: Tool for running isolated tasks reproducibly in any environment with a simple YAML configuration.
+---
+
 # Cirrus CLI

 Tart itself is only responsible for managing virtual machines, but we've built Tart support into a tool called Cirrus CLI
@@ -1,128 +0,0 @@
-# Cirrus Runners for GitHub Actions
-
-*Cirrus Runners* is the fastest and most cost-efficient way to get your current CI workflows to benefit from Apple Silicon hardware. No need to manage infrastructure or migrate to another CI provider.
-Your actions will be executed in clean macOS virtual machines with 4 Apple M2 cores.
-
-## Testimonials from customers
-
-Mitchell Hashimoto, HashiCorp co-founder:
-
-> I've been using "Cirrus Runners" since [that tweet](https://twitter.com/mitchellh/status/1731071326201561194) and it has been fantastic. Huge speed increase, huge cost decrease, zero maintenance, exactly what I wanted.
-
-Max Lapides, Senior Mobile Engineer at [Tonal](https://www.tonal.com/):
-
-> Previously, we were using the GitHub‑hosted macOS runners and our iOS build took ~30 minutes. Now with Cirrus Runners, the iOS build only takes ~12 minutes. That’s a huge boost to our productivity, and for only $150/month per runner it is much less expensive too.
-
-John A., Software Engineer at [GitKraken](https://www.gitkraken.com/):
-
-> GitHub Actions MacOS-x86 runners have become increasingly unreliable, so we're moving our Mac builds over to arm64 because Cirrus Labs' M1 runners are not only ~3 times faster, they've also been far more stable.
-
-Sebastian Jachec, Mobile Engineer at [Daybridge](https://www.daybridge.com/):
-
-> It’s been plain-sailing with the Cirrus Runners — they’ve been great! They’re consistently 60+% faster on workflows that we previously used Github Actions’ macOS runners for.
-
-## Pricing
-
-Each Cirrus Runner costs $150 a month and there is no limit on the amount of minutes for your actions.
-We recommend to purchase several Cirrus Runners depending on your team size, so you can run actions in
-parallel. Note that you can change your subscription at any time via [this page](https://billing.stripe.com/p/login/3cs7vNbzo92p7fy3cc)
-or by emailing [support@cirruslabs.org](mailto:support@cirruslabs.org).
-
-### Discounts
-
-We offer two mutually exclusive discounts:
-
- 10% "Volume Discount" for subscriptions of 10 or more Cirrus Runners.
- 15% "Annual Discount" for 12 months subscription commitment of any amount of Cirrus Runners.
-
-Please contact [support@cirruslabs.org](mailto:support@cirruslabs.org) after activating the subscription in order to get the discount applied.
-
-### Priority Support
-
-Subscriptions of 20 or more Cirrus Runners include access to [Priority Support](../licensing.md#priority-support).
-Please contact [sales@cirruslabs.org](mailto:sales@cirruslabs.org) in order to get all the details.
-
-### CPU and Memory resources of Cirrus Runners
-
-By default, a single Cirrus Runner is allocated with 4 M2 cores and 12 GB of unified memory which is enough for most of the workloads.
-For workloads that require more resources it is possible to use XL Cirrus Runners which have twice the resources: a full M2 chip with 8 cores
-and 24 GB of unified memory. Note that a single XL Cirrus Runner also uses twice the concurrency.
-
-In order to use an XL Cirrus Runner for a job please append `-xl` suffix to your `runs-on` property. More on that down below.
-
-## Installation
-
-Once you configure [Cirrus Runners App](https://github.com/apps/cirrus-runners) for your organization, you'll be redirected
-to a checkout page powered by Stripe. During the checkout process you'll be able to configure a subscription for
-a desired amount of parallel Cirrus Runners and try it for free for 10 days.
-
-Once configured, please follow instruction below. If you have any questions please contact [support@cirruslabs.org](mailto:support@cirruslabs.org).
-Subscriptions with more than 10 runners also include Priority Support 
-
-## Configuring Cirrus Runners
-
-In order for Cirrus Runners to be used by your GitHub Actions workflow jobs, specify a desired image in the `runs-on` property.
-
-=== "Default Cirrus Runner"
-
-    ```yaml
-    name: Tests
-    jobs:
-      test:
-        runs-on: ghcr.io/cirruslabs/macos-sonoma-xcode:latest
-    ```
-
-=== "XL Cirrus Runner"
-
-    ```yaml
-    name: Integration Tests
-    jobs:
-      test:
-        runs-on: ghcr.io/cirruslabs/macos-sonoma-xcode:latest-xl
-    ```
-
-List of all available images can be found in [this repository](https://github.com/cirruslabs/macos-image-templates).
-
-Note that Tart VM images don't have the same set of pre-installed packages as the official Intel GitHub runners.
-If something is missing please [create an issue within this repository](https://github.com/cirruslabs/macos-image-templates/issues/new).
-
-When workflows are executing you'll see Cirrus on-demand runners on your organization's settings page at `https://github.com/organizations/<ORGANIZATION>/settings/actions/runners`.
-Note that Cirrus Runners will get added to the default runner group.
-
-!!! tip "Using Cirrus Runners with public repositories"
-
-    By default, only private repositories can access runners in a default runner group, but you can override this in your organization's settings:
-
-    ```https://github.com/organizations/<YOUR ORGANIZATION NAME>/settings/actions/runner-groups/1```
-
-![](/assets/images/TartGHARunners.png)
-
-### Dashboard
-
-You can also see the status of your runners on the [Cirrus Runners Dashboard](https://cirrus-runners.app/). This dashboard
-also provides insights into price performance of your Cirrus Runners. Please check out [this blog post](/blog/2023/11/03/new-dashboard-with-insights-into-performance-of-cirrus-runners/)
-to learn more about what this dashboard can do for you.
-
-![](/assets/images/RunnersDashboard.png)
-
-## Data handling flow
-
-By design Cirrus Runners service never sees any of your secrets or source code and acts as compute platform with the lastest
-Apple Silicon hardware that can quickly allocate CPU/Memory resources for your jobs.
-
-Here is a high-level overview of how Cirrus Runners service manages runners for your organization:
-
- Cirrus Runner GitHub App is subscribed to [`workflow_job`](https://docs.github.com/en/webhooks/webhook-events-and-payloads#workflow_job).
- Upon receiving a new event targeting Cirrus Runners via `runs-on` property the following steps take place:
-
-    - Non-personal information about your job is saved to perform health checking of Cirrus Runners execution.
-    - Cirrus Runners GitHub App has only one permission that allows generating temporary registration tokens for
-      self-hosted GitHub Actions Runners. Note that Cirrus Runners GitHub App itself doesn't have access to contents of
-      repositories in your organization.
-    - Cirrus Runners Service creates a new single use Tart VM, generates a temporary registration tokens for self-hosted runners
-      and passes it without storing inside the VM for the GitHub Actions Runner service to [start a ephemeral runner](https://github.blog/changelog/2021-09-20-github-actions-ephemeral-self-hosted-runners-new-webhooks-for-auto-scaling/).
-
- Cirrus Runners service continuously monitors health of the Tart VM executing your job to make sure it runs to completion.
- After the job finishes the ephemeral Tart VM is getting destroyed with all the information of the job run.
-
-If you have any questions or concerns please feel free to reach out to [support@cirruslabs.org](mailto:support@cirruslabs.org).
@@ -1,3 +1,8 @@
+---
+title: GitLab Runner Executor
+description: Run jobs in isolated ephemeral Tart Virtual Machines.
+---
+
 # GitLab Runner Executor

 It is possible to run GitLab jobs in isolated ephemeral Tart Virtual Machines via [Tart Executor](https://github.com/cirruslabs/gitlab-tart-executor).
@@ -1,3 +1,8 @@
+---
+title: Managing Virtual Machine
+description: Use Packer to build custom VM images, configure VMs and work with remote OCI registries.
+---
+
 # Managing Virtual Machine

 ## Creating from scratch
@@ -15,7 +20,7 @@ tart create --from-ipsw=latest sonoma-vanilla
 tart run sonoma-vanilla
 ```

-After the initial booting of the VM you'll need to manually go through the macOS installation process. As a convention we recommend creating an `admin` user with an `admin` password. After the regular installation please do some additional modifications in the VM:
+After the initial booting of the VM, you'll need to manually go through the macOS installation process. As a convention we recommend creating an `admin` user with an `admin` password. After the regular installation please do some additional modifications in the VM:

 1. Enable Auto-Login. Users & Groups -> Login Options -> Automatic login -> admin.
 2. Allow SSH. Sharing -> Remote Login
@@ -48,7 +53,7 @@ sudo ufw allow ssh

 ## Configuring a VM

-By default, a tart VM uses 2 CPUs and 4 GB of memory with a `1024x768` display. This can be changed with `tart set` command.
+By default, a Tart VM uses 2 CPUs and 4 GB of memory with a `1024x768` display. This can be changed after VM creation with `tart set` command.
 Please refer to `tart set --help` for additional details.

 ## Building with Packer
@@ -92,18 +97,19 @@ Here is a [repository with Packer templates](https://github.com/cirruslabs/macos

 ## Working with a Remote OCI Container Registry

-<!-- markdownlint-disable MD034 -->
-For example, let's say you want to push/pull images to a registry hosted at https://acme.io/.
-<!-- markdownlint-enable MD034 -->
+Tart supports interacting with Open Container Initiative (OCI) registries, but only runs images created and pushed by Tart. This means images created for container engines, like Docker, can't be pulled. Instead, create a custom image as documented above.
+
+For example, let's say you want to push/pull images to an OCI registry hosted at `https://acme.io/`.

 ### Registry Authorization

-First, you need to log in and save credential for `acme.io` host via `tart login` command:
+First, you need to login to `acme.io` with the `tart login` command:

 ```bash
 tart login acme.io
 ```

+If you login to your registry with OAuth, you may need to create an access token to use as the password.
 Credentials are securely stored in Keychain.

 In addition, Tart supports [Docker credential helpers](https://docs.docker.com/engine/reference/commandline/login/#credential-helpers)
@@ -128,10 +134,10 @@ You can either pull an image:
 tart pull acme.io/remoteorg/name:latest
 ```

-...or instantiate a VM from a remote image:
+or create a VM from a remote image:

 ```bash
 tart clone acme.io/remoteorg/name:latest my-local-vm-name
 ```

-This invocation calls the `tart pull` implicitly (if the image is not being present) before doing the actual cloning.
+If the specified image is not already present, this invocation calls the `tart pull` implicitly before cloning.
@@ -0,0 +1,260 @@
+# Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# -----------------------------------------------------------------------------
+# Configuration
+# -----------------------------------------------------------------------------
+
+# Definitions
+definitions:
+
+  # Background image
+  - &background_image >-
+    {{ layout.background_image | x }}
+
+  # Background color (default: indigo)
+  - &background_color >-
+    {%- if layout.background_color -%}
+      {{ layout.background_color }}
+    {%- else -%}
+      {%- set palette = config.theme.palette or {} -%}
+      {%- if not palette is mapping -%}
+        {%- set list = palette | selectattr("accent") | list + palette -%}
+        {%- set palette = list | first -%}
+      {%- endif -%}
+      {%- set accent = palette.get("accent", "indigo") -%}
+      {%- set accent = accent.replace(" ", "-") -%}
+      {{ {
+        "red":         "#ff1a47",
+        "pink":        "#f50056",
+        "purple":      "#df41fb",
+        "deep-purple": "#7c4dff",
+        "indigo":      "#526cfe",
+        "blue":        "#4287ff",
+        "light-blue":  "#0091eb",
+        "cyan":        "#00bad6",
+        "teal":        "#00bda4",
+        "green":       "#00c753",
+        "light-green": "#63de17",
+        "lime":        "#b0eb00",
+        "yellow":      "#ffd500",
+        "amber":       "#ffaa00",
+        "orange":      "#ff9100",
+        "deep-orange": "#ff6e42"
+      }[accent] or "#4051b5" }}
+    {%- endif -%}
+
+  # Text color (default: white)
+  - &color >-
+    {%- if layout.color -%}
+      {{ layout.color }}
+    {%- else -%}
+      {%- set palette = config.theme.palette or {} -%}
+      {%- if not palette is mapping -%}
+        {%- set list = palette | selectattr("accent") | list + palette -%}
+        {%- set palette = list | first -%}
+      {%- endif -%}
+      {%- set accent = palette.get("accent", "indigo") -%}
+      {%- set accent = accent.replace(" ", "-") -%}
+      {{ {
+        "red":         "#ffffff",
+        "pink":        "#ffffff",
+        "purple":      "#ffffff",
+        "deep-purple": "#ffffff",
+        "indigo":      "#ffffff",
+        "blue":        "#ffffff",
+        "light-blue":  "#ffffff",
+        "cyan":        "#ffffff",
+        "teal":        "#ffffff",
+        "green":       "#ffffff",
+        "light-green": "#ffffff",
+        "lime":        "#000000",
+        "yellow":      "#000000",
+        "amber":       "#000000",
+        "orange":      "#000000",
+        "deep-orange": "#ffffff"
+      }[accent] or "#ffffff" }}
+    {%- endif -%}
+
+  # Font family (default: Roboto)
+  - &font_family >-
+    {%- if layout.font_family -%}
+      {{ layout.font_family }}
+    {%- elif config.theme.font != false -%}
+      {{ config.theme.font.get("text", "Roboto") }}
+    {%- else -%}
+      Roboto
+    {%- endif -%}
+
+  # Font variant
+  - &font_variant >-
+    {%- if layout.font_variant -%}
+      {{ layout.font_variant }}
+    {%- endif -%}
+
+  # Site name
+  - &site_name >-
+    {{ config.site_name }}
+
+  # Page title
+  - &page_title >-
+    {%- if page.meta.no_title_in_card -%}
+      {# do not show anything #}
+    {%- elif layout.title -%}
+      {{ layout.title }}
+    {%- else -%}
+      {{ page.meta.get("title", page.title) }}
+    {%- endif -%}
+
+  # Page title with site name
+  - &page_title_with_site_name >-
+    {%- if not page.is_homepage -%}
+      {{ page.meta.get("title", page.title) }} - {{ config.site_name }}
+    {%- else -%}
+      {{ page.meta.get("title", page.title) }}
+    {%- endif -%}
+
+  # Page description
+  - &page_description >-
+    {%- if layout.description -%}
+      {{ layout.description }}
+    {%- else -%}
+      {{ page.meta.get("description", config.site_description) | x }}
+    {%- endif -%}
+
+  # Page description for social card
+  - &page_description_social_card >-
+    {%- if layout.description -%}
+      {{ layout.description }}
+    {%- else -%}
+      {{ page.meta.get("description", config.site_description_social_card) | x }}
+    {%- endif -%}
+
+  # Logo
+  - &logo >-
+    {%- if layout.logo -%}
+      {{ layout.logo }}
+    {%- elif config.theme.logo -%}
+      {{ config.docs_dir }}/{{ config.theme.logo }}
+    {%- endif -%}
+
+  # Logo (icon)
+  - &logo_icon >-
+    {%- if not layout.logo -%}
+      {{ config.theme.icon.logo | x }}
+    {%- endif -%}
+
+# Meta tags
+tags:
+
+  # Open Graph
+  og:type: website
+  og:title: *page_title_with_site_name
+  og:description: *page_description
+  og:image: "{{ image.url }}"
+  og:image:type: "{{ image.type }}"
+  og:image:width: "{{ image.width }}"
+  og:image:height: "{{ image.height }}"
+  og:url: "{{ page.canonical_url }}"
+
+  # Twitter
+  twitter:card: summary_large_image
+  twitter:title: *page_title_with_site_name
+  twitter:description: *page_description
+  twitter:image: "{{ image.url }}"
+
+# -----------------------------------------------------------------------------
+# Specification
+# -----------------------------------------------------------------------------
+
+# Card size and layers
+size: { width: 1200, height: 630 }
+layers:
+
+  # Background
+  - background:
+      image: *background_image
+      color: *background_color
+
+  # Logo
+  - size: { width: 170, height: 192 }
+    offset: { x: 966, y: 64 }
+    background:
+      image: *logo
+    icon:
+      value: *logo_icon
+      color: *color
+
+  # Site name
+  - size: { width: 832, height: 42 }
+    offset: { x: 64, y: 64 }
+    typography:
+      content: *site_name
+      align: start center
+      color: *color
+      font:
+        family: *font_family
+        variant: *font_variant
+        style: Bold
+
+  # Motto
+  - size: { width: 832, height: 150 }
+    offset: { x: 64, y: 106 }
+    typography:
+      content: "{{ config.motto }}"
+      align: start center
+      color: *color
+      line:
+        amount: 2.5
+        height: 1.25
+      font:
+        family: *font_family
+        variant: *font_variant
+        style: Bold
+
+  # Page title
+  - size: { width: 1072, height: 256 }
+    offset: { x: 64, y: 256 }
+    typography:
+      content: *page_title
+      align: start center
+      color: *color
+      line:
+        amount: 3
+        height: 1.25
+      font:
+        family: *font_family
+        variant: *font_variant
+        style: Bold
+
+  # Page description
+  - size: { width: 832, height: 64 }
+    offset: { x: 64, y: 512 }
+    typography:
+      content: *page_description_social_card
+      align: start center
+      color: *color
+      line:
+        amount: 2
+        height: 1.5
+      font:
+        family: *font_family
+        variant: *font_variant
+        style: Regular
@@ -11,7 +11,7 @@ This page covers Terms of Service only for Cirrus Runners and Tart Documentation

 ### The Gist

-Cirrus Labs Inc ("Cirrus Labs") operates the [Cirrus Runners service](../integrations/github-actions.md) which we hope you use.
+Cirrus Labs Inc ("Cirrus Labs") operates the [Cirrus Runners service](https://cirrus-runners.app/) which we hope you use.
 If you use it, please use it responsibly. If you don't, we'll have to terminate your subscription.

 For paid plans, you'll be charged on a monthly basis. You can cancel anytime, but there are no refunds.
@@ -1,6 +1,8 @@
 ---
 hide:
  - navigation
+title: Licensing and Support
+description: Free Tier with 100 CPU core limit. Very affordable Tiers for larger enterprises.
 ---

 Both [Tart Virtualization](https://github.com/cirruslabs/tart) and [Orchard Orchestration](https://github.com/cirruslabs/orchard)
@@ -0,0 +1,65 @@
+## Architecture
+
+Orchard cluster consists of two components:
+
+* Controller — responsible for managing the cluster and scheduling of resources
+* Worker — responsible for executing the VMs
+* Client — responsible for creating, modifying and removing the resources on the Controller, can either be an Orchard CLI or [an API consumer](/orchard/integration-guide)
+
+Normally you deploy a single Controller that needs to be accessible to both the Clients and Workers. Then you can deploy the Workers, which can reside anywhere and be inaccessible to Clients directly, e.g. behind a NAT.
+
+## Security
+
+When an Orchard Client or a Worker connects to the Controller, they need to establish trust and verify that they're talking to the right Controller, so that no [man-in-the-middle attack](https://en.wikipedia.org/wiki/Man-in-the-middle_attack) is possible.
+
+Similarly to web-browsers (that rely on the [public key infrastructure](https://en.wikipedia.org/wiki/Public_key_infrastructure)) and SSH (which relies on semi-automated fingerprint verification), Orchard combines these two traits in a hybrid approach by defaulting to automatic PKI verification (can be disabled by [`--no-pki`](#--no-pki-override)) and falling-back to a manual verification for self-signed certificates.
+
+This hybrid approach is needed because the Controller can be configured in two ways:
+
+* *Controller with a publicly valid certificate*
+    * can be configured manually by passing `--controller-cert` and `--controller-key` command-line arguments to `orchard controller run`
+* *Controller with a self-signed certificate*
+    * configured automatically on first Controller start-up when no `--controller-cert` and `--controller-key` command-line arguments are passed
+
+Below we'll explain how Orchard client and Worker secure the connection when accessing these two Controller types.
+
+### Client
+
+Client is associated with the Controller using a `orchard context create` command, which works as follows:
+
+* Client attempts to connect to the Controller and validate its certificate using host's root CA set (can be disabled with [`--no-pki`](#--no-pki-override))
+* if the Client encounters a  *Controller with a publicly valid certificate*, that would be the last step and the association would succeed
+* if the Client is dealing with *Controller with a self-signed certificate*, the Client will do another connection attempt to probe the Controller's certificate
+* the probed Controller's certificate fingerprint is then presented to the user, and if the user agrees to trust it, the Client then considers that certificate to be trusted for a given context
+* Client finally connects to the Controller again with a trusted CA set containing only that certificate, executes the final API sanity checks, and if everything is OK then the association succeeds
+
+Afterward, each interaction with the Controller  (e.g. `orchard create vm` command) will stick to the chosen verification method and will re-verify the presented Controller's certificate against:
+
+* *Controller with a self-signed certificate*: a trusted certificate stored in the Orchard's configuration file
+* *Controller with a publicly valid certificate*: host's root CA set
+
+### Worker
+
+To make the Worker connect to the Controller, a Bootstrap Token needs to be obtained using the `orchard get bootstrap-token` command.
+
+While this approach provides a less ad-hoc experience than that you'd have with `orchard context create`, it allows one to mass-deploy workers non-interactively, using tools such as Ansible.
+
+This resulting Bootstrap Token will either include the Controller's certificate (when the current context is with a *Controller with a self-signed certificate*) or omit it (when the current context is with a *Controller with a publicly valid certificate*).
+
+The way Worker connects to the Controller using the `orchard worker run` command is as follows:
+
+* when the Bootstrap Token contains the Controller's certificate:
+    * the Orchard Worker will try to connect to the Controller with a trusted CA set containing only that certificate
+* when the Bootstrap Token has no Controller's certificate:
+    * the Orchard Worker will try the PKI approach (can be disabled with [`--no-pki`](#--no-pki-override) to effectively prevent the Worker from connecting) and fail if certificate verification using PKI is not possible
+
+### `--no-pki` override
+
+If you only intend to access the *Controller with a self-signed certificate* and want to additionally guard yourself against [CA compromises](https://en.wikipedia.org/wiki/Certificate_authority#CA_compromise) and other PKI-specific attacks, pass a `--no-pki` command-line argument to the following commands:
+
+* `orchard context create --no-pki`
+    * this will prevent the Client from using PKI and will let you interactively verify the Controller's certificate fingerprint before connecting, thus creating a non-PKI association
+* `orchard worker run --no-pki`
+    * this will prevent the Worker from trying to use PKI when connecting to the Controller using a Bootstrap Token that has no certificate included in it, thus failing fast and letting you know that you need to create a proper Bootstrap Token
+
+We've deliberately chosen not to use environment variables (e.g. `ORCHARD_NO_PKI`) because they fail silently (e.g. due to a typo), compared to command-line arguments, which will result in an error that is much easier to detect.
@@ -0,0 +1,195 @@
+## Introduction
+
+Compared to Worker, which can only be deployed on a macOS machine, Controller can be also deployed on Linux.
+
+In fact, we've made a [container image](https://github.com/cirruslabs/orchard/pkgs/container/orchard) to ease deploying the Controller in container-native environments such as Kubernetes.
+
+Another thing to keep in mind that Orchard API is secured by default: all requests must be authenticated with the credentials of a service account. When you first run Orchard Controller, a `bootstrap-admin` service account will be created automatically and credentials will be printed to the standard output.
+
+If you already have a token in mind that you want to use for the `bootstrap-admin` service account, or you've got locked out and want this service account with a well-known password back, you can set the `ORCHARD_BOOTSTRAP_ADMIN_TOKEN` when running the controller.
+
+For example to use a secure, random value:
+
+```bash
+ORCHARD_BOOTSTRAP_ADMIN_TOKEN=$(openssl rand -hex 32) orchard controller run
+```
+
+## Deployment Methods
+
+While you can always start `orchard controller run` manually with the required arguments, this method is not recommended due to lack of persistence.
+
+In the following sections you'll find several examples of how to run Orchard Controller in various environments in a more persistent way. Feel free to submit PRs with more examples.
+
+### Google Compute Engine
+
+An example below will deploy a single instance of Orchard Controller in Google Cloud Compute Engine in `us-central1` region.
+
+First, let's create a static IP address for our instance:
+
+```bash
+gcloud compute addresses create orchard-ip --region=us-central1
+export ORCHARD_IP=$(gcloud compute addresses describe orchard-ip --format='value(address)' --region=us-central1)
+```
+
+Once we have the IP address, we can create a new instance with Orchard Controller running inside a container:
+
+```bash
+gcloud compute instances create-with-container orchard-controller \
+  --machine-type=e2-micro \
+  --zone=us-central1-a \
+  --image-family cos-stable \
+  --image-project cos-cloud \
+  --tags=https-server \
+  --address=$ORCHARD_IP \
+  --container-image=ghcr.io/cirruslabs/orchard:latest \
+  --container-env=PORT=443 \
+  --container-env=ORCHARD_BOOTSTRAP_ADMIN_TOKEN=$ORCHARD_BOOTSTRAP_ADMIN_TOKEN \
+  --container-mount-host-path=host-path=/home/orchard-data,mode=rw,mount-path=/data
+```
+
+Now you can create a new context for your local client:
+
+```bash
+orchard context create --name production \
+  --service-account-name bootstrap-admin \
+  --service-account-token $ORCHARD_BOOTSTRAP_ADMIN_TOKEN \
+  https://$ORCHARD_IP:443
+```
+
+And select it as the default context:
+
+```bash
+orchard context default production
+```
+
+### Kubernetes (GKE, EKS, etc.)
+
+The easiest way to run Orchard Controller on Kubernetes is to expose it through the `LoadBalancer` service.
+
+This way no fiddling with the TLS certificates and HTTP proxying is needed, and most cloud providers will allocate a ready-to-use IP-address that can directly used in `orchard context create` and `orchard worker run` commands, or additionally assigned to a DNS domain name for a more memorable hostname.
+
+Do deploy on Kubernetes, only three resources are needed:
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: orchard-controller
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+  # Uncomment this when deploying on Amazon's EKS and
+  # change to the desired storage class name if needed
+  # storageClassName: gp2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: orchard-controller
+spec:
+  serviceName: orchard-controller
+  replicas: 1
+  selector:
+    matchLabels:
+      app: orchard-controller
+  template:
+    metadata:
+      labels:
+        app: orchard-controller
+    spec:
+      containers:
+        - name: orchard-controller
+          image: ghcr.io/cirruslabs/orchard:latest
+          volumeMounts:
+            - mountPath: /data
+              name: orchard-controller
+      volumes:
+        - name: orchard-controller
+          persistentVolumeClaim:
+            claimName: orchard-controller
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: orchard-controller
+spec:
+  selector:
+    app: orchard-controller
+  ports:
+    - protocol: TCP
+      port: 6120
+      targetPort: 6120
+  type: LoadBalancer
+```
+
+Once deployed, the bootstrap credentials will be printed to the standard output. You can inspect them by running `kubectl logs deployment/orchard-controller`.
+
+The resources above ensure that Controller's database is stored in a persistent storage and survives restats.
+
+You can further allocate a static IP address and use it by adding annotations to the `Service` resource. Here's how to do that:
+
+* on Google's GKE: <https://cloud.google.com/kubernetes-engine/docs/concepts/service-load-balancer-parameters#spd-static-ip>
+* on Amazon's EKS: <https://kubernetes.io/docs/reference/labels-annotations-taints/#service-beta-kubernetes-io-aws-load-balancer-eip-allocations>
+
+### systemd service on Debian-based distributions
+
+This should work for most Debian-based distributions like Debian, Ubuntu, etc.
+
+Firstly, make sure that the APT transport for downloading packages via HTTPS and common X.509 certificates are installed:
+
+```shell
+sudo apt-get update && sudo apt-get -y install apt-transport-https ca-certificates
+```
+
+Then, add the Cirrus Labs repository:
+
+```shell
+echo "deb [trusted=yes] https://apt.fury.io/cirruslabs/ /" | sudo tee /etc/apt/sources.list.d/cirruslabs.list
+```
+
+Update the package index files and install the Orchard Controller:
+
+```shell
+sudo apt-get update && sudo apt-get -y install orchard-controller
+```
+
+Finally, enable and start the Orchard Controller systemd service:
+
+```shell
+sudo systemctl enable orchard-controller
+sudo systemctl start orchard-controller
+```
+
+The bootstrap credentials will be printed to the standard output. You can inspect them by running `sudo systemctl status orhcard-controller` or `journalctl -u orchard-controller`.
+
+### systemd service on RPM-based distributions
+
+This should work for most RPM-based distributions like Fedora, CentOS, etc.
+
+First, create a `/etc/yum.repos.d/cirruslabs.repo` file with the following contents:
+
+```ini
+[cirruslabs]
+name=Cirrus Labs Repo
+baseurl=https://yum.fury.io/cirruslabs/
+enabled=1
+gpgcheck=0
+```
+
+Then, install the Orchard Controller:
+
+```shell
+sudo yum -y install orchard-controller
+```
+
+Finally, enable and start the Orchard Controller systemd service:
+
+```shell
+systemctl enable orchard-controller
+systemctl start orchard-controller
+```
+
+The bootstrap credentials will be printed to the standard output. You can inspect them by running `sudo systemctl status orhcard-controller` or `journalctl -u orchard-controller`.
@@ -0,0 +1,127 @@
+## Obtain a Boostrap Token
+
+First, create a service account with a minimal set of roles (`compute:read` and `compute:write`) required for proper Worker functioning:
+
+```bash
+orchard create service-account worker-pool-m1 --roles "compute:read" --roles "compute:write"
+```
+
+Then, generate a Bootstrap Token for this service account:
+
+```shell
+orchard get bootstrap-token worker-pool-m1
+```
+
+We will reference the value of the Bootstrap Token generated here as `${BOOTSTRAP_TOKEN}` below.
+
+Further, we assume that Orchard controller is available on `orchard.example.com`
+
+## Deployment Methods
+
+While you can always run `orchard worker run` manually with the required arguments, this method of deploying the Worker is not recommended.
+
+Instead, we've listed a more persistent methods of a Worker deployment below.
+
+### launchd
+
+[launchd](https://launchd.info/) is an init system for macOS that manages daemons, agents and other background processes.
+
+In this deployment method, we'll create a new job definition file for the launchd to manage on its behalf.
+
+To begin, first install Orchard:
+
+```shell
+brew install cirruslabs/cli/orchard
+```
+
+Ensure that the following command:
+
+```shell
+which orchard
+```
+
+...yields `/opt/homebrew/bin/orchard`. If not, you'll need to replace all of the occurences of `/opt/homebrew/bin/orchard` in the job definition below.
+
+Then, create a launchd job definition in `/Library/LaunchDaemons/org.cirruslabs.orchard.worker.plist` with the following contents:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>Label</key>
+    <string>org.cirruslabs.orchard.worker</string>
+    <key>UserName</key>
+    <string>admin</string>
+    <key>Program</key>
+    <string>/opt/homebrew/bin/orchard</string>
+    <key>ProgramArguments</key>
+    <array>
+      <string>/opt/homebrew/bin/orchard</string>
+      <string>worker</string>
+      <string>run</string>
+      <string>--bootstrap-token</string>
+      <string>${BOOTSTRAP_TOKEN}</string>
+      <string>orchard.example.com</string>
+    </array>
+    <key>EnvironmentVariables</key>
+    <dict>
+      <key>PATH</key>
+      <string>/bin:/usr/bin:/usr/local/bin:/opt/homebrew/bin</string>
+    </dict>
+    <key>WorkingDirectory</key>
+    <string>/var/empty</string>
+    <key>RunAtLoad</key>
+    <true/>
+    <key>KeepAlive</key>
+    <true/>
+    <key>StandardOutPath</key>
+    <string>/Users/admin/orchard-launchd.log</string>
+    <key>StandardErrorPath</key>
+    <string>/Users/admin/orchard-launchd.log</string>
+  </dict>
+</plist>
+```
+
+This assumes that your macOS user on the host is named `admin`. If not, change all occurrences of `admin` in the job definition above to `$USER`.
+
+Finally, change the `orchard.example.com` to the FQDN or an IP-address of your Orchard Controller.
+
+Now, you can start the job:
+
+```shell
+launchctl load -w /Library/LaunchDaemons/org.cirruslabs.orchard.worker.plist
+```
+
+### Ansible
+
+If you have a set of machines that you want to use as Orchard Workers, you can use [Ansible](https://docs.ansible.com/) to configure them.
+
+We've created the [cirruslabs/ansible-orchard](https://github.com/cirruslabs/ansible-orchard) repository with a basic Ansible playbook for convenient setup.
+
+To use it, clone it locally:
+
+```shell
+git clone https://github.com/cirruslabs/ansible-orchard.git
+cd ansible-orchard/
+```
+
+Make sure that the Ansible Galaxy dependencies are installed:
+
+```shell
+ansible-galaxy install -r requirements.yml
+```
+
+Then, edit the `production-pool` file and populate the following fields:
+
+* `hosts` — replace `worker-1.hosts.internal` with your worker FQDN or IP-address and add more hosts if needed
+* `ansible_user` — set it macOS user on the host for the SSH to work
+* `orchard_worker_user` — set it macOS user on the host under which the Worker will run, e.g. `admin`
+* `orchard_worker_controller_url` — set it to FQDN or an IP-address of your Orchard Controller, for example, `orchard.example.com`
+* `orchard_worker_bootstrap_token` — set it to `${BOOTSTRAP_TOKEN}` we've generated above
+
+Deploy the playbook:
+
+```shell
+ansible-playbook --inventory-file production-pool --ask-pass playbook-workers.yml
+```
@@ -0,0 +1,187 @@
+Orchard has a REST API that follows [OpenAPI specification](https://swagger.io/specification/) and is described in [`api/openapi.yaml`](https://github.com/cirruslabs/orchard/blob/main/api/openapi.yaml).
+
+You can run `orchard dev` locally and navigate to `http://127.0.0.1:6120/v1/` for interactive documentation.
+
+![](/assets/images/orchard/orchard-api-documentation-browser.png)
+
+## Using the API
+
+Below you'll find examples of using Orchard API via vanilla Python's request library and Golang package that Orchard CLI build on top of.
+
+### Authentication
+
+When running in non-development mode, Orchard API expects a [basic access authentication](https://en.wikipedia.org/wiki/Basic_access_authentication) to be provided for each API call.
+
+Below you'll find two snippets that retrieve controller's information and output its version:
+
+#### Authentication in Python
+
+```python
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+def main():
+  # Authentication
+  basic_auth = HTTPBasicAuth("service account name", "service account token")
+
+  response = requests.get("http://127.0.0.1:6120/v1/info", auth=basic_auth)
+
+  print(response.json()["version"])
+
+
+if __name__ == '__main__':
+  main()
+```
+
+#### Authentication in Golang
+
+```go
+package main
+
+import (
+  "context"
+  "fmt"
+  "github.com/cirruslabs/orchard/pkg/client"
+  "log"
+)
+
+func main() {
+  client, err := client.New()
+  if err != nil {
+    log.Fatalf("failed to initialize Orchard API client: %v", err)
+  }
+
+  controllerInfo, err := client.Controller().Info(context.Background())
+  if err != nil {
+    log.Fatalf("failed to retrieve controller's information: %v", err)
+  }
+
+  fmt.Println(controllerInfo.Version)
+}
+```
+
+Note that we don't provide any credentials for Golang's version of the snippet: this is because Orchard's Golang API client (`github.com/cirruslabs/orchard/pkg/client`) has the ability to read the current's user Orchard context automatically.
+
+### Creating a VM
+
+A more intricate example would be spinning off a VM with a startup script that outputs date, reading its logs and removing it from the controller:
+
+#### Creating a VM in Python
+
+```python
+import time
+import uuid
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+def main():
+  vm_name = str(uuid.uuid4())
+
+  basic_auth = HTTPBasicAuth("service account name", "service account token")
+
+  # Create VM
+  response = requests.post("http://127.0.0.1:6120/v1/vms", auth=basic_auth, json={
+    "name": vm_name,
+    "image": "ghcr.io/cirruslabs/macos-sonoma-base:latest",
+    "cpu": 4,
+    "memory": 4096,
+    "startup_script": {
+      "script_content": "date",
+    }
+  })
+  response.raise_for_status()
+
+  # Retrieve VM's logs
+  while True:
+    response = requests.get(f"http://127.0.0.1:6120/v1/vms/{vm_name}/events", auth=basic_auth)
+    response.raise_for_status()
+
+    result = response.json()
+
+    if isinstance(result, list) and len(result) != 0:
+      print(result[0]["payload"])
+      break
+
+    time.sleep(1)
+
+  # Delete VM
+  response = requests.delete(f"http://127.0.0.1:6120/v1/vms/{vm_name}", auth=basic_auth)
+  response.raise_for_status()
+
+
+if __name__ == '__main__':
+  main()
+```
+
+#### Creating a VM in Golang
+
+```go
+package main
+
+import (
+  "context"
+  "fmt"
+  "github.com/cirruslabs/orchard/pkg/client"
+  v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
+  "github.com/google/uuid"
+  "log"
+  "time"
+)
+
+func main() {
+  vmName := uuid.New().String()
+
+  client, err := client.New()
+  if err != nil {
+    log.Fatalf("failed to initialize Orchard API client: %v", err)
+  }
+
+  // Create VM
+  err = client.VMs().Create(context.Background(), &v1.VM{
+    Meta: v1.Meta{
+      Name: vmName,
+    },
+    Image:  "ghcr.io/cirruslabs/macos-sonoma-base:latest",
+    CPU:    4,
+    Memory: 4096,
+    StartupScript: &v1.VMScript{
+      ScriptContent: "date",
+    },
+  })
+  if err != nil {
+    log.Fatalf("failed to create VM: %v")
+  }
+
+  // Retrieve VM's logs
+  for {
+    vmLogs, err := client.VMs().Logs(context.Background(), vmName)
+    if err != nil {
+      log.Fatalf("failed to retrieve VM logs")
+    }
+
+    if len(vmLogs) != 0 {
+      fmt.Println(vmLogs[0])
+      break
+    }
+
+    time.Sleep(time.Second)
+  }
+
+  // Delete VM
+  if err := client.VMs().Delete(context.Background(), vmName); err != nil {
+    log.Fatalf("failed to delete VM: %v", err)
+  }
+}
+```
+
+## Resource management
+
+Some resources, such as `Worker` and `VM`, have a `resource` field which is a dictionary that maps between resource names and their amounts (amount requested or amount provided, depending on the resource) and is useful for scheduling.
+
+Well-known resources:
+
+* `org.cirruslabs.tart-vms` — number of Tart VM slots available on the machine or requested by the VM
+    * this number is `2` for workers and `1` for VMs by default
@@ -0,0 +1,30 @@
+## Backups
+
+In order to backup the Orchard Controller, simply copy its `ORCHARD_HOME` (which defaults to `~/.orchard/`) directory somewhere safe and restore it when needed.
+
+This directory contains a BadgerDB database that Controller uses to store state and an X.509 certificate with key.
+
+## Upgrades
+
+Since the Orchard's initial release, we've managed to maintain the backwards compatibility between versions up to this day, so generally, it doesn't matter whether you upgrade the Controller or Worker(s) first.
+
+In case a new functionality is introduced, you might be required to finish the upgrade of both the Controller and the Worker(s) to be able to use it fully.
+
+In case there will be backwards-incompatible changes introduced in the future, we will try to do our best and highlight this in the [release notes](https://github.com/cirruslabs/orchard/releases) accordingly.
+
+## Observability
+
+Both the Controller and Worker produce some useful OpenTelemetry metrics. Metrics are scoped with `org.cirruslabs.orchard` prefix and include information about resource utilization, statuses or Workers, scheduling/pull time and many more.
+
+By default, the telemetry is sent to `https://localhost:4317` using the gRPC protocol and to `http://localhost:4318` using the HTTP protocol.
+
+You can override this by setting the [standard OpenTelemetry environment variable](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) `OTEL_EXPORTER_OTLP_ENDPOINT`.
+
+Please refer to [OTEL Collector documentation](https://opentelemetry.io/docs/collector/) for instruction on how to setup a sidecar for the metrics collections or find out if your SaaS monitoring has an available OTEL endpoint (see [Honeycomb](https://docs.honeycomb.io/send-data/opentelemetry/) as an example).
+
+### Sending metrics to Google Cloud Platform
+
+There are two standard options of ingesting metrics procuded by Orchard Controller and Workers into the GCP:
+
+* [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) + [Google Cloud Exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/googlecloudexporter/README.md) — open-source solution that can be later re-purposed to send metrics to any OTLP-compatible endpoint by swapping a single [exporter](https://opentelemetry.io/docs/collector/configuration/#exporters)
+* [Ops Agent](https://cloud.google.com/monitoring/agent/ops-agent/otlp) — Google-backed solution with a syntax similar to OpenTelemetry Collector, but tied to GCP-only
@@ -0,0 +1,101 @@
+Tart is great for running workloads on a single machine, but what if you have more than one computer at your disposal
+and
+a couple of VMs is not enough anymore for your needs? This is where [Orchard](https://github.com/cirruslabs/orchard)
+comes in to play!
+
+It allows you to orchestrate multiple Tart-capable hosts from either an Orchard CLI (which we demonstrate below)
+or [through the API](/orchard/integration-guide).
+
+The easiest way to start is to run Orchard in local development mode:
+
+```shell
+brew install cirruslabs/cli/orchard
+orchard dev
+```
+
+This will run an Orchard Controller and an Orchard Worker in a single process on your local machine, allowing you to
+test both the CLI functionality and the API from a tool like cURL or programming language of choice, without the need to
+authenticate requests.
+
+Note that in production deployments, these two components are started separately and enable security by default. Please
+refer to [Deploying Controller](/orchard/deploying-controller) and [Deploying Workers](/orchard/deploying-workers) for
+more information.
+
+## Creating Virtual Machines
+
+Now, let's create a Virtual Machine:
+
+```shell
+orchard create vm --image ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
+```
+
+You can check a list of VM resources to see if the Virtual Machine we've created above is already running:
+
+```shell
+orchard list vms
+```
+
+## Accessing Virtual Machines
+
+Orchard has an ability to do port forwarding that `ssh` and `vnc` commands are built on top of. All port forwarding
+connections are done via the Orchard Controller instance which "proxies" a secure connection to the Orchard Workers.
+
+Therefore, your workers can be located under a stricter firewall that only allows connections to the Orchard Controller
+instance. Orchard Controller instance is secured by default and all API calls are authenticated and authorized.
+
+### SSH
+
+To SSH into a VM, use the `orchard ssh` command:
+
+```shell
+orchard ssh vm sonoma-base
+```
+
+You can specify the `--username` and `--password` flags to specify the username/password pair to use for the SSH
+protocol. By default, `admin`/`admin` is used.
+
+You can also execute remote commands instead of spawning a login shell, similarly to how OpenSSH's `ssh` command accepts
+a command argument:
+
+```shell
+orchard ssh vm sonoma-base "uname -a"
+```
+
+You can execute scripts remotely this way, by telling the remote command-line interpreter to read from the standard
+input and using the redirection operator as follows:
+
+```shell
+orchard ssh vm sonoma-base "bash -s" < script.sh
+```
+
+### VNC
+
+Similarly to `ssh` command, you can use `vnc` command to open Screen Sharing into a remote VM:
+
+```shell
+orchard vnc vm sonoma-base
+```
+
+You can specify the `--username` and `--password` flags to specify the username/password pair to use for the VNC
+protocol. By default, `admin`/`admin` is used.
+
+## Deleting Virtual Machines
+
+The following command will delete the VM we've created above and clean-up the resources associated with it:
+
+```shell
+orchard delete vm sonoma-base
+```
+
+## Environment variables
+
+In addition to controlling the Orchard via the CLI arguments, there are environment variables that may be beneficial
+both when automating Orchard and in daily use:
+
+| Variable name                   | Description                                                                                                                                                                                                                                                                                                                                                                  |
+|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `ORCHARD_HOME`                  | Override Orchard's home directory. Useful when running multiple Orchard instances on the same host and when testing.                                                                                                                                                                                                                                                         |
+| `ORCHARD_LICENSE_TIER`          | The default license limit only allows connecting 4 Orchard Workers to the Orchard Controller. If you've purchased a [Gold Tier License](/licensing/), set this variable to `gold` to increase the limit to 20 Orchard Workers. And if you've purchased a [Platinum Tier License](/licensing/), set this variable to `platinum` to increase the limit to 200 Orchard Workers. |
+| `ORCHARD_URL`                   | Override controller URL on per-command basis.                                                                                                                                                                                                                                                                                                                                |
+| `ORCHARD_SERVICE_ACCOUNT_NAME`  | Override service account name (used for controller API auth) on per-command basis.                                                                                                                                                                                                                                                                                           |
+| `ORCHARD_SERVICE_ACCOUNT_TOKEN` | Override service account token (used for controller API auth) on per-command basis.                                                                                                                                                                                                                                                                                          |
@@ -1,6 +1,8 @@
 ---
 hide:
  - navigation
+title: Quick Start
+description: Install Tart and run your first virtual machine on Apple Silicon in minutes.
 ---

 Try running a Tart VM on your Apple Silicon device running macOS 13.0 (Ventura) or later (will download a 25 GB image):
@@ -13,10 +15,10 @@ tart run sonoma-base

 ??? info "Manual installation from a release archive"
    It's also possible to manually install `tart` binary from the latest released archive:
-    
+
    ```bash
-    curl -LO https://github.com/cirruslabs/tart/releases/latest/download/tart.tar.gz
-    tar -xzvf tart.tar.gz
+    curl -LO https://github.com/cirruslabs/tart/releases/latest/download/tart-arm64.tar.gz
+    tar -xzvf tart-arm64.tar.gz
    ./tart.app/Contents/MacOS/tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
    ./tart.app/Contents/MacOS/tart run sonoma-base
    ```
@@ -129,7 +131,7 @@ Note: to use the directory mounting feature, the guest VM needs to run macOS 13.

 ??? tip "Changing mount location"
    It is possible to remount the directories after a virtual machine is started by running the following commands:
-    
+
    ```bash
    sudo umount "/Volumes/My Shared Files"
    mkdir ~/workspace
@@ -143,7 +145,15 @@ Note: to use the directory mounting feature, the guest VM needs to run macOS 13.
 To be able to access the shared directories from the Linux guest, you need to manually mount the virtual filesystem first:

 ```bash
-mount -t virtiofs com.apple.virtio-fs.automount /mnt/shared
+sudo mkdir /mnt/shared
+sudo mount -t virtiofs com.apple.virtio-fs.automount /mnt/shared
 ```

 The directory we've mounted above will be accessible from the `/mnt/shared/project` path inside a guest VM.
+
+??? info "Auto-mount at boot time"
+    To automatically mount this directory at boot time, add the following line to the `/etc/fstab` file:
+
+    ```shell
+    com.apple.virtio-fs.automount /mnt/shared virtiofs rw,relatime 0 0
+    ```
@@ -85,10 +85,10 @@
      <!-- landing image -->
      <div class="tx-landing__image">
        <dotlottie-player
-          src="/assets/animations/TartLogo.lottie"
-          mode="normal"
-          style="width: 75%; margin: auto"
-          autoplay
+            src="/assets/animations/TartLogo.lottie"
+            mode="normal"
+            style="width: 75%; margin: auto"
+            autoplay
        />
      </div>

@@ -114,10 +114,9 @@
      <header class="md-typeset">
        <h1 id="virtualization-and-beyond">
          Virtualization and beyond
-          <a
-            href="#virtualization-and-beyond"
-            class="headerlink"
-            title="Permanent link"
+          <a href="#virtualization-and-beyond"
+             class="headerlink"
+             title="Permanent link"
          >
            ¶
          </a>
@@ -125,12 +124,11 @@
      </header>
      <div class="mdx-spotlight">
        <figure class="mdx-spotlight__feature">
-          <img
-            src="assets/images/spotlight/virtualization-framework.webp"
-            alt="Apple’s native Virtualization.Framework"
-            loading="lazy"
-            width="500"
-            height="212"
+          <img src="assets/images/spotlight/virtualization-framework.webp"
+               alt="Apple’s native Virtualization.Framework"
+               loading="lazy"
+               width="500"
+               height="212"
          />
          <figcaption class="md-typeset">
            <h2>Native performance</h2>
@@ -144,12 +142,11 @@
          </figcaption>
        </figure>
        <figure class="mdx-spotlight__feature">
-          <img
-            src="assets/images/spotlight/supported-registries.webp"
-            alt="OCI-compatible container registries"
-            loading="lazy"
-            width="500"
-            height="160"
+          <img src="assets/images/spotlight/supported-registries.webp"
+               alt="OCI-compatible container registries"
+               loading="lazy"
+               width="500"
+               height="160"
          />
          <figcaption class="md-typeset">
            <h2>Remote storage for Virtual Machines</h2>
@@ -161,39 +158,39 @@
          </figcaption>
        </figure>
        <figure class="mdx-spotlight__feature">
-          <img
-            src="assets/images/spotlight/github-actions-runners.webp"
-            alt="GitHub Actions Runners"
-            loading="lazy"
-            width="500"
-            height="280"
+          <img src="assets/images/spotlight/github-actions-runners.webp"
+               alt="GitHub Actions Runners"
+               loading="lazy"
+               width="500"
+               height="280"
          />
          <figcaption class="md-typeset">
            <h2>Seamless integration with your existing automations</h2>
            <p>
              Tart integrates with many continuous integration systems, including a dedicated
              service of on-demand GitHub Actions Runners. With a single line change, you can cut your
-              CI/CD costs by up to <b>30 times</b> by using <a href="https://cirrus-runners.app/">Cirrus Runners</a>
+              CI/CD costs by up to <b>30 times</b> by using <a href="https://cirrus-runners.app/">Cirrus
+              Runners</a>
              to run your workflows.
            </p>
          </figcaption>
        </figure>
        <figure class="mdx-spotlight__feature">
          <div id="lottie-player">
-            <dotlottie-player
-              src="/assets/animations/Orchard.lottie"
-              mode="normal"
-              style="height: 280px; margin: auto"
-              autoplay
-              loop
+            <dotlottie-player src="/assets/animations/Orchard.lottie"
+                              mode="normal"
+                              style="height: 280px; margin: auto"
+                              autoplay
+                              loop
            />
          </div>
          <figcaption class="md-typeset">
            <h2>Run at scale with <a href="https://github.com/cirruslabs/orchard">Orchard</a></h2>
            <p>
-              Tart toolset includes Orchard Orchestration &mdash; tool to run and manage Tart virtual machines
-              at scale on a cluster of Apple Silicon hosts. An Orchard Cluster exposes a simple REST API to manage
-              thousands virtual machines. Orchard CLI allows accessing remote virtual machines like they run locally.
+              Tart toolset includes Orchard Orchestration &mdash; tool to run and manage Tart virtual
+              machines at scale on a cluster of Apple Silicon hosts. An Orchard Cluster exposes a simple REST API to
+              manage thousands virtual machines. Orchard CLI allows accessing remote virtual machines like they run
+              locally.
            </p>
          </figcaption>
        </figure>
@@ -208,38 +205,38 @@
      <header class="md-typeset">
        <h1 id="powerhouse">
          Automation Powerhouse
-          <a
-            href="#powerhouse"
-            class="headerlink"
-            title="Permanent link"
+          <a href="#powerhouse"
+             class="headerlink"
+             title="Permanent link"
          >
            ¶
          </a>
        </h1>
      </header>
      <script>
-        fetch("https://api.github.com/repos/cirruslabs/tart/releases?per_page=100")
-          .then((response) => response.json())
-          .then((releases) => {
-            let allDownloads = 0;
-            for (let release of releases) {
-              for (let asset of release.assets) {
-                if (asset && asset.content_type === "application/octet-stream") {
-                  allDownloads += asset.download_count || 0
-                }
-              }
-            }
-            let counterElement = document.getElementById('installation-counter');
-            if (counterElement) {
-              // Live installation count is available starting version 1.0.0
-              // Prior Tart was installed a little over 14,000 times, let's count them too
-              let installationPriorV1 = 14
-              counterElement.textContent = (installationPriorV1 + Math.round(allDownloads / 1000)) + ",000"
-            }
-          })
+          fetch("https://api.github.com/repos/cirruslabs/tart/releases?per_page=100")
+              .then((response) => response.json())
+              .then((releases) => {
+                  let allDownloads = 0;
+                  for (let release of releases) {
+                      for (let asset of release.assets) {
+                          if (asset && asset.content_type === "application/octet-stream") {
+                              allDownloads += asset.download_count || 0
+                          }
+                      }
+                  }
+                  let counterElement = document.getElementById('installation-counter');
+                  if (counterElement) {
+                      // Live installation count is available starting version 1.0.0
+                      // Prior Tart was installed a little over 14,000 times, let's count them too
+                      let installationPriorV1 = 14
+                      counterElement.textContent = (installationPriorV1 + Math.round(allDownloads / 1000)) + ",000"
+                  }
+              })
      </script>
      <h2>
-        With more than <strong id="installation-counter">25,000</strong> installations to date, Tart has been adopted for various scenarios.
+        With more than <strong id="installation-counter">25,000</strong> installations to date, Tart has been
+        adopted for various scenarios.
        Its applications range from powering CI/CD pipelines and reproducible local development environments,
        to helping in the testing of device management systems without actual physical devices.
      </h2>
@@ -254,10 +251,9 @@
      <header class="md-typeset">
        <h1 id="what-our-users-say">
          What our users say
-          <a
-            href="#what-our-users-say"
-            class="headerlink"
-            title="Permanent link"
+          <a href="#what-our-users-say"
+             class="headerlink"
+             title="Permanent link"
          >
            ¶
          </a>
@@ -265,70 +261,60 @@
      </header>
      <div class="mdx-users">
        <figure class="mdx-users__testimonial">
-          <img
-            src="assets/images/users/mitchell-hashimoto.webp"
-            alt="Mitchell Hashimoto"
-            loading="lazy"
-            width="200"
-            height="200"
+          <img src="assets/images/users/mikhail-tokarev.webp"
+               alt="Mikhail Tokarev"
+               loading="lazy"
+               width="200"
+               height="200"
          />
          <figcaption class="md-typeset">
-            <h2>Mitchell Hashimoto</h2>
            <h3>
-              <a href="https://www.hashicorp.com/" target="_blank">HashiCorp</a> co-founder
+              Mikhail Tokarev, CTO at <a href="https://codemagic.io/start/" target="_blank">Codemagic</a>
            </h3>
            <hr/>
            <cite>
-              I've been using "Cirrus Runners" since <a href="https://x.com/mitchellh/status/1731071326201561194" target="_blank">that tweet</a> and
-              it has been fantastic. Huge speed increase, huge cost decrease, zero maintenance, exactly what I wanted.
+              Thanks to the minimal overhead of using the Apple Virtualization
+              API, we’ve seen some performance improvements in booting new
+              virtual machines compared with Anka.
            </cite>
          </figcaption>
        </figure>
        <figure class="mdx-users__testimonial">
-          <img
-            src="assets/images/users/seb-jachec.webp"
-            alt="Sebastian Jachec"
-            loading="lazy"
-            width="200"
-            height="200"
+          <img src="assets/images/users/expo.webp"
+               alt="Expo"
+               loading="lazy"
+               width="200"
+               height="200"
          />
          <figcaption class="md-typeset">
-            <h2>Sebastian Jachec</h2>
            <h3>
-              Mobile Engineer at
-              <a href="https://daybridge.com/" target="_blank">Daybridge</a>
+              Infrastructure Team at <a href="https://expo.dev/" target="_blank">Expo</a>
            </h3>
            <hr/>
            <cite>
-              It&rsquo;s been plain-sailing with the
-              <a href="/integrations/github-actions">Cirrus Runners</a>&nbsp;&mdash;
-              they&rsquo;ve been great! They&rsquo;re consistently&nbsp;60+%
-              faster on&nbsp;workflows that we&nbsp;previously used Github
-              Actions&rsquo; macOS runners for.
+              Tart was the practical way for us to use the Virtualization framework. Cirrus Labs’
+              continued maintenance and support gives us confidence, and it is also important for us
+              to be able to read the source code when we need to understand an abstraction layer below.
            </cite>
          </figcaption>
        </figure>
        <figure class="mdx-users__testimonial">
-          <img
-            src="assets/images/users/max-lapides.webp"
-            alt="Max Lapides"
-            loading="lazy"
-            width="200"
-            height="200"
+          <img src="assets/images/users/snowflake.webp"
+               alt="Snowflake"
+               loading="lazy"
+               width="200"
+               height="200"
          />
          <figcaption class="md-typeset">
-            <h2>Max Lapides</h2>
            <h3>
-              Senior Mobile Engineer at
-              <a href="https://www.tonal.com/" target="_blank">Tonal</a>
+              Red Team at <a href="https://www.snowflake.com/" target="_blank">Snowflake</a>
            </h3>
            <hr/>
            <cite>
-              Previously, we were using the GitHub&#8209;hosted macOS runners
-              and our iOS build took ~30&nbsp;minutes. Now with
-              <a href="/integrations/github-actions">Cirrus Runners</a>, the iOS build only
-              takes ~12&nbsp;minutes. That’s a huge boost to our productivity,
-              and for only $150/month per runner it is much less expensive too.
+              The Snowflake Red Team had a need for macOS CI/CD and a segmented macOS development
+              environment. We solved this problem and shared our implementation with macOS EC2 and Tart.
+              We also automated this process with Terraform/Packer to simplify the deployment of our
+              infrastructure and machine images.
            </cite>
          </figcaption>
        </figure>
@@ -1,6 +1,6 @@
 pytest
 testcontainers
-requests
+requests == 2.31.0 # work around https://github.com/psf/requests/issues/6707
 bitmath
 pytest-dependency
 paramiko
@@ -1,16 +1,18 @@
 import uuid

+import pytest
 from paramiko.client import SSHClient, AutoAddPolicy


-def test_run(tart):
+@pytest.mark.parametrize("run_opts", [[], ["--no-graphics"]])
+def test_run(tart, run_opts):
    vm_name = f"integration-test-run-{uuid.uuid4()}"

    # Instantiate a VM with admin:admin SSH access
    tart.run(["clone", "ghcr.io/cirruslabs/macos-sonoma-base:latest", vm_name])

    # Run the VM asynchronously
-    tart_run_process = tart.run_async(["run", vm_name])
+    tart_run_process = tart.run_async(["run", vm_name] + run_opts)

    # Obtain the VM's IP
    stdout, _ = tart.run(["ip", vm_name, "--wait", "120"])
@@ -2,12 +2,11 @@ repo_url: https://github.com/cirruslabs/tart/
 site_url: https://tart.run/
 edit_uri: blob/main/docs/

-site_name: Tart
+site_name: Tart Virtualization
 site_author: Cirrus Labs
 copyright: © Cirrus Labs 2017-present
 site_description: >
  Tart is a virtualization toolset to build, run and manage macOS and Linux virtual machines (VMs) on Apple Silicon.
-  Built by CI engineers for your automation needs. 

 remote_branch: main

@@ -47,7 +46,10 @@ plugins:
      match_path: blog/posts/.*
      date_from_meta:
        as_creation: date
-  - social
+  - social:
+      cards_layout_dir: docs/layouts
+      cards_layout: custom
+      debug: true
  - search
  - minify

@@ -89,13 +91,19 @@ nav:
  - "Home": index.md
  - "Quick Start": quick-start.md
  - "Integrations":
-    - "GitHub Actions": integrations/github-actions.md
-    - "GitLab Runner": integrations/gitlab-runner.md
-    - "Buildkite": integrations/buildkite.md
-    - "Self-hosted CI": integrations/cirrus-cli.md
-    - "Managing VMs": integrations/vm-management.md
+      - "Self-hosted CI": integrations/cirrus-cli.md
+      - "GitHub Actions": https://cirrus-runners.app/
+      - "GitLab Runner": integrations/gitlab-runner.md
+      - "Buildkite": integrations/buildkite.md
+      - "Managing VMs": integrations/vm-management.md
  - "Support & Licensing": licensing.md
-  - "Orchestration": https://github.com/cirruslabs/orchard
+  - "Orchestration":
+      - "Quick Start": orchard/quick-start.md
+      - "Architecture and Security": orchard/architecture-and-security.md
+      - "Deploying Controller": orchard/deploying-controller.md
+      - "Deploying Workers": orchard/deploying-workers.md
+      - "Managing the Cluster": orchard/managing-cluster.md
+      - "Integrating with the API": orchard/integration-guide.md
  - "FAQ": faq.md
  - "Legal":
      - 'Terms of Service': legal/terms.md
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker run --pull=always --rm -it -p 8000:8000 -v ${PWD}:/docs ghcr.io/cirruslabs/mkdocs-material-insiders:latest build
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker run --pull=always --rm -it -p 8000:8000 -v ${PWD}:/docs ghcr.io/cirruslabs/mkdocs-material-insiders:latest
Author	SHA1	Message	Date
Nikolay Edigaryev	b98e23956b	Package.swift: bump Sentry SDK to 8.36.0 + upgrade other packages (#905 ) * Package.swift: bump Sentry SDK to 8.36.0 * $ swift package update	2024-09-19 19:06:37 +00:00
Fedor Korotkov	ce23f9c2a7	Completely disable audio devices in case of `--no-audio` (#904 ) This way VM won't have empty audio device at all. This should fix with an issue like that https://github.com/actions/runner-images/issues/9330	2024-09-17 09:40:37 +00:00
Nikolay Edigaryev	3da91e6518	tart run: provide a hint with names of other running VMs (#900 ) When VM limit gets exceeded.	2024-09-09 20:45:59 +04:00
Nikolay Edigaryev	7046886713	docs(orchard): document Kubernetes and systemd service deployment (#899 )	2024-09-09 16:40:17 +04:00
Nikolay EdigaryevandFedor Korotkov	3fde7d08dd	Orchard documentation (#897 ) * Orchard documentation * Fix typo Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com> * architecture-and-security.md: change list order --------- Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com>	2024-08-28 00:09:57 +04:00
Fedor Korotkov	227301436c	Revert "Drop Monterey Support (#843 )" (#893 ) This reverts commit `017592075f`.	2024-08-14 14:57:55 -04:00
Nikolay Edigaryev	106eb5a2c8	tart push: re-try when encountering errors when pushing disk layers (#888 ) * tart push: re-try when encountering errors when pushing disk layers * Only re-try on URLError	2024-08-10 13:06:39 -04:00
Nikolay Edigaryev	10bf706653	tart push: avoid uploading blobs if they are already present (#887 ) By issuing HEAD requests to the registry before doing the actual upload.	2024-08-09 17:26:20 +04:00
Fedor Korotkov	ff928ad77d	Optimize DiskV2 Deduplication (#878 ) * Revert "Lowercase `tart.app` (#751)" This reverts commit `a9e2a19015`. * Optimize DiskV2 deduplication logic In case we cloned `disk.img` from a local image, check if data at offset has the expected contents already. * Hole punch only if needed * Calculate hash only if needed * subdataChunks optimization * Reapply "Lowercase `tart.app` (#751)" This reverts commit `e74e9c845a`. * format * Save at least 1GB on deduplication logic * Build separately * Revert "subdataChunks optimization" This reverts commit `e59382aeba`. * Another optimization * Removed debug log * reformat * Revert "Hole punch only if needed" This reverts commit `8c569fc5`	2024-08-05 12:24:31 -04:00
Nikolay Edigaryev	33b5cfe2ed	tart run: delay tilde (~) expansion until we're dealing with local path (#880 )	2024-08-05 15:42:10 +04:00
Nikolay Edigaryev	3892cdb00d	tart run: replace --sync with --root-disk-opts (#879 ) * VZDiskImageSynchronizationMode's "description" field is a dead code * Re-use the VZDiskImageSynchronizationMode extension * tart run: replace --sync with --root-disk-opts * VM: support root disk synchronization mode on macOS	2024-08-05 15:17:58 +04:00
Nicholas FitzRoy-Dale	5f2199ef3e	Support setting root disk synchronization mode (#875 ) * Support setting root disk synchronization mode Adds a new VMConfig parameter (tart get / tart set) called 'sync' which can be set to 'full' (default), 'fsync', or 'none', corresponding with the values of VZDiskImageSynchronizationMode and allowing a tradeoff between data integrity and speed. * Remove unused import * Fix formatting * Make root disk sync behaviour a commandline option	2024-08-05 13:12:43 +04:00
Fedor Korotkov	3f26baa341	Update testimonials to focus on Tart (#876 ) Cirrus Runners have their own testimonials and a website now. No need to mix things together.	2024-08-01 17:09:08 +04:00
Nikolay Edigaryev	06cae1296e	tart run: support disabling disk synchronization for --disk (#872 )	2024-07-25 20:15:07 +04:00
Nikolay EdigaryevandFedor Korotkov	1b81b12760	tart pull: try to re-use APFS blocks by cloning the base image (#864 ) * tart pull: try to re-use APFS blocks by cloning the base image * Punch a hole when a zero chunk is detected * Properly retrieve errno when hole punching operation fails * tart pull: do not retry on RuntimeError * Ensure that the holes we're about to punch are FS block size-aligned * VMDirectory: remove unused static variables * tart pull: log if we've found an image to deduplicate against * Do not prematurely read contents from disk * Only consider candidates with deduplicatedBytes more than 0 * APFS reuse UX/DX improvements (#870) * Show how much deduplication happening Improvement to the APFS deduplication logic which checks whether a disk image file `mayShareFileContent` with some other file, and then we put a custom attribute to track the deduplication since there is no way to get this information from APFS itself. It's not 100% accurate but given that OCI cache is immutable the actual disk usage can only be lover than that. * Use string attribute * Update Sources/tart/URL+Prunable.swift Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> * Added SizeOnDisk colume --------- Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> --------- Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com>	2024-07-25 15:33:15 +00:00
Nikolay Edigaryev	4ed73bc775	--no-audio: only disable the source and sink (#869 ) To prevent crashes in the guest when playing or recording audio.	2024-07-18 15:10:44 +00:00
Nikolay Edigaryev	2dc25ce478	tart push: support --concurrency command-line argument (#868 ) * tart push: support --concurrency command-line argument * LayerizerTests: specify "concurrency" argument	2024-07-18 17:52:55 +04:00
Nikolay Edigaryev	1e74e268a5	DiskV2: change layer size to 512 * 1024 * 1024 bytes (#866 ) Needed to test https://github.com/cirruslabs/tart/pull/864.	2024-07-17 16:37:45 +00:00
Nikolay Edigaryev	bff344fb7f	tart login: better error when an improperly formatted host is provided (#863 ) * tart login: better error when an improperly formatted host is provided * Revert old behavior w.r.t. URLComponents()	2024-07-15 18:36:09 +04:00
Nikolay Edigaryev	ababe8cefc	tart pull: choose across multiple VM images to deduplicate against (#862 ) This is accomplished by saving the OCI VM image manifests on "tart pull" in "manifest.json" file and then using them on successive "tart pull"'s to find the best candidate that results in the most de-duplication, measured in bytes.	2024-07-15 18:36:01 +04:00
Fedor Korotkov	ea5313698e	Do not prune running VMs (#861 ) Also prevent pushing of a running VM Fixes #860	2024-07-15 07:10:51 -04:00
Fedor Korotkov	679289d7ab	Added Figma as a user (#858 ) See https://www.figma.com/open-source/ Plus reworked users section since now we can curate the best representative.	2024-07-10 15:06:01 +00:00
Nikolay Edigaryev	5eccdf7412	Support customizing VM disks and mounting remote VMs in `tart run` (#847 ) * Support remote VM names in --disk command-line argument * tart set: introduce "--disk" to support replacing VM's disk contents * Complete the code comment	2024-07-02 18:12:35 +04:00
Nikolay Edigaryev	63e3235d91	tart run: pick up --net-softnet-allow when using --net-host (#853 )	2024-07-02 16:39:51 +04:00
Nikolay Edigaryev	a760a431c3	Jumping through the hoops: SSH jump host functionality in Orchard (#844 )	2024-06-20 22:39:41 +00:00
Tor Arne Vestbø	bf5081b3d9	Hook SIGUSR2 to requestStop (#842 ) For macOS this brings up a dialog, asking the user if they are sure they want to shut down, which makes this less useful for automated graceful shutdowns, but it may behave better on Linux, and there might be ways to instruct macOS to not ask the user, so it's still a nice feature, and aligns with the SIGUSR1 for suspend, and SIGINT for non-graceful shutdown.	2024-06-17 12:11:00 -04:00
Fedor Korotkov	017592075f	Drop Monterey Support (#843 ) * Drop Monterey Support People will still be able to run and SSH into Monterey VMs or use VNC but pointing devices/keyboard won't work. Fixes #841 * Fixed x86 build	2024-06-17 15:20:21 +00:00
Fedor Korotkov	84e1ae2b38	Fixed GoReleaser 2.0.0 (#839 ) GoReleaser changes some flags	2024-06-05 19:09:56 +04:00
Fedor Korotkov	d50e113300	Rearrange companies To make the patter pretty	2024-06-05 08:32:38 -04:00
marc	fce52f1514	Add Atlassian as Tart user (#838 )	2024-06-05 08:31:23 -04:00
Fedor Korotkov	9484b8b2c9	Update Sentry Token (#836 ) The latest release has this error: > error: Project not found. Please check that you entered the project and organization slugs correctly. Which seems indicating that Auth is broken and we are getting 404? In the Sentry Settings I didn't find any token which I find strange. So I created one and re-encrypted.	2024-06-01 13:11:05 +00:00
Fedor Korotkov	dd46033812	Friendly decompression error message (#835 ) * Friendly decompression error message * Wrap FilterError	2024-05-31 14:47:41 +00:00
Fedor Korotkov	c655288de7	Fancy Social Cards (#830 )	2024-05-22 16:07:08 +04:00
Nikolay Edigaryev	204002f776	VMStorageOCI: percent-encode the colon in RemoteName's host (#828 ) * VMStorageOCI: percent-encode the colon in RemoteName's host * Do not use String extensions and add a comment	2024-05-21 11:21:28 -04:00
Nikolay Edigaryev	a0ae2f4e66	integration-tests: downgrade "requests" package to 2.31.0 (#829 ) To fix the build failing. See https://github.com/psf/requests/issues/6707 for more details.	2024-05-21 13:35:33 +00:00
Nikolay Edigaryev	7c386e3466	tart pull: try to re-use local VM image layers to speed-up the pulling (#825 ) * Remove unused pullFromRegistry() method with "reference" argument * tart pull: try to deduplicate disk layers to speed-up the pulling	2024-05-16 19:43:56 +04:00
Nikolay Edigaryev	dbbd716214	tart push: use fixed size chunks to allow for better deduplication (#821 )	2024-05-14 19:23:04 +04:00
William Theaker	13d5ddb4a4	Minor documentation improvements. (#819 ) * Minor documentation improvements. * Fix MD031 * Add sudo to mount instructions.	2024-05-13 12:44:26 -04:00
Fedor Korotkov	626316a4cd	Update manual installation instructions (#816 ) Fixes #815	2024-05-06 23:10:36 +04:00
Fedor Korotkov	fbe35302c2	Use warn images (#812 )	2024-05-05 12:07:00 +04:00
Fedor Korotkov	e1353f4540	[docs] fixed Cirrus Runners link (#813 )	2024-05-05 12:06:39 +04:00
Fedor Korotkov	985db24474	Introduce `--random-mac` and `--random-serial` flags for `tart set` (#809 ) To generate new MAC address and/or serial number for a given VM.	2024-05-02 18:27:49 +04:00
Nikolay Edigaryev	1d01bf63fb	tart run: resolve VM's IP using ARP when using --net-bridged and --vnc (#811 )	2024-05-02 18:04:26 +04:00
Andrew Malchuk	3ff3850da2	Add support pasting clipboard from host for Linux VMs (#806 ) * Added partial support pasting clipboard from host (only for Linux VMs) * Added option "--no-clipboard" to run command	2024-05-02 09:48:25 +04:00
Tor Arne Vestbø	c6e8d0bfd7	Gracefully stop vm on tart stop (#808 ) * Give Virtualization.framework a chance to stop the VM on tart stop We were letting the CancellationError bubble up all the way until it terminated app, which meant we didn't hit the shutdown code in run(), stopping the VM and the network. We now catch CancellationError and proceed to gracefully shut down. We only stop the VM if it's still running, as a VM that has been stopped via the menu can't be stopped again. * Gracefully shut down VM when Tart is quit via menu Normally the quit action will result in AppKit calling exit(), but we want to gracefully shut down the VM, so we use the same path as for closing of the VM window, namely signal our own process with SIGINT or SIGUSR1. If that doesn't work we let AppKit terminate as before. This fixes the "Warning: NSActivity <_NSActivityAssertion: 0x600001f785a0> was ended multiple times" warning seen on the console when quitting Tart via the menu. * Activate Tart after application finishes launching This ensures that the VM window has been shown by the time we activate, so that we consistently activate and bring the VM window to the front.	2024-04-30 09:27:41 -04:00
Fedor Korotkov	755aad4d7c	Check all VMs for MAC collision (#801 ) * Check all VMs for MAC collision Before only suspendable VMs were getting checked. Not sure why. It makes sense to check all. * Always acquire a lock	2024-04-25 09:26:42 -04:00
Nikolay Edigaryev	9f38441a42	Fix pathHasMode() and only check for S_IFBLK (#800 )	2024-04-23 11:12:35 -04:00
Fedor Korotkov	3d46c4e6c2	Support all NBD schemas (#799 ) See https://github.com/NetworkBlockDevice/nbd/blob/master/doc/uri.md#nbd-uri-scheme Fixes #792	2024-04-23 18:17:42 +04:00
Nikolay Edigaryev	e59221f6a0	tart run: do not require root to mount a block device (#798 )	2024-04-23 17:03:02 +04:00
Fedor Korotkov	c6e99345cd	Validate Suspendability (#797 ) And show "Suspend" menu item based on `--suspnedable` flag Fixes #796	2024-04-22 10:07:01 +00:00
Nikolay EdigaryevandFedor Korotkov	8bc2e99f63	Document how to mount the shared directory on Linux at boot time (#793 ) * Document how to mount the shared directory on Linux at boot time * Use admonition Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com> --------- Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com>	2024-04-17 23:34:01 -04:00
Fedor Korotkov	f36f86b61c	[docs] lint to Cirrus Runners site (#789 )	2024-04-12 21:33:15 +04:00
Nikolay Edigaryev	79084555f6	tart pull: retry VM pull with exponential backoff (#788 )	2024-04-12 21:32:03 +04:00
Fedor Korotkov	896d03ce0b	Fixed Swift Warning (#787 ) * Fixed Swift Warning Plus updated all the dependencies and Swift Tools. Fixes #785 * Fixed race condition	2024-04-11 19:53:10 +04:00
Fedor KorotkovandNikolay Edigaryev	99c91cbf87	Allow mounting NBD disks (#786 ) * Allow mounting NBD disks Fixes #759 * Apply suggestions from code review Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> * Removed unnecessary docs --------- Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com>	2024-04-11 17:20:58 +04:00
Bartek Pacia	da8afa1348	Add shell completions (#780 ) * add VM completion for run command * add VM completion for stop command * create ShellCompletions utilities * add shell completions to some commands * add shell completion for fqn command * run command: fix tiny typo * add shell completion for get command * more shell completions * remove unnecessary `try` * refactor ShellCompletions file	2024-04-11 06:22:50 -04:00
Nikolay Edigaryev	97b7ffef52	tart stop: throw RuntimeError.VMNotRunning consistently and use enumeration instead of strings (#784 ) * Use enumeration instead of just strings for VMDirectory state * tart stop: throw RuntimeError.VMNotRunning consistently	2024-04-10 14:53:04 +00:00
Tor Arne Vestbø	13e7794bfc	Generate shell completions by calling tart.app executable (#775 )	2024-04-03 01:22:52 +02:00
Nikolay Edigaryev	b7b3b702ac	Sentry: upgrade and attach command-line arguments (#774 ) * Sentry: upgrade and attach command-line arguments * Sentry's setContext(): explicitly pass a String	2024-04-02 18:31:16 +04:00
Tor Arne Vestbø	560dba79e4	Report operating system in tart get (#772 ) Can be useful to know from outside the VM.	2024-03-31 14:18:26 -04:00
Tor Arne Vestbø	2b33b8f9e6	Report progress when downloading IPSW files (#768 ) The URLSession async/await functions do not report progress through the normal URLSessionTaskDelegate callbacks, as reported in: https://developer.apple.com/forums/thread/723015 We don't want to use URLSession.bytes, as that results in a much slower download speed compared to URLSession.download, but we can work around the lack of progress callbacks by observing the progress on the URLSessionTask itself. Fixes #767	2024-03-28 19:15:06 +04:00
Tor Arne Vestbø	d8b010c79c	Support cancellation of installation process (#770 ) We wrap the installation with a withTaskCancellationHandler, which ensures that the SIGINT signal handling code in main() will trigger a cancellation of the installer. As the VZMacOSInstaller must be both created and interacted with on the VM's queue, which in our case is the main queue, we need to move the logic to a separate function tagged with @MainActor. This makes sense either way, as it cleans up the code a bit.	2024-03-28 00:14:33 +04:00
Nikolay Edigaryev	5cd83c38cd	Introduce Golang-based benchmarking utility (#769 ) * Introduce Golang-based benchmarking utility * benchmark fio: properly configure logger level * benchmark: properly terminate on Ctrl+C when initializing/running Tart * benchmark(fio): increase runtime to 30 seconds * benchmark(fio): IOPS are already per second * benchmark(fio): --numjobs 1 --iodepth 1 --end_fsync 1 * benchmark(README.md): add results	2024-03-27 18:45:01 +04:00
Bartek Pacia	1a3b862631	goreleaser: set up automatic installation of shell completion files (#766 )	2024-03-26 12:08:09 +04:00
Fedor Korotkov	ae2d59e5c2	Revert "Do not magically set --no-graphics when --vnc is passed (#763 ) * Revert "Do not magically set `--no-graphics` when `--vnc` is passed (#732)" This reverts commit `a48f4d4ec9`. * Mark `--graphics` as private	2024-03-19 08:28:27 +00:00
Evgeniy Baranov	7eac45702b	Fix the --insecure flag issue by disabling ATS in Info.plist (#760 )	2024-03-19 03:42:14 -04:00
Nikolay Edigaryev	e06d89f95d	integration-tests: test_run() with --no-graphics (#757 )	2024-03-12 15:12:02 +04:00
Fedor Korotkov	0602d6e0e1	Pack additional resources into brew releases (#756 ) To fix missing icon since #746	2024-03-12 10:46:56 +00:00
Fedor Korotkov	ac5d0baa0c	Fixed `--no-graphics` mode (#755 ) Regression introduced in #746	2024-03-12 10:42:35 +00:00
Nikolay Edigaryev	ee27cc57bb	tart run: introduce --net-softnet-allow command-line argument (#753 )	2024-03-11 22:17:34 +04:00